From 6910e70b663078b3d28bc7b6eb30cc176d30beb7 Mon Sep 17 00:00:00 2001 From: Bradlee Speice Date: Sat, 22 Oct 2016 22:29:18 -0400 Subject: [PATCH] Add a new Rust article! --- a-rustic-re-podcasting-server-part-1.html | 408 ++++++ archives.html | 2 + author/bradlee-speice.html | 2 + author/bradlee-speice2.html | 2 + authors.html | 2 + autocallable-bonds.html | 22 +- categories.html | 2 + category/blog.html | 2 + category/blog2.html | 2 + cloudy-in-seattle.html | 52 +- complaining-about-the-weather.html | 106 +- event-studies-and-earnings-releases.html | 340 ++--- feeds/all.atom.xml | 1283 ++++++++++------- feeds/blog.atom.xml | 1283 ++++++++++------- index.html | 4 +- index2.html | 2 + predicting-santander-customer-happiness.html | 36 +- ...tability-using-the-investment-formula.html | 58 +- tag/nutone.html | 123 ++ tag/rust.html | 123 ++ tags.html | 4 + testing-cramer.html | 134 +- the-unfair-casino.html | 78 +- tick-tock.html | 82 +- tweet-like-me.html | 36 +- welcome-and-an-algorithm.html | 60 +- 26 files changed, 2728 insertions(+), 1520 deletions(-) create mode 100644 a-rustic-re-podcasting-server-part-1.html create mode 100644 tag/nutone.html create mode 100644 tag/rust.html diff --git a/a-rustic-re-podcasting-server-part-1.html b/a-rustic-re-podcasting-server-part-1.html new file mode 100644 index 0000000..8aca65b --- /dev/null +++ b/a-rustic-re-podcasting-server-part-1.html @@ -0,0 +1,408 @@ + + + + + + + + + + + A Rustic Re-Podcasting Server (Part 1) - Bradlee Speice + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+ + +
+
+ + + + +
+
+
+
+

A Rustic Re-Podcasting Server (Part 1)

+

Bradlee Speice, Sat 22 October 2016, Blog

+
+
+

+ +nutone, Rust

+
+
+
+
+ + + +
+ + + + +
+

I listen to a lot of Drum and Bass music, because it's beautiful music. And +there's a particular site, Bassdrive.com that hosts +a lot of great content. Specifically, the +archives section of the site has a +list of the past shows that you can download and listen to. The issue is, it's +just a giant list of links to download. I'd really like +this in a podcast format to take with me on the road, etc.

+

So I wrote the elektricity web +application to actually accomplish all that. Whenever you request a feed, it +goes out to Bassdrive, processes all the links on a page, and serves up some +fresh, tasty RSS to satisfy your ears. I hosted it on Heroku using the free +tier because it's really not resource-intensive at all.

+

The issue so far is that I keep running out of free tier hours during a +month because my podcasting application likes to have a server scan for new +episodes constantly. Not sure why it's doing that, but I don't have a whole +lot of control over it. It's a phenomenal application otherwise.

+

My (over-engineered) solution: Re-write the application using the +Rust programming language. I'd like to run +this on a small hacker board I own, and doing this in Rust would allow me to +easily cross-compile it. Plus, I've been very interested in the Rust language +for a while and this would be a great opportunity to really learn it well. +The code is available here as development +progresses.

+

The Setup

+

We'll be using the iron library to handle the +server, and hyper to fetch the data we need from elsewhere +on the interwebs. HTML5Ever allows +us to ingest the content that will be coming from Bassdrive, and finally, +output is done with handlebars-rust.

+

It will ultimately be interesting to see how much more work must be done to +actually get this working over another language like Python. Coming from a +dynamic state of mind it's super easy to just chain stuff together, ship it out, +and call it a day. I think I'm going to end up getting much dirtier trying to +write all of this out.

+

Issue 1: Strings

+

Strings in Rust are hard. I acknowledge Python can get away with some things +that make strings super easy (and Python 3 has gotten better at cracking down +on some bad cases, str <-> bytes specifically), but Rust is hard.

+

Let's take for example the 404 error handler I'm trying to write. The result +should be incredibly simple: All I want is to echo back +Didn't find URL: <url>. Shouldn't be that hard right? In Python I'd just do +something like:

+
def echo_handler(request):
+    return "You're visiting: {}".format(request.uri)
+
+ + +

And we'd call it a day. Rust isn't so simple. Let's start with the trivial +examples people post online:

+
fn hello_world(req: &mut Request) -> IronResult<Response> {
+    Ok(Response::with((status::Ok, "You found the server!")))
+}
+
+ + +

Doesn't look too bad right? In fact, it's essentially the same as the Python +version! All we need to do is just send back a string of some form. So, we +look up the documentation for Request and see a url field that will contain +what we want. Let's try the first iteration:

+
fn hello_world(req: &mut Request) -> IronResult<Response> {
+    Ok(Response::with((status::Ok, "You found the URL: " + req.url)))
+}
+
+ + +

Which yields the error:

+
error[E0369]: binary operation `+` cannot be applied to type `&'static str`
+
+ + +

OK, what's going on here? Time to start Googling for "concatenate strings in Rust". That's what we +want to do right? Concatenate a static string and the URL.

+

After Googling, we come across a helpful concat! macro that looks really nice! Let's try that one:

+
fn hello_world(req: &mut Request) -> IronResult<Response> {
+    Ok(Response::with((status::Ok, concat!("You found the URL: ", req.url))))
+}
+
+ + +

And the error:

+

error: expected a literal

+

Turns out Rust actually blows up because the concat! macro expects us to know +at compile time what req.url is. Which, in my outsider opinion, is a bit +strange. println! and format!, etc., all handle values they don't know at +compile time. Why can't concat!? By any means, we need a new plan of attack. +How about we try formatting strings?

+
fn hello_world(req: &mut Request) -> IronResult<Response> {
+    Ok(Response::with((status::Ok, format!("You found the URL: {}", req.url))))
+}
+
+ + +

And at long last, it works. Onwards!

+

Issue 2: Fighting with the borrow checker

+

Rust's single coolest feature is how the compiler can guarantee safety in your +program. As long as you don't use unsafe pointers in Rust, you're guaranteed +safety. And not having truly manual memory management is really cool; I'm +totally OK with never having to write malloc() again.

+

That said, even the Rust documentation makes a specific note:

+
+

Many new users to Rust experience something we like to call +‘fighting with the borrow checker’, where the Rust compiler refuses to +compile a program that the author thinks is valid.

+
+

If you have to put it in the documentation, it's not a helpful note: +it's hazing.

+

So now that we have a handler which works with information from the request, we +want to start making something that looks like an actual web application. +The router provided by iron isn't terribly difficult so I won't cover it. +Instead, the thing that had me stumped for a couple hours was trying to +dynamically create routes.

+

The unfortunate thing with Rust (in my limited experience at the moment) is that +there is a severe lack of non-trivial examples. Using the router is easy when +you want to give an example of a static function. But how do you you start +working on things that are a bit more complex?

+

We're going to cover that here. Our first try: creating a function which returns +other functions. This is a principle called currying. We set up a function that allows us to keep some data in scope +for another function to come later.

+
fn build_handler(message: String) -> Fn(&mut Request) -> IronResult<Response> {
+    move |_: &mut Request| {
+        Ok(Response::with((status::Ok, message)))
+    }
+}
+
+ + +

We've simply set up a function that returns another anonymous function with the +message parameter scoped in. If you compile this, you get not 1, not 2, but 5 +new errors. 4 of them are the same though:

+
error[E0277]: the trait bound `for<'r, 'r, 'r> std::ops::Fn(&'r mut iron::Request<'r, 'r>) -> std::result::Result<iron::Response, iron::IronError> + 'static: std::marker::Sized` is not satisfied
+
+ + +

...oookay. I for one, am not going to spend time trying to figure out what's +going on there.

+

And it is here that I will save the audience many hours of frustrated effort. +At this point, I decided to switch from iron to pure hyper since using +hyper would give me a much simpler API. All I would have to do is build a +function that took two parameters as input, and we're done. That said, it +ultimately posed many more issues because I started getting into a weird fight +with the 'static lifetime +and being a Rust newbie I just gave up on trying to understand it.

+

Instead, we will abandon (mostly) the curried function attempt, and instead +take advantage of something Rust actually intends us to use: struct and +trait.

+

Remember when I talked about a lack of non-trivial examples on the Internet? +This is what I was talking about. I could only find one example of this +available online, and it was incredibly complex and contained code we honestly +don't need or care about. There was no documentation of how to build routes that +didn't use static functions, etc. But, I'm assuming you don't really care about +my whining, so let's get to it.

+

The iron documentation mentions the Handler trait as being something we can implement. +Does the function signature for that handle() method look familiar? It's what +we've been working with so far.

+

The principle is that we need to define a new struct to hold our data, then +implement that handle() method to return the result. Something that looks +like this might do:

+
struct EchoHandler {
+    message: String
+}
+
+impl Handler for EchoHandler {
+    fn handle(&self, _: &mut Request) -> IronResult<Response> {
+        Ok(Response::with((status::Ok, self.message)))
+    }
+}
+
+// Later in the code when we set up the router...
+let echo = EchoHandler {
+    message: "Is it working yet?"
+}
+router.get("/", echo.handle, "index");
+
+ + +

We attempt to build a struct, and give its handle method off to the router +so the router knows what to do.

+

You guessed it, more errors:

+
error: attempted to take value of method `handle` on type `EchoHandler`
+
+ + +

Now, the Rust compiler is actually a really nice fellow, and offers us help:

+
help: maybe a `()` to call it is missing? If not, try an anonymous function
+
+ + +

We definitely don't want to call that function, so maybe try an anonymous +function as it recommends?

+
router.get("/", |req: &mut Request| echo.handle(req), "index");
+
+ + +

Another error:

+
error[E0373]: closure may outlive the current function, but it borrows `echo`, which is owned by the current function
+
+ + +

Another helpful message:

+
help: to force the closure to take ownership of `echo` (and any other referenced variables), use the `move` keyword
+
+ + +

We're getting closer though! Let's implement this change:

+
router.get("/", move |req: &mut Request| echo.handle(req), "index");
+
+ + +

And here's where things get strange:

+
error[E0507]: cannot move out of borrowed content
+  --> src/main.rs:18:40
+   |
+18 |         Ok(Response::with((status::Ok, self.message)))
+   |                                        ^^^^ cannot move out of borrowed content
+
+ + +

Now, this took me another couple hours to figure out. I'm going to explain it, +but keep this in mind: Rust only allows one reference at a time (exceptions +apply of course).

+

When we attempt to use self.message as it has been created in the earlier +struct, we essentially are trying to give it away to another piece of code. +Rust's semantics then state that we may no longer access it unless it is +returned to us (which iron's code does not do). There are two ways to fix +this:

+
    +
  1. Only give away references (i.e. &self.message instead of self.message) +instead of transferring ownership
  2. +
  3. Make a copy of the underlying value which will be safe to give away
  4. +
+

I didn't know these were the two options originally, so I hope this helps the +audience out. Because iron won't accept a reference, we are forced into the +second option: making a copy. To do so, we just need to change the function +to look like this:

+
Ok(Response::with((status::Ok, self.message.clone())))
+
+ + +

Not so bad, huh? My only complaint is that it took so long to figure out exactly +what was going on.

+

And now we have a small server that we can configure dynamically. At long last.

+
+

Final sidenote: You can actually do this without anonymous functions. Just +change the router line to: + router.get("/", echo, "index");

+

Rust's type system seems to figure out that we want to use the handle() method.

+
+

Conclusion

+

After a good long days' work, we now have the routing functionality set up on +our application. We should be able to scale this pretty well in the future: +the RSS content we need to deliver in the future can be treated as a string, so +the building blocks are in place.

+

There are two important things I learned starting with Rust today:

+
    +
  1. Rust is a new language, and while the code is high-quality, the mindshare is coming.
  2. +
  3. I'm a terrible programmer.
  4. +
+

Number 1 is pretty obvious and not surprising to anyone. Number two caught me +off guard. I've gotten used to having either a garbage collector (Java, Python, +etc.) or playing a little fast and loose with scoping rules (C, C++). You don't +have to worry about object lifetime there. With Rust, it's forcing me to fully +understand and use well the memory in my applications. In the final mistake I +fixed (using .clone()) I would have been fine in C++ to just give away that +reference and never use it again. I wouldn't have run into a "use-after-free" +error, but I would have potentially been leaking memory. Rust forced me to be +incredibly precise about how I use it.

+

All said I'm excited for using Rust more. I think it's super cool, it's just +going to take me a lot longer to do this than I originally thought.

+ + +
+
+ + +
+ +
+ + + + + + + \ No newline at end of file diff --git a/archives.html b/archives.html index cd0d38f..35c80d0 100644 --- a/archives.html +++ b/archives.html @@ -82,6 +82,8 @@

+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
Wed 08 June 2016
Event Studies and Earnings Releases
Sun 15 May 2016
diff --git a/author/bradlee-speice.html b/author/bradlee-speice.html index 93b41e9..d323b2c 100644 --- a/author/bradlee-speice.html +++ b/author/bradlee-speice.html @@ -82,6 +82,8 @@

+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
Wed 08 June 2016
Event Studies and Earnings Releases
Sun 15 May 2016
diff --git a/author/bradlee-speice2.html b/author/bradlee-speice2.html index 93b41e9..d323b2c 100644 --- a/author/bradlee-speice2.html +++ b/author/bradlee-speice2.html @@ -82,6 +82,8 @@

+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
Wed 08 June 2016
Event Studies and Earnings Releases
Sun 15 May 2016
diff --git a/authors.html b/authors.html index 96e8e13..e47ea18 100644 --- a/authors.html +++ b/authors.html @@ -82,6 +82,8 @@

Bradlee Speice

+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
Wed 08 June 2016
Event Studies and Earnings Releases
Sun 15 May 2016
diff --git a/autocallable-bonds.html b/autocallable-bonds.html index b359b30..152d4b9 100644 --- a/autocallable-bonds.html +++ b/autocallable-bonds.html @@ -134,7 +134,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at
S0 = 102.2
 nominal = 100
 q = 2.84 / 100
-σ = 15.37 / 100
+σ = 15.37 / 100
 term = [0, .49, .9, 1.21, 1.45, 1.69] / 100 + 1
 
 ###
@@ -187,7 +187,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at
 
In [3]:
-
simulate_gbm = function(S0, μ, σ, T, n)
+
simulate_gbm = function(S0, μ, σ, T, n)
     # Set the initial state
     m = length(S0)
     t = T / n
@@ -196,7 +196,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at
     
     # Build out all states
     for i=1:(n-1)
-        motion[:,i+1] = motion[:,i] .* exp((μ - σ^2/2)*t) .* exp(sqrt(t) * σ .* randn(m))
+        motion[:,i+1] = motion[:,i] .* exp((μ - σ^2/2)*t) .* exp(sqrt(t) * σ .* randn(m))
     end
     
     return motion
@@ -260,7 +260,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at
     
initial = ones(5) * S0
 # Using μ=0, T=.25 for now, we'll use the proper values later
-motion = simulate_gbm(initial, 0, σ, .25, 200) 
+motion = simulate_gbm(initial, 0, σ, .25, 200) 
 
 display_motion(motion, .25)
 
@@ -1943,8 +1943,8 @@ fig.select("#fig-3a6dd25ad25c4037a166889ee51bb151-element-20")
full_motion = ones(5) * S0
 full_term = vcat(term[1], forward_yield)
 for i=1:T
-    μ = (full_term[i] - 1 - q)
-    year_motion = simulate_gbm(full_motion[:,end], μ, σ, 1, n)
+    μ = (full_term[i] - 1 - q)
+    year_motion = simulate_gbm(full_motion[:,end], μ, σ, 1, n)
     full_motion = hcat(full_motion, year_motion)
 end
 
@@ -3507,8 +3507,8 @@ fig.select("#fig-0378e04b897742b597befd2e8e1c169e-element-20")
     # And an S0 to kick things off.
     final_motion = ones(m) * S0
     for i=1:T
-        μ = (forward[i] - 1 - q)
-        year_motion = simulate_gbm(final_motion[:,end], μ, σ, 1, n)
+        μ = (forward[i] - 1 - q)
+        year_motion = simulate_gbm(final_motion[:,end], μ, σ, 1, n)
         final_motion = hcat(final_motion, year_motion)
     end
     return final_motion
@@ -3576,7 +3576,7 @@ fig.select("#fig-0378e04b897742b597befd2e8e1c169e-element-20")
 
     for i=1:t
         price = year_prices[i]
-        if price  call_barrier
+        if price  call_barrier
             return (nominal + coupon*i) * exp((prod(forward_structure[i:end])-1)*(t-i))
         end
     end
@@ -3686,9 +3686,9 @@ Present value of Athena note: $95.00, notional: $100.00
for i=1:t price = year_prices[i] - if price call_barrier + if price call_barrier return (nominal + coupon + total_coupons)*exp((prod(forward_structure[i:end])-1)*(t-i)) - elseif price coupon_barrier + elseif price coupon_barrier total_coupons = total_coupons * exp(forward_structure[i]-1) + coupon else total_coupons *= exp(forward_structure[i]-1) diff --git a/categories.html b/categories.html index 77aebb2..808adaa 100644 --- a/categories.html +++ b/categories.html @@ -82,6 +82,8 @@

Blog

+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
Wed 08 June 2016
Event Studies and Earnings Releases
Sun 15 May 2016
diff --git a/category/blog.html b/category/blog.html index 782a8a4..a9e60a5 100644 --- a/category/blog.html +++ b/category/blog.html @@ -83,6 +83,8 @@

Blog

+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
Wed 08 June 2016
Event Studies and Earnings Releases
Sun 15 May 2016
diff --git a/category/blog2.html b/category/blog2.html index 782a8a4..a9e60a5 100644 --- a/category/blog2.html +++ b/category/blog2.html @@ -83,6 +83,8 @@

Blog

+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
Wed 08 June 2016
Event Studies and Earnings Releases
Sun 15 May 2016
diff --git a/cloudy-in-seattle.html b/cloudy-in-seattle.html index a9ae5c5..6f61324 100644 --- a/cloudy-in-seattle.html +++ b/cloudy-in-seattle.html @@ -4,7 +4,7 @@ - + @@ -359,7 +359,7 @@
In [2]:
-
city_forecasts = pickle.load(open('city_forecasts.p', 'rb'))
+
city_forecasts = pickle.load(open('city_forecasts.p', 'rb'))
 forecasts_df = pd.DataFrame.from_dict(city_forecasts)
 
@@ -373,24 +373,24 @@
In [3]:
-
cities = ['binghamton', 'cary', 'nyc', 'seattle']
+
cities = ['binghamton', 'cary', 'nyc', 'seattle']
 city_colors = {cities[i]: Palette[i] for i in range(0, 4)}
 
 def safe_cover(frame):
-    if frame and 'cloudCover' in frame:
-        return frame['cloudCover']
+    if frame and 'cloudCover' in frame:
+        return frame['cloudCover']
     else:
         return np.NaN
 
 def monthly_avg_cloudcover(city, year, month):
     dates = pd.DatetimeIndex(start=datetime(year, month, 1, 12),
                              end=datetime(year, month + 1, 1, 12),
-                             freq='D', closed='left')
-    cloud_cover_vals = list(map(lambda x: safe_cover(forecasts_df[city][x]['currently']), dates))
+                             freq='D', closed='left')
+    cloud_cover_vals = list(map(lambda x: safe_cover(forecasts_df[city][x]['currently']), dates))
     cloud_cover_samples = len(list(filter(lambda x: x is not np.NaN, cloud_cover_vals)))
-    # Ignore an issue with nanmean having all NaN values. We'll discuss the data issues below.
+    # Ignore an issue with nanmean having all NaN values. We'll discuss the data issues below.
     with warnings.catch_warnings():
-        warnings.simplefilter('ignore')
+        warnings.simplefilter('ignore')
         return np.nanmean(cloud_cover_vals), cloud_cover_samples
 
@@ -409,18 +409,18 @@ return [monthly_avg_cloudcover(city, y, month) for y in years] months = [ - ('July', 7), - ('August', 8), - ('September', 9), - ('October', 10), - ('November', 11) + ('July', 7), + ('August', 8), + ('September', 9), + ('October', 10), + ('November', 11) ] for month, month_id in months: month_averages = {city: city_avg_cc(city, month_id) for city in cities} - f = figure(title="{} Average Cloud Cover".format(month), - x_axis_label='Year', - y_axis_label='Cloud Cover Percentage') + f = figure(title="{} Average Cloud Cover".format(month), + x_axis_label='Year', + y_axis_label='Cloud Cover Percentage') for city in cities: f.line(years, [x[0] for x in month_averages[city]], legend=city, color=city_colors[city]) @@ -610,20 +610,20 @@
def safe_precip(frame):
-    if frame and 'precipProbability' in frame:
-        return frame['precipProbability']
+    if frame and 'precipProbability' in frame:
+        return frame['precipProbability']
     else:
         return np.NaN
 
 def monthly_avg_precip(city, year, month):
     dates = pd.DatetimeIndex(start=datetime(year, month, 1, 12),
                              end=datetime(year, month + 1, 1, 12),
-                             freq='D', closed='left')
-    precip_vals = list(map(lambda x: safe_precip(forecasts_df[city][x]['currently']), dates))
+                             freq='D', closed='left')
+    precip_vals = list(map(lambda x: safe_precip(forecasts_df[city][x]['currently']), dates))
     precip_samples = len(list(filter(lambda x: x is not np.NaN, precip_vals)))
-    # Ignore an issue with nanmean having all NaN values. We'll discuss the data issues below.
+    # Ignore an issue with nanmean having all NaN values. We'll discuss the data issues below.
     with warnings.catch_warnings():
-        warnings.simplefilter('ignore')
+        warnings.simplefilter('ignore')
         return np.nanmean(precip_vals), precip_samples
 
 def city_avg_precip(city, month):
@@ -631,9 +631,9 @@
 
 for month, month_id in months:
     month_averages = {city: city_avg_cc(city, month_id) for city in cities}
-    f = figure(title="{} Average Precipitation Chance".format(month),
-               x_axis_label='Year',
-               y_axis_label='Precipitation Chance Percentage')
+    f = figure(title="{} Average Precipitation Chance".format(month),
+               x_axis_label='Year',
+               y_axis_label='Precipitation Chance Percentage')
     for city in cities:
         f.line(years, [x[0] for x in month_averages[city]],
                legend=city, color=city_colors[city])
diff --git a/complaining-about-the-weather.html b/complaining-about-the-weather.html
index 08d7adc..5b2c289 100644
--- a/complaining-about-the-weather.html
+++ b/complaining-about-the-weather.html
@@ -352,7 +352,7 @@
 
In [2]:
-
city_forecasts = pickle.load(open('city_forecasts.p', 'rb'))
+
city_forecasts = pickle.load(open('city_forecasts.p', 'rb'))
 forecast_df = pd.DataFrame.from_dict(city_forecasts)
 
@@ -366,31 +366,31 @@
In [3]:
-
cary_forecast = forecast_df['cary']
+
cary_forecast = forecast_df['cary']
 years = range(1990, 2016)
 months = range(7, 12)
-months_str = ['July', 'August', 'September', 'October', 'November']
+months_str = ['July', 'August', 'September', 'October', 'November']
 
 def safe_cover(frame):
-    if frame and 'cloudCover' in frame:
-        return frame['cloudCover']
+    if frame and 'cloudCover' in frame:
+        return frame['cloudCover']
     else:
         return np.NaN
 
 def monthly_avg_cloudcover(year, month):
     dates = pd.DatetimeIndex(start=datetime(year, month, 1, 12),
                              end=datetime(year, month + 1, 1, 12),
-                             freq='D', closed='left')
-    cloud_cover_vals = list(map(lambda x: safe_cover(cary_forecast[x]['currently']), dates))
+                             freq='D', closed='left')
+    cloud_cover_vals = list(map(lambda x: safe_cover(cary_forecast[x]['currently']), dates))
     cloud_cover_samples = len(list(filter(lambda x: x is not np.NaN, cloud_cover_vals)))
     return np.nanmean(cloud_cover_vals), cloud_cover_samples
 
 
 monthly_cover_vals = [[monthly_avg_cloudcover(y, m)[0] for y in years] for m in months]
 
-f = figure(title='Monthly Average Cloud Cover',
+f = figure(title='Monthly Average Cloud Cover',
            x_range=(1990, 2015),
-          x_axis_label='Year')
+          x_axis_label='Year')
 for x in range(0, len(months)):
     f.line(years, monthly_cover_vals[x], legend=months_str[x], color=Palette[x])
 show(f)
@@ -457,8 +457,8 @@
 
def monthly_cloudy_days(year, month):
     dates = pd.DatetimeIndex(start=datetime(year, month, 1, 12),
                              end=datetime(year, month + 1, 1, 12),
-                             freq='D', closed='left')
-    cloud_cover_vals = list(map(lambda x: safe_cover(cary_forecast[x]['currently']), dates))
+                             freq='D', closed='left')
+    cloud_cover_vals = list(map(lambda x: safe_cover(cary_forecast[x]['currently']), dates))
     cloud_cover_samples = len(list(filter(lambda x: x is not np.NaN, cloud_cover_vals)))
     cloudy_days = [cover > .5 for cover in cloud_cover_vals]
     return np.count_nonzero(cloudy_days), cloud_cover_samples
@@ -466,16 +466,16 @@
 monthly_days_vals = [[monthly_cloudy_days(y, m)[0] for y in years] for m in months]
 monthly_cover_samples = [[monthly_cloudy_days(y, m)[1] for y in years] for m in months]
 
-f = figure(title='Monthly Cloudy Days',
+f = figure(title='Monthly Cloudy Days',
            x_range=(1990, 2015),
-          x_axis_label='Year')
+          x_axis_label='Year')
 for x in range(0, len(months)):
     f.line(years, monthly_days_vals[x], legend=months_str[x], color=Palette[x])
 show(f)
 
-f = figure(title='Monthly Cloud Cover Samples',
+f = figure(title='Monthly Cloud Cover Samples',
           x_range=(1990, 2015),
-          x_axis_label='Year',
+          x_axis_label='Year',
           height=300)
 for x in range(0, len(months)):
     f.line(years, monthly_cover_samples[x], legend=months_str[x], color=Palette[x])
@@ -571,24 +571,24 @@
 
def safe_precip(frame):
-    if frame and 'precipProbability' in frame:
-        return frame['precipProbability']
+    if frame and 'precipProbability' in frame:
+        return frame['precipProbability']
     else:
         return np.NaN
 
 def monthly_avg_precip(year, month):
     dates = pd.DatetimeIndex(start=datetime(year, month, 1, 12),
                              end=datetime(year, month + 1, 1, 12),
-                             freq='D', closed='left')
-    precip_vals = list(map(lambda x: safe_precip(cary_forecast[x]['currently']), dates))
+                             freq='D', closed='left')
+    precip_vals = list(map(lambda x: safe_precip(cary_forecast[x]['currently']), dates))
     precip_samples = len(list(filter(lambda x: x is not np.NaN, precip_vals)))
     return np.nanmean(precip_vals), precip_samples
 
 monthly_avg_precip_vals = [[monthly_avg_precip(y, m)[0] for y in years] for m in months]
 
-f = figure(title='Monthly Average Precipitation Chance',
+f = figure(title='Monthly Average Precipitation Chance',
            x_range=(1990, 2015),
-          x_axis_label='Year')
+          x_axis_label='Year')
 for x in range(0, len(months)):
     f.line(years, monthly_avg_precip_vals[x], legend=months_str[x], color=Palette[x])
 show(f)
@@ -654,8 +654,8 @@
 
def monthly_rainy_days(year, month):
     dates = pd.DatetimeIndex(start=datetime(year, month, 1, 12),
                              end=datetime(year, month + 1, 1, 12),
-                             freq='D', closed='left')
-    precip_prob_vals = list(map(lambda x: safe_precip(cary_forecast[x]['currently']), dates))
+                             freq='D', closed='left')
+    precip_prob_vals = list(map(lambda x: safe_precip(cary_forecast[x]['currently']), dates))
     precip_prob_samples = len(list(filter(lambda x: x is not np.NaN, precip_prob_vals)))
     precip_days = [prob > .25 for prob in precip_prob_vals]
     return np.count_nonzero(precip_days), precip_prob_samples
@@ -663,16 +663,16 @@
 monthly_precip_days_vals = [[monthly_rainy_days(y, m)[0] for y in years] for m in months]
 monthly_precip_samples = [[monthly_rainy_days(y, m)[1] for y in years] for m in months]
 
-f = figure(title='Monthly Rainy Days',
+f = figure(title='Monthly Rainy Days',
            x_range=(1990, 2015),
-          x_axis_label='Year')
+          x_axis_label='Year')
 for x in range(0, len(months)):
     f.line(years, monthly_precip_days_vals[x], legend=months_str[x], color=Palette[x])
 show(f)
 
-f = figure(title='Monthly Rainy Days Samples',
+f = figure(title='Monthly Rainy Days Samples',
           x_range=(1990, 2015),
-          x_axis_label='Year',
+          x_axis_label='Year',
           height=300)
 for x in range(0, len(months)):
     f.line(years, monthly_precip_samples[x], legend=months_str[x], color=Palette[x])
@@ -775,64 +775,64 @@
 import requests
 from datetime import datetime
 
-# Coordinate data from http://itouchmap.com/latlong.html
+# Coordinate data from http://itouchmap.com/latlong.html
 cary_loc = (35.79154,-78.781117)
 nyc_loc = (40.78306,-73.971249)
 seattle_loc = (47.60621,-122.332071)
 binghamton_loc = (42.098687,-75.917974)
 cities = {
-    'cary': cary_loc,
-    'nyc': nyc_loc,
-    'seattle': seattle_loc,
-    'binghamton': binghamton_loc
+    'cary': cary_loc,
+    'nyc': nyc_loc,
+    'seattle': seattle_loc,
+    'binghamton': binghamton_loc
 }
 
-apikey = '' # My super-secret API Key
+apikey = '' # My super-secret API Key
 
 def get_forecast(lat, long, date=None):
-    forecast_base = "https://api.forecast.io/forecast/"
+    forecast_base = "https://api.forecast.io/forecast/"
     if date is None:
-        url = forecast_base + apikey + '/{},{}'.format(lat, long)
+        url = forecast_base + apikey + '/{},{}'.format(lat, long)
     else:
         epoch = int(date.timestamp())
-        url = forecast_base + apikey + '/{},{},{}'.format(lat, long, epoch)
+        url = forecast_base + apikey + '/{},{},{}'.format(lat, long, epoch)
 
     return requests.get(url).json()
 
 years = range(1990,2016)
-# For datetimes, the 12 is for getting the weather at noon.
-# We're doing this over midnight because we're more concerned
-# with what people see, and people don't typically see the weather
-# at midnight.
+# For datetimes, the 12 is for getting the weather at noon.
+# We're doing this over midnight because we're more concerned
+# with what people see, and people don't typically see the weather
+# at midnight.
 dt_indices = [pd.date_range(start=datetime(year, 7, 1, 12),
                           end=datetime(year, 11, 30, 12))
               for year in years]
 dt_merge = reduce(lambda x, y: x.union(y), dt_indices)
 
-# Because we have to pay a little bit to use the API, we use for loops here
-# instead of a comprehension - if something breaks, we want to preserve the
-# data already retrieved
+# Because we have to pay a little bit to use the API, we use for loops here
+# instead of a comprehension - if something breaks, we want to preserve the
+# data already retrieved
 city_forecasts = {}
 for city, loc in cities.items():
-    print("Retrieving data for {} starting at {}".format(city,
-                                                         datetime.now().strftime("%I:%M:%S %p")))
+    print("Retrieving data for {} starting at {}".format(city,
+                                                         datetime.now().strftime("%I:%M:%S %p")))
     for dt in dt_merge:
         try:
             city_forecasts[(city, dt)] = get_forecast(*loc, dt)
         except Exception as e:
             print(e)
             city_forecasts[(city, dt)] = None
-print("End forecast retrieval: {}".format(datetime.now().strftime("%I:%M:%S %p")))
+print("End forecast retrieval: {}".format(datetime.now().strftime("%I:%M:%S %p")))
 
 import pickle
-pickle.dump(city_forecasts, open('city_forecasts.p', 'wb'))
+pickle.dump(city_forecasts, open('city_forecasts.p', 'wb'))
 
-### Output:
-# Retrieving data for binghamton starting at 05:13:42 PM
-# Retrieving data for seattle starting at 05:30:51 PM
-# Retrieving data for nyc starting at 05:48:30 PM
-# Retrieving data for cary starting at 06:08:32 PM
-# End forecast retrieval: 06:25:21 PM
+### Output:
+# Retrieving data for binghamton starting at 05:13:42 PM
+# Retrieving data for seattle starting at 05:30:51 PM
+# Retrieving data for nyc starting at 05:48:30 PM
+# Retrieving data for cary starting at 06:08:32 PM
+# End forecast retrieval: 06:25:21 PM
 
diff --git a/event-studies-and-earnings-releases.html b/event-studies-and-earnings-releases.html index fc5ebe9..bf9b6fc 100644 --- a/event-studies-and-earnings-releases.html +++ b/event-studies-and-earnings-releases.html @@ -4,7 +4,7 @@ - + @@ -108,19 +108,19 @@
from IPython.display import HTML
 
-HTML('''<script>
-code_show=true; 
-function code_toggle() {
- if (code_show){
- $('div.input').hide();
- } else {
- $('div.input').show();
- }
- code_show = !code_show
-} 
-$( document ).ready(code_toggle);
-</script>
-<form action="javascript:code_toggle()"><input type="submit" value="Click here to toggle on/off the raw code."></form>''')
+HTML('''<script>
+code_show=true; 
+function code_toggle() {
+ if (code_show){
+ $('div.input').hide();
+ } else {
+ $('div.input').show();
+ }
+ code_show = !code_show
+} 
+$( document ).ready(code_toggle);
+</script>
+<form action="javascript:code_toggle()"><input type="submit" value="Click here to toggle on/off the raw code."></form>''')
 
@@ -182,20 +182,20 @@ $( document ).ready(code_toggle); %matplotlib inline def fetch_ticker(ticker, start, end): - # Quandl is currently giving me issues with returning - # the entire dataset and not slicing server-side. - # So instead, we'll do it client-side! - q_format = '%Y-%m-%d' - ticker_data = quandl.get('YAHOO/' + ticker, + # Quandl is currently giving me issues with returning + # the entire dataset and not slicing server-side. + # So instead, we'll do it client-side! + q_format = '%Y-%m-%d' + ticker_data = quandl.get('YAHOO/' + ticker, start_date=start.strftime(q_format), end_date=end.strftime(q_format), authtoken=QUANDL_KEY) return ticker_data -def ohlc_dataframe(data, ax=None): - # Much of this code re-used from: - # http://matplotlib.org/examples/pylab_examples/finance_demo.html - if ax is None: +def ohlc_dataframe(data, ax=None): + # Much of this code re-used from: + # http://matplotlib.org/examples/pylab_examples/finance_demo.html + if ax is None: f, ax = plt.subplots() vals = [(date2num(date), *(data.loc[date])) @@ -204,20 +204,20 @@ $( document ).ready(code_toggle); mondays = WeekdayLocator(MONDAY) alldays = DayLocator() - weekFormatter = DateFormatter('%b %d') + weekFormatter = DateFormatter('%b %d') ax.xaxis.set_major_locator(mondays) ax.xaxis.set_minor_locator(alldays) ax.xaxis.set_major_formatter(weekFormatter) return ax -AAPL = fetch_ticker('AAPL', datetime(2016, 3, 1), datetime(2016, 5, 1)) +AAPL = fetch_ticker('AAPL', datetime(2016, 3, 1), datetime(2016, 5, 1)) ax = ohlc_dataframe(AAPL) plt.vlines(date2num(datetime(2016, 4, 26, 12)), ax.get_ylim()[0], ax.get_ylim()[1], - color='b', - label='Earnings Release') + color='b', + label='Earnings Release') plt.legend(loc=3) -plt.title("Apple Price 3/1/2016 - 5/1/2016"); +plt.title("Apple Price 3/1/2016 - 5/1/2016");
@@ -439,12 +439,12 @@ rgAAAABJRU5ErkJggg==
In [3]:
-
FB = fetch_ticker('FB', datetime(2016, 3, 1), datetime(2016, 5, 5))
+
FB = fetch_ticker('FB', datetime(2016, 3, 1), datetime(2016, 5, 5))
 ax = ohlc_dataframe(FB)
 plt.vlines(date2num(datetime(2016, 4, 27, 12)),
            ax.get_ylim()[0], ax.get_ylim()[1],
-           color='b', label='Earnings Release')
-plt.title('Facebook Price 3/5/2016 - 5/5/2016')
+           color='b', label='Earnings Release')
+plt.title('Facebook Price 3/5/2016 - 5/5/2016')
 plt.legend(loc=2);
 
@@ -698,36 +698,36 @@ czHjid0552LGE7tzzsXM/wfgLII59KYqMQAAAABJRU5ErkJggg==
def plot_hilo(ax, start, end, data):
     ax.plot([date2num(start), date2num(end)],
-            [data.loc[start]['High'], data.loc[end]['High']],
-            color='b')
+            [data.loc[start]['High'], data.loc[end]['High']],
+            color='b')
     ax.plot([date2num(start), date2num(end)],
-            [data.loc[start]['Low'], data.loc[end]['Low']],
-            color='b')
+            [data.loc[start]['Low'], data.loc[end]['Low']],
+            color='b')
 
 f, axarr = plt.subplots(1, 2)
 
 ax_aapl = axarr[0]
 ax_fb = axarr[1]
 
-# Plot the AAPL trend up and down
+# Plot the AAPL trend up and down
 ohlc_dataframe(AAPL, ax=ax_aapl)
 plot_hilo(ax_aapl, datetime(2016, 3, 1), datetime(2016, 4, 15), AAPL)
 plot_hilo(ax_aapl, datetime(2016, 4, 18), datetime(2016, 4, 26), AAPL)
 ax_aapl.vlines(date2num(datetime(2016, 4, 26, 12)),
                ax_aapl.get_ylim()[0], ax_aapl.get_ylim()[1],
-               color='g', label='Earnings Release')
+               color='g', label='Earnings Release')
 ax_aapl.legend(loc=2)
-ax_aapl.set_title('AAPL Price History')
+ax_aapl.set_title('AAPL Price History')
 
-# Plot the FB trend down and up
+# Plot the FB trend down and up
 ohlc_dataframe(FB, ax=ax_fb)
 plot_hilo(ax_fb, datetime(2016, 3, 30), datetime(2016, 4, 27), FB)
 plot_hilo(ax_fb, datetime(2016, 4, 28), datetime(2016, 5, 5), FB)
 ax_fb.vlines(date2num(datetime(2016, 4, 27, 12)),
              ax_fb.get_ylim()[0], ax_fb.get_ylim()[1],
-             color='g', label='Earnings Release')
+             color='g', label='Earnings Release')
 ax_fb.legend(loc=2)
-ax_fb.set_title('FB Price History')
+ax_fb.set_title('FB Price History')
 
 f.set_size_inches(18, 6)
 
@@ -1385,17 +1385,17 @@ cAw4PwAAAABJRU5ErkJggg== from pandas.tseries.offsets import CustomBusinessDay from datetime import datetime, timedelta -# If you remove rules, it removes them from *all* calendars -# To ensure we don't pop rules we don't want to, first make -# sure to fully copy the object +# If you remove rules, it removes them from *all* calendars +# To ensure we don't pop rules we don't want to, first make +# sure to fully copy the object trade_calendar = USFederalHolidayCalendar() -trade_calendar.rules.pop(6) # Remove Columbus day -trade_calendar.rules.pop(7) # Remove Veteran's day +trade_calendar.rules.pop(6) # Remove Columbus day +trade_calendar.rules.pop(7) # Remove Veteran's day TradeDay = lambda days: CustomBusinessDay(days, calendar=trade_calendar) def plot_study(array): - # Given a 2-d array, we assume the event happens at index `lookback`, - # and create all of our summary statistics from there. + # Given a 2-d array, we assume the event happens at index `lookback`, + # and create all of our summary statistics from there. lookback = int((array.shape[1] - 1) / 2) norm_factor = np.repeat(array[:,lookback].reshape(-1, 1), array.shape[1], axis=1) centered_data = array / norm_factor - 1 @@ -1414,17 +1414,17 @@ cAw4PwAAAABJRU5ErkJggg== axarr[0].fill_between(range(0, range_end), means[-lookforward:] + std_dev, means[-lookforward:] - std_dev, - alpha=.5, label="$\pm$ 1 s.d.") + alpha=.5, label="$\pm$ 1 s.d.") axarr[1].fill_between(range(0, range_end), means[-lookforward:] + std_dev, means[-lookforward:] - std_dev, - alpha=.5, label="$\pm$ 1 s.d.") + alpha=.5, label="$\pm$ 1 s.d.") max_err = maxes - means[-lookforward+1:] min_err = means[-lookforward+1:] - mins axarr[0].errorbar(range(1, range_end), means[-lookforward+1:], - yerr=[min_err, max_err], label='Max & Min') + yerr=[min_err, max_err], label='Max & Min') axarr[0].legend(loc=2) axarr[1].legend(loc=2) @@ -1432,8 +1432,8 @@ cAw4PwAAAABJRU5ErkJggg== axarr[1].set_xlim((-lookback-1, lookback+1)) def plot_study_small(array): - # Given a 2-d array, we assume the event happens at index `lookback`, - # and create all of our summary statistics from there. + # Given a 2-d array, we assume the event happens at index `lookback`, + # and create all of our summary statistics from there. lookback = int((array.shape[1] - 1) / 2) norm_factor = np.repeat(array[:,lookback].reshape(-1, 1), array.shape[1], axis=1) centered_data = array / norm_factor - 1 @@ -1450,24 +1450,24 @@ cAw4PwAAAABJRU5ErkJggg== plt.fill_between(range(0, range_end), means[-lookforward:] + std_dev, means[-lookforward:] - std_dev, - alpha=.5, label="$\pm$ 1 s.d.") + alpha=.5, label="$\pm$ 1 s.d.") max_err = maxes - means[-lookforward+1:] min_err = means[-lookforward+1:] - mins plt.errorbar(range(1, range_end), means[-lookforward+1:], - yerr=[min_err, max_err], label='Max & Min') + yerr=[min_err, max_err], label='Max & Min') plt.legend(loc=2) plt.xlim((-lookback-1, lookback+1)) def fetch_event_data(ticker, events, horizon=5): - # Use horizon+1 to account for including the day of the event, - # and half-open interval - that is, for a horizon of 5, - # we should be including 11 events. Additionally, using the - # CustomBusinessDay means we automatically handle issues if - # for example a company reports Friday afternoon - the date - # calculator will turn this into a "Saturday" release, but - # we effectively shift that to Monday with the logic below. + # Use horizon+1 to account for including the day of the event, + # and half-open interval - that is, for a horizon of 5, + # we should be including 11 events. Additionally, using the + # CustomBusinessDay means we automatically handle issues if + # for example a company reports Friday afternoon - the date + # calculator will turn this into a "Saturday" release, but + # we effectively shift that to Monday with the logic below. td_back = TradeDay(horizon+1) td_forward = TradeDay(horizon+1) @@ -1476,28 +1476,28 @@ cAw4PwAAAABJRU5ErkJggg== total_data = fetch_ticker(ticker, start_date, end_date) event_data = [total_data.ix[event-td_back:event+td_forward]\ [0:horizon*2+1]\ - ['Adjusted Close'] + ['Adjusted Close'] for event in events] return np.array(event_data) -# Generate a couple of random events +# Generate a couple of random events event_dates = [datetime(2016, 5, 27) - timedelta(days=1) - TradeDay(x*20) for x in range(1, 40)] -data = fetch_event_data('CELG', event_dates) +data = fetch_event_data('CELG', event_dates) plot_study_small(data) plt.legend(loc=3) plt.gcf().set_size_inches(12, 6); -plt.annotate('Mean price for days leading up to each event', +plt.annotate('Mean price for days leading up to each event', (-5, -.01), (-4.5, .025), - arrowprops=dict(facecolor='black', shrink=0.05)) -plt.annotate('', (-.1, .005), (-.5, .02), - arrowprops={'facecolor': 'black', 'shrink': .05}) -plt.annotate('$\pm$ 1 std. dev. each day', (5, .055), (2.5, .085), - arrowprops={'facecolor': 'black', 'shrink': .05}) -plt.annotate('Min/Max each day', (.9, -.07), (-1, -.1), - arrowprops={'facecolor': 'black', 'shrink': .05}); + arrowprops=dict(facecolor='black', shrink=0.05)) +plt.annotate('', (-.1, .005), (-.5, .02), + arrowprops={'facecolor': 'black', 'shrink': .05}) +plt.annotate('$\pm$ 1 std. dev. each day', (5, .055), (2.5, .085), + arrowprops={'facecolor': 'black', 'shrink': .05}) +plt.annotate('Min/Max each day', (.9, -.07), (-1, -.1), + arrowprops={'facecolor': 'black', 'shrink': .05});
@@ -1957,24 +1957,24 @@ PlISLSIiIiLiIyXRIiIiIiI+UhItIiIiIuIjJdEiIiIiIj76P+bhd3LjvwIqAAAAAElFTkSuQmCC ohlc_dataframe(FB[datetime(2016, 4, 18):], FB_plot) FB_truncated = FB[datetime(2016, 4, 18):datetime(2016, 4, 27)] -midpoint = FB_truncated['Open']/2 + FB_truncated['Close']/2 -FB_plot.plot(FB_truncated.index, midpoint, label='Midpoint') +midpoint = FB_truncated['Open']/2 + FB_truncated['Close']/2 +FB_plot.plot(FB_truncated.index, midpoint, label='Midpoint') FB_plot.vlines(date2num(datetime(2016, 4, 27, 12)), ax_fb.get_ylim()[0], ax_fb.get_ylim()[1], - color='g', label='Earnings Release') + color='g', label='Earnings Release') FB_plot.legend(loc=2) -FB_plot.set_title('FB Midpoint Plot') +FB_plot.set_title('FB Midpoint Plot') AAPL_plot = axarr[1] ohlc_dataframe(AAPL[datetime(2016, 4, 10):], AAPL_plot) AAPL_truncated = AAPL[datetime(2016, 4, 10):datetime(2016, 4, 26)] -midpoint = AAPL_truncated['Open']/2 + AAPL_truncated['Close']/2 -AAPL_plot.plot(AAPL_truncated.index, midpoint, label='Midpoint') +midpoint = AAPL_truncated['Open']/2 + AAPL_truncated['Close']/2 +AAPL_plot.plot(AAPL_truncated.index, midpoint, label='Midpoint') AAPL_plot.vlines(date2num(datetime(2016, 4, 26, 12)), ax_aapl.get_ylim()[0], ax_aapl.get_ylim()[1], - color='g', label='Earnings Release') + color='g', label='Earnings Release') AAPL_plot.legend(loc=3) -AAPL_plot.set_title('AAPL Midpoint Plot'); +AAPL_plot.set_title('AAPL Midpoint Plot');
@@ -2453,74 +2453,74 @@ TkSuQmCC
In [7]:
-
# Read in the events for each stock;
-# The file was created using the first code block in the Appendix
+
# Read in the events for each stock;
+# The file was created using the first code block in the Appendix
 import yaml
 from dateutil.parser import parse
 from progressbar import ProgressBar
 
-data_str = open('earnings_dates.yaml', 'r').read()
-# Need to remove invalid lines
-filtered = filter(lambda x: '{' not in x, data_str.split('\n'))
-earnings_data = yaml.load('\n'.join(filtered))
+data_str = open('earnings_dates.yaml', 'r').read()
+# Need to remove invalid lines
+filtered = filter(lambda x: '{' not in x, data_str.split('\n'))
+earnings_data = yaml.load('\n'.join(filtered))
 
-# Convert our earnings data into a list of (ticker, date) pairs
-# to make it easy to work with.
-# This is horribly inefficient, but should get us what we need
+# Convert our earnings data into a list of (ticker, date) pairs
+# to make it easy to work with.
+# This is horribly inefficient, but should get us what we need
 ticker_dates = []
 for ticker, date_list in earnings_data.items():
     for iso_str in date_list:
         ticker_dates.append((ticker, parse(iso_str)))
 
 def does_trend_down(ticker, event, horizon):
-    # Figure out if the `event` has a downtrend for
-    # the `horizon` days preceding it
-    # As an interpretation note: it is assumed that
-    # the closing price of day `event` is the reference
-    # point, and we want `horizon` days before that.
-    # The price_data.hdf was created in the second appendix code block
+    # Figure out if the `event` has a downtrend for
+    # the `horizon` days preceding it
+    # As an interpretation note: it is assumed that
+    # the closing price of day `event` is the reference
+    # point, and we want `horizon` days before that.
+    # The price_data.hdf was created in the second appendix code block
     try:
-        ticker_data = pd.read_hdf('price_data.hdf', ticker)
+        ticker_data = pd.read_hdf('price_data.hdf', ticker)
         data = ticker_data[event-TradeDay(horizon):event]
-        midpoints = data['Open']/2 + data['Close']/2
+        midpoints = data['Open']/2 + data['Close']/2
 
-        # Shift dates one forward into the future and subtract
-        # Effectively: do we trend down over all days?
+        # Shift dates one forward into the future and subtract
+        # Effectively: do we trend down over all days?
         elems = midpoints - midpoints.shift(1)
         return len(elems)-1 == len(elems.dropna()[elems <= 0])
     except KeyError:
-        # If the stock doesn't exist, it doesn't qualify as trending down
-        # Mostly this is here to make sure the entire analysis doesn't 
-        # blow up if there were issues in data retrieval
-        return False
+        # If the stock doesn't exist, it doesn't qualify as trending down
+        # Mostly this is here to make sure the entire analysis doesn't 
+        # blow up if there were issues in data retrieval
+        return False
 
 def study_trend(horizon, trend_function):
     five_day_events = np.zeros((1, horizon*2 + 1))
     invalid_events = []
     for ticker, event in ProgressBar()(ticker_dates):
         if trend_function(ticker, event, horizon):
-            ticker_data = pd.read_hdf('price_data.hdf', ticker)
-            event_data = ticker_data[event-TradeDay(horizon):event+TradeDay(horizon)]['Close']
+            ticker_data = pd.read_hdf('price_data.hdf', ticker)
+            event_data = ticker_data[event-TradeDay(horizon):event+TradeDay(horizon)]['Close']
 
             try:
                 five_day_events = np.vstack([five_day_events, event_data])
             except ValueError:
-                # Sometimes we don't get exactly the right number of values due to calendar
-                # issues. I've fixed most everything I can, and the few issues that are left
-                # I assume don't systemically bias the results (i.e. data could be missing
-                # because it doesn't exist, etc.). After running through, ~1% of events get
-                # discarded this way
+                # Sometimes we don't get exactly the right number of values due to calendar
+                # issues. I've fixed most everything I can, and the few issues that are left
+                # I assume don't systemically bias the results (i.e. data could be missing
+                # because it doesn't exist, etc.). After running through, ~1% of events get
+                # discarded this way
                 invalid_events.append((ticker, event))
             
 
-    # Remove our initial zero row
+    # Remove our initial zero row
     five_day_events = five_day_events[1:,:]
     plot_study(five_day_events)
-    plt.gcf().suptitle('Action over {} days: {} events'
+    plt.gcf().suptitle('Action over {} days: {} events'
                        .format(horizon,five_day_events.shape[0]))
     plt.gcf().set_size_inches(18, 6)
     
-# Start with a 5 day study
+# Start with a 5 day study
 study_trend(5, does_trend_down)
 
@@ -3000,7 +3000,7 @@ ADBuCBoAAAAAAMC4+f+DLBbJVWQG2gAAAABJRU5ErkJggg==
In [8]:
-
# 8 day study next
+
# 8 day study next
 study_trend(8, does_trend_down)
 
@@ -3479,7 +3479,7 @@ AAAAKMr/B9pAH9lY/+K6AAAAAElFTkSuQmCC
In [9]:
-
# 3 day study after that
+
# 3 day study after that
 study_trend(3, does_trend_down)
 
@@ -3892,26 +3892,26 @@ STfmKmb+uQ4AAAAASUVORK5CYII=
def does_trend_up(ticker, event, horizon):
-    # Figure out if the `event` has an uptrend for
-    # the `horizon` days preceding it
-    # As an interpretation note: it is assumed that
-    # the closing price of day `event` is the reference
-    # point, and we want `horizon` days before that.
-    # The price_data.hdf was created in the second appendix code block
+    # Figure out if the `event` has an uptrend for
+    # the `horizon` days preceding it
+    # As an interpretation note: it is assumed that
+    # the closing price of day `event` is the reference
+    # point, and we want `horizon` days before that.
+    # The price_data.hdf was created in the second appendix code block
     try:
-        ticker_data = pd.read_hdf('price_data.hdf', ticker)
+        ticker_data = pd.read_hdf('price_data.hdf', ticker)
         data = ticker_data[event-TradeDay(horizon):event]
-        midpoints = data['Open']/2 + data['Close']/2
+        midpoints = data['Open']/2 + data['Close']/2
 
-        # Shift dates one forward into the future and subtract
-        # Effectively: do we trend down over all days?
+        # Shift dates one forward into the future and subtract
+        # Effectively: do we trend down over all days?
         elems = midpoints - midpoints.shift(1)
         return len(elems)-1 == len(elems.dropna()[elems >= 0])
     except KeyError:
-        # If the stock doesn't exist, it doesn't qualify as trending down
-        # Mostly this is here to make sure the entire analysis doesn't 
-        # blow up if there were issues in data retrieval
-        return False
+        # If the stock doesn't exist, it doesn't qualify as trending down
+        # Mostly this is here to make sure the entire analysis doesn't 
+        # blow up if there were issues in data retrieval
+        return False
 
 study_trend(5, does_trend_up)
 
@@ -5192,38 +5192,38 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== self.dates = [] def handle_starttag(self, tag, attrs): - if tag == 'table': + if tag == 'table': self.store_dates = True def handle_data(self, data): if self.store_dates: - match = re.match(r'\d+/\d+/\d+', data) + match = re.match(r'\d+/\d+/\d+', data) if match: self.dates.append(match.group(0)) - # If a company reports before the bell, record the earnings date - # being at midnight the day before. Ex: WMT reports 5/19/2016, - # but we want the reference point to be the closing price on 5/18/2016 - if 'After Close' in data: + # If a company reports before the bell, record the earnings date + # being at midnight the day before. Ex: WMT reports 5/19/2016, + # but we want the reference point to be the closing price on 5/18/2016 + if 'After Close' in data: self.earnings_offset = timedelta(days=0) - elif 'Before Open' in data: + elif 'Before Open' in data: self.earnings_offset = timedelta(days=-1) def handle_endtag(self, tag): - if tag == 'table': + if tag == 'table': self.store_dates = False def earnings_releases(ticker): - #print("Looking up ticker {}".format(ticker)) - user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) '\ - 'Gecko/20100101 Firefox/46.0' - headers = {'user-agent': user_agent} - base_url = 'http://www.streetinsider.com/ec_earnings.php?q={}'\ + #print("Looking up ticker {}".format(ticker)) + user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) '\ + 'Gecko/20100101 Firefox/46.0' + headers = {'user-agent': user_agent} + base_url = 'http://www.streetinsider.com/ec_earnings.php?q={}'\ .format(ticker) e = EarningsParser() s = requests.Session() a = requests.adapters.HTTPAdapter(max_retries=0) - s.mount('http://', a) + s.mount('http://', a) e.feed(str(s.get(base_url, headers=headers).content)) if e.earnings_offset is not None: @@ -5231,18 +5231,18 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== past = filter(lambda x: x < datetime.now(), dates) return list(map(lambda d: d.isoformat(), past)) -# Use a Russell-3000 ETF tracker (ticker IWV) to get a list of holdings -r3000 = pd.read_csv('https://www.ishares.com/us/products/239714/' - 'ishares-russell-3000-etf/1449138789749.ajax?' - 'fileType=csv&fileName=IWV_holdings&dataType=fund', +# Use a Russell-3000 ETF tracker (ticker IWV) to get a list of holdings +r3000 = pd.read_csv('https://www.ishares.com/us/products/239714/' + 'ishares-russell-3000-etf/1449138789749.ajax?' + 'fileType=csv&fileName=IWV_holdings&dataType=fund', header=10) -r3000_equities = r3000[(r3000['Exchange'] == 'NASDAQ') | - (r3000['Exchange'] == 'New York Stock Exchange Inc.')] +r3000_equities = r3000[(r3000['Exchange'] == 'NASDAQ') | + (r3000['Exchange'] == 'New York Stock Exchange Inc.')] -dates_file = open('earnings_dates.yaml', 'w') +dates_file = open('earnings_dates.yaml', 'w') with futures.ThreadPoolExecutor(max_workers=8) as pool: - fs = {pool.submit(earnings_releases, r3000_equities.ix[t]['Ticker']): t + fs = {pool.submit(earnings_releases, r3000_equities.ix[t]['Ticker']): t for t in r3000_equities.index} pbar = progressbar.ProgressBar(term_width=80, max_value=r3000_equities.index.max()) @@ -5250,7 +5250,7 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== for future in futures.as_completed(fs): i = fs[future] pbar.update(i) - dates_file.write(yaml.dump({r3000_equities.ix[i]['Ticker']: + dates_file.write(yaml.dump({r3000_equities.ix[i]['Ticker']: future.result()}))

Downloading stock price data needed for the event studies:

@@ -5263,31 +5263,31 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== from progressbar import ProgressBar def fetch_ticker(ticker, start, end): - # Quandl is currently giving me issues with returning - # the entire dataset and not slicing server-side. - # So instead, we'll do it client-side! - q_format = '%Y-%m-%d' - ticker_data = quandl.get('YAHOO/' + ticker, + # Quandl is currently giving me issues with returning + # the entire dataset and not slicing server-side. + # So instead, we'll do it client-side! + q_format = '%Y-%m-%d' + ticker_data = quandl.get('YAHOO/' + ticker, start_date=start.strftime(q_format), end_date=end.strftime(q_format), authtoken=QUANDL_KEY) return ticker_data -data_str = open('earnings_dates.yaml', 'r').read() -# Need to remove invalid lines -filtered = filter(lambda x: '{' not in x, data_str.split('\n')) -earnings_data = yaml.load('\n'.join(filtered)) +data_str = open('earnings_dates.yaml', 'r').read() +# Need to remove invalid lines +filtered = filter(lambda x: '{' not in x, data_str.split('\n')) +earnings_data = yaml.load('\n'.join(filtered)) -# Get the first 1500 keys - split up into two statements -# because of Quandl rate limits +# Get the first 1500 keys - split up into two statements +# because of Quandl rate limits tickers = list(earnings_data.keys()) price_dict = {} invalid_tickers = [] for ticker in ProgressBar()(tickers[0:1500]): try: - # Replace '.' with '-' in name for some tickers - fixed = ticker.replace('.', '-') + # Replace '.' with '-' in name for some tickers + fixed = ticker.replace('.', '-') event_strs = earnings_data[ticker] events = [parse(event) for event in event_strs] td = timedelta(days=20) @@ -5296,11 +5296,11 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== except quandl.NotFoundError: invalid_tickers.append(ticker) -# Execute this after 10 minutes have passed +# Execute this after 10 minutes have passed for ticker in ProgressBar()(tickers[1500:]): try: - # Replace '.' with '-' in name for some tickers - fixed = ticker.replace('.', '-') + # Replace '.' with '-' in name for some tickers + fixed = ticker.replace('.', '-') event_strs = earnings_data[ticker] events = [parse(event) for event in event_strs] td = timedelta(days=20) @@ -5309,7 +5309,7 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== except quandl.NotFoundError: invalid_tickers.append(ticker) -prices_store = pd.HDFStore('price_data.hdf') +prices_store = pd.HDFStore('price_data.hdf') for ticker, prices in price_dict.items(): prices_store[ticker] = prices
diff --git a/feeds/all.atom.xml b/feeds/all.atom.xml index 90913f7..01c3c26 100644 --- a/feeds/all.atom.xml +++ b/feeds/all.atom.xml @@ -1,5 +1,272 @@ -Bradlee Speicehttps://bspeice.github.io/2016-06-08T00:00:00-04:00Event Studies and Earnings Releases2016-06-08T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-06-08:event-studies-and-earnings-releases.html<script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/3.0.0/jquery.min.js"></script> +Bradlee Speicehttps://bspeice.github.io/2016-10-22T00:00:00-04:00A Rustic Re-Podcasting Server (Part 1)2016-10-22T00:00:00-04:002016-10-22T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-10-22:a-rustic-re-podcasting-server-part-1.html<p>I listen to a lot of Drum and Bass music, because it's beautiful music. And +there's a particular site, <a href="http://bassdrive.com/">Bassdrive.com</a> that hosts +a lot of great content. Specifically, the +<a href="http://archives.bassdrivearchive.com/">archives</a> section of the site has a +list of the past shows that you can download and listen to. The issue is, it's +just a <a href="http://archives.bassdrivearchive.com/6%20-%20Saturday/Electronic%20Warfare%20-%20The%20Overfiend/">giant list of links to download</a>. I'd really like +this in a podcast format to take with me on the road, etc.</p> +<p>So I wrote the <a href="https://github.com/bspeice/elektricity">elektricity</a> web +application to actually accomplish all that. Whenever you request a feed, it +goes out to Bassdrive, processes all the links on a page, and serves up some +fresh, tasty RSS to satisfy your ears. I hosted it on Heroku using the free +tier because it's really not resource-intensive at all.</p> +<p><strong>The issue so far</strong> is that I keep running out of free tier hours during a +month because my podcasting application likes to have a server scan for new +episodes constantly. Not sure why it's doing that, but I don't have a whole +lot of control over it. It's a phenomenal application otherwise.</p> +<p><strong>My (over-engineered) solution</strong>: Re-write the application using the +<a href="https://www.rust-lang.org/en-US/">Rust</a> programming language. I'd like to run +this on a small hacker board I own, and doing this in Rust would allow me to +easily cross-compile it. Plus, I've been very interested in the Rust language +for a while and this would be a great opportunity to really learn it well. +The code is available <a href="https://github.com/bspeice/nutone">here</a> as development +progresses.</p> +<h1>The Setup</h1> +<p>We'll be using the <a href="http://ironframework.io/">iron</a> library to handle the +server, and <a href="http://hyper.rs/">hyper</a> to fetch the data we need from elsewhere +on the interwebs. <a href="http://doc.servo.org/html5ever/index.html">HTML5Ever</a> allows +us to ingest the content that will be coming from Bassdrive, and finally, +output is done with <a href="http://sunng87.github.io/handlebars-rust/handlebars/index.html">handlebars-rust</a>.</p> +<p>It will ultimately be interesting to see how much more work must be done to +actually get this working over another language like Python. Coming from a +dynamic state of mind it's super easy to just chain stuff together, ship it out, +and call it a day. I think I'm going to end up getting much dirtier trying to +write all of this out.</p> +<h1>Issue 1: Strings</h1> +<p>Strings in Rust are hard. I acknowledge Python can get away with some things +that make strings super easy (and Python 3 has gotten better at cracking down +on some bad cases, <code>str &lt;-&gt; bytes</code> specifically), but Rust is hard.</p> +<p>Let's take for example the <code>404</code> error handler I'm trying to write. The result +should be incredibly simple: All I want is to echo back +<code>Didn't find URL: &lt;url&gt;</code>. Shouldn't be that hard right? In Python I'd just do +something like:</p> +<div class="highlight"><pre><span class="k">def</span> <span class="nf">echo_handler</span><span class="p">(</span><span class="n">request</span><span class="p">):</span> + <span class="k">return</span> <span class="s2">&quot;You&#39;re visiting: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">uri</span><span class="p">)</span> +</pre></div> + + +<p>And we'd call it a day. Rust isn't so simple. Let's start with the trivial +examples people post online:</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">hello_world</span><span class="p">(</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;You found the server!&quot;</span><span class="p">)))</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>Doesn't look too bad right? In fact, it's essentially the same as the Python +version! All we need to do is just send back a string of some form. So, we +look up the documentation for <a href="http://ironframework.io/doc/iron/request/struct.Request.html"><code>Request</code></a> and see a <code>url</code> field that will contain +what we want. Let's try the first iteration:</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">hello_world</span><span class="p">(</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;You found the URL: &quot;</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">req</span><span class="p">.</span><span class="n">url</span><span class="p">)))</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>Which yields the error:</p> +<div class="highlight"><pre>error[E0369]: binary operation `+` cannot be applied to type `&amp;&#39;static str` +</pre></div> + + +<p>OK, what's going on here? Time to start Googling for <a href="https://www.google.com/#q=concatenate+strings+in+rust">"concatenate strings in Rust"</a>. That's what we +want to do right? Concatenate a static string and the URL.</p> +<p>After Googling, we come across a helpful <a href="https://doc.rust-lang.org/std/macro.concat!.html"><code>concat!</code></a> macro that looks really nice! Let's try that one:</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">hello_world</span><span class="p">(</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="n">concat</span><span class="o">!</span><span class="p">(</span><span class="s">&quot;You found the URL: &quot;</span><span class="p">,</span><span class="w"> </span><span class="n">req</span><span class="p">.</span><span class="n">url</span><span class="p">))))</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>And the error:</p> +<p><code>error: expected a literal</code></p> +<p>Turns out Rust actually blows up because the <code>concat!</code> macro expects us to know +at compile time what <code>req.url</code> is. Which, in my outsider opinion, is a bit +strange. <code>println!</code> and <code>format!</code>, etc., all handle values they don't know at +compile time. Why can't <code>concat!</code>? By any means, we need a new plan of attack. +How about we try formatting strings?</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">hello_world</span><span class="p">(</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="n">format</span><span class="o">!</span><span class="p">(</span><span class="s">&quot;You found the URL: {}&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">req</span><span class="p">.</span><span class="n">url</span><span class="p">))))</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>And at long last, it works. Onwards!</p> +<h1>Issue 2: Fighting with the borrow checker</h1> +<p>Rust's single coolest feature is how the compiler can guarantee safety in your +program. As long as you don't use <code>unsafe</code> pointers in Rust, you're guaranteed +safety. And not having truly manual memory management is really cool; I'm +totally OK with never having to write <code>malloc()</code> again.</p> +<p>That said, even <a href="https://doc.rust-lang.org/book/ownership.html">the Rust documentation</a> makes a specific note:</p> +<blockquote> +<p>Many new users to Rust experience something we like to call +‘fighting with the borrow checker’, where the Rust compiler refuses to +compile a program that the author thinks is valid.</p> +</blockquote> +<p>If you have to put it in the documentation, it's not a helpful note: +it's hazing.</p> +<p>So now that we have a handler which works with information from the request, we +want to start making something that looks like an actual web application. +The router provided by <code>iron</code> isn't terribly difficult so I won't cover it. +Instead, the thing that had me stumped for a couple hours was trying to +dynamically create routes.</p> +<p>The unfortunate thing with Rust (in my limited experience at the moment) is that +there is a severe lack of non-trivial examples. Using the router is easy when +you want to give an example of a static function. But how do you you start +working on things that are a bit more complex?</p> +<p>We're going to cover that here. Our first try: creating a function which returns +other functions. This is a principle called <a href="http://stackoverflow.com/a/36321/1454178">currying</a>. We set up a function that allows us to keep some data in scope +for another function to come later.</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">build_handler</span><span class="p">(</span><span class="n">message</span><span class="o">:</span><span class="w"> </span><span class="nb">String</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="nb">Fn</span><span class="p">(</span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="n">move</span><span class="w"> </span><span class="o">|</span><span class="n">_</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="o">|</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="n">message</span><span class="p">)))</span><span class="w"></span> +<span class="w"> </span><span class="p">}</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>We've simply set up a function that returns another anonymous function with the +<code>message</code> parameter scoped in. If you compile this, you get not 1, not 2, but 5 +new errors. 4 of them are the same though:</p> +<div class="highlight"><pre>error[E0277]: the trait bound `for&lt;&#39;r, &#39;r, &#39;r&gt; std::ops::Fn(&amp;&#39;r mut iron::Request&lt;&#39;r, &#39;r&gt;) -&gt; std::result::Result&lt;iron::Response, iron::IronError&gt; + &#39;static: std::marker::Sized` is not satisfied +</pre></div> + + +<p>...oookay. I for one, am not going to spend time trying to figure out what's +going on there.</p> +<p>And it is here that I will save the audience many hours of frustrated effort. +At this point, I decided to switch from <code>iron</code> to pure <code>hyper</code> since using +<code>hyper</code> would give me a much simpler API. All I would have to do is build a +function that took two parameters as input, and we're done. That said, it +ultimately posed many more issues because I started getting into a weird fight +with the <code>'static</code> <a href="https://doc.rust-lang.org/book/lifetimes.html">lifetime</a> +and being a Rust newbie I just gave up on trying to understand it.</p> +<p>Instead, we will abandon (mostly) the curried function attempt, and instead +take advantage of something Rust actually intends us to use: <code>struct</code> and +<code>trait</code>.</p> +<p>Remember when I talked about a lack of non-trivial examples on the Internet? +This is what I was talking about. I could only find <em>one</em> example of this +available online, and it was incredibly complex and contained code we honestly +don't need or care about. There was no documentation of how to build routes that +didn't use static functions, etc. But, I'm assuming you don't really care about +my whining, so let's get to it.</p> +<p>The <code>iron</code> documentation mentions the <a href="http://ironframework.io/doc/iron/middleware/trait.Handler.html"><code>Handler</code></a> trait as being something we can implement. +Does the function signature for that <code>handle()</code> method look familiar? It's what +we've been working with so far.</p> +<p>The principle is that we need to define a new <code>struct</code> to hold our data, then +implement that <code>handle()</code> method to return the result. Something that looks +like this might do:</p> +<div class="highlight"><pre><span class="k">struct</span><span class="w"> </span><span class="n">EchoHandler</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="n">message</span><span class="o">:</span><span class="w"> </span><span class="nb">String</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> + +<span class="k">impl</span><span class="w"> </span><span class="n">Handler</span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="n">EchoHandler</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="n">handle</span><span class="p">(</span><span class="o">&amp;</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">_</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">message</span><span class="p">)))</span><span class="w"></span> +<span class="w"> </span><span class="p">}</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> + +<span class="c1">// Later in the code when we set up the router...</span> +<span class="kd">let</span><span class="w"> </span><span class="n">echo</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">EchoHandler</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="n">message</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;Is it working yet?&quot;</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +<span class="n">router</span><span class="p">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;/&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">echo</span><span class="p">.</span><span class="n">handle</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;index&quot;</span><span class="p">);</span><span class="w"></span> +</pre></div> + + +<p>We attempt to build a struct, and give its <code>handle</code> method off to the router +so the router knows what to do.</p> +<p>You guessed it, more errors:</p> +<div class="highlight"><pre><span class="n">error</span><span class="o">:</span> <span class="n">attempted</span> <span class="n">to</span> <span class="n">take</span> <span class="n">value</span> <span class="n">of</span> <span class="n">method</span> <span class="err">`</span><span class="n">handle</span><span class="err">`</span> <span class="n">on</span> <span class="n">type</span> <span class="err">`</span><span class="n">EchoHandler</span><span class="err">`</span> +</pre></div> + + +<p>Now, the Rust compiler is actually a really nice fellow, and offers us help:</p> +<div class="highlight"><pre><span class="n">help</span><span class="o">:</span> <span class="n">maybe</span> <span class="n">a</span> <span class="err">`</span><span class="o">()</span><span class="err">`</span> <span class="n">to</span> <span class="n">call</span> <span class="n">it</span> <span class="k">is</span> <span class="n">missing</span><span class="o">?</span> <span class="n">If</span> <span class="n">not</span><span class="o">,</span> <span class="k">try</span> <span class="n">an</span> <span class="n">anonymous</span> <span class="kd">function</span> +</pre></div> + + +<p>We definitely don't want to call that function, so maybe try an anonymous +function as it recommends?</p> +<div class="highlight"><pre><span class="n">router</span><span class="p">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;/&quot;</span><span class="p">,</span><span class="w"> </span><span class="o">|</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="o">|</span><span class="w"> </span><span class="n">echo</span><span class="p">.</span><span class="n">handle</span><span class="p">(</span><span class="n">req</span><span class="p">),</span><span class="w"> </span><span class="s">&quot;index&quot;</span><span class="p">);</span><span class="w"></span> +</pre></div> + + +<p>Another error:</p> +<div class="highlight"><pre>error[E0373]: closure may outlive the current function, but it borrows `echo`, which is owned by the current function +</pre></div> + + +<p>Another helpful message:</p> +<div class="highlight"><pre><span class="n">help</span><span class="o">:</span> <span class="n">to</span> <span class="n">force</span> <span class="n">the</span> <span class="n">closure</span> <span class="n">to</span> <span class="n">take</span> <span class="n">ownership</span> <span class="n">of</span> <span class="err">`</span><span class="n">echo</span><span class="err">`</span> <span class="o">(</span><span class="n">and</span> <span class="n">any</span> <span class="n">other</span> <span class="n">referenced</span> <span class="n">variables</span><span class="o">),</span> <span class="n">use</span> <span class="n">the</span> <span class="err">`</span><span class="n">move</span><span class="err">`</span> <span class="n">keyword</span> +</pre></div> + + +<p>We're getting closer though! Let's implement this change:</p> +<div class="highlight"><pre><span class="n">router</span><span class="p">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;/&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">move</span><span class="w"> </span><span class="o">|</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="o">|</span><span class="w"> </span><span class="n">echo</span><span class="p">.</span><span class="n">handle</span><span class="p">(</span><span class="n">req</span><span class="p">),</span><span class="w"> </span><span class="s">&quot;index&quot;</span><span class="p">);</span><span class="w"></span> +</pre></div> + + +<p>And here's where things get strange:</p> +<div class="highlight"><pre>error[E0507]: cannot move out of borrowed content + --&gt; src/main.rs:18:40 + | +18 | Ok(Response::with((status::Ok, self.message))) + | ^^^^ cannot move out of borrowed content +</pre></div> + + +<p>Now, this took me another couple hours to figure out. I'm going to explain it, +but <strong>keep this in mind: Rust only allows one reference at a time</strong> (exceptions +apply of course).</p> +<p>When we attempt to use <code>self.message</code> as it has been created in the earlier +<code>struct</code>, we essentially are trying to give it away to another piece of code. +Rust's semantics then state that <em>we may no longer access it</em> unless it is +returned to us (which <code>iron</code>'s code does not do). There are two ways to fix +this:</p> +<ol> +<li>Only give away references (i.e. <code>&amp;self.message</code> instead of <code>self.message</code>) +instead of transferring ownership</li> +<li>Make a copy of the underlying value which will be safe to give away</li> +</ol> +<p>I didn't know these were the two options originally, so I hope this helps the +audience out. Because <code>iron</code> won't accept a reference, we are forced into the +second option: making a copy. To do so, we just need to change the function +to look like this:</p> +<div class="highlight"><pre><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">message</span><span class="p">.</span><span class="n">clone</span><span class="p">())))</span><span class="w"></span> +</pre></div> + + +<p>Not so bad, huh? My only complaint is that it took so long to figure out exactly +what was going on.</p> +<p>And now we have a small server that we can configure dynamically. At long last.</p> +<blockquote> +<p>Final sidenote: You can actually do this without anonymous functions. Just +change the router line to: + <code>router.get("/", echo, "index");</code></p> +<p>Rust's type system seems to figure out that we want to use the <code>handle()</code> method.</p> +</blockquote> +<h1>Conclusion</h1> +<p>After a good long days' work, we now have the routing functionality set up on +our application. We should be able to scale this pretty well in the future: +the RSS content we need to deliver in the future can be treated as a string, so +the building blocks are in place.</p> +<p>There are two important things I learned starting with Rust today:</p> +<ol> +<li>Rust is a new language, and while the code is high-quality, the mindshare is coming.</li> +<li>I'm a terrible programmer.</li> +</ol> +<p>Number 1 is pretty obvious and not surprising to anyone. Number two caught me +off guard. I've gotten used to having either a garbage collector (Java, Python, +etc.) or playing a little fast and loose with scoping rules (C, C++). You don't +have to worry about object lifetime there. With Rust, it's forcing me to fully +understand and use well the memory in my applications. In the final mistake I +fixed (using <code>.clone()</code>) I would have been fine in C++ to just give away that +reference and never use it again. I wouldn't have run into a "use-after-free" +error, but I would have potentially been leaking memory. Rust forced me to be +incredibly precise about how I use it.</p> +<p>All said I'm excited for using Rust more. I think it's super cool, it's just +going to take me a lot longer to do this than I originally thought.</p>Event Studies and Earnings Releases2016-06-08T00:00:00-04:002016-06-08T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-06-08:event-studies-and-earnings-releases.html<script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/3.0.0/jquery.min.js"></script> <p> <div class="cell border-box-sizing text_cell rendered"> @@ -21,19 +288,19 @@ <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="kn">from</span> <span class="nn">IPython.display</span> <span class="k">import</span> <span class="n">HTML</span> -<span class="n">HTML</span><span class="p">(</span><span class="s">&#39;&#39;&#39;&lt;script&gt;</span> -<span class="s">code_show=true; </span> -<span class="s">function code_toggle() {</span> -<span class="s"> if (code_show){</span> -<span class="s"> $(&#39;div.input&#39;).hide();</span> -<span class="s"> } else {</span> -<span class="s"> $(&#39;div.input&#39;).show();</span> -<span class="s"> }</span> -<span class="s"> code_show = !code_show</span> -<span class="s">} </span> -<span class="s">$( document ).ready(code_toggle);</span> -<span class="s">&lt;/script&gt;</span> -<span class="s">&lt;form action=&quot;javascript:code_toggle()&quot;&gt;&lt;input type=&quot;submit&quot; value=&quot;Click here to toggle on/off the raw code.&quot;&gt;&lt;/form&gt;&#39;&#39;&#39;</span><span class="p">)</span> +<span class="n">HTML</span><span class="p">(</span><span class="s1">&#39;&#39;&#39;&lt;script&gt;</span> +<span class="s1">code_show=true; </span> +<span class="s1">function code_toggle() {</span> +<span class="s1"> if (code_show){</span> +<span class="s1"> $(&#39;div.input&#39;).hide();</span> +<span class="s1"> } else {</span> +<span class="s1"> $(&#39;div.input&#39;).show();</span> +<span class="s1"> }</span> +<span class="s1"> code_show = !code_show</span> +<span class="s1">} </span> +<span class="s1">$( document ).ready(code_toggle);</span> +<span class="s1">&lt;/script&gt;</span> +<span class="s1">&lt;form action=&quot;javascript:code_toggle()&quot;&gt;&lt;input type=&quot;submit&quot; value=&quot;Click here to toggle on/off the raw code.&quot;&gt;&lt;/form&gt;&#39;&#39;&#39;</span><span class="p">)</span> </pre></div> </div> @@ -95,20 +362,20 @@ $( document ).ready(code_toggle); <span class="o">%</span><span class="k">matplotlib</span> inline <span class="k">def</span> <span class="nf">fetch_ticker</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">):</span> - <span class="c"># Quandl is currently giving me issues with returning</span> - <span class="c"># the entire dataset and not slicing server-side.</span> - <span class="c"># So instead, we&#39;ll do it client-side!</span> - <span class="n">q_format</span> <span class="o">=</span> <span class="s">&#39;%Y-%m-%d&#39;</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&#39;YAHOO/&#39;</span> <span class="o">+</span> <span class="n">ticker</span><span class="p">,</span> + <span class="c1"># Quandl is currently giving me issues with returning</span> + <span class="c1"># the entire dataset and not slicing server-side.</span> + <span class="c1"># So instead, we&#39;ll do it client-side!</span> + <span class="n">q_format</span> <span class="o">=</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;YAHOO/&#39;</span> <span class="o">+</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">start_date</span><span class="o">=</span><span class="n">start</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">q_format</span><span class="p">),</span> <span class="n">end_date</span><span class="o">=</span><span class="n">end</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">q_format</span><span class="p">),</span> <span class="n">authtoken</span><span class="o">=</span><span class="n">QUANDL_KEY</span><span class="p">)</span> <span class="k">return</span> <span class="n">ticker_data</span> -<span class="k">def</span> <span class="nf">ohlc_dataframe</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="k">None</span><span class="p">):</span> - <span class="c"># Much of this code re-used from:</span> - <span class="c"># http://matplotlib.org/examples/pylab_examples/finance_demo.html</span> - <span class="k">if</span> <span class="n">ax</span> <span class="ow">is</span> <span class="k">None</span><span class="p">:</span> +<span class="k">def</span> <span class="nf">ohlc_dataframe</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> + <span class="c1"># Much of this code re-used from:</span> + <span class="c1"># http://matplotlib.org/examples/pylab_examples/finance_demo.html</span> + <span class="k">if</span> <span class="n">ax</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> <span class="n">f</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">()</span> <span class="n">vals</span> <span class="o">=</span> <span class="p">[(</span><span class="n">date2num</span><span class="p">(</span><span class="n">date</span><span class="p">),</span> <span class="o">*</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">date</span><span class="p">]))</span> @@ -117,20 +384,20 @@ $( document ).ready(code_toggle); <span class="n">mondays</span> <span class="o">=</span> <span class="n">WeekdayLocator</span><span class="p">(</span><span class="n">MONDAY</span><span class="p">)</span> <span class="n">alldays</span> <span class="o">=</span> <span class="n">DayLocator</span><span class="p">()</span> - <span class="n">weekFormatter</span> <span class="o">=</span> <span class="n">DateFormatter</span><span class="p">(</span><span class="s">&#39;%b %d&#39;</span><span class="p">)</span> + <span class="n">weekFormatter</span> <span class="o">=</span> <span class="n">DateFormatter</span><span class="p">(</span><span class="s1">&#39;%b </span><span class="si">%d</span><span class="s1">&#39;</span><span class="p">)</span> <span class="n">ax</span><span class="o">.</span><span class="n">xaxis</span><span class="o">.</span><span class="n">set_major_locator</span><span class="p">(</span><span class="n">mondays</span><span class="p">)</span> <span class="n">ax</span><span class="o">.</span><span class="n">xaxis</span><span class="o">.</span><span class="n">set_minor_locator</span><span class="p">(</span><span class="n">alldays</span><span class="p">)</span> <span class="n">ax</span><span class="o">.</span><span class="n">xaxis</span><span class="o">.</span><span class="n">set_major_formatter</span><span class="p">(</span><span class="n">weekFormatter</span><span class="p">)</span> <span class="k">return</span> <span class="n">ax</span> -<span class="n">AAPL</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="s">&#39;AAPL&#39;</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> +<span class="n">AAPL</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="s1">&#39;AAPL&#39;</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">AAPL</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">,</span> - <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;b&#39;</span><span class="p">,</span> + <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&quot;Apple Price 3/1/2016 - 5/1/2016&quot;</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s2">&quot;Apple Price 3/1/2016 - 5/1/2016&quot;</span><span class="p">);</span> </pre></div> </div> @@ -352,12 +619,12 @@ rgAAAABJRU5ErkJggg== <div class="prompt input_prompt">In&nbsp;[3]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">FB</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="s">&#39;FB&#39;</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span> +<div class=" highlight hl-ipython3"><pre><span class="n">FB</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="s1">&#39;FB&#39;</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">FB</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;Facebook Price 3/5/2016 - 5/5/2016&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;Facebook Price 3/5/2016 - 5/5/2016&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">);</span> </pre></div> @@ -611,36 +878,36 @@ czHjid0552LGE7tzzsXM/wfgLII59KYqMQAAAABJRU5ErkJggg== <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">plot_hilo</span><span class="p">(</span><span class="n">ax</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span> <span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">([</span><span class="n">date2num</span><span class="p">(</span><span class="n">start</span><span class="p">),</span> <span class="n">date2num</span><span class="p">(</span><span class="n">end</span><span class="p">)],</span> - <span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">start</span><span class="p">][</span><span class="s">&#39;High&#39;</span><span class="p">],</span> <span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">end</span><span class="p">][</span><span class="s">&#39;High&#39;</span><span class="p">]],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">)</span> + <span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">start</span><span class="p">][</span><span class="s1">&#39;High&#39;</span><span class="p">],</span> <span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">end</span><span class="p">][</span><span class="s1">&#39;High&#39;</span><span class="p">]],</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;b&#39;</span><span class="p">)</span> <span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">([</span><span class="n">date2num</span><span class="p">(</span><span class="n">start</span><span class="p">),</span> <span class="n">date2num</span><span class="p">(</span><span class="n">end</span><span class="p">)],</span> - <span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">start</span><span class="p">][</span><span class="s">&#39;Low&#39;</span><span class="p">],</span> <span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">end</span><span class="p">][</span><span class="s">&#39;Low&#39;</span><span class="p">]],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">)</span> + <span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">start</span><span class="p">][</span><span class="s1">&#39;Low&#39;</span><span class="p">],</span> <span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">end</span><span class="p">][</span><span class="s1">&#39;Low&#39;</span><span class="p">]],</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;b&#39;</span><span class="p">)</span> <span class="n">f</span><span class="p">,</span> <span class="n">axarr</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span> <span class="n">ax_aapl</span> <span class="o">=</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="n">ax_fb</span> <span class="o">=</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> -<span class="c"># Plot the AAPL trend up and down</span> +<span class="c1"># Plot the AAPL trend up and down</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">AAPL</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="n">ax_aapl</span><span class="p">)</span> <span class="n">plot_hilo</span><span class="p">(</span><span class="n">ax_aapl</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="n">AAPL</span><span class="p">)</span> <span class="n">plot_hilo</span><span class="p">(</span><span class="n">ax_aapl</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">18</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">),</span> <span class="n">AAPL</span><span class="p">)</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> -<span class="n">ax_aapl</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s">&#39;AAPL Price History&#39;</span><span class="p">)</span> +<span class="n">ax_aapl</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;AAPL Price History&#39;</span><span class="p">)</span> -<span class="c"># Plot the FB trend down and up</span> +<span class="c1"># Plot the FB trend down and up</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">FB</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="n">ax_fb</span><span class="p">)</span> <span class="n">plot_hilo</span><span class="p">(</span><span class="n">ax_fb</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">30</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">),</span> <span class="n">FB</span><span class="p">)</span> <span class="n">plot_hilo</span><span class="p">(</span><span class="n">ax_fb</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">28</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span> <span class="n">FB</span><span class="p">)</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> -<span class="n">ax_fb</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s">&#39;FB Price History&#39;</span><span class="p">)</span> +<span class="n">ax_fb</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;FB Price History&#39;</span><span class="p">)</span> <span class="n">f</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> </pre></div> @@ -1298,17 +1565,17 @@ cAw4PwAAAABJRU5ErkJggg== <span class="kn">from</span> <span class="nn">pandas.tseries.offsets</span> <span class="k">import</span> <span class="n">CustomBusinessDay</span> <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span> -<span class="c"># If you remove rules, it removes them from *all* calendars</span> -<span class="c"># To ensure we don&#39;t pop rules we don&#39;t want to, first make</span> -<span class="c"># sure to fully copy the object</span> +<span class="c1"># If you remove rules, it removes them from *all* calendars</span> +<span class="c1"># To ensure we don&#39;t pop rules we don&#39;t want to, first make</span> +<span class="c1"># sure to fully copy the object</span> <span class="n">trade_calendar</span> <span class="o">=</span> <span class="n">USFederalHolidayCalendar</span><span class="p">()</span> -<span class="n">trade_calendar</span><span class="o">.</span><span class="n">rules</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span> <span class="c"># Remove Columbus day</span> -<span class="n">trade_calendar</span><span class="o">.</span><span class="n">rules</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">7</span><span class="p">)</span> <span class="c"># Remove Veteran&#39;s day</span> +<span class="n">trade_calendar</span><span class="o">.</span><span class="n">rules</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span> <span class="c1"># Remove Columbus day</span> +<span class="n">trade_calendar</span><span class="o">.</span><span class="n">rules</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">7</span><span class="p">)</span> <span class="c1"># Remove Veteran&#39;s day</span> <span class="n">TradeDay</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">days</span><span class="p">:</span> <span class="n">CustomBusinessDay</span><span class="p">(</span><span class="n">days</span><span class="p">,</span> <span class="n">calendar</span><span class="o">=</span><span class="n">trade_calendar</span><span class="p">)</span> <span class="k">def</span> <span class="nf">plot_study</span><span class="p">(</span><span class="n">array</span><span class="p">):</span> - <span class="c"># Given a 2-d array, we assume the event happens at index `lookback`,</span> - <span class="c"># and create all of our summary statistics from there.</span> + <span class="c1"># Given a 2-d array, we assume the event happens at index `lookback`,</span> + <span class="c1"># and create all of our summary statistics from there.</span> <span class="n">lookback</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">array</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="mi">2</span><span class="p">)</span> <span class="n">norm_factor</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="n">array</span><span class="p">[:,</span><span class="n">lookback</span><span class="p">]</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">array</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="n">centered_data</span> <span class="o">=</span> <span class="n">array</span> <span class="o">/</span> <span class="n">norm_factor</span> <span class="o">-</span> <span class="mi">1</span> @@ -1327,17 +1594,17 @@ cAw4PwAAAABJRU5ErkJggg== <span class="n">axarr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">fill_between</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">+</span> <span class="n">std_dev</span><span class="p">,</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">-</span> <span class="n">std_dev</span><span class="p">,</span> - <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> + <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">fill_between</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">+</span> <span class="n">std_dev</span><span class="p">,</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">-</span> <span class="n">std_dev</span><span class="p">,</span> - <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> + <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> <span class="n">max_err</span> <span class="o">=</span> <span class="n">maxes</span> <span class="o">-</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span> <span class="n">min_err</span> <span class="o">=</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span> <span class="o">-</span> <span class="n">mins</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">errorbar</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:],</span> - <span class="n">yerr</span><span class="o">=</span><span class="p">[</span><span class="n">min_err</span><span class="p">,</span> <span class="n">max_err</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Max &amp; Min&#39;</span><span class="p">)</span> + <span class="n">yerr</span><span class="o">=</span><span class="p">[</span><span class="n">min_err</span><span class="p">,</span> <span class="n">max_err</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Max &amp; Min&#39;</span><span class="p">)</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> @@ -1345,8 +1612,8 @@ cAw4PwAAAABJRU5ErkJggg== <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">set_xlim</span><span class="p">((</span><span class="o">-</span><span class="n">lookback</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">lookback</span><span class="o">+</span><span class="mi">1</span><span class="p">))</span> <span class="k">def</span> <span class="nf">plot_study_small</span><span class="p">(</span><span class="n">array</span><span class="p">):</span> - <span class="c"># Given a 2-d array, we assume the event happens at index `lookback`,</span> - <span class="c"># and create all of our summary statistics from there.</span> + <span class="c1"># Given a 2-d array, we assume the event happens at index `lookback`,</span> + <span class="c1"># and create all of our summary statistics from there.</span> <span class="n">lookback</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">array</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="mi">2</span><span class="p">)</span> <span class="n">norm_factor</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="n">array</span><span class="p">[:,</span><span class="n">lookback</span><span class="p">]</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">array</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="n">centered_data</span> <span class="o">=</span> <span class="n">array</span> <span class="o">/</span> <span class="n">norm_factor</span> <span class="o">-</span> <span class="mi">1</span> @@ -1363,24 +1630,24 @@ cAw4PwAAAABJRU5ErkJggg== <span class="n">plt</span><span class="o">.</span><span class="n">fill_between</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">+</span> <span class="n">std_dev</span><span class="p">,</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">-</span> <span class="n">std_dev</span><span class="p">,</span> - <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> + <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> <span class="n">max_err</span> <span class="o">=</span> <span class="n">maxes</span> <span class="o">-</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span> <span class="n">min_err</span> <span class="o">=</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span> <span class="o">-</span> <span class="n">mins</span> <span class="n">plt</span><span class="o">.</span><span class="n">errorbar</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:],</span> - <span class="n">yerr</span><span class="o">=</span><span class="p">[</span><span class="n">min_err</span><span class="p">,</span> <span class="n">max_err</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Max &amp; Min&#39;</span><span class="p">)</span> + <span class="n">yerr</span><span class="o">=</span><span class="p">[</span><span class="n">min_err</span><span class="p">,</span> <span class="n">max_err</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Max &amp; Min&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">xlim</span><span class="p">((</span><span class="o">-</span><span class="n">lookback</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">lookback</span><span class="o">+</span><span class="mi">1</span><span class="p">))</span> <span class="k">def</span> <span class="nf">fetch_event_data</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">events</span><span class="p">,</span> <span class="n">horizon</span><span class="o">=</span><span class="mi">5</span><span class="p">):</span> - <span class="c"># Use horizon+1 to account for including the day of the event,</span> - <span class="c"># and half-open interval - that is, for a horizon of 5,</span> - <span class="c"># we should be including 11 events. Additionally, using the</span> - <span class="c"># CustomBusinessDay means we automatically handle issues if</span> - <span class="c"># for example a company reports Friday afternoon - the date</span> - <span class="c"># calculator will turn this into a &quot;Saturday&quot; release, but</span> - <span class="c"># we effectively shift that to Monday with the logic below.</span> + <span class="c1"># Use horizon+1 to account for including the day of the event,</span> + <span class="c1"># and half-open interval - that is, for a horizon of 5,</span> + <span class="c1"># we should be including 11 events. Additionally, using the</span> + <span class="c1"># CustomBusinessDay means we automatically handle issues if</span> + <span class="c1"># for example a company reports Friday afternoon - the date</span> + <span class="c1"># calculator will turn this into a &quot;Saturday&quot; release, but</span> + <span class="c1"># we effectively shift that to Monday with the logic below.</span> <span class="n">td_back</span> <span class="o">=</span> <span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span> <span class="n">td_forward</span> <span class="o">=</span> <span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span> @@ -1389,28 +1656,28 @@ cAw4PwAAAABJRU5ErkJggg== <span class="n">total_data</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="p">)</span> <span class="n">event_data</span> <span class="o">=</span> <span class="p">[</span><span class="n">total_data</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">td_back</span><span class="p">:</span><span class="n">event</span><span class="o">+</span><span class="n">td_forward</span><span class="p">]</span>\ <span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="n">horizon</span><span class="o">*</span><span class="mi">2</span><span class="o">+</span><span class="mi">1</span><span class="p">]</span>\ - <span class="p">[</span><span class="s">&#39;Adjusted Close&#39;</span><span class="p">]</span> + <span class="p">[</span><span class="s1">&#39;Adjusted Close&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">events</span><span class="p">]</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">event_data</span><span class="p">)</span> -<span class="c"># Generate a couple of random events</span> +<span class="c1"># Generate a couple of random events</span> <span class="n">event_dates</span> <span class="o">=</span> <span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">27</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="n">TradeDay</span><span class="p">(</span><span class="n">x</span><span class="o">*</span><span class="mi">20</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">40</span><span class="p">)]</span> -<span class="n">data</span> <span class="o">=</span> <span class="n">fetch_event_data</span><span class="p">(</span><span class="s">&#39;CELG&#39;</span><span class="p">,</span> <span class="n">event_dates</span><span class="p">)</span> +<span class="n">data</span> <span class="o">=</span> <span class="n">fetch_event_data</span><span class="p">(</span><span class="s1">&#39;CELG&#39;</span><span class="p">,</span> <span class="n">event_dates</span><span class="p">)</span> <span class="n">plot_study_small</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span> <span class="mi">6</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s">&#39;Mean price for days leading up to each event&#39;</span><span class="p">,</span> +<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s1">&#39;Mean price for days leading up to each event&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">-</span><span class="mi">5</span><span class="p">,</span> <span class="o">-.</span><span class="mi">01</span><span class="p">),</span> <span class="p">(</span><span class="o">-</span><span class="mf">4.5</span><span class="p">,</span> <span class="o">.</span><span class="mi">025</span><span class="p">),</span> - <span class="n">arrowprops</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">facecolor</span><span class="o">=</span><span class="s">&#39;black&#39;</span><span class="p">,</span> <span class="n">shrink</span><span class="o">=</span><span class="mf">0.05</span><span class="p">))</span> -<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s">&#39;&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">-.</span><span class="mi">1</span><span class="p">,</span> <span class="o">.</span><span class="mi">005</span><span class="p">),</span> <span class="p">(</span><span class="o">-.</span><span class="mi">5</span><span class="p">,</span> <span class="o">.</span><span class="mi">02</span><span class="p">),</span> - <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s">&#39;black&#39;</span><span class="p">,</span> <span class="s">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">})</span> -<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s">&#39;$\pm$ 1 std. dev. each day&#39;</span><span class="p">,</span> <span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="o">.</span><span class="mi">055</span><span class="p">),</span> <span class="p">(</span><span class="mf">2.5</span><span class="p">,</span> <span class="o">.</span><span class="mi">085</span><span class="p">),</span> - <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s">&#39;black&#39;</span><span class="p">,</span> <span class="s">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">})</span> -<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s">&#39;Min/Max each day&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">.</span><span class="mi">9</span><span class="p">,</span> <span class="o">-.</span><span class="mi">07</span><span class="p">),</span> <span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-.</span><span class="mi">1</span><span class="p">),</span> - <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s">&#39;black&#39;</span><span class="p">,</span> <span class="s">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">});</span> + <span class="n">arrowprops</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">facecolor</span><span class="o">=</span><span class="s1">&#39;black&#39;</span><span class="p">,</span> <span class="n">shrink</span><span class="o">=</span><span class="mf">0.05</span><span class="p">))</span> +<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">-.</span><span class="mi">1</span><span class="p">,</span> <span class="o">.</span><span class="mi">005</span><span class="p">),</span> <span class="p">(</span><span class="o">-.</span><span class="mi">5</span><span class="p">,</span> <span class="o">.</span><span class="mi">02</span><span class="p">),</span> + <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s1">&#39;black&#39;</span><span class="p">,</span> <span class="s1">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">})</span> +<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s1">&#39;$\pm$ 1 std. dev. each day&#39;</span><span class="p">,</span> <span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="o">.</span><span class="mi">055</span><span class="p">),</span> <span class="p">(</span><span class="mf">2.5</span><span class="p">,</span> <span class="o">.</span><span class="mi">085</span><span class="p">),</span> + <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s1">&#39;black&#39;</span><span class="p">,</span> <span class="s1">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">})</span> +<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s1">&#39;Min/Max each day&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">.</span><span class="mi">9</span><span class="p">,</span> <span class="o">-.</span><span class="mi">07</span><span class="p">),</span> <span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-.</span><span class="mi">1</span><span class="p">),</span> + <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s1">&#39;black&#39;</span><span class="p">,</span> <span class="s1">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">});</span> </pre></div> </div> @@ -1870,24 +2137,24 @@ PlISLSIiIiLiIyXRIiIiIiI+UhItIiIiIuIjJdEiIiIiIj76P+bhd3LjvwIqAAAAAElFTkSuQmCC <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">FB</span><span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">18</span><span class="p">):],</span> <span class="n">FB_plot</span><span class="p">)</span> <span class="n">FB_truncated</span> <span class="o">=</span> <span class="n">FB</span><span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">18</span><span class="p">):</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">)]</span> -<span class="n">midpoint</span> <span class="o">=</span> <span class="n">FB_truncated</span><span class="p">[</span><span class="s">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">FB_truncated</span><span class="p">[</span><span class="s">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> -<span class="n">FB_plot</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">FB_truncated</span><span class="o">.</span><span class="n">index</span><span class="p">,</span> <span class="n">midpoint</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Midpoint&#39;</span><span class="p">)</span> +<span class="n">midpoint</span> <span class="o">=</span> <span class="n">FB_truncated</span><span class="p">[</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">FB_truncated</span><span class="p">[</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> +<span class="n">FB_plot</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">FB_truncated</span><span class="o">.</span><span class="n">index</span><span class="p">,</span> <span class="n">midpoint</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Midpoint&#39;</span><span class="p">)</span> <span class="n">FB_plot</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">FB_plot</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> -<span class="n">FB_plot</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s">&#39;FB Midpoint Plot&#39;</span><span class="p">)</span> +<span class="n">FB_plot</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;FB Midpoint Plot&#39;</span><span class="p">)</span> <span class="n">AAPL_plot</span> <span class="o">=</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">AAPL</span><span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">10</span><span class="p">):],</span> <span class="n">AAPL_plot</span><span class="p">)</span> <span class="n">AAPL_truncated</span> <span class="o">=</span> <span class="n">AAPL</span><span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">10</span><span class="p">):</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">)]</span> -<span class="n">midpoint</span> <span class="o">=</span> <span class="n">AAPL_truncated</span><span class="p">[</span><span class="s">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">AAPL_truncated</span><span class="p">[</span><span class="s">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> -<span class="n">AAPL_plot</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">AAPL_truncated</span><span class="o">.</span><span class="n">index</span><span class="p">,</span> <span class="n">midpoint</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Midpoint&#39;</span><span class="p">)</span> +<span class="n">midpoint</span> <span class="o">=</span> <span class="n">AAPL_truncated</span><span class="p">[</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">AAPL_truncated</span><span class="p">[</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> +<span class="n">AAPL_plot</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">AAPL_truncated</span><span class="o">.</span><span class="n">index</span><span class="p">,</span> <span class="n">midpoint</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Midpoint&#39;</span><span class="p">)</span> <span class="n">AAPL_plot</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">AAPL_plot</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span> -<span class="n">AAPL_plot</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s">&#39;AAPL Midpoint Plot&#39;</span><span class="p">);</span> +<span class="n">AAPL_plot</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;AAPL Midpoint Plot&#39;</span><span class="p">);</span> </pre></div> </div> @@ -2366,74 +2633,74 @@ TkSuQmCC <div class="prompt input_prompt">In&nbsp;[7]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># Read in the events for each stock;</span> -<span class="c"># The file was created using the first code block in the Appendix</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># Read in the events for each stock;</span> +<span class="c1"># The file was created using the first code block in the Appendix</span> <span class="kn">import</span> <span class="nn">yaml</span> <span class="kn">from</span> <span class="nn">dateutil.parser</span> <span class="k">import</span> <span class="n">parse</span> <span class="kn">from</span> <span class="nn">progressbar</span> <span class="k">import</span> <span class="n">ProgressBar</span> -<span class="n">data_str</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> -<span class="c"># Need to remove invalid lines</span> -<span class="n">filtered</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s">&#39;{&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">x</span><span class="p">,</span> <span class="n">data_str</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">&#39;</span><span class="se">\n</span><span class="s">&#39;</span><span class="p">))</span> -<span class="n">earnings_data</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s">&#39;</span><span class="se">\n</span><span class="s">&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">filtered</span><span class="p">))</span> +<span class="n">data_str</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> +<span class="c1"># Need to remove invalid lines</span> +<span class="n">filtered</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s1">&#39;{&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">x</span><span class="p">,</span> <span class="n">data_str</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">))</span> +<span class="n">earnings_data</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">filtered</span><span class="p">))</span> -<span class="c"># Convert our earnings data into a list of (ticker, date) pairs</span> -<span class="c"># to make it easy to work with.</span> -<span class="c"># This is horribly inefficient, but should get us what we need</span> +<span class="c1"># Convert our earnings data into a list of (ticker, date) pairs</span> +<span class="c1"># to make it easy to work with.</span> +<span class="c1"># This is horribly inefficient, but should get us what we need</span> <span class="n">ticker_dates</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">date_list</span> <span class="ow">in</span> <span class="n">earnings_data</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> <span class="k">for</span> <span class="n">iso_str</span> <span class="ow">in</span> <span class="n">date_list</span><span class="p">:</span> <span class="n">ticker_dates</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">ticker</span><span class="p">,</span> <span class="n">parse</span><span class="p">(</span><span class="n">iso_str</span><span class="p">)))</span> <span class="k">def</span> <span class="nf">does_trend_down</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">event</span><span class="p">,</span> <span class="n">horizon</span><span class="p">):</span> - <span class="c"># Figure out if the `event` has a downtrend for</span> - <span class="c"># the `horizon` days preceding it</span> - <span class="c"># As an interpretation note: it is assumed that</span> - <span class="c"># the closing price of day `event` is the reference</span> - <span class="c"># point, and we want `horizon` days before that.</span> - <span class="c"># The price_data.hdf was created in the second appendix code block</span> + <span class="c1"># Figure out if the `event` has a downtrend for</span> + <span class="c1"># the `horizon` days preceding it</span> + <span class="c1"># As an interpretation note: it is assumed that</span> + <span class="c1"># the closing price of day `event` is the reference</span> + <span class="c1"># point, and we want `horizon` days before that.</span> + <span class="c1"># The price_data.hdf was created in the second appendix code block</span> <span class="k">try</span><span class="p">:</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s1">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> <span class="n">data</span> <span class="o">=</span> <span class="n">ticker_data</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">):</span><span class="n">event</span><span class="p">]</span> - <span class="n">midpoints</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">data</span><span class="p">[</span><span class="s">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> + <span class="n">midpoints</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> - <span class="c"># Shift dates one forward into the future and subtract</span> - <span class="c"># Effectively: do we trend down over all days?</span> + <span class="c1"># Shift dates one forward into the future and subtract</span> + <span class="c1"># Effectively: do we trend down over all days?</span> <span class="n">elems</span> <span class="o">=</span> <span class="n">midpoints</span> <span class="o">-</span> <span class="n">midpoints</span><span class="o">.</span><span class="n">shift</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">elems</span><span class="p">)</span><span class="o">-</span><span class="mi">1</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">elems</span><span class="o">.</span><span class="n">dropna</span><span class="p">()[</span><span class="n">elems</span> <span class="o">&lt;=</span> <span class="mi">0</span><span class="p">])</span> <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span> - <span class="c"># If the stock doesn&#39;t exist, it doesn&#39;t qualify as trending down</span> - <span class="c"># Mostly this is here to make sure the entire analysis doesn&#39;t </span> - <span class="c"># blow up if there were issues in data retrieval</span> - <span class="k">return</span> <span class="k">False</span> + <span class="c1"># If the stock doesn&#39;t exist, it doesn&#39;t qualify as trending down</span> + <span class="c1"># Mostly this is here to make sure the entire analysis doesn&#39;t </span> + <span class="c1"># blow up if there were issues in data retrieval</span> + <span class="k">return</span> <span class="kc">False</span> <span class="k">def</span> <span class="nf">study_trend</span><span class="p">(</span><span class="n">horizon</span><span class="p">,</span> <span class="n">trend_function</span><span class="p">):</span> <span class="n">five_day_events</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span> <span class="n">horizon</span><span class="o">*</span><span class="mi">2</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span> <span class="n">invalid_events</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">ProgressBar</span><span class="p">()(</span><span class="n">ticker_dates</span><span class="p">):</span> <span class="k">if</span> <span class="n">trend_function</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">event</span><span class="p">,</span> <span class="n">horizon</span><span class="p">):</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> - <span class="n">event_data</span> <span class="o">=</span> <span class="n">ticker_data</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">):</span><span class="n">event</span><span class="o">+</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">)][</span><span class="s">&#39;Close&#39;</span><span class="p">]</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s1">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> + <span class="n">event_data</span> <span class="o">=</span> <span class="n">ticker_data</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">):</span><span class="n">event</span><span class="o">+</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">)][</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span> <span class="k">try</span><span class="p">:</span> <span class="n">five_day_events</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">([</span><span class="n">five_day_events</span><span class="p">,</span> <span class="n">event_data</span><span class="p">])</span> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> - <span class="c"># Sometimes we don&#39;t get exactly the right number of values due to calendar</span> - <span class="c"># issues. I&#39;ve fixed most everything I can, and the few issues that are left</span> - <span class="c"># I assume don&#39;t systemically bias the results (i.e. data could be missing</span> - <span class="c"># because it doesn&#39;t exist, etc.). After running through, ~1% of events get</span> - <span class="c"># discarded this way</span> + <span class="c1"># Sometimes we don&#39;t get exactly the right number of values due to calendar</span> + <span class="c1"># issues. I&#39;ve fixed most everything I can, and the few issues that are left</span> + <span class="c1"># I assume don&#39;t systemically bias the results (i.e. data could be missing</span> + <span class="c1"># because it doesn&#39;t exist, etc.). After running through, ~1% of events get</span> + <span class="c1"># discarded this way</span> <span class="n">invalid_events</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">ticker</span><span class="p">,</span> <span class="n">event</span><span class="p">))</span> - <span class="c"># Remove our initial zero row</span> + <span class="c1"># Remove our initial zero row</span> <span class="n">five_day_events</span> <span class="o">=</span> <span class="n">five_day_events</span><span class="p">[</span><span class="mi">1</span><span class="p">:,:]</span> <span class="n">plot_study</span><span class="p">(</span><span class="n">five_day_events</span><span class="p">)</span> - <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&#39;Action over {} days: {} events&#39;</span> + <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s1">&#39;Action over {} days: {} events&#39;</span> <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">horizon</span><span class="p">,</span><span class="n">five_day_events</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> -<span class="c"># Start with a 5 day study</span> +<span class="c1"># Start with a 5 day study</span> <span class="n">study_trend</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">does_trend_down</span><span class="p">)</span> </pre></div> @@ -2913,7 +3180,7 @@ ADBuCBoAAAAAAMC4+f+DLBbJVWQG2gAAAABJRU5ErkJggg== <div class="prompt input_prompt">In&nbsp;[8]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># 8 day study next</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># 8 day study next</span> <span class="n">study_trend</span><span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="n">does_trend_down</span><span class="p">)</span> </pre></div> @@ -3392,7 +3659,7 @@ AAAAKMr/B9pAH9lY/+K6AAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[9]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># 3 day study after that</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># 3 day study after that</span> <span class="n">study_trend</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">does_trend_down</span><span class="p">)</span> </pre></div> @@ -3805,26 +4072,26 @@ STfmKmb+uQ4AAAAASUVORK5CYII= <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">does_trend_up</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">event</span><span class="p">,</span> <span class="n">horizon</span><span class="p">):</span> - <span class="c"># Figure out if the `event` has an uptrend for</span> - <span class="c"># the `horizon` days preceding it</span> - <span class="c"># As an interpretation note: it is assumed that</span> - <span class="c"># the closing price of day `event` is the reference</span> - <span class="c"># point, and we want `horizon` days before that.</span> - <span class="c"># The price_data.hdf was created in the second appendix code block</span> + <span class="c1"># Figure out if the `event` has an uptrend for</span> + <span class="c1"># the `horizon` days preceding it</span> + <span class="c1"># As an interpretation note: it is assumed that</span> + <span class="c1"># the closing price of day `event` is the reference</span> + <span class="c1"># point, and we want `horizon` days before that.</span> + <span class="c1"># The price_data.hdf was created in the second appendix code block</span> <span class="k">try</span><span class="p">:</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s1">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> <span class="n">data</span> <span class="o">=</span> <span class="n">ticker_data</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">):</span><span class="n">event</span><span class="p">]</span> - <span class="n">midpoints</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">data</span><span class="p">[</span><span class="s">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> + <span class="n">midpoints</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> - <span class="c"># Shift dates one forward into the future and subtract</span> - <span class="c"># Effectively: do we trend down over all days?</span> + <span class="c1"># Shift dates one forward into the future and subtract</span> + <span class="c1"># Effectively: do we trend down over all days?</span> <span class="n">elems</span> <span class="o">=</span> <span class="n">midpoints</span> <span class="o">-</span> <span class="n">midpoints</span><span class="o">.</span><span class="n">shift</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">elems</span><span class="p">)</span><span class="o">-</span><span class="mi">1</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">elems</span><span class="o">.</span><span class="n">dropna</span><span class="p">()[</span><span class="n">elems</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">])</span> <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span> - <span class="c"># If the stock doesn&#39;t exist, it doesn&#39;t qualify as trending down</span> - <span class="c"># Mostly this is here to make sure the entire analysis doesn&#39;t </span> - <span class="c"># blow up if there were issues in data retrieval</span> - <span class="k">return</span> <span class="k">False</span> + <span class="c1"># If the stock doesn&#39;t exist, it doesn&#39;t qualify as trending down</span> + <span class="c1"># Mostly this is here to make sure the entire analysis doesn&#39;t </span> + <span class="c1"># blow up if there were issues in data retrieval</span> + <span class="k">return</span> <span class="kc">False</span> <span class="n">study_trend</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">does_trend_up</span><span class="p">)</span> </pre></div> @@ -5105,38 +5372,38 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="bp">self</span><span class="o">.</span><span class="n">dates</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">def</span> <span class="nf">handle_starttag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span> - <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;table&#39;</span><span class="p">:</span> + <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;table&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_dates</span> <span class="o">=</span> <span class="bp">True</span> <span class="k">def</span> <span class="nf">handle_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_dates</span><span class="p">:</span> - <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s">r&#39;\d+/\d+/\d+&#39;</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span> + <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">r&#39;\d+/\d+/\d+&#39;</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span> <span class="k">if</span> <span class="n">match</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">dates</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span> - <span class="c"># If a company reports before the bell, record the earnings date</span> - <span class="c"># being at midnight the day before. Ex: WMT reports 5/19/2016,</span> - <span class="c"># but we want the reference point to be the closing price on 5/18/2016</span> - <span class="k">if</span> <span class="s">&#39;After Close&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span> + <span class="c1"># If a company reports before the bell, record the earnings date</span> + <span class="c1"># being at midnight the day before. Ex: WMT reports 5/19/2016,</span> + <span class="c1"># but we want the reference point to be the closing price on 5/18/2016</span> + <span class="k">if</span> <span class="s1">&#39;After Close&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">earnings_offset</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> - <span class="k">elif</span> <span class="s">&#39;Before Open&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span> + <span class="k">elif</span> <span class="s1">&#39;Before Open&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">earnings_offset</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span> <span class="k">def</span> <span class="nf">handle_endtag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">):</span> - <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;table&#39;</span><span class="p">:</span> + <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;table&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_dates</span> <span class="o">=</span> <span class="bp">False</span> <span class="k">def</span> <span class="nf">earnings_releases</span><span class="p">(</span><span class="n">ticker</span><span class="p">):</span> - <span class="c">#print(&quot;Looking up ticker {}&quot;.format(ticker))</span> - <span class="n">user_agent</span> <span class="o">=</span> <span class="s">&#39;Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) &#39;</span>\ - <span class="s">&#39;Gecko/20100101 Firefox/46.0&#39;</span> - <span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s">&#39;user-agent&#39;</span><span class="p">:</span> <span class="n">user_agent</span><span class="p">}</span> - <span class="n">base_url</span> <span class="o">=</span> <span class="s">&#39;http://www.streetinsider.com/ec_earnings.php?q={}&#39;</span>\ + <span class="c1">#print(&quot;Looking up ticker {}&quot;.format(ticker))</span> + <span class="n">user_agent</span> <span class="o">=</span> <span class="s1">&#39;Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) &#39;</span>\ + <span class="s1">&#39;Gecko/20100101 Firefox/46.0&#39;</span> + <span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;user-agent&#39;</span><span class="p">:</span> <span class="n">user_agent</span><span class="p">}</span> + <span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;http://www.streetinsider.com/ec_earnings.php?q={}&#39;</span>\ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ticker</span><span class="p">)</span> <span class="n">e</span> <span class="o">=</span> <span class="n">EarningsParser</span><span class="p">()</span> <span class="n">s</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span> <span class="n">a</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">adapters</span><span class="o">.</span><span class="n">HTTPAdapter</span><span class="p">(</span><span class="n">max_retries</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> - <span class="n">s</span><span class="o">.</span><span class="n">mount</span><span class="p">(</span><span class="s">&#39;http://&#39;</span><span class="p">,</span> <span class="n">a</span><span class="p">)</span> + <span class="n">s</span><span class="o">.</span><span class="n">mount</span><span class="p">(</span><span class="s1">&#39;http://&#39;</span><span class="p">,</span> <span class="n">a</span><span class="p">)</span> <span class="n">e</span><span class="o">.</span><span class="n">feed</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">s</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">base_url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span><span class="o">.</span><span class="n">content</span><span class="p">))</span> <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">earnings_offset</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span> @@ -5144,18 +5411,18 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="n">past</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="o">&lt;</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(),</span> <span class="n">dates</span><span class="p">)</span> <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span><span class="o">.</span><span class="n">isoformat</span><span class="p">(),</span> <span class="n">past</span><span class="p">))</span> -<span class="c"># Use a Russell-3000 ETF tracker (ticker IWV) to get a list of holdings</span> -<span class="n">r3000</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s">&#39;https://www.ishares.com/us/products/239714/&#39;</span> - <span class="s">&#39;ishares-russell-3000-etf/1449138789749.ajax?&#39;</span> - <span class="s">&#39;fileType=csv&amp;fileName=IWV_holdings&amp;dataType=fund&#39;</span><span class="p">,</span> +<span class="c1"># Use a Russell-3000 ETF tracker (ticker IWV) to get a list of holdings</span> +<span class="n">r3000</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;https://www.ishares.com/us/products/239714/&#39;</span> + <span class="s1">&#39;ishares-russell-3000-etf/1449138789749.ajax?&#39;</span> + <span class="s1">&#39;fileType=csv&amp;fileName=IWV_holdings&amp;dataType=fund&#39;</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span> -<span class="n">r3000_equities</span> <span class="o">=</span> <span class="n">r3000</span><span class="p">[(</span><span class="n">r3000</span><span class="p">[</span><span class="s">&#39;Exchange&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;NASDAQ&#39;</span><span class="p">)</span> <span class="o">|</span> - <span class="p">(</span><span class="n">r3000</span><span class="p">[</span><span class="s">&#39;Exchange&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;New York Stock Exchange Inc.&#39;</span><span class="p">)]</span> +<span class="n">r3000_equities</span> <span class="o">=</span> <span class="n">r3000</span><span class="p">[(</span><span class="n">r3000</span><span class="p">[</span><span class="s1">&#39;Exchange&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;NASDAQ&#39;</span><span class="p">)</span> <span class="o">|</span> + <span class="p">(</span><span class="n">r3000</span><span class="p">[</span><span class="s1">&#39;Exchange&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;New York Stock Exchange Inc.&#39;</span><span class="p">)]</span> -<span class="n">dates_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s">&#39;w&#39;</span><span class="p">)</span> +<span class="n">dates_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">with</span> <span class="n">futures</span><span class="o">.</span><span class="n">ThreadPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="mi">8</span><span class="p">)</span> <span class="k">as</span> <span class="n">pool</span><span class="p">:</span> - <span class="n">fs</span> <span class="o">=</span> <span class="p">{</span><span class="n">pool</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">earnings_releases</span><span class="p">,</span> <span class="n">r3000_equities</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">t</span><span class="p">][</span><span class="s">&#39;Ticker&#39;</span><span class="p">]):</span> <span class="n">t</span> + <span class="n">fs</span> <span class="o">=</span> <span class="p">{</span><span class="n">pool</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">earnings_releases</span><span class="p">,</span> <span class="n">r3000_equities</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">t</span><span class="p">][</span><span class="s1">&#39;Ticker&#39;</span><span class="p">]):</span> <span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">r3000_equities</span><span class="o">.</span><span class="n">index</span><span class="p">}</span> <span class="n">pbar</span> <span class="o">=</span> <span class="n">progressbar</span><span class="o">.</span><span class="n">ProgressBar</span><span class="p">(</span><span class="n">term_width</span><span class="o">=</span><span class="mi">80</span><span class="p">,</span> <span class="n">max_value</span><span class="o">=</span><span class="n">r3000_equities</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">max</span><span class="p">())</span> @@ -5163,7 +5430,7 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="o">.</span><span class="n">as_completed</span><span class="p">(</span><span class="n">fs</span><span class="p">):</span> <span class="n">i</span> <span class="o">=</span> <span class="n">fs</span><span class="p">[</span><span class="n">future</span><span class="p">]</span> <span class="n">pbar</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> - <span class="n">dates_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">yaml</span><span class="o">.</span><span class="n">dump</span><span class="p">({</span><span class="n">r3000_equities</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s">&#39;Ticker&#39;</span><span class="p">]:</span> + <span class="n">dates_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">yaml</span><span class="o">.</span><span class="n">dump</span><span class="p">({</span><span class="n">r3000_equities</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">&#39;Ticker&#39;</span><span class="p">]:</span> <span class="n">future</span><span class="o">.</span><span class="n">result</span><span class="p">()}))</span> </pre></div> <p>Downloading stock price data needed for the event studies:</p> @@ -5176,31 +5443,31 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="kn">from</span> <span class="nn">progressbar</span> <span class="kn">import</span> <span class="n">ProgressBar</span> <span class="k">def</span> <span class="nf">fetch_ticker</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">):</span> - <span class="c"># Quandl is currently giving me issues with returning</span> - <span class="c"># the entire dataset and not slicing server-side.</span> - <span class="c"># So instead, we&#39;ll do it client-side!</span> - <span class="n">q_format</span> <span class="o">=</span> <span class="s">&#39;%Y-%m-</span><span class="si">%d</span><span class="s">&#39;</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&#39;YAHOO/&#39;</span> <span class="o">+</span> <span class="n">ticker</span><span class="p">,</span> + <span class="c1"># Quandl is currently giving me issues with returning</span> + <span class="c1"># the entire dataset and not slicing server-side.</span> + <span class="c1"># So instead, we&#39;ll do it client-side!</span> + <span class="n">q_format</span> <span class="o">=</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;YAHOO/&#39;</span> <span class="o">+</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">start_date</span><span class="o">=</span><span class="n">start</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">q_format</span><span class="p">),</span> <span class="n">end_date</span><span class="o">=</span><span class="n">end</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">q_format</span><span class="p">),</span> <span class="n">authtoken</span><span class="o">=</span><span class="n">QUANDL_KEY</span><span class="p">)</span> <span class="k">return</span> <span class="n">ticker_data</span> -<span class="n">data_str</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> -<span class="c"># Need to remove invalid lines</span> -<span class="n">filtered</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s">&#39;{&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">x</span><span class="p">,</span> <span class="n">data_str</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">&#39;</span><span class="se">\n</span><span class="s">&#39;</span><span class="p">))</span> -<span class="n">earnings_data</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s">&#39;</span><span class="se">\n</span><span class="s">&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">filtered</span><span class="p">))</span> +<span class="n">data_str</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> +<span class="c1"># Need to remove invalid lines</span> +<span class="n">filtered</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s1">&#39;{&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">x</span><span class="p">,</span> <span class="n">data_str</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">))</span> +<span class="n">earnings_data</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">filtered</span><span class="p">))</span> -<span class="c"># Get the first 1500 keys - split up into two statements</span> -<span class="c"># because of Quandl rate limits</span> +<span class="c1"># Get the first 1500 keys - split up into two statements</span> +<span class="c1"># because of Quandl rate limits</span> <span class="n">tickers</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">earnings_data</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> <span class="n">price_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">invalid_tickers</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">ticker</span> <span class="ow">in</span> <span class="n">ProgressBar</span><span class="p">()(</span><span class="n">tickers</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="mi">1500</span><span class="p">]):</span> <span class="k">try</span><span class="p">:</span> - <span class="c"># Replace &#39;.&#39; with &#39;-&#39; in name for some tickers</span> - <span class="n">fixed</span> <span class="o">=</span> <span class="n">ticker</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s">&#39;.&#39;</span><span class="p">,</span> <span class="s">&#39;-&#39;</span><span class="p">)</span> + <span class="c1"># Replace &#39;.&#39; with &#39;-&#39; in name for some tickers</span> + <span class="n">fixed</span> <span class="o">=</span> <span class="n">ticker</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;-&#39;</span><span class="p">)</span> <span class="n">event_strs</span> <span class="o">=</span> <span class="n">earnings_data</span><span class="p">[</span><span class="n">ticker</span><span class="p">]</span> <span class="n">events</span> <span class="o">=</span> <span class="p">[</span><span class="n">parse</span><span class="p">(</span><span class="n">event</span><span class="p">)</span> <span class="k">for</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">event_strs</span><span class="p">]</span> <span class="n">td</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span> @@ -5209,11 +5476,11 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="k">except</span> <span class="n">quandl</span><span class="o">.</span><span class="n">NotFoundError</span><span class="p">:</span> <span class="n">invalid_tickers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ticker</span><span class="p">)</span> -<span class="c"># Execute this after 10 minutes have passed</span> +<span class="c1"># Execute this after 10 minutes have passed</span> <span class="k">for</span> <span class="n">ticker</span> <span class="ow">in</span> <span class="n">ProgressBar</span><span class="p">()(</span><span class="n">tickers</span><span class="p">[</span><span class="mi">1500</span><span class="p">:]):</span> <span class="k">try</span><span class="p">:</span> - <span class="c"># Replace &#39;.&#39; with &#39;-&#39; in name for some tickers</span> - <span class="n">fixed</span> <span class="o">=</span> <span class="n">ticker</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s">&#39;.&#39;</span><span class="p">,</span> <span class="s">&#39;-&#39;</span><span class="p">)</span> + <span class="c1"># Replace &#39;.&#39; with &#39;-&#39; in name for some tickers</span> + <span class="n">fixed</span> <span class="o">=</span> <span class="n">ticker</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;-&#39;</span><span class="p">)</span> <span class="n">event_strs</span> <span class="o">=</span> <span class="n">earnings_data</span><span class="p">[</span><span class="n">ticker</span><span class="p">]</span> <span class="n">events</span> <span class="o">=</span> <span class="p">[</span><span class="n">parse</span><span class="p">(</span><span class="n">event</span><span class="p">)</span> <span class="k">for</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">event_strs</span><span class="p">]</span> <span class="n">td</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span> @@ -5222,7 +5489,7 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="k">except</span> <span class="n">quandl</span><span class="o">.</span><span class="n">NotFoundError</span><span class="p">:</span> <span class="n">invalid_tickers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ticker</span><span class="p">)</span> -<span class="n">prices_store</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">HDFStore</span><span class="p">(</span><span class="s">&#39;price_data.hdf&#39;</span><span class="p">)</span> +<span class="n">prices_store</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">HDFStore</span><span class="p">(</span><span class="s1">&#39;price_data.hdf&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">prices</span> <span class="ow">in</span> <span class="n">price_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> <span class="n">prices_store</span><span class="p">[</span><span class="n">ticker</span><span class="p">]</span> <span class="o">=</span> <span class="n">prices</span> </pre></div> @@ -5235,7 +5502,7 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>The Unfair Casino2016-05-15T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-05-15:the-unfair-casino.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>The Unfair Casino2016-05-15T00:00:00-04:002016-05-15T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-05-15:the-unfair-casino.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -5263,8 +5530,8 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <span class="n">fair_1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> <span class="n">fair_2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> -<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">fair_1</span> <span class="o">+</span> <span class="n">fair_2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;Fair Distribution&#39;</span><span class="p">);</span> +<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">fair_1</span> <span class="o">+</span> <span class="n">fair_2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s1">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;Fair Distribution&#39;</span><span class="p">);</span> </pre></div> </div> @@ -5475,8 +5742,8 @@ D_2 = \left\{ <span class="n">x1</span> <span class="o">=</span> <span class="n">unfair_die</span><span class="p">(</span><span class="n">d1</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> <span class="n">x2</span> <span class="o">=</span> <span class="n">unfair_die</span><span class="p">(</span><span class="n">d2</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> -<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">x1</span> <span class="o">+</span> <span class="n">x2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;$D_1$ biased low, $D_2$ biased high&#39;</span><span class="p">);</span> +<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">x1</span> <span class="o">+</span> <span class="n">x2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s1">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;$D_1$ biased low, $D_2$ biased high&#39;</span><span class="p">);</span> </pre></div> </div> @@ -5674,8 +5941,8 @@ D_2 = \left\{ <span class="n">x1</span> <span class="o">=</span> <span class="n">unfair_die</span><span class="p">(</span><span class="n">d1</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> <span class="n">x2</span> <span class="o">=</span> <span class="n">unfair_die</span><span class="p">(</span><span class="n">d2</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> -<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">x1</span> <span class="o">+</span> <span class="n">x2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&quot;$D_1$ and $D_2$ biased to 1 and 6&quot;</span><span class="p">);</span> +<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">x1</span> <span class="o">+</span> <span class="n">x2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s1">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s2">&quot;$D_1$ and $D_2$ biased to 1 and 6&quot;</span><span class="p">);</span> </pre></div> </div> @@ -5895,18 +6162,18 @@ and continue the next iteration.</li> <span class="kn">from</span> <span class="nn">numpy</span> <span class="k">import</span> <span class="n">polynomial</span> <span class="k">def</span> <span class="nf">density_coef</span><span class="p">(</span><span class="n">d1_density</span><span class="p">,</span> <span class="n">d2_density</span><span class="p">):</span> - <span class="c"># Calculating the probabilities of each outcome was taken</span> - <span class="c"># from this author: http://math.stackexchange.com/a/1710392/320784</span> + <span class="c1"># Calculating the probabilities of each outcome was taken</span> + <span class="c1"># from this author: http://math.stackexchange.com/a/1710392/320784</span> <span class="n">d1_p</span> <span class="o">=</span> <span class="n">polynomial</span><span class="o">.</span><span class="n">Polynomial</span><span class="p">(</span><span class="n">d1_density</span><span class="p">)</span> <span class="n">d2_p</span> <span class="o">=</span> <span class="n">polynomial</span><span class="o">.</span><span class="n">Polynomial</span><span class="p">(</span><span class="n">d2_density</span><span class="p">)</span> <span class="n">coefs</span> <span class="o">=</span> <span class="p">(</span><span class="n">d1_p</span> <span class="o">*</span> <span class="n">d2_p</span><span class="p">)</span><span class="o">.</span><span class="n">coef</span> <span class="k">return</span> <span class="n">coefs</span> <span class="k">def</span> <span class="nf">score</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">d1_density</span><span class="p">,</span> <span class="n">d2_density</span><span class="p">):</span> - <span class="c"># We&#39;ve now got the probabilities of each event, but we need</span> - <span class="c"># to shift the array a bit so we can use the x values to actually</span> - <span class="c"># index into it. This will allow us to do all the calculations</span> - <span class="c"># incredibly quickly</span> + <span class="c1"># We&#39;ve now got the probabilities of each event, but we need</span> + <span class="c1"># to shift the array a bit so we can use the x values to actually</span> + <span class="c1"># index into it. This will allow us to do all the calculations</span> + <span class="c1"># incredibly quickly</span> <span class="n">coefs</span> <span class="o">=</span> <span class="n">density_coef</span><span class="p">(</span><span class="n">d1_density</span><span class="p">,</span> <span class="n">d2_density</span><span class="p">)</span> <span class="n">coefs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">hstack</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">coefs</span><span class="p">))</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">coefs</span><span class="p">[</span><span class="n">x</span><span class="p">])</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> @@ -5933,23 +6200,23 @@ and continue the next iteration.</li> <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">permute</span><span class="p">(</span><span class="n">d1_density</span><span class="p">,</span> <span class="n">d2_density</span><span class="p">):</span> - <span class="c"># To ensure we have legitimate densities, we will randomly</span> - <span class="c"># increase one die face probability by `change`,</span> - <span class="c"># and decrease one by `change`.</span> - <span class="c"># This means there are something less than (1/`change`)^12 possibilities</span> - <span class="c"># we are trying to search over.</span> + <span class="c1"># To ensure we have legitimate densities, we will randomly</span> + <span class="c1"># increase one die face probability by `change`,</span> + <span class="c1"># and decrease one by `change`.</span> + <span class="c1"># This means there are something less than (1/`change`)^12 possibilities</span> + <span class="c1"># we are trying to search over.</span> <span class="n">change</span> <span class="o">=</span> <span class="o">.</span><span class="mi">01</span> <span class="n">d1_index1</span><span class="p">,</span> <span class="n">d1_index2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span> <span class="n">d2_index1</span><span class="p">,</span> <span class="n">d2_index2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span> - <span class="c"># Also make sure to copy. I&#39;ve had some weird aliasing issues</span> - <span class="c"># in the past that made everything blow up.</span> + <span class="c1"># Also make sure to copy. I&#39;ve had some weird aliasing issues</span> + <span class="c1"># in the past that made everything blow up.</span> <span class="n">new_d1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">d1_density</span><span class="p">))</span> <span class="n">new_d2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">d2_density</span><span class="p">))</span> - <span class="c"># While this doesn&#39;t account for the possibility that some</span> - <span class="c"># values go negative, in practice this never happens</span> + <span class="c1"># While this doesn&#39;t account for the possibility that some</span> + <span class="c1"># values go negative, in practice this never happens</span> <span class="n">new_d1</span><span class="p">[</span><span class="n">d1_index1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">change</span> <span class="n">new_d1</span><span class="p">[</span><span class="n">d1_index2</span><span class="p">]</span> <span class="o">-=</span> <span class="n">change</span> <span class="n">new_d2</span><span class="p">[</span><span class="n">d2_index1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">change</span> @@ -5982,24 +6249,24 @@ and continue the next iteration.</li> <span class="n">switch_failures</span> <span class="o">=</span> <span class="mi">0</span> <span class="n">iter_count</span> <span class="o">=</span> <span class="mi">0</span> - <span class="c"># Start with guessing fair dice</span> + <span class="c1"># Start with guessing fair dice</span> <span class="n">cur_d1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="mi">1</span><span class="o">/</span><span class="mi">6</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> <span class="n">cur_d2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="mi">1</span><span class="o">/</span><span class="mi">6</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> <span class="n">cur_score</span> <span class="o">=</span> <span class="n">score</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">cur_d1</span><span class="p">,</span> <span class="n">cur_d2</span><span class="p">)</span> - <span class="c"># Keep track of our best guesses - may not be</span> - <span class="c"># what we end with</span> + <span class="c1"># Keep track of our best guesses - may not be</span> + <span class="c1"># what we end with</span> <span class="n">max_score</span> <span class="o">=</span> <span class="n">cur_score</span> <span class="n">max_d1</span> <span class="o">=</span> <span class="n">cur_d1</span> <span class="n">max_d2</span> <span class="o">=</span> <span class="n">cur_d2</span> - <span class="c"># Optimization stops when we have failed to switch `conv_count`</span> - <span class="c"># times (presumably because we have a great guess), or we reach</span> - <span class="c"># the maximum number of iterations.</span> + <span class="c1"># Optimization stops when we have failed to switch `conv_count`</span> + <span class="c1"># times (presumably because we have a great guess), or we reach</span> + <span class="c1"># the maximum number of iterations.</span> <span class="k">while</span> <span class="n">switch_failures</span> <span class="o">&lt;</span> <span class="n">conv_count</span> <span class="ow">and</span> <span class="n">iter_count</span> <span class="o">&lt;</span> <span class="n">max_iter</span><span class="p">:</span> <span class="n">iter_count</span> <span class="o">+=</span> <span class="mi">1</span> <span class="k">if</span> <span class="n">iter_count</span> <span class="o">%</span> <span class="p">(</span><span class="n">max_iter</span> <span class="o">/</span> <span class="mi">10</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> - <span class="nb">print</span><span class="p">(</span><span class="s">&#39;Iteration: {}; Current score (higher is better): {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> + <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Iteration: {}; Current score (higher is better): {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> <span class="n">iter_count</span><span class="p">,</span> <span class="n">cur_score</span><span class="p">))</span> <span class="n">new_d1</span><span class="p">,</span> <span class="n">new_d2</span> <span class="o">=</span> <span class="n">permute</span><span class="p">(</span><span class="n">cur_d1</span><span class="p">,</span> <span class="n">cur_d2</span><span class="p">)</span> @@ -6011,19 +6278,19 @@ and continue the next iteration.</li> <span class="n">max_d2</span> <span class="o">=</span> <span class="n">new_d2</span> <span class="k">if</span> <span class="n">new_score</span> <span class="o">&gt;</span> <span class="n">cur_score</span><span class="p">:</span> - <span class="c"># If the new permutation beats the old one,</span> - <span class="c"># automatically select it.</span> + <span class="c1"># If the new permutation beats the old one,</span> + <span class="c1"># automatically select it.</span> <span class="n">cur_score</span> <span class="o">=</span> <span class="n">new_score</span> <span class="n">cur_d1</span> <span class="o">=</span> <span class="n">new_d1</span> <span class="n">cur_d2</span> <span class="o">=</span> <span class="n">new_d2</span> <span class="n">switch_failures</span> <span class="o">=</span> <span class="mi">0</span> <span class="k">else</span><span class="p">:</span> - <span class="c"># We didn&#39;t beat the current score, but allow</span> - <span class="c"># for possibly switching anyways.</span> + <span class="c1"># We didn&#39;t beat the current score, but allow</span> + <span class="c1"># for possibly switching anyways.</span> <span class="n">accept_prob</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">new_score</span> <span class="o">-</span> <span class="n">cur_score</span><span class="p">)</span> <span class="n">coin_toss</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">()</span> <span class="k">if</span> <span class="n">coin_toss</span> <span class="o">&lt;</span> <span class="n">accept_prob</span><span class="p">:</span> - <span class="c"># We randomly switch to the new distribution</span> + <span class="c1"># We randomly switch to the new distribution</span> <span class="n">cur_score</span> <span class="o">=</span> <span class="n">new_score</span> <span class="n">cur_d1</span> <span class="o">=</span> <span class="n">new_d1</span> <span class="n">cur_d2</span> <span class="o">=</span> <span class="n">new_d2</span> @@ -6031,7 +6298,7 @@ and continue the next iteration.</li> <span class="k">else</span><span class="p">:</span> <span class="n">switch_failures</span> <span class="o">+=</span> <span class="mi">1</span> - <span class="c"># Return both our best guess, and the ending guess</span> + <span class="c1"># Return both our best guess, and the ending guess</span> <span class="k">return</span> <span class="n">max_d1</span><span class="p">,</span> <span class="n">max_d2</span><span class="p">,</span> <span class="n">cur_d1</span><span class="p">,</span> <span class="n">cur_d2</span> </pre></div> @@ -6065,20 +6332,20 @@ and continue the next iteration.</li> <span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">perf_counter</span><span class="p">()</span> <span class="n">max_d1</span><span class="p">,</span> <span class="n">max_d2</span><span class="p">,</span> <span class="n">final_d1</span><span class="p">,</span> <span class="n">final_d2</span> <span class="o">=</span> <span class="n">optimize</span><span class="p">(</span><span class="n">d1_vals</span> <span class="o">+</span> <span class="n">d2_vals</span><span class="p">)</span> <span class="n">end</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">perf_counter</span><span class="p">()</span> - <span class="nb">print</span><span class="p">(</span><span class="s">&quot;Simulated Annealing time: {:.02f}s&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">start</span><span class="p">))</span> + <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Simulated Annealing time: {:.02f}s&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">start</span><span class="p">))</span> <span class="n">coef_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">13</span><span class="p">)</span> <span class="o">-</span> <span class="o">.</span><span class="mi">5</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">221</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">bar</span><span class="p">(</span><span class="n">coef_range</span><span class="p">,</span> <span class="n">density_coef</span><span class="p">(</span><span class="n">d1_dist</span><span class="p">,</span> <span class="n">d2_dist</span><span class="p">),</span> <span class="n">width</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> - <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;True Distribution&#39;</span><span class="p">)</span> + <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;True Distribution&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">222</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">hist</span><span class="p">(</span><span class="n">d1_vals</span> <span class="o">+</span> <span class="n">d2_vals</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">)</span> - <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;Empirical Distribution&#39;</span><span class="p">)</span> + <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;Empirical Distribution&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">223</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">bar</span><span class="p">(</span><span class="n">coef_range</span><span class="p">,</span> <span class="n">density_coef</span><span class="p">(</span><span class="n">max_d1</span><span class="p">,</span> <span class="n">max_d2</span><span class="p">),</span> <span class="n">width</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> - <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;Recovered Distribution&#39;</span><span class="p">)</span> + <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;Recovered Distribution&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span> @@ -7963,7 +8230,7 @@ JDVg6JIkSWrA0CVJktSAoUuSJKkBQ5ckSVIDhi5JkqQG/gdFMjKH5D5DewAAAABJRU5ErkJggg== MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Tick Tock...2016-04-06T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-04-06:tick-tock.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Tick Tock...2016-04-06T00:00:00-04:002016-04-06T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-04-06:tick-tock.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -8014,7 +8281,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <div class="prompt input_prompt">In&nbsp;[2]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># Import all the OAuth secret information from a local file</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># Import all the OAuth secret information from a local file</span> <span class="kn">from</span> <span class="nn">secrets</span> <span class="k">import</span> <span class="n">CLIENT_SECRET</span><span class="p">,</span> <span class="n">CLIENT_ID</span><span class="p">,</span> <span class="n">CALLBACK_URL</span> </pre></div> @@ -8055,17 +8322,17 @@ Application: Thank you, now give me all the data.</code></pre> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="kn">import</span> <span class="nn">urllib</span> -<span class="n">FITBIT_URI</span> <span class="o">=</span> <span class="s">&#39;https://www.fitbit.com/oauth2/authorize&#39;</span> +<span class="n">FITBIT_URI</span> <span class="o">=</span> <span class="s1">&#39;https://www.fitbit.com/oauth2/authorize&#39;</span> <span class="n">params</span> <span class="o">=</span> <span class="p">{</span> - <span class="c"># If we need more than one scope, must be a CSV string</span> - <span class="s">&#39;scope&#39;</span><span class="p">:</span> <span class="s">&#39;heartrate&#39;</span><span class="p">,</span> - <span class="s">&#39;response_type&#39;</span><span class="p">:</span> <span class="s">&#39;token&#39;</span><span class="p">,</span> - <span class="s">&#39;expires_in&#39;</span><span class="p">:</span> <span class="mi">86400</span><span class="p">,</span> <span class="c"># 1 day</span> - <span class="s">&#39;redirect_uri&#39;</span><span class="p">:</span> <span class="n">CALLBACK_URL</span><span class="p">,</span> - <span class="s">&#39;client_id&#39;</span><span class="p">:</span> <span class="n">CLIENT_ID</span> + <span class="c1"># If we need more than one scope, must be a CSV string</span> + <span class="s1">&#39;scope&#39;</span><span class="p">:</span> <span class="s1">&#39;heartrate&#39;</span><span class="p">,</span> + <span class="s1">&#39;response_type&#39;</span><span class="p">:</span> <span class="s1">&#39;token&#39;</span><span class="p">,</span> + <span class="s1">&#39;expires_in&#39;</span><span class="p">:</span> <span class="mi">86400</span><span class="p">,</span> <span class="c1"># 1 day</span> + <span class="s1">&#39;redirect_uri&#39;</span><span class="p">:</span> <span class="n">CALLBACK_URL</span><span class="p">,</span> + <span class="s1">&#39;client_id&#39;</span><span class="p">:</span> <span class="n">CLIENT_ID</span> <span class="p">}</span> -<span class="n">request_url</span> <span class="o">=</span> <span class="n">FITBIT_URI</span> <span class="o">+</span> <span class="s">&#39;?&#39;</span> <span class="o">+</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlencode</span><span class="p">(</span><span class="n">params</span><span class="p">)</span> +<span class="n">request_url</span> <span class="o">=</span> <span class="n">FITBIT_URI</span> <span class="o">+</span> <span class="s1">&#39;?&#39;</span> <span class="o">+</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlencode</span><span class="p">(</span><span class="n">params</span><span class="p">)</span> </pre></div> </div> @@ -8088,10 +8355,10 @@ Application: Thank you, now give me all the data.</code></pre> <div class="prompt input_prompt">In&nbsp;[6]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># The `response_url` variable contains the full URL that</span> -<span class="c"># FitBit sent back to us, but most importantly,</span> -<span class="c"># contains the token we need for authorization.</span> -<span class="n">access_token</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">parse_qsl</span><span class="p">(</span><span class="n">response_url</span><span class="p">))[</span><span class="s">&#39;access_token&#39;</span><span class="p">]</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># The `response_url` variable contains the full URL that</span> +<span class="c1"># FitBit sent back to us, but most importantly,</span> +<span class="c1"># contains the token we need for authorization.</span> +<span class="n">access_token</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">parse_qsl</span><span class="p">(</span><span class="n">response_url</span><span class="p">))[</span><span class="s1">&#39;access_token&#39;</span><span class="p">]</span> </pre></div> </div> @@ -8120,41 +8387,41 @@ Application: Thank you, now give me all the data.</code></pre> <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span> <span class="n">session</span> <span class="o">=</span> <span class="n">OAuth2Session</span><span class="p">(</span><span class="n">token</span><span class="o">=</span><span class="p">{</span> - <span class="s">&#39;access_token&#39;</span><span class="p">:</span> <span class="n">access_token</span><span class="p">,</span> - <span class="s">&#39;token_type&#39;</span><span class="p">:</span> <span class="s">&#39;Bearer&#39;</span> + <span class="s1">&#39;access_token&#39;</span><span class="p">:</span> <span class="n">access_token</span><span class="p">,</span> + <span class="s1">&#39;token_type&#39;</span><span class="p">:</span> <span class="s1">&#39;Bearer&#39;</span> <span class="p">})</span> -<span class="n">format_str</span> <span class="o">=</span> <span class="s">&#39;%Y-%m-%d&#39;</span> +<span class="n">format_str</span> <span class="o">=</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span> <span class="n">start_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="n">end_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">31</span><span class="p">)</span> <span class="n">dr</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">date_range</span><span class="p">(</span><span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="p">)</span> -<span class="n">url</span> <span class="o">=</span> <span class="s">&#39;https://api.fitbit.com/1/user/-/activities/heart/date/{0}/1d/1min.json&#39;</span> +<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://api.fitbit.com/1/user/-/activities/heart/date/{0}/1d/1min.json&#39;</span> <span class="n">hr_responses</span> <span class="o">=</span> <span class="p">[</span><span class="n">session</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">format_str</span><span class="p">)))</span> <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">dr</span><span class="p">]</span> <span class="k">def</span> <span class="nf">record_to_df</span><span class="p">(</span><span class="n">record</span><span class="p">):</span> - <span class="k">if</span> <span class="s">&#39;activities-heart&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">record</span><span class="p">:</span> - <span class="k">return</span> <span class="k">None</span> - <span class="n">date_str</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s">&#39;activities-heart&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">][</span><span class="s">&#39;dateTime&#39;</span><span class="p">]</span> - <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s">&#39;activities-heart-intraday&#39;</span><span class="p">][</span><span class="s">&#39;dataset&#39;</span><span class="p">])</span> + <span class="k">if</span> <span class="s1">&#39;activities-heart&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">record</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">None</span> + <span class="n">date_str</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s1">&#39;activities-heart&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;dateTime&#39;</span><span class="p">]</span> + <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s1">&#39;activities-heart-intraday&#39;</span><span class="p">][</span><span class="s1">&#39;dataset&#39;</span><span class="p">])</span> - <span class="n">df</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s">&#39;time&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span> - <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">date_str</span> <span class="o">+</span> <span class="s">&#39; &#39;</span> <span class="o">+</span> <span class="n">x</span><span class="p">,</span> <span class="s">&#39;%Y-%m-%d %H:%M:%S&#39;</span><span class="p">))</span> + <span class="n">df</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s1">&#39;time&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span> + <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">date_str</span> <span class="o">+</span> <span class="s1">&#39; &#39;</span> <span class="o">+</span> <span class="n">x</span><span class="p">,</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1"> %H:%M:%S&#39;</span><span class="p">))</span> <span class="k">return</span> <span class="n">df</span> <span class="n">hr_dataframes</span> <span class="o">=</span> <span class="p">[</span><span class="n">record_to_df</span><span class="p">(</span><span class="n">record</span><span class="o">.</span><span class="n">json</span><span class="p">())</span> <span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">hr_responses</span><span class="p">]</span> <span class="n">hr_df_concat</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">(</span><span class="n">hr_dataframes</span><span class="p">)</span> -<span class="c"># There are some minutes with missing data, so we need to correct that</span> +<span class="c1"># There are some minutes with missing data, so we need to correct that</span> <span class="n">full_daterange</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">date_range</span><span class="p">(</span><span class="n">hr_df_concat</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">hr_df_concat</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;min&#39;</span><span class="p">)</span> -<span class="n">hr_df_full</span> <span class="o">=</span> <span class="n">hr_df_concat</span><span class="o">.</span><span class="n">reindex</span><span class="p">(</span><span class="n">full_daterange</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s">&#39;nearest&#39;</span><span class="p">)</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;min&#39;</span><span class="p">)</span> +<span class="n">hr_df_full</span> <span class="o">=</span> <span class="n">hr_df_concat</span><span class="o">.</span><span class="n">reindex</span><span class="p">(</span><span class="n">full_daterange</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s1">&#39;nearest&#39;</span><span class="p">)</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Heartbeats from {} to {}: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Heartbeats from {} to {}: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> - <span class="n">hr_df_full</span><span class="p">[</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()))</span> + <span class="n">hr_df_full</span><span class="p">[</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()))</span> </pre></div> </div> @@ -8202,9 +8469,9 @@ hr_q \cdot n - hr_d \cdot (n-m) <div class="prompt input_prompt">In&nbsp;[8]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">quarterly_count</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">quarterly_count</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> <span class="n">leap_day_count</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">month</span> <span class="o">==</span> <span class="mi">2</span><span class="p">)</span> <span class="o">&amp;</span> - <span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">day</span> <span class="o">==</span> <span class="mi">29</span><span class="p">)][</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> + <span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">day</span> <span class="o">==</span> <span class="mi">29</span><span class="p">)][</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> <span class="n">num_quarters</span> <span class="o">=</span> <span class="mi">96</span> <span class="n">leap_days</span> <span class="o">=</span> <span class="mi">6</span> @@ -8249,17 +8516,17 @@ hr_q \cdot n - hr_d \cdot (n-m) <div class="prompt input_prompt">In&nbsp;[9]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">minute_mean</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> -<span class="c"># Don&#39;t you wish you knew?</span> -<span class="c"># birthday_minutes = ???</span> +<div class=" highlight hl-ipython3"><pre><span class="n">minute_mean</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> +<span class="c1"># Don&#39;t you wish you knew?</span> +<span class="c1"># birthday_minutes = ???</span> <span class="n">birthday_heartbeats</span> <span class="o">=</span> <span class="n">birthday_minutes</span> <span class="o">*</span> <span class="n">minute_mean</span> <span class="n">heartbeats_until_2016</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">birthday_heartbeats</span> <span class="o">+</span> <span class="n">jan_92_jan_16</span><span class="p">)</span> <span class="n">remaining_2016</span> <span class="o">=</span> <span class="n">total_heartbeats</span> <span class="o">-</span> <span class="n">heartbeats_until_2016</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Heartbeats so far: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">heartbeats_until_2016</span><span class="p">))</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Remaining heartbeats: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_2016</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Heartbeats so far: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">heartbeats_until_2016</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Remaining heartbeats: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_2016</span><span class="p">))</span> </pre></div> </div> @@ -8350,8 +8617,8 @@ hr_q \cdot n - hr_d \cdot (n - m) <div class=" highlight hl-ipython3"><pre><span class="n">four_year_periods</span> <span class="o">=</span> <span class="n">remaining_2016</span> <span class="o">//</span> <span class="n">heartbeats_4year</span> <span class="n">remaining_4y</span> <span class="o">=</span> <span class="n">remaining_2016</span> <span class="o">-</span> <span class="n">four_year_periods</span> <span class="o">*</span> <span class="n">heartbeats_4year</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Four year periods remaining: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">four_year_periods</span><span class="p">))</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Remaining heartbeats after 4 year periods: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_4y</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Four year periods remaining: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">four_year_periods</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Remaining heartbeats after 4 year periods: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_4y</span><span class="p">))</span> </pre></div> </div> @@ -8390,14 +8657,14 @@ Remaining heartbeats after 4 year periods: 48041640 <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="n">remaining_leap</span> <span class="o">=</span> <span class="n">remaining_4y</span> <span class="o">-</span> <span class="n">leap_day_count</span> -<span class="c"># Ignore leap day in the data set</span> +<span class="c1"># Ignore leap day in the data set</span> <span class="n">heartbeats_quarter</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">month</span> <span class="o">!=</span> <span class="mi">2</span><span class="p">)</span> <span class="o">&amp;</span> - <span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">day</span> <span class="o">!=</span> <span class="mi">29</span><span class="p">)][</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> + <span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">day</span> <span class="o">!=</span> <span class="mi">29</span><span class="p">)][</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> <span class="n">quarters_left</span> <span class="o">=</span> <span class="n">remaining_leap</span> <span class="o">//</span> <span class="n">heartbeats_quarter</span> <span class="n">remaining_year</span> <span class="o">=</span> <span class="n">remaining_leap</span> <span class="o">-</span> <span class="n">quarters_left</span> <span class="o">*</span> <span class="n">heartbeats_quarter</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Quarters left starting 2068: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">quarters_left</span><span class="p">))</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Remaining heartbeats after that: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_year</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Quarters left starting 2068: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">quarters_left</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Remaining heartbeats after that: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_year</span><span class="p">))</span> </pre></div> </div> @@ -8483,7 +8750,7 @@ Remaining heartbeats after that: 4760716 MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Tweet Like Me2016-03-28T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-03-28:tweet-like-me.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Tweet Like Me2016-03-28T00:00:00-04:002016-03-28T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-03-28:tweet-like-me.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -8525,11 +8792,11 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span> -<span class="n">tweets</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s">&#39;tweets.csv&#39;</span><span class="p">)</span> +<span class="n">tweets</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;tweets.csv&#39;</span><span class="p">)</span> <span class="n">text</span> <span class="o">=</span> <span class="n">tweets</span><span class="o">.</span><span class="n">text</span> -<span class="c"># Don&#39;t include tweets in reply to or mentioning people</span> -<span class="n">replies</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span><span class="s">&#39;@&#39;</span><span class="p">)</span> +<span class="c1"># Don&#39;t include tweets in reply to or mentioning people</span> +<span class="n">replies</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span><span class="s1">&#39;@&#39;</span><span class="p">)</span> <span class="n">text_norep</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="o">~</span><span class="n">replies</span><span class="p">]</span> </pre></div> @@ -8584,10 +8851,10 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="kn">from</span> <span class="nn">functools</span> <span class="k">import</span> <span class="n">reduce</span> -<span class="c"># Get all possible words</span> +<span class="c1"># Get all possible words</span> <span class="n">all_words</span> <span class="o">=</span> <span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">+</span><span class="n">y</span><span class="p">,</span> <span class="n">tokens</span><span class="p">,</span> <span class="p">[])</span> <span class="n">unique_words</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">all_words</span><span class="p">)</span> -<span class="n">actual_words</span> <span class="o">=</span> <span class="nb">set</span><span class="p">([</span><span class="n">x</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s">&#39;.&#39;</span> <span class="k">else</span> <span class="k">None</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">unique_words</span><span class="p">])</span> +<span class="n">actual_words</span> <span class="o">=</span> <span class="nb">set</span><span class="p">([</span><span class="n">x</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;.&#39;</span> <span class="k">else</span> <span class="kc">None</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">unique_words</span><span class="p">])</span> <span class="n">word_dist</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">for</span> <span class="n">word</span> <span class="ow">in</span> <span class="nb">iter</span><span class="p">(</span><span class="n">actual_words</span><span class="p">):</span> @@ -8619,9 +8886,9 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <div class=" highlight hl-ipython3"><pre><span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span> <span class="o">%</span><span class="k">matplotlib</span> inline -<span class="n">hashtags</span> <span class="o">=</span> <span class="n">text_norep</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s">&#39;#&#39;</span><span class="p">)</span> +<span class="n">hashtags</span> <span class="o">=</span> <span class="n">text_norep</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;#&#39;</span><span class="p">)</span> <span class="n">bins</span> <span class="o">=</span> <span class="n">hashtags</span><span class="o">.</span><span class="n">unique</span><span class="p">()</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> -<span class="n">hashtags</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="n">bins</span><span class="p">)</span> +<span class="n">hashtags</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s1">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="n">bins</span><span class="p">)</span> </pre></div> </div> @@ -8793,12 +9060,12 @@ AElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[6]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">hashtags</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">all_words</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;#&#39;</span><span class="p">]</span> +<div class=" highlight hl-ipython3"><pre><span class="n">hashtags</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">all_words</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;#&#39;</span><span class="p">]</span> <span class="n">n_hashtags</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">hashtags</span><span class="p">)</span> -<span class="n">unique_hashtags</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">unique_words</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;#&#39;</span><span class="p">]))</span> -<span class="n">hashtag_dist</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s">&#39;hashtags&#39;</span><span class="p">:</span> <span class="n">unique_hashtags</span><span class="p">,</span> - <span class="s">&#39;prob&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">all_words</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="n">h</span><span class="p">)</span> <span class="o">/</span> <span class="n">n_hashtags</span> +<span class="n">unique_hashtags</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">unique_words</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;#&#39;</span><span class="p">]))</span> +<span class="n">hashtag_dist</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;hashtags&#39;</span><span class="p">:</span> <span class="n">unique_hashtags</span><span class="p">,</span> + <span class="s1">&#39;prob&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">all_words</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="n">h</span><span class="p">)</span> <span class="o">/</span> <span class="n">n_hashtags</span> <span class="k">for</span> <span class="n">h</span> <span class="ow">in</span> <span class="n">unique_hashtags</span><span class="p">]})</span> <span class="nb">len</span><span class="p">(</span><span class="n">hashtag_dist</span><span class="p">)</span> </pre></div> @@ -8856,7 +9123,7 @@ z: .17</code></pre> <span class="k">def</span> <span class="nf">multinom_sim</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">vals</span><span class="p">,</span> <span class="n">probs</span><span class="p">):</span> <span class="n">occurrences</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">multinomial</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">probs</span><span class="p">)</span> <span class="n">results</span> <span class="o">=</span> <span class="n">occurrences</span> <span class="o">*</span> <span class="n">vals</span> - <span class="k">return</span> <span class="s">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">results</span><span class="p">[</span><span class="n">results</span> <span class="o">!=</span> <span class="s">&#39;&#39;</span><span class="p">])</span> + <span class="k">return</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">results</span><span class="p">[</span><span class="n">results</span> <span class="o">!=</span> <span class="s1">&#39;&#39;</span><span class="p">])</span> <span class="k">def</span> <span class="nf">sim_n_hashtags</span><span class="p">(</span><span class="n">hashtag_freq</span><span class="p">):</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">poisson</span><span class="p">(</span><span class="n">hashtag_freq</span><span class="p">)</span> @@ -8866,7 +9133,7 @@ z: .17</code></pre> <span class="k">def</span> <span class="nf">sim_first_word</span><span class="p">(</span><span class="n">first_word_dist</span><span class="p">):</span> <span class="n">probs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">(</span><span class="n">first_word_dist</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> - <span class="k">return</span> <span class="n">multinom_sim</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">first_word_dist</span><span class="o">.</span><span class="n">reset_index</span><span class="p">()[</span><span class="s">&#39;index&#39;</span><span class="p">],</span> <span class="n">probs</span><span class="p">)</span> + <span class="k">return</span> <span class="n">multinom_sim</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">first_word_dist</span><span class="o">.</span><span class="n">reset_index</span><span class="p">()[</span><span class="s1">&#39;index&#39;</span><span class="p">],</span> <span class="n">probs</span><span class="p">)</span> <span class="k">def</span> <span class="nf">sim_next_word</span><span class="p">(</span><span class="n">current</span><span class="p">,</span> <span class="n">word_dist</span><span class="p">):</span> <span class="n">dist</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">word_dist</span><span class="p">[</span><span class="n">current</span><span class="p">])</span> @@ -8901,7 +9168,7 @@ z: .17</code></pre> <span class="n">fifth</span> <span class="o">=</span> <span class="n">sim_next_word</span><span class="p">(</span><span class="n">fourth</span><span class="p">,</span> <span class="n">word_dist</span><span class="p">)</span> <span class="n">hashtag</span> <span class="o">=</span> <span class="n">sim_hashtags</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">hashtag_dist</span><span class="p">)</span> -<span class="s">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">first</span><span class="p">,</span> <span class="n">second</span><span class="p">,</span> <span class="n">third</span><span class="p">,</span> <span class="n">fourth</span><span class="p">,</span> <span class="n">fifth</span><span class="p">,</span> <span class="n">hashtag</span><span class="p">))</span> +<span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">first</span><span class="p">,</span> <span class="n">second</span><span class="p">,</span> <span class="n">third</span><span class="p">,</span> <span class="n">fourth</span><span class="p">,</span> <span class="n">fifth</span><span class="p">,</span> <span class="n">hashtag</span><span class="p">))</span> </pre></div> </div> @@ -8950,13 +9217,13 @@ z: .17</code></pre> <span class="n">tweet</span> <span class="o">=</span> <span class="n">first</span> <span class="n">current</span> <span class="o">=</span> <span class="n">first</span> - <span class="k">while</span> <span class="n">chars_remaining</span> <span class="o">&gt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">tweet</span><span class="p">)</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="n">hashtags</span><span class="p">)</span> <span class="ow">and</span> <span class="n">current</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s">&#39;.&#39;</span> <span class="ow">and</span> <span class="n">current</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s">&#39;!&#39;</span><span class="p">:</span> + <span class="k">while</span> <span class="n">chars_remaining</span> <span class="o">&gt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">tweet</span><span class="p">)</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="n">hashtags</span><span class="p">)</span> <span class="ow">and</span> <span class="n">current</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;.&#39;</span> <span class="ow">and</span> <span class="n">current</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;!&#39;</span><span class="p">:</span> <span class="n">current</span> <span class="o">=</span> <span class="n">sim_next_word</span><span class="p">(</span><span class="n">current</span><span class="p">,</span> <span class="n">word_dist</span><span class="p">)</span> - <span class="n">tweet</span> <span class="o">+=</span> <span class="s">&#39; &#39;</span> <span class="o">+</span> <span class="n">current</span> + <span class="n">tweet</span> <span class="o">+=</span> <span class="s1">&#39; &#39;</span> <span class="o">+</span> <span class="n">current</span> <span class="n">tweet</span> <span class="o">=</span> <span class="n">tweet</span><span class="p">[:</span><span class="o">-</span><span class="mi">2</span><span class="p">]</span> <span class="o">+</span> <span class="n">tweet</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> - <span class="k">return</span> <span class="s">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">tweet</span><span class="p">,</span> <span class="n">hashtags</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> + <span class="k">return</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">tweet</span><span class="p">,</span> <span class="n">hashtags</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> </pre></div> </div> @@ -9058,7 +9325,7 @@ Aamazing conference when you married #DepartmentOfRedundancyDepartment Yep , but MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Predicting Santander Customer Happiness2016-03-05T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-03-05:predicting-santander-customer-happiness.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Predicting Santander Customer Happiness2016-03-05T00:00:00-05:002016-03-05T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-03-05:predicting-santander-customer-happiness.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -9080,13 +9347,13 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span> <span class="o">%</span><span class="k">matplotlib</span> inline -<span class="c"># Record how long it takes to run the notebook - I&#39;m curious.</span> +<span class="c1"># Record how long it takes to run the notebook - I&#39;m curious.</span> <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span> <span class="n">start</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> -<span class="n">dataset</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s">&#39;split_train.csv&#39;</span><span class="p">)</span> +<span class="n">dataset</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;split_train.csv&#39;</span><span class="p">)</span> <span class="n">dataset</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">ID</span> -<span class="n">X</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">drop</span><span class="p">([</span><span class="s">&#39;TARGET&#39;</span><span class="p">,</span> <span class="s">&#39;ID&#39;</span><span class="p">,</span> <span class="s">&#39;ID.1&#39;</span><span class="p">],</span> <span class="mi">1</span><span class="p">)</span> +<span class="n">X</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">drop</span><span class="p">([</span><span class="s1">&#39;TARGET&#39;</span><span class="p">,</span> <span class="s1">&#39;ID&#39;</span><span class="p">,</span> <span class="s1">&#39;ID.1&#39;</span><span class="p">],</span> <span class="mi">1</span><span class="p">)</span> <span class="n">y</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">TARGET</span> </pre></div> @@ -9216,13 +9483,13 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[5]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># First we need to `binarize` the data to 0-1; some of the labels are {0, 1},</span> -<span class="c"># some are {0, 3}, etc.</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># First we need to `binarize` the data to 0-1; some of the labels are {0, 1},</span> +<span class="c1"># some are {0, 3}, etc.</span> <span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="k">import</span> <span class="n">binarize</span> <span class="n">X_bin</span> <span class="o">=</span> <span class="n">binarize</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="n">b_class</span><span class="p">])</span> <span class="n">accuracy</span> <span class="o">=</span> <span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">X_bin</span><span class="p">[:,</span><span class="n">i</span><span class="p">]</span> <span class="o">==</span> <span class="n">y</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">b_class</span><span class="p">))]</span> -<span class="n">acc_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s">&quot;Accuracy&quot;</span><span class="p">:</span> <span class="n">accuracy</span><span class="p">},</span> <span class="n">index</span><span class="o">=</span><span class="n">b_class</span><span class="p">)</span> +<span class="n">acc_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;Accuracy&quot;</span><span class="p">:</span> <span class="n">accuracy</span><span class="p">},</span> <span class="n">index</span><span class="o">=</span><span class="n">b_class</span><span class="p">)</span> <span class="n">acc_df</span><span class="o">.</span><span class="n">describe</span><span class="p">()</span> </pre></div> @@ -9306,9 +9573,9 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="n">unsat</span> <span class="o">=</span> <span class="n">y</span><span class="p">[</span><span class="n">y</span> <span class="o">==</span> <span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">count</span><span class="p">()</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Satisfied customers: {}; Unsatisfied customers: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">y</span><span class="p">)</span> <span class="o">-</span> <span class="n">unsat</span><span class="p">,</span> <span class="n">unsat</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Satisfied customers: {}; Unsatisfied customers: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">y</span><span class="p">)</span> <span class="o">-</span> <span class="n">unsat</span><span class="p">,</span> <span class="n">unsat</span><span class="p">))</span> <span class="n">naive_guess</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span> <span class="o">==</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">y</span><span class="p">)))</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Naive guess accuracy: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Naive guess accuracy: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">))</span> </pre></div> </div> @@ -9369,9 +9636,9 @@ Naive guess accuracy: 0.9608561656706882 <span class="k">return</span> <span class="n">gnb</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_xform</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> <span class="n">dim_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">21</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">dim_range</span><span class="p">,</span> <span class="p">[</span><span class="n">evaluate_gnb</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="k">for</span> <span class="n">dim</span> <span class="ow">in</span> <span class="n">dim_range</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Gaussian NB Accuracy&quot;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s">&#39;k&#39;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Inverse Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s">&#39;k&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">dim_range</span><span class="p">,</span> <span class="p">[</span><span class="n">evaluate_gnb</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="k">for</span> <span class="n">dim</span> <span class="ow">in</span> <span class="n">dim_range</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Gaussian NB Accuracy&quot;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s1">&#39;k&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Inverse Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s1">&#39;k&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">();</span> </pre></div> @@ -9655,9 +9922,9 @@ rkJggg== <span class="k">return</span> <span class="n">gnb</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_xform</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> <span class="n">dim_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">21</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">dim_range</span><span class="p">,</span> <span class="p">[</span><span class="n">evaluate_gnb</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="k">for</span> <span class="n">dim</span> <span class="ow">in</span> <span class="n">dim_range</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Gaussian NB Accuracy&quot;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s">&#39;k&#39;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Inverse Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s">&#39;k&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">dim_range</span><span class="p">,</span> <span class="p">[</span><span class="n">evaluate_gnb</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="k">for</span> <span class="n">dim</span> <span class="ow">in</span> <span class="n">dim_range</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Gaussian NB Accuracy&quot;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s1">&#39;k&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Inverse Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s1">&#39;k&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">();</span> </pre></div> @@ -9933,7 +10200,7 @@ rkJggg== <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="n">end</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Running time: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">start</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Running time: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">start</span><span class="p">))</span> </pre></div> </div> @@ -9962,14 +10229,14 @@ rkJggg== <div class="text_cell_render border-box-sizing rendered_html"> <h1 id="Appendix">Appendix<a class="anchor-link" href="#Appendix">&#182;</a></h1><p>Code used to split the initial training data:</p> <div class="highlight"><pre><span class="kn">from</span> <span class="nn">sklearn.cross_validation</span> <span class="kn">import</span> <span class="n">train_test_split</span> -<span class="n">data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s">&#39;train.csv&#39;</span><span class="p">)</span> +<span class="n">data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;train.csv&#39;</span><span class="p">)</span> <span class="n">data</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">ID</span> <span class="n">data_train</span><span class="p">,</span> <span class="n">data_validate</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span> <span class="n">data</span><span class="p">,</span> <span class="n">train_size</span><span class="o">=.</span><span class="mi">7</span><span class="p">)</span> -<span class="n">data_train</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s">&#39;split_train.csv&#39;</span><span class="p">)</span> -<span class="n">data_validate</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s">&#39;split_validate.csv&#39;</span><span class="p">)</span> +<span class="n">data_train</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s1">&#39;split_train.csv&#39;</span><span class="p">)</span> +<span class="n">data_validate</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s1">&#39;split_validate.csv&#39;</span><span class="p">)</span> </pre></div> </div> @@ -9980,7 +10247,7 @@ rkJggg== MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Profitability using the Investment Formula2016-02-26T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-02-26:profitability-using-the-investment-formula.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Profitability using the Investment Formula2016-02-26T00:00:00-05:002016-02-26T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-02-26:profitability-using-the-investment-formula.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -10017,7 +10284,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span> <span class="kn">from</span> <span class="nn">Quandl</span> <span class="k">import</span> <span class="n">get</span> <span class="k">as</span> <span class="n">qget</span> <span class="o">%</span><span class="k">matplotlib</span> inline -<span class="n">api_key</span> <span class="o">=</span> <span class="s">&#39;&#39;</span> +<span class="n">api_key</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span> <span class="n">profitability</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">p</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">m</span><span class="p">,</span> <span class="n">d</span><span class="p">:</span> <span class="mi">1000</span><span class="o">*</span><span class="n">p</span> <span class="o">/</span> <span class="p">(</span><span class="n">m</span> <span class="o">+</span> <span class="n">i</span><span class="o">*</span><span class="n">d</span><span class="p">)</span> @@ -10041,15 +10308,15 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <span class="n">expected_close</span> <span class="o">=</span> <span class="n">open_vals</span><span class="p">[</span><span class="n">day</span><span class="p">]</span> <span class="o">*</span> <span class="n">expected</span> <span class="n">todays_purchase</span> <span class="o">=</span> <span class="n">daily_investment</span><span class="p">(</span><span class="n">open_vals</span><span class="p">[</span><span class="n">day</span><span class="p">],</span> <span class="n">expected_close</span><span class="p">,</span> <span class="n">invested</span><span class="p">,</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="n">day</span><span class="p">])</span> <span class="n">invested</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">invested</span><span class="p">,</span> <span class="n">todays_purchase</span> <span class="o">+</span> <span class="n">bias</span><span class="p">)</span> - <span class="c"># expected_profit = expected_close * (invested / open_vals[0:len(invested)]).sum() - invested.sum()</span> + <span class="c1"># expected_profit = expected_close * (invested / open_vals[0:len(invested)]).sum() - invested.sum()</span> <span class="n">day</span> <span class="o">+=</span> <span class="mi">1</span> <span class="n">profitable</span> <span class="o">=</span> <span class="n">is_profitable</span><span class="p">(</span><span class="n">close_vals</span><span class="p">[</span><span class="n">day</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">invested</span><span class="p">,</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="nb">len</span><span class="p">(</span><span class="n">invested</span><span class="p">)])</span> \ <span class="ow">or</span> <span class="n">is_profitable</span><span class="p">(</span><span class="n">open_vals</span><span class="p">[</span><span class="n">day</span><span class="p">],</span> <span class="n">invested</span><span class="p">,</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="nb">len</span><span class="p">(</span><span class="n">invested</span><span class="p">)])</span> <span class="n">shares</span> <span class="o">=</span> <span class="p">(</span><span class="n">invested</span> <span class="o">/</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="nb">len</span><span class="p">(</span><span class="n">invested</span><span class="p">)])</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> - <span class="c"># Make sure we can&#39;t see into the future - we know either today&#39;s close or tomorrow&#39;s open</span> - <span class="c"># will be profitable, but we need to check which one.</span> + <span class="c1"># Make sure we can&#39;t see into the future - we know either today&#39;s close or tomorrow&#39;s open</span> + <span class="c1"># will be profitable, but we need to check which one.</span> <span class="k">if</span> <span class="n">is_profitable</span><span class="p">(</span><span class="n">close_vals</span><span class="p">[</span><span class="n">day</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">invested</span><span class="p">,</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="nb">len</span><span class="p">(</span><span class="n">invested</span><span class="p">)]):</span> <span class="n">ending_price</span> <span class="o">=</span> <span class="n">close_vals</span><span class="p">[</span><span class="n">day</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> @@ -10062,21 +10329,21 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <span class="n">ticker_info</span> <span class="o">=</span> <span class="n">qget</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start</span><span class="p">,</span> <span class="n">api_key</span><span class="o">=</span><span class="n">api_key</span><span class="p">)</span> <span class="n">evaluation_times</span> <span class="o">=</span> <span class="n">ticker_info</span><span class="p">[:</span><span class="n">end</span><span class="p">]</span><span class="o">.</span><span class="n">index</span> - <span class="c"># Handle Google vs. YFinance data</span> - <span class="k">if</span> <span class="s">&quot;Adjusted Close&quot;</span> <span class="ow">in</span> <span class="n">ticker_info</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span> - <span class="n">close_column</span> <span class="o">=</span> <span class="s">&quot;Adjusted Close&quot;</span> + <span class="c1"># Handle Google vs. YFinance data</span> + <span class="k">if</span> <span class="s2">&quot;Adjusted Close&quot;</span> <span class="ow">in</span> <span class="n">ticker_info</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span> + <span class="n">close_column</span> <span class="o">=</span> <span class="s2">&quot;Adjusted Close&quot;</span> <span class="k">else</span><span class="p">:</span> - <span class="n">close_column</span> <span class="o">=</span> <span class="s">&quot;Close&quot;</span> + <span class="n">close_column</span> <span class="o">=</span> <span class="s2">&quot;Close&quot;</span> - <span class="n">sim</span> <span class="o">=</span> <span class="p">{</span><span class="n">d</span><span class="p">:</span> <span class="n">simulate_day</span><span class="p">(</span><span class="n">ticker_info</span><span class="p">[</span><span class="n">d</span><span class="p">:][</span><span class="s">&quot;Open&quot;</span><span class="p">],</span> <span class="n">ticker_info</span><span class="p">[</span><span class="n">d</span><span class="p">:][</span><span class="n">close_column</span><span class="p">],</span> + <span class="n">sim</span> <span class="o">=</span> <span class="p">{</span><span class="n">d</span><span class="p">:</span> <span class="n">simulate_day</span><span class="p">(</span><span class="n">ticker_info</span><span class="p">[</span><span class="n">d</span><span class="p">:][</span><span class="s2">&quot;Open&quot;</span><span class="p">],</span> <span class="n">ticker_info</span><span class="p">[</span><span class="n">d</span><span class="p">:][</span><span class="n">close_column</span><span class="p">],</span> <span class="mi">100</span><span class="p">,</span> <span class="mf">1.02</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span> <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">evaluation_times</span><span class="p">}</span> <span class="n">sim_series</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">sim</span><span class="p">)</span> <span class="n">result</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">()</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;profit&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;max&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">max</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;days&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;score&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">profitability</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">],</span> <span class="nb">max</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])))</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;investments&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;profit&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;max&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">max</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;days&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;score&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">profitability</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">],</span> <span class="nb">max</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])))</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;investments&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="k">return</span> <span class="n">result</span> @@ -10111,7 +10378,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <div class="prompt input_prompt">In&nbsp;[7]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s">&quot;YAHOO/FB&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/AAPL&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/NFLX&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/GOOG&quot;</span><span class="p">])</span> +<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s2">&quot;YAHOO/FB&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/AAPL&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/NFLX&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/GOOG&quot;</span><span class="p">])</span> </pre></div> </div> @@ -10124,9 +10391,9 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <div class="prompt input_prompt">In&nbsp;[8]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">8</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -10598,9 +10865,9 @@ kiSpNScYJEmSJElSa04wSJIkSZKk1pxgkCRJkiRJrf1/uiw9D8O0d6EAAAAASUVORK5CYII= <div class="prompt input_prompt">In&nbsp;[10]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -12237,7 +12504,7 @@ EZIkSZIkSZIkSZIk6RIpIiRJkiRJkiRJkiRJ0iX+D91jLOlExGw8AAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[13]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s">&quot;YAHOO/X&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/CAT&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/NFLX&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/GOOG&quot;</span><span class="p">])</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s2">&quot;YAHOO/X&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/CAT&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/NFLX&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/GOOG&quot;</span><span class="p">])</span> </pre></div> </div> @@ -12250,9 +12517,9 @@ EZIkSZIkSZIkSZIk6RIpIiRJkiRJkiRJkiRJ0iX+D91jLOlExGw8AAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[14]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">8</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -12738,9 +13005,9 @@ JEmSJEnSwGwwSJIkSZKkgdlgkCRJkiRJA7PBIEmSJEmSBvb/AfKAew4+oiojAAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[15]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -14523,7 +14790,7 @@ hUKhUCgUpor/A9AnO0PoIItTAAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[21]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s">&#39;YAHOO/REGN&#39;</span><span class="p">,</span> <span class="s">&#39;YAHOO/CELG&#39;</span><span class="p">,</span> <span class="s">&#39;GOOG/NASDAQ_BIB&#39;</span><span class="p">,</span> <span class="s">&#39;GOOG/NASDAQ_IBB&#39;</span><span class="p">])</span> +<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s1">&#39;YAHOO/REGN&#39;</span><span class="p">,</span> <span class="s1">&#39;YAHOO/CELG&#39;</span><span class="p">,</span> <span class="s1">&#39;GOOG/NASDAQ_BIB&#39;</span><span class="p">,</span> <span class="s1">&#39;GOOG/NASDAQ_IBB&#39;</span><span class="p">])</span> </pre></div> </div> @@ -14536,9 +14803,9 @@ hUKhUCgUpor/A9AnO0PoIItTAAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[22]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">8</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -15089,9 +15356,9 @@ WKBTQVcCAAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[23]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -16853,7 +17120,7 @@ JBKJRCJrRTQiRCKRSCQSiUQikUgkElkr/h9yZ17EmpnoFwAAAABJRU5ErkJggg== MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Guaranteed Money Maker2016-02-03T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-02-03:guaranteed-money-maker.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Guaranteed Money Maker2016-02-03T00:00:00-05:002016-02-03T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-02-03:guaranteed-money-maker.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -17116,7 +17383,7 @@ Profit: 0.08304813163696423 MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Cloudy in Seattle2016-01-23T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-01-23:cloudy-in-seattle.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Cloudy in Seattle2016-01-23T00:00:00-05:002016-01-23T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-01-23:cloudy-in-seattle.html<p> <div class="cell border-box-sizing code_cell rendered"> <div class="input"> <div class="prompt input_prompt">In&nbsp;[1]:</div> @@ -17389,7 +17656,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[2]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">city_forecasts</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="s">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s">&#39;rb&#39;</span><span class="p">))</span> +<div class=" highlight hl-ipython3"><pre><span class="n">city_forecasts</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="s1">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">))</span> <span class="n">forecasts_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">city_forecasts</span><span class="p">)</span> </pre></div> @@ -17403,24 +17670,24 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[3]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cities</span> <span class="o">=</span> <span class="p">[</span><span class="s">&#39;binghamton&#39;</span><span class="p">,</span> <span class="s">&#39;cary&#39;</span><span class="p">,</span> <span class="s">&#39;nyc&#39;</span><span class="p">,</span> <span class="s">&#39;seattle&#39;</span><span class="p">]</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cities</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;binghamton&#39;</span><span class="p">,</span> <span class="s1">&#39;cary&#39;</span><span class="p">,</span> <span class="s1">&#39;nyc&#39;</span><span class="p">,</span> <span class="s1">&#39;seattle&#39;</span><span class="p">]</span> <span class="n">city_colors</span> <span class="o">=</span> <span class="p">{</span><span class="n">cities</span><span class="p">[</span><span class="n">i</span><span class="p">]:</span> <span class="n">Palette</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">4</span><span class="p">)}</span> <span class="k">def</span> <span class="nf">safe_cover</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s">&#39;cloudCover&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> - <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s">&#39;cloudCover&#39;</span><span class="p">]</span> + <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s1">&#39;cloudCover&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> + <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s1">&#39;cloudCover&#39;</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">monthly_avg_cloudcover</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">forecasts_df</span><span class="p">[</span><span class="n">city</span><span class="p">][</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">forecasts_df</span><span class="p">[</span><span class="n">city</span><span class="p">][</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">cloud_cover_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">cloud_cover_vals</span><span class="p">)))</span> - <span class="c"># Ignore an issue with nanmean having all NaN values. We&#39;ll discuss the data issues below.</span> + <span class="c1"># Ignore an issue with nanmean having all NaN values. We&#39;ll discuss the data issues below.</span> <span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span> - <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s">&#39;ignore&#39;</span><span class="p">)</span> + <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">nanmean</span><span class="p">(</span><span class="n">cloud_cover_vals</span><span class="p">),</span> <span class="n">cloud_cover_samples</span> </pre></div> @@ -17439,18 +17706,18 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="k">return</span> <span class="p">[</span><span class="n">monthly_avg_cloudcover</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">month</span><span class="p">)</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="n">months</span> <span class="o">=</span> <span class="p">[</span> - <span class="p">(</span><span class="s">&#39;July&#39;</span><span class="p">,</span> <span class="mi">7</span><span class="p">),</span> - <span class="p">(</span><span class="s">&#39;August&#39;</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> - <span class="p">(</span><span class="s">&#39;September&#39;</span><span class="p">,</span> <span class="mi">9</span><span class="p">),</span> - <span class="p">(</span><span class="s">&#39;October&#39;</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span> - <span class="p">(</span><span class="s">&#39;November&#39;</span><span class="p">,</span> <span class="mi">11</span><span class="p">)</span> + <span class="p">(</span><span class="s1">&#39;July&#39;</span><span class="p">,</span> <span class="mi">7</span><span class="p">),</span> + <span class="p">(</span><span class="s1">&#39;August&#39;</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> + <span class="p">(</span><span class="s1">&#39;September&#39;</span><span class="p">,</span> <span class="mi">9</span><span class="p">),</span> + <span class="p">(</span><span class="s1">&#39;October&#39;</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span> + <span class="p">(</span><span class="s1">&#39;November&#39;</span><span class="p">,</span> <span class="mi">11</span><span class="p">)</span> <span class="p">]</span> <span class="k">for</span> <span class="n">month</span><span class="p">,</span> <span class="n">month_id</span> <span class="ow">in</span> <span class="n">months</span><span class="p">:</span> <span class="n">month_averages</span> <span class="o">=</span> <span class="p">{</span><span class="n">city</span><span class="p">:</span> <span class="n">city_avg_cc</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">month_id</span><span class="p">)</span> <span class="k">for</span> <span class="n">city</span> <span class="ow">in</span> <span class="n">cities</span><span class="p">}</span> - <span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&quot;{} Average Cloud Cover&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">month</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">,</span> - <span class="n">y_axis_label</span><span class="o">=</span><span class="s">&#39;Cloud Cover Percentage&#39;</span><span class="p">)</span> + <span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s2">&quot;{} Average Cloud Cover&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">month</span><span class="p">),</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">,</span> + <span class="n">y_axis_label</span><span class="o">=</span><span class="s1">&#39;Cloud Cover Percentage&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">city</span> <span class="ow">in</span> <span class="n">cities</span><span class="p">:</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="p">[</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">month_averages</span><span class="p">[</span><span class="n">city</span><span class="p">]],</span> <span class="n">legend</span><span class="o">=</span><span class="n">city</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="n">city_colors</span><span class="p">[</span><span class="n">city</span><span class="p">])</span> @@ -17640,20 +17907,20 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">safe_precip</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s">&#39;precipProbability&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> - <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s">&#39;precipProbability&#39;</span><span class="p">]</span> + <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s1">&#39;precipProbability&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> + <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s1">&#39;precipProbability&#39;</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">monthly_avg_precip</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">precip_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">forecasts_df</span><span class="p">[</span><span class="n">city</span><span class="p">][</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">precip_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">forecasts_df</span><span class="p">[</span><span class="n">city</span><span class="p">][</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">precip_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">precip_vals</span><span class="p">)))</span> - <span class="c"># Ignore an issue with nanmean having all NaN values. We&#39;ll discuss the data issues below.</span> + <span class="c1"># Ignore an issue with nanmean having all NaN values. We&#39;ll discuss the data issues below.</span> <span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span> - <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s">&#39;ignore&#39;</span><span class="p">)</span> + <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">nanmean</span><span class="p">(</span><span class="n">precip_vals</span><span class="p">),</span> <span class="n">precip_samples</span> <span class="k">def</span> <span class="nf">city_avg_precip</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> @@ -17661,9 +17928,9 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="k">for</span> <span class="n">month</span><span class="p">,</span> <span class="n">month_id</span> <span class="ow">in</span> <span class="n">months</span><span class="p">:</span> <span class="n">month_averages</span> <span class="o">=</span> <span class="p">{</span><span class="n">city</span><span class="p">:</span> <span class="n">city_avg_cc</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">month_id</span><span class="p">)</span> <span class="k">for</span> <span class="n">city</span> <span class="ow">in</span> <span class="n">cities</span><span class="p">}</span> - <span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&quot;{} Average Precipitation Chance&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">month</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">,</span> - <span class="n">y_axis_label</span><span class="o">=</span><span class="s">&#39;Precipitation Chance Percentage&#39;</span><span class="p">)</span> + <span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s2">&quot;{} Average Precipitation Chance&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">month</span><span class="p">),</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">,</span> + <span class="n">y_axis_label</span><span class="o">=</span><span class="s1">&#39;Precipitation Chance Percentage&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">city</span> <span class="ow">in</span> <span class="n">cities</span><span class="p">:</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="p">[</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">month_averages</span><span class="p">[</span><span class="n">city</span><span class="p">]],</span> <span class="n">legend</span><span class="o">=</span><span class="n">city</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="n">city_colors</span><span class="p">[</span><span class="n">city</span><span class="p">])</span> @@ -17852,7 +18119,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Complaining about the Weather2016-01-01T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-01-01:complaining-about-the-weather.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Complaining about the Weather2016-01-01T00:00:00-05:002016-01-01T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-01-01:complaining-about-the-weather.html<p> <div class="cell border-box-sizing code_cell rendered"> <div class="input"> <div class="prompt input_prompt">In&nbsp;[1]:</div> @@ -18118,7 +18385,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[2]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">city_forecasts</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="s">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s">&#39;rb&#39;</span><span class="p">))</span> +<div class=" highlight hl-ipython3"><pre><span class="n">city_forecasts</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="s1">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">))</span> <span class="n">forecast_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">city_forecasts</span><span class="p">)</span> </pre></div> @@ -18132,31 +18399,31 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[3]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cary_forecast</span> <span class="o">=</span> <span class="n">forecast_df</span><span class="p">[</span><span class="s">&#39;cary&#39;</span><span class="p">]</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cary_forecast</span> <span class="o">=</span> <span class="n">forecast_df</span><span class="p">[</span><span class="s1">&#39;cary&#39;</span><span class="p">]</span> <span class="n">years</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2016</span><span class="p">)</span> <span class="n">months</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="mi">7</span><span class="p">,</span> <span class="mi">12</span><span class="p">)</span> -<span class="n">months_str</span> <span class="o">=</span> <span class="p">[</span><span class="s">&#39;July&#39;</span><span class="p">,</span> <span class="s">&#39;August&#39;</span><span class="p">,</span> <span class="s">&#39;September&#39;</span><span class="p">,</span> <span class="s">&#39;October&#39;</span><span class="p">,</span> <span class="s">&#39;November&#39;</span><span class="p">]</span> +<span class="n">months_str</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;July&#39;</span><span class="p">,</span> <span class="s1">&#39;August&#39;</span><span class="p">,</span> <span class="s1">&#39;September&#39;</span><span class="p">,</span> <span class="s1">&#39;October&#39;</span><span class="p">,</span> <span class="s1">&#39;November&#39;</span><span class="p">]</span> <span class="k">def</span> <span class="nf">safe_cover</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s">&#39;cloudCover&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> - <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s">&#39;cloudCover&#39;</span><span class="p">]</span> + <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s1">&#39;cloudCover&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> + <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s1">&#39;cloudCover&#39;</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">monthly_avg_cloudcover</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">cloud_cover_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">cloud_cover_vals</span><span class="p">)))</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">nanmean</span><span class="p">(</span><span class="n">cloud_cover_vals</span><span class="p">),</span> <span class="n">cloud_cover_samples</span> <span class="n">monthly_cover_vals</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_avg_cloudcover</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Average Cloud Cover&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Average Cloud Cover&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">)</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_cover_vals</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> <span class="n">show</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> @@ -18223,8 +18490,8 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">monthly_cloudy_days</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">cloud_cover_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">cloud_cover_vals</span><span class="p">)))</span> <span class="n">cloudy_days</span> <span class="o">=</span> <span class="p">[</span><span class="n">cover</span> <span class="o">&gt;</span> <span class="o">.</span><span class="mi">5</span> <span class="k">for</span> <span class="n">cover</span> <span class="ow">in</span> <span class="n">cloud_cover_vals</span><span class="p">]</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">count_nonzero</span><span class="p">(</span><span class="n">cloudy_days</span><span class="p">),</span> <span class="n">cloud_cover_samples</span> @@ -18232,16 +18499,16 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">monthly_days_vals</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_cloudy_days</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> <span class="n">monthly_cover_samples</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_cloudy_days</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Cloudy Days&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Cloudy Days&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">)</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_days_vals</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> <span class="n">show</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Cloud Cover Samples&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Cloud Cover Samples&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">,</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">300</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_cover_samples</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> @@ -18337,24 +18604,24 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">safe_precip</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s">&#39;precipProbability&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> - <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s">&#39;precipProbability&#39;</span><span class="p">]</span> + <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s1">&#39;precipProbability&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> + <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s1">&#39;precipProbability&#39;</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">monthly_avg_precip</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">precip_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">precip_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">precip_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">precip_vals</span><span class="p">)))</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">nanmean</span><span class="p">(</span><span class="n">precip_vals</span><span class="p">),</span> <span class="n">precip_samples</span> <span class="n">monthly_avg_precip_vals</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_avg_precip</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Average Precipitation Chance&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Average Precipitation Chance&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">)</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_avg_precip_vals</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> <span class="n">show</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> @@ -18420,8 +18687,8 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">monthly_rainy_days</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">precip_prob_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">precip_prob_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">precip_prob_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">precip_prob_vals</span><span class="p">)))</span> <span class="n">precip_days</span> <span class="o">=</span> <span class="p">[</span><span class="n">prob</span> <span class="o">&gt;</span> <span class="o">.</span><span class="mi">25</span> <span class="k">for</span> <span class="n">prob</span> <span class="ow">in</span> <span class="n">precip_prob_vals</span><span class="p">]</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">count_nonzero</span><span class="p">(</span><span class="n">precip_days</span><span class="p">),</span> <span class="n">precip_prob_samples</span> @@ -18429,16 +18696,16 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">monthly_precip_days_vals</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_rainy_days</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> <span class="n">monthly_precip_samples</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_rainy_days</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Rainy Days&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Rainy Days&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">)</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_precip_days_vals</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> <span class="n">show</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Rainy Days Samples&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Rainy Days Samples&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">,</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">300</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_precip_samples</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> @@ -18541,64 +18808,64 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="kn">import</span> <span class="nn">requests</span> <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span> -<span class="c"># Coordinate data from http://itouchmap.com/latlong.html</span> +<span class="c1"># Coordinate data from http://itouchmap.com/latlong.html</span> <span class="n">cary_loc</span> <span class="o">=</span> <span class="p">(</span><span class="mf">35.79154</span><span class="p">,</span><span class="o">-</span><span class="mf">78.781117</span><span class="p">)</span> <span class="n">nyc_loc</span> <span class="o">=</span> <span class="p">(</span><span class="mf">40.78306</span><span class="p">,</span><span class="o">-</span><span class="mf">73.971249</span><span class="p">)</span> <span class="n">seattle_loc</span> <span class="o">=</span> <span class="p">(</span><span class="mf">47.60621</span><span class="p">,</span><span class="o">-</span><span class="mf">122.332071</span><span class="p">)</span> <span class="n">binghamton_loc</span> <span class="o">=</span> <span class="p">(</span><span class="mf">42.098687</span><span class="p">,</span><span class="o">-</span><span class="mf">75.917974</span><span class="p">)</span> <span class="n">cities</span> <span class="o">=</span> <span class="p">{</span> - <span class="s">&#39;cary&#39;</span><span class="p">:</span> <span class="n">cary_loc</span><span class="p">,</span> - <span class="s">&#39;nyc&#39;</span><span class="p">:</span> <span class="n">nyc_loc</span><span class="p">,</span> - <span class="s">&#39;seattle&#39;</span><span class="p">:</span> <span class="n">seattle_loc</span><span class="p">,</span> - <span class="s">&#39;binghamton&#39;</span><span class="p">:</span> <span class="n">binghamton_loc</span> + <span class="s1">&#39;cary&#39;</span><span class="p">:</span> <span class="n">cary_loc</span><span class="p">,</span> + <span class="s1">&#39;nyc&#39;</span><span class="p">:</span> <span class="n">nyc_loc</span><span class="p">,</span> + <span class="s1">&#39;seattle&#39;</span><span class="p">:</span> <span class="n">seattle_loc</span><span class="p">,</span> + <span class="s1">&#39;binghamton&#39;</span><span class="p">:</span> <span class="n">binghamton_loc</span> <span class="p">}</span> -<span class="n">apikey</span> <span class="o">=</span> <span class="s">&#39;&#39;</span> <span class="c"># My super-secret API Key</span> +<span class="n">apikey</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span> <span class="c1"># My super-secret API Key</span> <span class="k">def</span> <span class="nf">get_forecast</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="n">date</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span> - <span class="n">forecast_base</span> <span class="o">=</span> <span class="s">&quot;https://api.forecast.io/forecast/&quot;</span> + <span class="n">forecast_base</span> <span class="o">=</span> <span class="s2">&quot;https://api.forecast.io/forecast/&quot;</span> <span class="k">if</span> <span class="n">date</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span> - <span class="n">url</span> <span class="o">=</span> <span class="n">forecast_base</span> <span class="o">+</span> <span class="n">apikey</span> <span class="o">+</span> <span class="s">&#39;/{},{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">)</span> + <span class="n">url</span> <span class="o">=</span> <span class="n">forecast_base</span> <span class="o">+</span> <span class="n">apikey</span> <span class="o">+</span> <span class="s1">&#39;/{},{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> <span class="n">epoch</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">())</span> - <span class="n">url</span> <span class="o">=</span> <span class="n">forecast_base</span> <span class="o">+</span> <span class="n">apikey</span> <span class="o">+</span> <span class="s">&#39;/{},{},{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="n">epoch</span><span class="p">)</span> + <span class="n">url</span> <span class="o">=</span> <span class="n">forecast_base</span> <span class="o">+</span> <span class="n">apikey</span> <span class="o">+</span> <span class="s1">&#39;/{},{},{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="n">epoch</span><span class="p">)</span> <span class="k">return</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> <span class="n">years</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span><span class="mi">2016</span><span class="p">)</span> -<span class="c"># For datetimes, the 12 is for getting the weather at noon.</span> -<span class="c"># We&#39;re doing this over midnight because we&#39;re more concerned</span> -<span class="c"># with what people see, and people don&#39;t typically see the weather</span> -<span class="c"># at midnight.</span> +<span class="c1"># For datetimes, the 12 is for getting the weather at noon.</span> +<span class="c1"># We&#39;re doing this over midnight because we&#39;re more concerned</span> +<span class="c1"># with what people see, and people don&#39;t typically see the weather</span> +<span class="c1"># at midnight.</span> <span class="n">dt_indices</span> <span class="o">=</span> <span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">date_range</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="mi">11</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">12</span><span class="p">))</span> <span class="k">for</span> <span class="n">year</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="n">dt_merge</span> <span class="o">=</span> <span class="nb">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">union</span><span class="p">(</span><span class="n">y</span><span class="p">),</span> <span class="n">dt_indices</span><span class="p">)</span> -<span class="c"># Because we have to pay a little bit to use the API, we use for loops here</span> -<span class="c"># instead of a comprehension - if something breaks, we want to preserve the</span> -<span class="c"># data already retrieved</span> +<span class="c1"># Because we have to pay a little bit to use the API, we use for loops here</span> +<span class="c1"># instead of a comprehension - if something breaks, we want to preserve the</span> +<span class="c1"># data already retrieved</span> <span class="n">city_forecasts</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">for</span> <span class="n">city</span><span class="p">,</span> <span class="n">loc</span> <span class="ow">in</span> <span class="n">cities</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> - <span class="k">print</span><span class="p">(</span><span class="s">&quot;Retrieving data for {} starting at {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> - <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s">&quot;%I:%M:%S %p&quot;</span><span class="p">)))</span> + <span class="k">print</span><span class="p">(</span><span class="s2">&quot;Retrieving data for {} starting at {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> + <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%I:%M:%S %p&quot;</span><span class="p">)))</span> <span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="n">dt_merge</span><span class="p">:</span> <span class="k">try</span><span class="p">:</span> <span class="n">city_forecasts</span><span class="p">[(</span><span class="n">city</span><span class="p">,</span> <span class="n">dt</span><span class="p">)]</span> <span class="o">=</span> <span class="n">get_forecast</span><span class="p">(</span><span class="o">*</span><span class="n">loc</span><span class="p">,</span> <span class="n">dt</span><span class="p">)</span> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="n">city_forecasts</span><span class="p">[(</span><span class="n">city</span><span class="p">,</span> <span class="n">dt</span><span class="p">)]</span> <span class="o">=</span> <span class="bp">None</span> -<span class="k">print</span><span class="p">(</span><span class="s">&quot;End forecast retrieval: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s">&quot;%I:%M:%S %p&quot;</span><span class="p">)))</span> +<span class="k">print</span><span class="p">(</span><span class="s2">&quot;End forecast retrieval: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%I:%M:%S %p&quot;</span><span class="p">)))</span> <span class="kn">import</span> <span class="nn">pickle</span> -<span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">city_forecasts</span><span class="p">,</span> <span class="nb">open</span><span class="p">(</span><span class="s">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s">&#39;wb&#39;</span><span class="p">))</span> +<span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">city_forecasts</span><span class="p">,</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s1">&#39;wb&#39;</span><span class="p">))</span> -<span class="c">### Output:</span> -<span class="c"># Retrieving data for binghamton starting at 05:13:42 PM</span> -<span class="c"># Retrieving data for seattle starting at 05:30:51 PM</span> -<span class="c"># Retrieving data for nyc starting at 05:48:30 PM</span> -<span class="c"># Retrieving data for cary starting at 06:08:32 PM</span> -<span class="c"># End forecast retrieval: 06:25:21 PM</span> +<span class="c1">### Output:</span> +<span class="c1"># Retrieving data for binghamton starting at 05:13:42 PM</span> +<span class="c1"># Retrieving data for seattle starting at 05:30:51 PM</span> +<span class="c1"># Retrieving data for nyc starting at 05:48:30 PM</span> +<span class="c1"># Retrieving data for cary starting at 06:08:32 PM</span> +<span class="c1"># End forecast retrieval: 06:25:21 PM</span> </pre></div> </div> @@ -18608,7 +18875,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Testing Cramer2015-12-26T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-12-26:testing-cramer.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Testing Cramer2015-12-26T00:00:00-05:002015-12-26T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-12-26:testing-cramer.html<p> <div class="cell border-box-sizing code_cell rendered"> <div class="input"> <div class="prompt input_prompt">In&nbsp;[1]:</div> @@ -18655,20 +18922,20 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">articles</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">def</span> <span class="nf">handle_starttag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span> - <span class="c">#if tag == &#39;div&#39; and (&quot;id&quot;, &quot;author_articles_wrapper&quot;) in attrs:</span> - <span class="c"># self.fetch_links = True</span> - <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;a&#39;</span> <span class="ow">and</span> <span class="p">(</span><span class="s">&#39;class&#39;</span><span class="p">,</span> <span class="s">&#39;dashboard_article_link&#39;</span><span class="p">)</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">:</span> - <span class="n">href</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;href&#39;</span><span class="p">,</span> <span class="n">attrs</span><span class="p">))[</span><span class="mi">0</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span> + <span class="c1">#if tag == &#39;div&#39; and (&quot;id&quot;, &quot;author_articles_wrapper&quot;) in attrs:</span> + <span class="c1"># self.fetch_links = True</span> + <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;a&#39;</span> <span class="ow">and</span> <span class="p">(</span><span class="s1">&#39;class&#39;</span><span class="p">,</span> <span class="s1">&#39;dashboard_article_link&#39;</span><span class="p">)</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">:</span> + <span class="n">href</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;href&#39;</span><span class="p">,</span> <span class="n">attrs</span><span class="p">))[</span><span class="mi">0</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span> <span class="bp">self</span><span class="o">.</span><span class="n">articles</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">href</span><span class="p">)</span> -<span class="n">base_url</span> <span class="o">=</span> <span class="s">&quot;http://seekingalpha.com/author/wall-street-breakfast/articles&quot;</span> -<span class="n">article_page_urls</span> <span class="o">=</span> <span class="p">[</span><span class="n">base_url</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="n">base_url</span> <span class="o">+</span> <span class="s">&#39;/{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">20</span><span class="p">)]</span> +<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;http://seekingalpha.com/author/wall-street-breakfast/articles&quot;</span> +<span class="n">article_page_urls</span> <span class="o">=</span> <span class="p">[</span><span class="n">base_url</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="n">base_url</span> <span class="o">+</span> <span class="s1">&#39;/{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">20</span><span class="p">)]</span> <span class="n">global_articles</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="n">article_page_urls</span><span class="p">:</span> - <span class="c"># We need to switch the user agent, as SA blocks the standard requests agent</span> + <span class="c1"># We need to switch the user agent, as SA blocks the standard requests agent</span> <span class="n">articles_html</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">page</span><span class="p">,</span> - <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="s">&quot;Wget/1.13.4&quot;</span><span class="p">})</span> + <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="s2">&quot;Wget/1.13.4&quot;</span><span class="p">})</span> <span class="n">parser</span> <span class="o">=</span> <span class="n">ArticleListParser</span><span class="p">()</span> <span class="n">parser</span><span class="o">.</span><span class="n">feed</span><span class="p">(</span><span class="n">articles_html</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="n">global_articles</span> <span class="o">+=</span> <span class="p">(</span><span class="n">parser</span><span class="o">.</span><span class="n">articles</span><span class="p">)</span> @@ -18685,24 +18952,24 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">class</span> <span class="nc">ArticleReturnParser</span><span class="p">(</span><span class="n">HTMLParser</span><span class="p">):</span> - <span class="s">&quot;Given an article, parse out the futures returns in it&quot;</span> + <span class="s2">&quot;Given an article, parse out the futures returns in it&quot;</span> - <span class="n">record_font_tags</span> <span class="o">=</span> <span class="k">False</span> - <span class="n">in_font_tag</span> <span class="o">=</span> <span class="k">False</span> + <span class="n">record_font_tags</span> <span class="o">=</span> <span class="kc">False</span> + <span class="n">in_font_tag</span> <span class="o">=</span> <span class="kc">False</span> <span class="n">counter</span> <span class="o">=</span> <span class="mi">0</span> - <span class="c"># data = {} # See __init__</span> + <span class="c1"># data = {} # See __init__</span> <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">def</span> <span class="nf">handle_starttag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span> - <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;span&#39;</span> <span class="ow">and</span> <span class="p">(</span><span class="s">&#39;itemprop&#39;</span><span class="p">,</span> <span class="s">&#39;datePublished&#39;</span><span class="p">)</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">:</span> - <span class="n">date_string</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;content&#39;</span><span class="p">,</span> <span class="n">attrs</span><span class="p">))[</span><span class="mi">0</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span> + <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;span&#39;</span> <span class="ow">and</span> <span class="p">(</span><span class="s1">&#39;itemprop&#39;</span><span class="p">,</span> <span class="s1">&#39;datePublished&#39;</span><span class="p">)</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">:</span> + <span class="n">date_string</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;content&#39;</span><span class="p">,</span> <span class="n">attrs</span><span class="p">))[</span><span class="mi">0</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span> <span class="n">date</span> <span class="o">=</span> <span class="n">dtparser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;date&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">date</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;date&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">date</span> - <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span> <span class="o">=</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;font&#39;</span> + <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span> <span class="o">=</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;font&#39;</span> <span class="k">def</span> <span class="nf">safe_float</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string</span><span class="p">):</span> <span class="k">try</span><span class="p">:</span> @@ -18711,43 +18978,43 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">handle_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">content</span><span class="p">):</span> - <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="ow">and</span> <span class="s">&quot;Futures at 6&quot;</span> <span class="ow">in</span> <span class="n">content</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="o">=</span> <span class="k">True</span> + <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="ow">and</span> <span class="s2">&quot;Futures at 6&quot;</span> <span class="ow">in</span> <span class="n">content</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="o">=</span> <span class="kc">True</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span><span class="p">:</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;DOW&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;DOW&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;S&amp;P&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;S&amp;P&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;NASDAQ&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;NASDAQ&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">3</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;Crude&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;Crude&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">4</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;Gold&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;Gold&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">+=</span> <span class="mi">1</span> <span class="k">def</span> <span class="nf">handle_endtag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span> <span class="o">=</span> <span class="k">False</span> + <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span> <span class="o">=</span> <span class="kc">False</span> <span class="k">def</span> <span class="nf">retrieve_data</span><span class="p">(</span><span class="n">url</span><span class="p">):</span> - <span class="n">sa</span> <span class="o">=</span> <span class="s">&quot;http://seekingalpha.com&quot;</span> + <span class="n">sa</span> <span class="o">=</span> <span class="s2">&quot;http://seekingalpha.com&quot;</span> <span class="n">article_html</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sa</span> <span class="o">+</span> <span class="n">url</span><span class="p">,</span> - <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="s">&quot;Wget/1.13.4&quot;</span><span class="p">})</span> + <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="s2">&quot;Wget/1.13.4&quot;</span><span class="p">})</span> <span class="n">parser</span> <span class="o">=</span> <span class="n">ArticleReturnParser</span><span class="p">()</span> <span class="n">parser</span><span class="o">.</span><span class="n">feed</span><span class="p">(</span><span class="n">article_html</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> - <span class="n">parser</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s">&quot;url&quot;</span><span class="p">:</span> <span class="n">url</span><span class="p">})</span> - <span class="n">parser</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s">&quot;text&quot;</span><span class="p">:</span> <span class="n">article_html</span><span class="o">.</span><span class="n">text</span><span class="p">})</span> + <span class="n">parser</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;url&quot;</span><span class="p">:</span> <span class="n">url</span><span class="p">})</span> + <span class="n">parser</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;text&quot;</span><span class="p">:</span> <span class="n">article_html</span><span class="o">.</span><span class="n">text</span><span class="p">})</span> <span class="k">return</span> <span class="n">parser</span><span class="o">.</span><span class="n">data</span> -<span class="c"># This copy **MUST** be in place. I&#39;m not sure why,</span> -<span class="c"># as you&#39;d think that the data being returned would already</span> -<span class="c"># represent a different memory location. Even so, it blows up</span> -<span class="c"># if you don&#39;t do this.</span> +<span class="c1"># This copy **MUST** be in place. I&#39;m not sure why,</span> +<span class="c1"># as you&#39;d think that the data being returned would already</span> +<span class="c1"># represent a different memory location. Even so, it blows up</span> +<span class="c1"># if you don&#39;t do this.</span> <span class="n">article_list</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">global_articles</span><span class="p">))</span> <span class="n">article_data</span> <span class="o">=</span> <span class="p">[</span><span class="n">copy</span><span class="p">(</span><span class="n">retrieve_data</span><span class="p">(</span><span class="n">url</span><span class="p">))</span> <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">article_list</span><span class="p">]</span> -<span class="c"># If there&#39;s an issue downloading the article, drop it.</span> +<span class="c1"># If there&#39;s an issue downloading the article, drop it.</span> <span class="n">article_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">article_data</span><span class="p">)</span><span class="o">.</span><span class="n">dropna</span><span class="p">()</span> </pre></div> @@ -18771,12 +19038,12 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[4]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># article_df is sorted by date, so we get the first row.</span> -<span class="n">start_date</span> <span class="o">=</span> <span class="n">article_df</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="n">by</span><span class="o">=</span><span class="s">&#39;date&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s">&#39;date&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> -<span class="n">SPY</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;GOOG/NYSE_SPY&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> -<span class="n">DJIA</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;GOOG/AMS_DIA&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> -<span class="n">RUSS</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;GOOG/AMEX_IWM&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> -<span class="n">NASDAQ</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;GOOG/EPA_QQQ&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># article_df is sorted by date, so we get the first row.</span> +<span class="n">start_date</span> <span class="o">=</span> <span class="n">article_df</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="n">by</span><span class="o">=</span><span class="s1">&#39;date&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;date&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> +<span class="n">SPY</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;GOOG/NYSE_SPY&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> +<span class="n">DJIA</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;GOOG/AMS_DIA&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> +<span class="n">RUSS</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;GOOG/AMEX_IWM&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> +<span class="n">NASDAQ</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;GOOG/EPA_QQQ&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> </pre></div> </div> @@ -18801,13 +19068,13 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">calculate_opening_ret</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="c"># I&#39;m not a huge fan of the appending for loop,</span> - <span class="c"># but it&#39;s a bit verbose for a comprehension</span> + <span class="c1"># I&#39;m not a huge fan of the appending for loop,</span> + <span class="c1"># but it&#39;s a bit verbose for a comprehension</span> <span class="n">data</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">frame</span><span class="p">)):</span> <span class="n">date</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">.</span><span class="n">name</span> - <span class="n">prior_close</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s">&#39;Close&#39;</span><span class="p">]</span> - <span class="n">open_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s">&#39;Open&#39;</span><span class="p">]</span> + <span class="n">prior_close</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span> + <span class="n">open_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span> <span class="n">data</span><span class="p">[</span><span class="n">date</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">open_val</span> <span class="o">-</span> <span class="n">prior_close</span><span class="p">)</span> <span class="o">/</span> <span class="n">prior_close</span> <span class="k">return</span> <span class="n">data</span> @@ -18818,8 +19085,8 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">NASDAQ_open_ret</span> <span class="o">=</span> <span class="n">calculate_opening_ret</span><span class="p">(</span><span class="n">NASDAQ</span><span class="p">)</span> <span class="k">def</span> <span class="nf">signs_match</span><span class="p">(</span><span class="n">list_1</span><span class="p">,</span> <span class="n">list_2</span><span class="p">):</span> - <span class="c"># This is a surprisingly difficult task - we have to match</span> - <span class="c"># up the dates in order to check if opening returns actually match</span> + <span class="c1"># This is a surprisingly difficult task - we have to match</span> + <span class="c1"># up the dates in order to check if opening returns actually match</span> <span class="n">index_dict_dt</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="o">.</span><span class="n">to_datetime</span><span class="p">():</span> <span class="n">list_2</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">list_2</span><span class="o">.</span><span class="n">keys</span><span class="p">()}</span> <span class="n">matches</span> <span class="o">=</span> <span class="p">[]</span> @@ -18835,26 +19102,26 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">matches</span> <span class="o">+=</span> <span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="n">matches</span> <span class="o">+=</span> <span class="p">[</span><span class="mi">0</span><span class="p">]</span> - <span class="c">#print(&quot;{}&quot;.format(list_2[index_dt]))</span> + <span class="c1">#print(&quot;{}&quot;.format(list_2[index_dt]))</span> <span class="k">return</span> <span class="n">matches</span> <span class="n">prediction_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">matches_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">count_dict</span> <span class="o">=</span> <span class="p">{}</span> -<span class="n">index_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s">&quot;SPY&quot;</span><span class="p">:</span> <span class="n">SPY_open_ret</span><span class="p">,</span> <span class="s">&quot;DJIA&quot;</span><span class="p">:</span> <span class="n">DJIA_open_ret</span><span class="p">,</span> <span class="s">&quot;RUSS&quot;</span><span class="p">:</span> <span class="n">RUSS_open_ret</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">:</span> <span class="n">NASDAQ_open_ret</span><span class="p">}</span> -<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="s">&quot;SPY&quot;</span><span class="p">,</span> <span class="s">&quot;DJIA&quot;</span><span class="p">,</span> <span class="s">&quot;RUSS&quot;</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">]</span> -<span class="n">futures</span> <span class="o">=</span> <span class="p">[</span><span class="s">&quot;Crude&quot;</span><span class="p">,</span> <span class="s">&quot;Gold&quot;</span><span class="p">,</span> <span class="s">&quot;DOW&quot;</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">,</span> <span class="s">&quot;S&amp;P&quot;</span><span class="p">]</span> +<span class="n">index_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;SPY&quot;</span><span class="p">:</span> <span class="n">SPY_open_ret</span><span class="p">,</span> <span class="s2">&quot;DJIA&quot;</span><span class="p">:</span> <span class="n">DJIA_open_ret</span><span class="p">,</span> <span class="s2">&quot;RUSS&quot;</span><span class="p">:</span> <span class="n">RUSS_open_ret</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">:</span> <span class="n">NASDAQ_open_ret</span><span class="p">}</span> +<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;SPY&quot;</span><span class="p">,</span> <span class="s2">&quot;DJIA&quot;</span><span class="p">,</span> <span class="s2">&quot;RUSS&quot;</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">]</span> +<span class="n">futures</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;Crude&quot;</span><span class="p">,</span> <span class="s2">&quot;Gold&quot;</span><span class="p">,</span> <span class="s2">&quot;DOW&quot;</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">,</span> <span class="s2">&quot;S&amp;P&quot;</span><span class="p">]</span> <span class="k">for</span> <span class="n">index</span> <span class="ow">in</span> <span class="n">indices</span><span class="p">:</span> - <span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">signs_match</span><span class="p">(</span><span class="n">article_df</span><span class="p">[[</span><span class="n">future</span><span class="p">,</span> <span class="s">&#39;date&#39;</span><span class="p">]],</span> + <span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">signs_match</span><span class="p">(</span><span class="n">article_df</span><span class="p">[[</span><span class="n">future</span><span class="p">,</span> <span class="s1">&#39;date&#39;</span><span class="p">]],</span> <span class="n">index_dict</span><span class="p">[</span><span class="n">index</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> <span class="n">count_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="n">future</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> <span class="n">prediction_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="n">future</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Articles Checked: &quot;</span><span class="p">)</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Articles Checked: &quot;</span><span class="p">)</span> <span class="nb">print</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">count_dict</span><span class="p">))</span> <span class="nb">print</span><span class="p">()</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Prediction Accuracy:&quot;</span><span class="p">)</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Prediction Accuracy:&quot;</span><span class="p">)</span> <span class="nb">print</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">prediction_dict</span><span class="p">))</span> </pre></div> @@ -18914,13 +19181,13 @@ S&amp;P 0.604478 0.597015 0.811808 0.848708 <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">calculate_closing_ret</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="c"># I&#39;m not a huge fan of the appending for loop,</span> - <span class="c"># but it&#39;s a bit verbose for a comprehension</span> + <span class="c1"># I&#39;m not a huge fan of the appending for loop,</span> + <span class="c1"># but it&#39;s a bit verbose for a comprehension</span> <span class="n">data</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">frame</span><span class="p">)):</span> <span class="n">date</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">.</span><span class="n">name</span> - <span class="n">open_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s">&#39;Open&#39;</span><span class="p">]</span> - <span class="n">close_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s">&#39;Close&#39;</span><span class="p">]</span> + <span class="n">open_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span> + <span class="n">close_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span> <span class="n">data</span><span class="p">[</span><span class="n">date</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">close_val</span> <span class="o">-</span> <span class="n">open_val</span><span class="p">)</span> <span class="o">/</span> <span class="n">open_val</span> <span class="k">return</span> <span class="n">data</span> @@ -18931,8 +19198,8 @@ S&amp;P 0.604478 0.597015 0.811808 0.848708 <span class="n">NASDAQ_close_ret</span> <span class="o">=</span> <span class="n">calculate_closing_ret</span><span class="p">(</span><span class="n">NASDAQ</span><span class="p">)</span> <span class="k">def</span> <span class="nf">signs_match</span><span class="p">(</span><span class="n">list_1</span><span class="p">,</span> <span class="n">list_2</span><span class="p">):</span> - <span class="c"># This is a surprisingly difficult task - we have to match</span> - <span class="c"># up the dates in order to check if opening returns actually match</span> + <span class="c1"># This is a surprisingly difficult task - we have to match</span> + <span class="c1"># up the dates in order to check if opening returns actually match</span> <span class="n">index_dict_dt</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="o">.</span><span class="n">to_datetime</span><span class="p">():</span> <span class="n">list_2</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">list_2</span><span class="o">.</span><span class="n">keys</span><span class="p">()}</span> <span class="n">matches</span> <span class="o">=</span> <span class="p">[]</span> @@ -18948,28 +19215,28 @@ S&amp;P 0.604478 0.597015 0.811808 0.848708 <span class="n">matches</span> <span class="o">+=</span> <span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="n">matches</span> <span class="o">+=</span> <span class="p">[</span><span class="mi">0</span><span class="p">]</span> - <span class="c">#print(&quot;{}&quot;.format(list_2[index_dt]))</span> + <span class="c1">#print(&quot;{}&quot;.format(list_2[index_dt]))</span> <span class="k">return</span> <span class="n">matches</span> <span class="n">matches_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">count_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">prediction_dict</span> <span class="o">=</span> <span class="p">{}</span> -<span class="n">index_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s">&quot;SPY&quot;</span><span class="p">:</span> <span class="n">SPY_close_ret</span><span class="p">,</span> <span class="s">&quot;DJIA&quot;</span><span class="p">:</span> <span class="n">DJIA_close_ret</span><span class="p">,</span> - <span class="s">&quot;RUSS&quot;</span><span class="p">:</span> <span class="n">RUSS_close_ret</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">:</span> <span class="n">NASDAQ_close_ret</span><span class="p">}</span> -<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="s">&quot;SPY&quot;</span><span class="p">,</span> <span class="s">&quot;DJIA&quot;</span><span class="p">,</span> <span class="s">&quot;RUSS&quot;</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">]</span> -<span class="n">futures</span> <span class="o">=</span> <span class="p">[</span><span class="s">&quot;Crude&quot;</span><span class="p">,</span> <span class="s">&quot;Gold&quot;</span><span class="p">,</span> <span class="s">&quot;DOW&quot;</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">,</span> <span class="s">&quot;S&amp;P&quot;</span><span class="p">]</span> +<span class="n">index_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;SPY&quot;</span><span class="p">:</span> <span class="n">SPY_close_ret</span><span class="p">,</span> <span class="s2">&quot;DJIA&quot;</span><span class="p">:</span> <span class="n">DJIA_close_ret</span><span class="p">,</span> + <span class="s2">&quot;RUSS&quot;</span><span class="p">:</span> <span class="n">RUSS_close_ret</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">:</span> <span class="n">NASDAQ_close_ret</span><span class="p">}</span> +<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;SPY&quot;</span><span class="p">,</span> <span class="s2">&quot;DJIA&quot;</span><span class="p">,</span> <span class="s2">&quot;RUSS&quot;</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">]</span> +<span class="n">futures</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;Crude&quot;</span><span class="p">,</span> <span class="s2">&quot;Gold&quot;</span><span class="p">,</span> <span class="s2">&quot;DOW&quot;</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">,</span> <span class="s2">&quot;S&amp;P&quot;</span><span class="p">]</span> <span class="k">for</span> <span class="n">index</span> <span class="ow">in</span> <span class="n">indices</span><span class="p">:</span> - <span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">signs_match</span><span class="p">(</span><span class="n">article_df</span><span class="p">[[</span><span class="n">future</span><span class="p">,</span> <span class="s">&#39;date&#39;</span><span class="p">]],</span> + <span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">signs_match</span><span class="p">(</span><span class="n">article_df</span><span class="p">[[</span><span class="n">future</span><span class="p">,</span> <span class="s1">&#39;date&#39;</span><span class="p">]],</span> <span class="n">index_dict</span><span class="p">[</span><span class="n">index</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> <span class="n">count_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="n">future</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> <span class="n">prediction_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="n">future</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Articles Checked:&quot;</span><span class="p">)</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Articles Checked:&quot;</span><span class="p">)</span> <span class="nb">print</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">count_dict</span><span class="p">))</span> <span class="nb">print</span><span class="p">()</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Prediction Accuracy:&quot;</span><span class="p">)</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Prediction Accuracy:&quot;</span><span class="p">)</span> <span class="nb">print</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">prediction_dict</span><span class="p">))</span> </pre></div> @@ -19031,7 +19298,7 @@ S&amp;P 0.585821 0.608209 0.535055 0.535055 MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Autocallable Bonds2015-11-27T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-11-27:autocallable-bonds.html<p>My only non-core class this semester has been in Structure Products. We've been surveying a wide variety of products, and the final project was to pick one to report on. +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Autocallable Bonds2015-11-27T00:00:00-05:002015-11-27T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-11-27:autocallable-bonds.html<p>My only non-core class this semester has been in Structure Products. We've been surveying a wide variety of products, and the final project was to pick one to report on. Because these are all very similar, we decided to demonstrate all 3 products at once.</p> <p>What follows below is a notebook demonstrating the usage of <a href="http://julialang.com">Julia</a> for Monte-Carlo simulation of some exotic products.</p> <p> @@ -19079,7 +19346,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at <div class=" highlight hl-julia"><pre><span class="n">S0</span> <span class="o">=</span> <span class="mf">102.2</span> <span class="n">nominal</span> <span class="o">=</span> <span class="mi">100</span> <span class="n">q</span> <span class="o">=</span> <span class="mf">2.84</span> <span class="o">/</span> <span class="mi">100</span> -<span class="err">σ</span> <span class="o">=</span> <span class="mf">15.37</span> <span class="o">/</span> <span class="mi">100</span> +<span class="n">σ</span> <span class="o">=</span> <span class="mf">15.37</span> <span class="o">/</span> <span class="mi">100</span> <span class="n">term</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">.</span><span class="mi">49</span><span class="p">,</span> <span class="o">.</span><span class="mi">9</span><span class="p">,</span> <span class="mf">1.21</span><span class="p">,</span> <span class="mf">1.45</span><span class="p">,</span> <span class="mf">1.69</span><span class="p">]</span> <span class="o">/</span> <span class="mi">100</span> <span class="o">+</span> <span class="mi">1</span> <span class="c">###</span> @@ -19132,7 +19399,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at <div class="prompt input_prompt">In&nbsp;[3]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-julia"><pre><span class="n">simulate_gbm</span> <span class="o">=</span> <span class="n">function</span><span class="p">(</span><span class="n">S0</span><span class="p">,</span> <span class="err">μ</span><span class="p">,</span> <span class="err">σ</span><span class="p">,</span> <span class="n">T</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> +<div class=" highlight hl-julia"><pre><span class="n">simulate_gbm</span> <span class="o">=</span> <span class="n">function</span><span class="p">(</span><span class="n">S0</span><span class="p">,</span> <span class="n">μ</span><span class="p">,</span> <span class="n">σ</span><span class="p">,</span> <span class="n">T</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> <span class="c"># Set the initial state</span> <span class="n">m</span> <span class="o">=</span> <span class="n">length</span><span class="p">(</span><span class="n">S0</span><span class="p">)</span> <span class="n">t</span> <span class="o">=</span> <span class="n">T</span> <span class="o">/</span> <span class="n">n</span> @@ -19141,7 +19408,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at <span class="c"># Build out all states</span> <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:(</span><span class="n">n</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> - <span class="n">motion</span><span class="p">[:,</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">motion</span><span class="p">[:,</span><span class="n">i</span><span class="p">]</span> <span class="o">.*</span> <span class="n">exp</span><span class="p">((</span><span class="err">μ</span> <span class="o">-</span> <span class="err">σ</span><span class="o">^</span><span class="mi">2</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span><span class="o">*</span><span class="n">t</span><span class="p">)</span> <span class="o">.*</span> <span class="n">exp</span><span class="p">(</span><span class="n">sqrt</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="o">*</span> <span class="err">σ</span> <span class="o">.*</span> <span class="n">randn</span><span class="p">(</span><span class="n">m</span><span class="p">))</span> + <span class="n">motion</span><span class="p">[:,</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">motion</span><span class="p">[:,</span><span class="n">i</span><span class="p">]</span> <span class="o">.*</span> <span class="n">exp</span><span class="p">((</span><span class="n">μ</span> <span class="o">-</span> <span class="n">σ</span><span class="o">^</span><span class="mi">2</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span><span class="o">*</span><span class="n">t</span><span class="p">)</span> <span class="o">.*</span> <span class="n">exp</span><span class="p">(</span><span class="n">sqrt</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="o">*</span> <span class="n">σ</span> <span class="o">.*</span> <span class="n">randn</span><span class="p">(</span><span class="n">m</span><span class="p">))</span> <span class="k">end</span> <span class="k">return</span> <span class="n">motion</span> @@ -19205,7 +19472,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at <div class="input_area"> <div class=" highlight hl-julia"><pre><span class="n">initial</span> <span class="o">=</span> <span class="n">ones</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span> <span class="o">*</span> <span class="n">S0</span> <span class="c"># Using μ=0, T=.25 for now, we&#39;ll use the proper values later</span> -<span class="n">motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">initial</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="err">σ</span><span class="p">,</span> <span class="o">.</span><span class="mi">25</span><span class="p">,</span> <span class="mi">200</span><span class="p">)</span> +<span class="n">motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">initial</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">σ</span><span class="p">,</span> <span class="o">.</span><span class="mi">25</span><span class="p">,</span> <span class="mi">200</span><span class="p">)</span> <span class="n">display_motion</span><span class="p">(</span><span class="n">motion</span><span class="p">,</span> <span class="o">.</span><span class="mi">25</span><span class="p">)</span> </pre></div> @@ -20888,8 +21155,8 @@ fig.select("#fig-3a6dd25ad25c4037a166889ee51bb151-element-20") <div class=" highlight hl-julia"><pre><span class="n">full_motion</span> <span class="o">=</span> <span class="n">ones</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span> <span class="o">*</span> <span class="n">S0</span> <span class="n">full_term</span> <span class="o">=</span> <span class="n">vcat</span><span class="p">(</span><span class="n">term</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">forward_yield</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:</span><span class="n">T</span> - <span class="err">μ</span> <span class="o">=</span> <span class="p">(</span><span class="n">full_term</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">q</span><span class="p">)</span> - <span class="n">year_motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">full_motion</span><span class="p">[:,</span><span class="k">end</span><span class="p">],</span> <span class="err">μ</span><span class="p">,</span> <span class="err">σ</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> + <span class="n">μ</span> <span class="o">=</span> <span class="p">(</span><span class="n">full_term</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">q</span><span class="p">)</span> + <span class="n">year_motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">full_motion</span><span class="p">[:,</span><span class="k">end</span><span class="p">],</span> <span class="n">μ</span><span class="p">,</span> <span class="n">σ</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> <span class="n">full_motion</span> <span class="o">=</span> <span class="n">hcat</span><span class="p">(</span><span class="n">full_motion</span><span class="p">,</span> <span class="n">year_motion</span><span class="p">)</span> <span class="k">end</span> @@ -22452,8 +22719,8 @@ fig.select("#fig-0378e04b897742b597befd2e8e1c169e-element-20") <span class="c"># And an S0 to kick things off.</span> <span class="n">final_motion</span> <span class="o">=</span> <span class="n">ones</span><span class="p">(</span><span class="n">m</span><span class="p">)</span> <span class="o">*</span> <span class="n">S0</span> <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:</span><span class="n">T</span> - <span class="err">μ</span> <span class="o">=</span> <span class="p">(</span><span class="n">forward</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">q</span><span class="p">)</span> - <span class="n">year_motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">final_motion</span><span class="p">[:,</span><span class="k">end</span><span class="p">],</span> <span class="err">μ</span><span class="p">,</span> <span class="err">σ</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> + <span class="n">μ</span> <span class="o">=</span> <span class="p">(</span><span class="n">forward</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">q</span><span class="p">)</span> + <span class="n">year_motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">final_motion</span><span class="p">[:,</span><span class="k">end</span><span class="p">],</span> <span class="n">μ</span><span class="p">,</span> <span class="n">σ</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> <span class="n">final_motion</span> <span class="o">=</span> <span class="n">hcat</span><span class="p">(</span><span class="n">final_motion</span><span class="p">,</span> <span class="n">year_motion</span><span class="p">)</span> <span class="k">end</span> <span class="k">return</span> <span class="n">final_motion</span> @@ -22521,7 +22788,7 @@ fig.select("#fig-0378e04b897742b597befd2e8e1c169e-element-20") <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:</span><span class="n">t</span> <span class="n">price</span> <span class="o">=</span> <span class="n">year_prices</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> - <span class="k">if</span> <span class="n">price</span> <span class="err">≥</span> <span class="n">call_barrier</span> + <span class="k">if</span> <span class="n">price</span> <span class="n">≥</span> <span class="n">call_barrier</span> <span class="k">return</span> <span class="p">(</span><span class="n">nominal</span> <span class="o">+</span> <span class="n">coupon</span><span class="o">*</span><span class="n">i</span><span class="p">)</span> <span class="o">*</span> <span class="n">exp</span><span class="p">((</span><span class="n">prod</span><span class="p">(</span><span class="n">forward_structure</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="k">end</span><span class="p">])</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">t</span><span class="o">-</span><span class="n">i</span><span class="p">))</span> <span class="k">end</span> <span class="k">end</span> @@ -22631,9 +22898,9 @@ Present value of Athena note: $95.00, notional: $100.00</pre> <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:</span><span class="n">t</span> <span class="n">price</span> <span class="o">=</span> <span class="n">year_prices</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> - <span class="k">if</span> <span class="n">price</span> <span class="err">≥</span> <span class="n">call_barrier</span> + <span class="k">if</span> <span class="n">price</span> <span class="n">≥</span> <span class="n">call_barrier</span> <span class="k">return</span> <span class="p">(</span><span class="n">nominal</span> <span class="o">+</span> <span class="n">coupon</span> <span class="o">+</span> <span class="n">total_coupons</span><span class="p">)</span><span class="o">*</span><span class="n">exp</span><span class="p">((</span><span class="n">prod</span><span class="p">(</span><span class="n">forward_structure</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="k">end</span><span class="p">])</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">t</span><span class="o">-</span><span class="n">i</span><span class="p">))</span> - <span class="k">elseif</span> <span class="n">price</span> <span class="err">≥</span> <span class="n">coupon_barrier</span> + <span class="k">elseif</span> <span class="n">price</span> <span class="n">≥</span> <span class="n">coupon_barrier</span> <span class="n">total_coupons</span> <span class="o">=</span> <span class="n">total_coupons</span> <span class="o">*</span> <span class="n">exp</span><span class="p">(</span><span class="n">forward_structure</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="n">coupon</span> <span class="k">else</span> <span class="n">total_coupons</span> <span class="o">*=</span> <span class="n">exp</span><span class="p">(</span><span class="n">forward_structure</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> @@ -22810,7 +23077,7 @@ Present value of Phoenix with memory note: $100.09</pre> MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Welcome, and an algorithm2015-12-05T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-11-19:welcome-and-an-algorithm.html<p>Hello! Glad to meet you. I'm currently a student at Columbia University +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Welcome, and an algorithm2015-12-05T00:00:00-05:002015-12-05T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-11-19:welcome-and-an-algorithm.html<p>Hello! Glad to meet you. I'm currently a student at Columbia University studying Financial Engineering, and want to give an overview of the projects I'm working on!</p> <p>To start things off, Columbia has been hosting a trading competition that @@ -22885,12 +23152,12 @@ to make some tweaks over the coming weeks, and do another forward test in Januar <span class="kn">import</span> <span class="nn">Quandl</span> <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span> -<span class="n">tickers</span> <span class="o">=</span> <span class="p">[</span><span class="s">&#39;XOM&#39;</span><span class="p">,</span> <span class="s">&#39;CVX&#39;</span><span class="p">,</span> <span class="s">&#39;CLB&#39;</span><span class="p">,</span> <span class="s">&#39;OXY&#39;</span><span class="p">,</span> <span class="s">&#39;SLB&#39;</span><span class="p">]</span> -<span class="n">market_ticker</span> <span class="o">=</span> <span class="s">&#39;GOOG/NYSE_VOO&#39;</span> +<span class="n">tickers</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;XOM&#39;</span><span class="p">,</span> <span class="s1">&#39;CVX&#39;</span><span class="p">,</span> <span class="s1">&#39;CLB&#39;</span><span class="p">,</span> <span class="s1">&#39;OXY&#39;</span><span class="p">,</span> <span class="s1">&#39;SLB&#39;</span><span class="p">]</span> +<span class="n">market_ticker</span> <span class="o">=</span> <span class="s1">&#39;GOOG/NYSE_VOO&#39;</span> <span class="n">lookback</span> <span class="o">=</span> <span class="mi">30</span> -<span class="n">d_col</span> <span class="o">=</span> <span class="s">&#39;Close&#39;</span> +<span class="n">d_col</span> <span class="o">=</span> <span class="s1">&#39;Close&#39;</span> -<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="n">tick</span><span class="p">:</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&#39;YAHOO/{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tick</span><span class="p">))[</span><span class="o">-</span><span class="n">lookback</span><span class="p">:]</span> <span class="k">for</span> <span class="n">tick</span> <span class="ow">in</span> <span class="n">tickers</span><span class="p">}</span> +<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="n">tick</span><span class="p">:</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;YAHOO/{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tick</span><span class="p">))[</span><span class="o">-</span><span class="n">lookback</span><span class="p">:]</span> <span class="k">for</span> <span class="n">tick</span> <span class="ow">in</span> <span class="n">tickers</span><span class="p">}</span> <span class="n">market</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">market_ticker</span><span class="p">)</span> </pre></div> @@ -23064,22 +23331,22 @@ s.t.\ \ & \vec{1} \omega = 1\\ <div class=" highlight hl-ipython3"><pre><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> <span class="kn">from</span> <span class="nn">scipy.optimize</span> <span class="k">import</span> <span class="n">minimize</span> -<span class="c">#sharpe_limit = .1</span> +<span class="c1">#sharpe_limit = .1</span> <span class="n">drawdown_limit</span> <span class="o">=</span> <span class="o">.</span><span class="mi">05</span> <span class="n">leverage</span> <span class="o">=</span> <span class="mi">250</span> -<span class="c"># Use the map so we can guarantee we maintain the correct order</span> -<span class="c"># sharpe_a = np.array(list(map(lambda tick: sharpes[tick], tickers))) * -1 # So we can write as upper-bound</span> +<span class="c1"># Use the map so we can guarantee we maintain the correct order</span> +<span class="c1"># sharpe_a = np.array(list(map(lambda tick: sharpes[tick], tickers))) * -1 # So we can write as upper-bound</span> <span class="n">dd_a</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">tick</span><span class="p">:</span> <span class="n">drawdowns</span><span class="p">[</span><span class="n">tick</span><span class="p">],</span> <span class="n">tickers</span><span class="p">)))</span> -<span class="n">returns_a</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">tick</span><span class="p">:</span> <span class="n">returns</span><span class="p">[</span><span class="n">tick</span><span class="p">]</span><span class="o">.</span><span class="n">mean</span><span class="p">(),</span> <span class="n">tickers</span><span class="p">)))</span> <span class="c"># Because minimizing</span> +<span class="n">returns_a</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">tick</span><span class="p">:</span> <span class="n">returns</span><span class="p">[</span><span class="n">tick</span><span class="p">]</span><span class="o">.</span><span class="n">mean</span><span class="p">(),</span> <span class="n">tickers</span><span class="p">)))</span> <span class="c1"># Because minimizing</span> <span class="n">meets_sharpe</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">*</span> <span class="n">sharpe_a</span><span class="p">)</span> <span class="o">-</span> <span class="n">sharpe_limit</span> <span class="k">def</span> <span class="nf">meets_dd</span><span class="p">(</span><span class="n">x</span><span class="p">):</span> <span class="n">portfolio</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">))</span> <span class="k">if</span> <span class="n">portfolio</span> <span class="o">&lt;</span> <span class="o">.</span><span class="mi">1</span><span class="p">:</span> - <span class="c"># If there are no stocks in the portfolio,</span> - <span class="c"># we can accidentally induce division by 0,</span> - <span class="c"># or division by something small enough to cause infinity</span> + <span class="c1"># If there are no stocks in the portfolio,</span> + <span class="c1"># we can accidentally induce division by 0,</span> + <span class="c1"># or division by something small enough to cause infinity</span> <span class="k">return</span> <span class="mi">0</span> <span class="k">return</span> <span class="n">drawdown_limit</span> <span class="o">-</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">*</span> <span class="n">dd_a</span><span class="p">)</span> <span class="o">/</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">))</span> @@ -23089,41 +23356,41 @@ s.t.\ \ & \vec{1} \omega = 1\\ <span class="k">def</span> <span class="nf">within_leverage</span><span class="p">(</span><span class="n">x</span><span class="p">):</span> <span class="k">return</span> <span class="n">leverage</span> <span class="o">-</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">))</span> -<span class="n">objective</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">sum</span><span class="p">(</span><span class="n">x</span> <span class="o">*</span> <span class="n">returns_a</span><span class="p">)</span> <span class="o">*</span> <span class="o">-</span><span class="mi">1</span> <span class="c"># Because we&#39;re minimizing</span> -<span class="n">bounds</span> <span class="o">=</span> <span class="p">((</span><span class="k">None</span><span class="p">,</span> <span class="k">None</span><span class="p">),)</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">tickers</span><span class="p">)</span> +<span class="n">objective</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">sum</span><span class="p">(</span><span class="n">x</span> <span class="o">*</span> <span class="n">returns_a</span><span class="p">)</span> <span class="o">*</span> <span class="o">-</span><span class="mi">1</span> <span class="c1"># Because we&#39;re minimizing</span> +<span class="n">bounds</span> <span class="o">=</span> <span class="p">((</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">),)</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">tickers</span><span class="p">)</span> <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">tickers</span><span class="p">))</span> <span class="n">constraints</span> <span class="o">=</span> <span class="p">[</span> <span class="p">{</span> - <span class="s">&#39;type&#39;</span><span class="p">:</span> <span class="s">&#39;eq&#39;</span><span class="p">,</span> - <span class="s">&#39;fun&#39;</span><span class="p">:</span> <span class="n">is_portfolio</span> + <span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;eq&#39;</span><span class="p">,</span> + <span class="s1">&#39;fun&#39;</span><span class="p">:</span> <span class="n">is_portfolio</span> <span class="p">},</span> <span class="p">{</span> - <span class="s">&#39;type&#39;</span><span class="p">:</span> <span class="s">&#39;ineq&#39;</span><span class="p">,</span> - <span class="s">&#39;fun&#39;</span><span class="p">:</span> <span class="n">within_leverage</span> - <span class="c">#}, {</span> - <span class="c"># &#39;type&#39;: &#39;ineq&#39;,</span> - <span class="c"># &#39;fun&#39;: meets_sharpe</span> + <span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;ineq&#39;</span><span class="p">,</span> + <span class="s1">&#39;fun&#39;</span><span class="p">:</span> <span class="n">within_leverage</span> + <span class="c1">#}, {</span> + <span class="c1"># &#39;type&#39;: &#39;ineq&#39;,</span> + <span class="c1"># &#39;fun&#39;: meets_sharpe</span> <span class="p">},</span> <span class="p">{</span> - <span class="s">&#39;type&#39;</span><span class="p">:</span> <span class="s">&#39;ineq&#39;</span><span class="p">,</span> - <span class="s">&#39;fun&#39;</span><span class="p">:</span> <span class="n">meets_dd</span> + <span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;ineq&#39;</span><span class="p">,</span> + <span class="s1">&#39;fun&#39;</span><span class="p">:</span> <span class="n">meets_dd</span> <span class="p">}</span> <span class="p">]</span> <span class="n">optimal</span> <span class="o">=</span> <span class="n">minimize</span><span class="p">(</span><span class="n">objective</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">bounds</span><span class="o">=</span><span class="n">bounds</span><span class="p">,</span> <span class="n">constraints</span><span class="o">=</span><span class="n">constraints</span><span class="p">,</span> - <span class="n">options</span><span class="o">=</span><span class="p">{</span><span class="s">&#39;maxiter&#39;</span><span class="p">:</span> <span class="mi">500</span><span class="p">})</span> + <span class="n">options</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;maxiter&#39;</span><span class="p">:</span> <span class="mi">500</span><span class="p">})</span> -<span class="c"># Optimization time!</span> +<span class="c1"># Optimization time!</span> <span class="n">display</span><span class="p">(</span><span class="n">optimal</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> -<span class="n">display</span><span class="p">(</span><span class="s">&quot;Holdings: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">tickers</span><span class="p">,</span> <span class="n">optimal</span><span class="o">.</span><span class="n">x</span><span class="p">))))</span> +<span class="n">display</span><span class="p">(</span><span class="s2">&quot;Holdings: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">tickers</span><span class="p">,</span> <span class="n">optimal</span><span class="o">.</span><span class="n">x</span><span class="p">))))</span> -<span class="n">expected_return</span> <span class="o">=</span> <span class="n">optimal</span><span class="o">.</span><span class="n">fun</span> <span class="o">*</span> <span class="o">-</span><span class="mi">100</span> <span class="c"># multiply by -100 to scale, and compensate for minimizing</span> -<span class="n">display</span><span class="p">(</span><span class="s">&quot;Expected Return: {:.3f}%&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">expected_return</span><span class="p">))</span> +<span class="n">expected_return</span> <span class="o">=</span> <span class="n">optimal</span><span class="o">.</span><span class="n">fun</span> <span class="o">*</span> <span class="o">-</span><span class="mi">100</span> <span class="c1"># multiply by -100 to scale, and compensate for minimizing</span> +<span class="n">display</span><span class="p">(</span><span class="s2">&quot;Expected Return: {:.3f}%&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">expected_return</span><span class="p">))</span> <span class="n">expected_drawdown</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">optimal</span><span class="o">.</span><span class="n">x</span><span class="p">)</span> <span class="o">*</span> <span class="n">dd_a</span><span class="p">)</span> <span class="o">/</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">optimal</span><span class="o">.</span><span class="n">x</span><span class="p">))</span> <span class="o">*</span> <span class="mi">100</span> -<span class="n">display</span><span class="p">(</span><span class="s">&quot;Expected Max Drawdown: {0:.2f}%&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">expected_drawdown</span><span class="p">))</span> +<span class="n">display</span><span class="p">(</span><span class="s2">&quot;Expected Max Drawdown: {0:.2f}%&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">expected_drawdown</span><span class="p">))</span> -<span class="c"># TODO: Calculate expected Sharpe</span> +<span class="c1"># TODO: Calculate expected Sharpe</span> </pre></div> </div> diff --git a/feeds/blog.atom.xml b/feeds/blog.atom.xml index b233e8a..50eb49b 100644 --- a/feeds/blog.atom.xml +++ b/feeds/blog.atom.xml @@ -1,5 +1,272 @@ -Bradlee Speicehttps://bspeice.github.io/2016-06-08T00:00:00-04:00Event Studies and Earnings Releases2016-06-08T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-06-08:event-studies-and-earnings-releases.html<script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/3.0.0/jquery.min.js"></script> +Bradlee Speicehttps://bspeice.github.io/2016-10-22T00:00:00-04:00A Rustic Re-Podcasting Server (Part 1)2016-10-22T00:00:00-04:002016-10-22T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-10-22:a-rustic-re-podcasting-server-part-1.html<p>I listen to a lot of Drum and Bass music, because it's beautiful music. And +there's a particular site, <a href="http://bassdrive.com/">Bassdrive.com</a> that hosts +a lot of great content. Specifically, the +<a href="http://archives.bassdrivearchive.com/">archives</a> section of the site has a +list of the past shows that you can download and listen to. The issue is, it's +just a <a href="http://archives.bassdrivearchive.com/6%20-%20Saturday/Electronic%20Warfare%20-%20The%20Overfiend/">giant list of links to download</a>. I'd really like +this in a podcast format to take with me on the road, etc.</p> +<p>So I wrote the <a href="https://github.com/bspeice/elektricity">elektricity</a> web +application to actually accomplish all that. Whenever you request a feed, it +goes out to Bassdrive, processes all the links on a page, and serves up some +fresh, tasty RSS to satisfy your ears. I hosted it on Heroku using the free +tier because it's really not resource-intensive at all.</p> +<p><strong>The issue so far</strong> is that I keep running out of free tier hours during a +month because my podcasting application likes to have a server scan for new +episodes constantly. Not sure why it's doing that, but I don't have a whole +lot of control over it. It's a phenomenal application otherwise.</p> +<p><strong>My (over-engineered) solution</strong>: Re-write the application using the +<a href="https://www.rust-lang.org/en-US/">Rust</a> programming language. I'd like to run +this on a small hacker board I own, and doing this in Rust would allow me to +easily cross-compile it. Plus, I've been very interested in the Rust language +for a while and this would be a great opportunity to really learn it well. +The code is available <a href="https://github.com/bspeice/nutone">here</a> as development +progresses.</p> +<h1>The Setup</h1> +<p>We'll be using the <a href="http://ironframework.io/">iron</a> library to handle the +server, and <a href="http://hyper.rs/">hyper</a> to fetch the data we need from elsewhere +on the interwebs. <a href="http://doc.servo.org/html5ever/index.html">HTML5Ever</a> allows +us to ingest the content that will be coming from Bassdrive, and finally, +output is done with <a href="http://sunng87.github.io/handlebars-rust/handlebars/index.html">handlebars-rust</a>.</p> +<p>It will ultimately be interesting to see how much more work must be done to +actually get this working over another language like Python. Coming from a +dynamic state of mind it's super easy to just chain stuff together, ship it out, +and call it a day. I think I'm going to end up getting much dirtier trying to +write all of this out.</p> +<h1>Issue 1: Strings</h1> +<p>Strings in Rust are hard. I acknowledge Python can get away with some things +that make strings super easy (and Python 3 has gotten better at cracking down +on some bad cases, <code>str &lt;-&gt; bytes</code> specifically), but Rust is hard.</p> +<p>Let's take for example the <code>404</code> error handler I'm trying to write. The result +should be incredibly simple: All I want is to echo back +<code>Didn't find URL: &lt;url&gt;</code>. Shouldn't be that hard right? In Python I'd just do +something like:</p> +<div class="highlight"><pre><span class="k">def</span> <span class="nf">echo_handler</span><span class="p">(</span><span class="n">request</span><span class="p">):</span> + <span class="k">return</span> <span class="s2">&quot;You&#39;re visiting: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">uri</span><span class="p">)</span> +</pre></div> + + +<p>And we'd call it a day. Rust isn't so simple. Let's start with the trivial +examples people post online:</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">hello_world</span><span class="p">(</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;You found the server!&quot;</span><span class="p">)))</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>Doesn't look too bad right? In fact, it's essentially the same as the Python +version! All we need to do is just send back a string of some form. So, we +look up the documentation for <a href="http://ironframework.io/doc/iron/request/struct.Request.html"><code>Request</code></a> and see a <code>url</code> field that will contain +what we want. Let's try the first iteration:</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">hello_world</span><span class="p">(</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;You found the URL: &quot;</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="n">req</span><span class="p">.</span><span class="n">url</span><span class="p">)))</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>Which yields the error:</p> +<div class="highlight"><pre>error[E0369]: binary operation `+` cannot be applied to type `&amp;&#39;static str` +</pre></div> + + +<p>OK, what's going on here? Time to start Googling for <a href="https://www.google.com/#q=concatenate+strings+in+rust">"concatenate strings in Rust"</a>. That's what we +want to do right? Concatenate a static string and the URL.</p> +<p>After Googling, we come across a helpful <a href="https://doc.rust-lang.org/std/macro.concat!.html"><code>concat!</code></a> macro that looks really nice! Let's try that one:</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">hello_world</span><span class="p">(</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="n">concat</span><span class="o">!</span><span class="p">(</span><span class="s">&quot;You found the URL: &quot;</span><span class="p">,</span><span class="w"> </span><span class="n">req</span><span class="p">.</span><span class="n">url</span><span class="p">))))</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>And the error:</p> +<p><code>error: expected a literal</code></p> +<p>Turns out Rust actually blows up because the <code>concat!</code> macro expects us to know +at compile time what <code>req.url</code> is. Which, in my outsider opinion, is a bit +strange. <code>println!</code> and <code>format!</code>, etc., all handle values they don't know at +compile time. Why can't <code>concat!</code>? By any means, we need a new plan of attack. +How about we try formatting strings?</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">hello_world</span><span class="p">(</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="n">format</span><span class="o">!</span><span class="p">(</span><span class="s">&quot;You found the URL: {}&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">req</span><span class="p">.</span><span class="n">url</span><span class="p">))))</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>And at long last, it works. Onwards!</p> +<h1>Issue 2: Fighting with the borrow checker</h1> +<p>Rust's single coolest feature is how the compiler can guarantee safety in your +program. As long as you don't use <code>unsafe</code> pointers in Rust, you're guaranteed +safety. And not having truly manual memory management is really cool; I'm +totally OK with never having to write <code>malloc()</code> again.</p> +<p>That said, even <a href="https://doc.rust-lang.org/book/ownership.html">the Rust documentation</a> makes a specific note:</p> +<blockquote> +<p>Many new users to Rust experience something we like to call +‘fighting with the borrow checker’, where the Rust compiler refuses to +compile a program that the author thinks is valid.</p> +</blockquote> +<p>If you have to put it in the documentation, it's not a helpful note: +it's hazing.</p> +<p>So now that we have a handler which works with information from the request, we +want to start making something that looks like an actual web application. +The router provided by <code>iron</code> isn't terribly difficult so I won't cover it. +Instead, the thing that had me stumped for a couple hours was trying to +dynamically create routes.</p> +<p>The unfortunate thing with Rust (in my limited experience at the moment) is that +there is a severe lack of non-trivial examples. Using the router is easy when +you want to give an example of a static function. But how do you you start +working on things that are a bit more complex?</p> +<p>We're going to cover that here. Our first try: creating a function which returns +other functions. This is a principle called <a href="http://stackoverflow.com/a/36321/1454178">currying</a>. We set up a function that allows us to keep some data in scope +for another function to come later.</p> +<div class="highlight"><pre><span class="k">fn</span><span class="w"> </span><span class="n">build_handler</span><span class="p">(</span><span class="n">message</span><span class="o">:</span><span class="w"> </span><span class="nb">String</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="nb">Fn</span><span class="p">(</span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="n">move</span><span class="w"> </span><span class="o">|</span><span class="n">_</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="o">|</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="n">message</span><span class="p">)))</span><span class="w"></span> +<span class="w"> </span><span class="p">}</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +</pre></div> + + +<p>We've simply set up a function that returns another anonymous function with the +<code>message</code> parameter scoped in. If you compile this, you get not 1, not 2, but 5 +new errors. 4 of them are the same though:</p> +<div class="highlight"><pre>error[E0277]: the trait bound `for&lt;&#39;r, &#39;r, &#39;r&gt; std::ops::Fn(&amp;&#39;r mut iron::Request&lt;&#39;r, &#39;r&gt;) -&gt; std::result::Result&lt;iron::Response, iron::IronError&gt; + &#39;static: std::marker::Sized` is not satisfied +</pre></div> + + +<p>...oookay. I for one, am not going to spend time trying to figure out what's +going on there.</p> +<p>And it is here that I will save the audience many hours of frustrated effort. +At this point, I decided to switch from <code>iron</code> to pure <code>hyper</code> since using +<code>hyper</code> would give me a much simpler API. All I would have to do is build a +function that took two parameters as input, and we're done. That said, it +ultimately posed many more issues because I started getting into a weird fight +with the <code>'static</code> <a href="https://doc.rust-lang.org/book/lifetimes.html">lifetime</a> +and being a Rust newbie I just gave up on trying to understand it.</p> +<p>Instead, we will abandon (mostly) the curried function attempt, and instead +take advantage of something Rust actually intends us to use: <code>struct</code> and +<code>trait</code>.</p> +<p>Remember when I talked about a lack of non-trivial examples on the Internet? +This is what I was talking about. I could only find <em>one</em> example of this +available online, and it was incredibly complex and contained code we honestly +don't need or care about. There was no documentation of how to build routes that +didn't use static functions, etc. But, I'm assuming you don't really care about +my whining, so let's get to it.</p> +<p>The <code>iron</code> documentation mentions the <a href="http://ironframework.io/doc/iron/middleware/trait.Handler.html"><code>Handler</code></a> trait as being something we can implement. +Does the function signature for that <code>handle()</code> method look familiar? It's what +we've been working with so far.</p> +<p>The principle is that we need to define a new <code>struct</code> to hold our data, then +implement that <code>handle()</code> method to return the result. Something that looks +like this might do:</p> +<div class="highlight"><pre><span class="k">struct</span><span class="w"> </span><span class="n">EchoHandler</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="n">message</span><span class="o">:</span><span class="w"> </span><span class="nb">String</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> + +<span class="k">impl</span><span class="w"> </span><span class="n">Handler</span><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="n">EchoHandler</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="k">fn</span><span class="w"> </span><span class="n">handle</span><span class="p">(</span><span class="o">&amp;</span><span class="bp">self</span><span class="p">,</span><span class="w"> </span><span class="n">_</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="p">)</span><span class="w"> </span><span class="o">-&gt;</span><span class="w"> </span><span class="n">IronResult</span><span class="o">&lt;</span><span class="n">Response</span><span class="o">&gt;</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">message</span><span class="p">)))</span><span class="w"></span> +<span class="w"> </span><span class="p">}</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> + +<span class="c1">// Later in the code when we set up the router...</span> +<span class="kd">let</span><span class="w"> </span><span class="n">echo</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">EchoHandler</span><span class="w"> </span><span class="p">{</span><span class="w"></span> +<span class="w"> </span><span class="n">message</span><span class="o">:</span><span class="w"> </span><span class="s">&quot;Is it working yet?&quot;</span><span class="w"></span> +<span class="p">}</span><span class="w"></span> +<span class="n">router</span><span class="p">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;/&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">echo</span><span class="p">.</span><span class="n">handle</span><span class="p">,</span><span class="w"> </span><span class="s">&quot;index&quot;</span><span class="p">);</span><span class="w"></span> +</pre></div> + + +<p>We attempt to build a struct, and give its <code>handle</code> method off to the router +so the router knows what to do.</p> +<p>You guessed it, more errors:</p> +<div class="highlight"><pre><span class="n">error</span><span class="o">:</span> <span class="n">attempted</span> <span class="n">to</span> <span class="n">take</span> <span class="n">value</span> <span class="n">of</span> <span class="n">method</span> <span class="err">`</span><span class="n">handle</span><span class="err">`</span> <span class="n">on</span> <span class="n">type</span> <span class="err">`</span><span class="n">EchoHandler</span><span class="err">`</span> +</pre></div> + + +<p>Now, the Rust compiler is actually a really nice fellow, and offers us help:</p> +<div class="highlight"><pre><span class="n">help</span><span class="o">:</span> <span class="n">maybe</span> <span class="n">a</span> <span class="err">`</span><span class="o">()</span><span class="err">`</span> <span class="n">to</span> <span class="n">call</span> <span class="n">it</span> <span class="k">is</span> <span class="n">missing</span><span class="o">?</span> <span class="n">If</span> <span class="n">not</span><span class="o">,</span> <span class="k">try</span> <span class="n">an</span> <span class="n">anonymous</span> <span class="kd">function</span> +</pre></div> + + +<p>We definitely don't want to call that function, so maybe try an anonymous +function as it recommends?</p> +<div class="highlight"><pre><span class="n">router</span><span class="p">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;/&quot;</span><span class="p">,</span><span class="w"> </span><span class="o">|</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="o">|</span><span class="w"> </span><span class="n">echo</span><span class="p">.</span><span class="n">handle</span><span class="p">(</span><span class="n">req</span><span class="p">),</span><span class="w"> </span><span class="s">&quot;index&quot;</span><span class="p">);</span><span class="w"></span> +</pre></div> + + +<p>Another error:</p> +<div class="highlight"><pre>error[E0373]: closure may outlive the current function, but it borrows `echo`, which is owned by the current function +</pre></div> + + +<p>Another helpful message:</p> +<div class="highlight"><pre><span class="n">help</span><span class="o">:</span> <span class="n">to</span> <span class="n">force</span> <span class="n">the</span> <span class="n">closure</span> <span class="n">to</span> <span class="n">take</span> <span class="n">ownership</span> <span class="n">of</span> <span class="err">`</span><span class="n">echo</span><span class="err">`</span> <span class="o">(</span><span class="n">and</span> <span class="n">any</span> <span class="n">other</span> <span class="n">referenced</span> <span class="n">variables</span><span class="o">),</span> <span class="n">use</span> <span class="n">the</span> <span class="err">`</span><span class="n">move</span><span class="err">`</span> <span class="n">keyword</span> +</pre></div> + + +<p>We're getting closer though! Let's implement this change:</p> +<div class="highlight"><pre><span class="n">router</span><span class="p">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;/&quot;</span><span class="p">,</span><span class="w"> </span><span class="n">move</span><span class="w"> </span><span class="o">|</span><span class="n">req</span><span class="o">:</span><span class="w"> </span><span class="o">&amp;</span><span class="k">mut</span><span class="w"> </span><span class="n">Request</span><span class="o">|</span><span class="w"> </span><span class="n">echo</span><span class="p">.</span><span class="n">handle</span><span class="p">(</span><span class="n">req</span><span class="p">),</span><span class="w"> </span><span class="s">&quot;index&quot;</span><span class="p">);</span><span class="w"></span> +</pre></div> + + +<p>And here's where things get strange:</p> +<div class="highlight"><pre>error[E0507]: cannot move out of borrowed content + --&gt; src/main.rs:18:40 + | +18 | Ok(Response::with((status::Ok, self.message))) + | ^^^^ cannot move out of borrowed content +</pre></div> + + +<p>Now, this took me another couple hours to figure out. I'm going to explain it, +but <strong>keep this in mind: Rust only allows one reference at a time</strong> (exceptions +apply of course).</p> +<p>When we attempt to use <code>self.message</code> as it has been created in the earlier +<code>struct</code>, we essentially are trying to give it away to another piece of code. +Rust's semantics then state that <em>we may no longer access it</em> unless it is +returned to us (which <code>iron</code>'s code does not do). There are two ways to fix +this:</p> +<ol> +<li>Only give away references (i.e. <code>&amp;self.message</code> instead of <code>self.message</code>) +instead of transferring ownership</li> +<li>Make a copy of the underlying value which will be safe to give away</li> +</ol> +<p>I didn't know these were the two options originally, so I hope this helps the +audience out. Because <code>iron</code> won't accept a reference, we are forced into the +second option: making a copy. To do so, we just need to change the function +to look like this:</p> +<div class="highlight"><pre><span class="nb">Ok</span><span class="p">(</span><span class="n">Response</span><span class="o">::</span><span class="n">with</span><span class="p">((</span><span class="n">status</span><span class="o">::</span><span class="nb">Ok</span><span class="p">,</span><span class="w"> </span><span class="bp">self</span><span class="p">.</span><span class="n">message</span><span class="p">.</span><span class="n">clone</span><span class="p">())))</span><span class="w"></span> +</pre></div> + + +<p>Not so bad, huh? My only complaint is that it took so long to figure out exactly +what was going on.</p> +<p>And now we have a small server that we can configure dynamically. At long last.</p> +<blockquote> +<p>Final sidenote: You can actually do this without anonymous functions. Just +change the router line to: + <code>router.get("/", echo, "index");</code></p> +<p>Rust's type system seems to figure out that we want to use the <code>handle()</code> method.</p> +</blockquote> +<h1>Conclusion</h1> +<p>After a good long days' work, we now have the routing functionality set up on +our application. We should be able to scale this pretty well in the future: +the RSS content we need to deliver in the future can be treated as a string, so +the building blocks are in place.</p> +<p>There are two important things I learned starting with Rust today:</p> +<ol> +<li>Rust is a new language, and while the code is high-quality, the mindshare is coming.</li> +<li>I'm a terrible programmer.</li> +</ol> +<p>Number 1 is pretty obvious and not surprising to anyone. Number two caught me +off guard. I've gotten used to having either a garbage collector (Java, Python, +etc.) or playing a little fast and loose with scoping rules (C, C++). You don't +have to worry about object lifetime there. With Rust, it's forcing me to fully +understand and use well the memory in my applications. In the final mistake I +fixed (using <code>.clone()</code>) I would have been fine in C++ to just give away that +reference and never use it again. I wouldn't have run into a "use-after-free" +error, but I would have potentially been leaking memory. Rust forced me to be +incredibly precise about how I use it.</p> +<p>All said I'm excited for using Rust more. I think it's super cool, it's just +going to take me a lot longer to do this than I originally thought.</p>Event Studies and Earnings Releases2016-06-08T00:00:00-04:002016-06-08T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-06-08:event-studies-and-earnings-releases.html<script type="text/javascript" src="https://cdn.jsdelivr.net/jquery/3.0.0/jquery.min.js"></script> <p> <div class="cell border-box-sizing text_cell rendered"> @@ -21,19 +288,19 @@ <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="kn">from</span> <span class="nn">IPython.display</span> <span class="k">import</span> <span class="n">HTML</span> -<span class="n">HTML</span><span class="p">(</span><span class="s">&#39;&#39;&#39;&lt;script&gt;</span> -<span class="s">code_show=true; </span> -<span class="s">function code_toggle() {</span> -<span class="s"> if (code_show){</span> -<span class="s"> $(&#39;div.input&#39;).hide();</span> -<span class="s"> } else {</span> -<span class="s"> $(&#39;div.input&#39;).show();</span> -<span class="s"> }</span> -<span class="s"> code_show = !code_show</span> -<span class="s">} </span> -<span class="s">$( document ).ready(code_toggle);</span> -<span class="s">&lt;/script&gt;</span> -<span class="s">&lt;form action=&quot;javascript:code_toggle()&quot;&gt;&lt;input type=&quot;submit&quot; value=&quot;Click here to toggle on/off the raw code.&quot;&gt;&lt;/form&gt;&#39;&#39;&#39;</span><span class="p">)</span> +<span class="n">HTML</span><span class="p">(</span><span class="s1">&#39;&#39;&#39;&lt;script&gt;</span> +<span class="s1">code_show=true; </span> +<span class="s1">function code_toggle() {</span> +<span class="s1"> if (code_show){</span> +<span class="s1"> $(&#39;div.input&#39;).hide();</span> +<span class="s1"> } else {</span> +<span class="s1"> $(&#39;div.input&#39;).show();</span> +<span class="s1"> }</span> +<span class="s1"> code_show = !code_show</span> +<span class="s1">} </span> +<span class="s1">$( document ).ready(code_toggle);</span> +<span class="s1">&lt;/script&gt;</span> +<span class="s1">&lt;form action=&quot;javascript:code_toggle()&quot;&gt;&lt;input type=&quot;submit&quot; value=&quot;Click here to toggle on/off the raw code.&quot;&gt;&lt;/form&gt;&#39;&#39;&#39;</span><span class="p">)</span> </pre></div> </div> @@ -95,20 +362,20 @@ $( document ).ready(code_toggle); <span class="o">%</span><span class="k">matplotlib</span> inline <span class="k">def</span> <span class="nf">fetch_ticker</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">):</span> - <span class="c"># Quandl is currently giving me issues with returning</span> - <span class="c"># the entire dataset and not slicing server-side.</span> - <span class="c"># So instead, we&#39;ll do it client-side!</span> - <span class="n">q_format</span> <span class="o">=</span> <span class="s">&#39;%Y-%m-%d&#39;</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&#39;YAHOO/&#39;</span> <span class="o">+</span> <span class="n">ticker</span><span class="p">,</span> + <span class="c1"># Quandl is currently giving me issues with returning</span> + <span class="c1"># the entire dataset and not slicing server-side.</span> + <span class="c1"># So instead, we&#39;ll do it client-side!</span> + <span class="n">q_format</span> <span class="o">=</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;YAHOO/&#39;</span> <span class="o">+</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">start_date</span><span class="o">=</span><span class="n">start</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">q_format</span><span class="p">),</span> <span class="n">end_date</span><span class="o">=</span><span class="n">end</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">q_format</span><span class="p">),</span> <span class="n">authtoken</span><span class="o">=</span><span class="n">QUANDL_KEY</span><span class="p">)</span> <span class="k">return</span> <span class="n">ticker_data</span> -<span class="k">def</span> <span class="nf">ohlc_dataframe</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="k">None</span><span class="p">):</span> - <span class="c"># Much of this code re-used from:</span> - <span class="c"># http://matplotlib.org/examples/pylab_examples/finance_demo.html</span> - <span class="k">if</span> <span class="n">ax</span> <span class="ow">is</span> <span class="k">None</span><span class="p">:</span> +<span class="k">def</span> <span class="nf">ohlc_dataframe</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> + <span class="c1"># Much of this code re-used from:</span> + <span class="c1"># http://matplotlib.org/examples/pylab_examples/finance_demo.html</span> + <span class="k">if</span> <span class="n">ax</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> <span class="n">f</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">()</span> <span class="n">vals</span> <span class="o">=</span> <span class="p">[(</span><span class="n">date2num</span><span class="p">(</span><span class="n">date</span><span class="p">),</span> <span class="o">*</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">date</span><span class="p">]))</span> @@ -117,20 +384,20 @@ $( document ).ready(code_toggle); <span class="n">mondays</span> <span class="o">=</span> <span class="n">WeekdayLocator</span><span class="p">(</span><span class="n">MONDAY</span><span class="p">)</span> <span class="n">alldays</span> <span class="o">=</span> <span class="n">DayLocator</span><span class="p">()</span> - <span class="n">weekFormatter</span> <span class="o">=</span> <span class="n">DateFormatter</span><span class="p">(</span><span class="s">&#39;%b %d&#39;</span><span class="p">)</span> + <span class="n">weekFormatter</span> <span class="o">=</span> <span class="n">DateFormatter</span><span class="p">(</span><span class="s1">&#39;%b </span><span class="si">%d</span><span class="s1">&#39;</span><span class="p">)</span> <span class="n">ax</span><span class="o">.</span><span class="n">xaxis</span><span class="o">.</span><span class="n">set_major_locator</span><span class="p">(</span><span class="n">mondays</span><span class="p">)</span> <span class="n">ax</span><span class="o">.</span><span class="n">xaxis</span><span class="o">.</span><span class="n">set_minor_locator</span><span class="p">(</span><span class="n">alldays</span><span class="p">)</span> <span class="n">ax</span><span class="o">.</span><span class="n">xaxis</span><span class="o">.</span><span class="n">set_major_formatter</span><span class="p">(</span><span class="n">weekFormatter</span><span class="p">)</span> <span class="k">return</span> <span class="n">ax</span> -<span class="n">AAPL</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="s">&#39;AAPL&#39;</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> +<span class="n">AAPL</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="s1">&#39;AAPL&#39;</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">AAPL</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">,</span> - <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;b&#39;</span><span class="p">,</span> + <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&quot;Apple Price 3/1/2016 - 5/1/2016&quot;</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s2">&quot;Apple Price 3/1/2016 - 5/1/2016&quot;</span><span class="p">);</span> </pre></div> </div> @@ -352,12 +619,12 @@ rgAAAABJRU5ErkJggg== <div class="prompt input_prompt">In&nbsp;[3]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">FB</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="s">&#39;FB&#39;</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span> +<div class=" highlight hl-ipython3"><pre><span class="n">FB</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="s1">&#39;FB&#39;</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">FB</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;Facebook Price 3/5/2016 - 5/5/2016&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;Facebook Price 3/5/2016 - 5/5/2016&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">);</span> </pre></div> @@ -611,36 +878,36 @@ czHjid0552LGE7tzzsXM/wfgLII59KYqMQAAAABJRU5ErkJggg== <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">plot_hilo</span><span class="p">(</span><span class="n">ax</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span> <span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">([</span><span class="n">date2num</span><span class="p">(</span><span class="n">start</span><span class="p">),</span> <span class="n">date2num</span><span class="p">(</span><span class="n">end</span><span class="p">)],</span> - <span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">start</span><span class="p">][</span><span class="s">&#39;High&#39;</span><span class="p">],</span> <span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">end</span><span class="p">][</span><span class="s">&#39;High&#39;</span><span class="p">]],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">)</span> + <span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">start</span><span class="p">][</span><span class="s1">&#39;High&#39;</span><span class="p">],</span> <span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">end</span><span class="p">][</span><span class="s1">&#39;High&#39;</span><span class="p">]],</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;b&#39;</span><span class="p">)</span> <span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">([</span><span class="n">date2num</span><span class="p">(</span><span class="n">start</span><span class="p">),</span> <span class="n">date2num</span><span class="p">(</span><span class="n">end</span><span class="p">)],</span> - <span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">start</span><span class="p">][</span><span class="s">&#39;Low&#39;</span><span class="p">],</span> <span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">end</span><span class="p">][</span><span class="s">&#39;Low&#39;</span><span class="p">]],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;b&#39;</span><span class="p">)</span> + <span class="p">[</span><span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">start</span><span class="p">][</span><span class="s1">&#39;Low&#39;</span><span class="p">],</span> <span class="n">data</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">end</span><span class="p">][</span><span class="s1">&#39;Low&#39;</span><span class="p">]],</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;b&#39;</span><span class="p">)</span> <span class="n">f</span><span class="p">,</span> <span class="n">axarr</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span> <span class="n">ax_aapl</span> <span class="o">=</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="n">ax_fb</span> <span class="o">=</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> -<span class="c"># Plot the AAPL trend up and down</span> +<span class="c1"># Plot the AAPL trend up and down</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">AAPL</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="n">ax_aapl</span><span class="p">)</span> <span class="n">plot_hilo</span><span class="p">(</span><span class="n">ax_aapl</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">15</span><span class="p">),</span> <span class="n">AAPL</span><span class="p">)</span> <span class="n">plot_hilo</span><span class="p">(</span><span class="n">ax_aapl</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">18</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">),</span> <span class="n">AAPL</span><span class="p">)</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> -<span class="n">ax_aapl</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s">&#39;AAPL Price History&#39;</span><span class="p">)</span> +<span class="n">ax_aapl</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;AAPL Price History&#39;</span><span class="p">)</span> -<span class="c"># Plot the FB trend down and up</span> +<span class="c1"># Plot the FB trend down and up</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">FB</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="n">ax_fb</span><span class="p">)</span> <span class="n">plot_hilo</span><span class="p">(</span><span class="n">ax_fb</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">30</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">),</span> <span class="n">FB</span><span class="p">)</span> <span class="n">plot_hilo</span><span class="p">(</span><span class="n">ax_fb</span><span class="p">,</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">28</span><span class="p">),</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span> <span class="n">FB</span><span class="p">)</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> -<span class="n">ax_fb</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s">&#39;FB Price History&#39;</span><span class="p">)</span> +<span class="n">ax_fb</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;FB Price History&#39;</span><span class="p">)</span> <span class="n">f</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> </pre></div> @@ -1298,17 +1565,17 @@ cAw4PwAAAABJRU5ErkJggg== <span class="kn">from</span> <span class="nn">pandas.tseries.offsets</span> <span class="k">import</span> <span class="n">CustomBusinessDay</span> <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span> -<span class="c"># If you remove rules, it removes them from *all* calendars</span> -<span class="c"># To ensure we don&#39;t pop rules we don&#39;t want to, first make</span> -<span class="c"># sure to fully copy the object</span> +<span class="c1"># If you remove rules, it removes them from *all* calendars</span> +<span class="c1"># To ensure we don&#39;t pop rules we don&#39;t want to, first make</span> +<span class="c1"># sure to fully copy the object</span> <span class="n">trade_calendar</span> <span class="o">=</span> <span class="n">USFederalHolidayCalendar</span><span class="p">()</span> -<span class="n">trade_calendar</span><span class="o">.</span><span class="n">rules</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span> <span class="c"># Remove Columbus day</span> -<span class="n">trade_calendar</span><span class="o">.</span><span class="n">rules</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">7</span><span class="p">)</span> <span class="c"># Remove Veteran&#39;s day</span> +<span class="n">trade_calendar</span><span class="o">.</span><span class="n">rules</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">6</span><span class="p">)</span> <span class="c1"># Remove Columbus day</span> +<span class="n">trade_calendar</span><span class="o">.</span><span class="n">rules</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">7</span><span class="p">)</span> <span class="c1"># Remove Veteran&#39;s day</span> <span class="n">TradeDay</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">days</span><span class="p">:</span> <span class="n">CustomBusinessDay</span><span class="p">(</span><span class="n">days</span><span class="p">,</span> <span class="n">calendar</span><span class="o">=</span><span class="n">trade_calendar</span><span class="p">)</span> <span class="k">def</span> <span class="nf">plot_study</span><span class="p">(</span><span class="n">array</span><span class="p">):</span> - <span class="c"># Given a 2-d array, we assume the event happens at index `lookback`,</span> - <span class="c"># and create all of our summary statistics from there.</span> + <span class="c1"># Given a 2-d array, we assume the event happens at index `lookback`,</span> + <span class="c1"># and create all of our summary statistics from there.</span> <span class="n">lookback</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">array</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="mi">2</span><span class="p">)</span> <span class="n">norm_factor</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="n">array</span><span class="p">[:,</span><span class="n">lookback</span><span class="p">]</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">array</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="n">centered_data</span> <span class="o">=</span> <span class="n">array</span> <span class="o">/</span> <span class="n">norm_factor</span> <span class="o">-</span> <span class="mi">1</span> @@ -1327,17 +1594,17 @@ cAw4PwAAAABJRU5ErkJggg== <span class="n">axarr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">fill_between</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">+</span> <span class="n">std_dev</span><span class="p">,</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">-</span> <span class="n">std_dev</span><span class="p">,</span> - <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> + <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">fill_between</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">+</span> <span class="n">std_dev</span><span class="p">,</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">-</span> <span class="n">std_dev</span><span class="p">,</span> - <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> + <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> <span class="n">max_err</span> <span class="o">=</span> <span class="n">maxes</span> <span class="o">-</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span> <span class="n">min_err</span> <span class="o">=</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span> <span class="o">-</span> <span class="n">mins</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">errorbar</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:],</span> - <span class="n">yerr</span><span class="o">=</span><span class="p">[</span><span class="n">min_err</span><span class="p">,</span> <span class="n">max_err</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Max &amp; Min&#39;</span><span class="p">)</span> + <span class="n">yerr</span><span class="o">=</span><span class="p">[</span><span class="n">min_err</span><span class="p">,</span> <span class="n">max_err</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Max &amp; Min&#39;</span><span class="p">)</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> @@ -1345,8 +1612,8 @@ cAw4PwAAAABJRU5ErkJggg== <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">set_xlim</span><span class="p">((</span><span class="o">-</span><span class="n">lookback</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">lookback</span><span class="o">+</span><span class="mi">1</span><span class="p">))</span> <span class="k">def</span> <span class="nf">plot_study_small</span><span class="p">(</span><span class="n">array</span><span class="p">):</span> - <span class="c"># Given a 2-d array, we assume the event happens at index `lookback`,</span> - <span class="c"># and create all of our summary statistics from there.</span> + <span class="c1"># Given a 2-d array, we assume the event happens at index `lookback`,</span> + <span class="c1"># and create all of our summary statistics from there.</span> <span class="n">lookback</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">array</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="mi">2</span><span class="p">)</span> <span class="n">norm_factor</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="n">array</span><span class="p">[:,</span><span class="n">lookback</span><span class="p">]</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">array</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="n">centered_data</span> <span class="o">=</span> <span class="n">array</span> <span class="o">/</span> <span class="n">norm_factor</span> <span class="o">-</span> <span class="mi">1</span> @@ -1363,24 +1630,24 @@ cAw4PwAAAABJRU5ErkJggg== <span class="n">plt</span><span class="o">.</span><span class="n">fill_between</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">+</span> <span class="n">std_dev</span><span class="p">,</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="p">:]</span> <span class="o">-</span> <span class="n">std_dev</span><span class="p">,</span> - <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> + <span class="n">alpha</span><span class="o">=.</span><span class="mi">5</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;$\pm$ 1 s.d.&quot;</span><span class="p">)</span> <span class="n">max_err</span> <span class="o">=</span> <span class="n">maxes</span> <span class="o">-</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span> <span class="n">min_err</span> <span class="o">=</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span> <span class="o">-</span> <span class="n">mins</span> <span class="n">plt</span><span class="o">.</span><span class="n">errorbar</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">range_end</span><span class="p">),</span> <span class="n">means</span><span class="p">[</span><span class="o">-</span><span class="n">lookforward</span><span class="o">+</span><span class="mi">1</span><span class="p">:],</span> - <span class="n">yerr</span><span class="o">=</span><span class="p">[</span><span class="n">min_err</span><span class="p">,</span> <span class="n">max_err</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Max &amp; Min&#39;</span><span class="p">)</span> + <span class="n">yerr</span><span class="o">=</span><span class="p">[</span><span class="n">min_err</span><span class="p">,</span> <span class="n">max_err</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Max &amp; Min&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">xlim</span><span class="p">((</span><span class="o">-</span><span class="n">lookback</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">lookback</span><span class="o">+</span><span class="mi">1</span><span class="p">))</span> <span class="k">def</span> <span class="nf">fetch_event_data</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">events</span><span class="p">,</span> <span class="n">horizon</span><span class="o">=</span><span class="mi">5</span><span class="p">):</span> - <span class="c"># Use horizon+1 to account for including the day of the event,</span> - <span class="c"># and half-open interval - that is, for a horizon of 5,</span> - <span class="c"># we should be including 11 events. Additionally, using the</span> - <span class="c"># CustomBusinessDay means we automatically handle issues if</span> - <span class="c"># for example a company reports Friday afternoon - the date</span> - <span class="c"># calculator will turn this into a &quot;Saturday&quot; release, but</span> - <span class="c"># we effectively shift that to Monday with the logic below.</span> + <span class="c1"># Use horizon+1 to account for including the day of the event,</span> + <span class="c1"># and half-open interval - that is, for a horizon of 5,</span> + <span class="c1"># we should be including 11 events. Additionally, using the</span> + <span class="c1"># CustomBusinessDay means we automatically handle issues if</span> + <span class="c1"># for example a company reports Friday afternoon - the date</span> + <span class="c1"># calculator will turn this into a &quot;Saturday&quot; release, but</span> + <span class="c1"># we effectively shift that to Monday with the logic below.</span> <span class="n">td_back</span> <span class="o">=</span> <span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span> <span class="n">td_forward</span> <span class="o">=</span> <span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span> @@ -1389,28 +1656,28 @@ cAw4PwAAAABJRU5ErkJggg== <span class="n">total_data</span> <span class="o">=</span> <span class="n">fetch_ticker</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="p">)</span> <span class="n">event_data</span> <span class="o">=</span> <span class="p">[</span><span class="n">total_data</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">td_back</span><span class="p">:</span><span class="n">event</span><span class="o">+</span><span class="n">td_forward</span><span class="p">]</span>\ <span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="n">horizon</span><span class="o">*</span><span class="mi">2</span><span class="o">+</span><span class="mi">1</span><span class="p">]</span>\ - <span class="p">[</span><span class="s">&#39;Adjusted Close&#39;</span><span class="p">]</span> + <span class="p">[</span><span class="s1">&#39;Adjusted Close&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">events</span><span class="p">]</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">event_data</span><span class="p">)</span> -<span class="c"># Generate a couple of random events</span> +<span class="c1"># Generate a couple of random events</span> <span class="n">event_dates</span> <span class="o">=</span> <span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">27</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="n">TradeDay</span><span class="p">(</span><span class="n">x</span><span class="o">*</span><span class="mi">20</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">40</span><span class="p">)]</span> -<span class="n">data</span> <span class="o">=</span> <span class="n">fetch_event_data</span><span class="p">(</span><span class="s">&#39;CELG&#39;</span><span class="p">,</span> <span class="n">event_dates</span><span class="p">)</span> +<span class="n">data</span> <span class="o">=</span> <span class="n">fetch_event_data</span><span class="p">(</span><span class="s1">&#39;CELG&#39;</span><span class="p">,</span> <span class="n">event_dates</span><span class="p">)</span> <span class="n">plot_study_small</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span> <span class="mi">6</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s">&#39;Mean price for days leading up to each event&#39;</span><span class="p">,</span> +<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s1">&#39;Mean price for days leading up to each event&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">-</span><span class="mi">5</span><span class="p">,</span> <span class="o">-.</span><span class="mi">01</span><span class="p">),</span> <span class="p">(</span><span class="o">-</span><span class="mf">4.5</span><span class="p">,</span> <span class="o">.</span><span class="mi">025</span><span class="p">),</span> - <span class="n">arrowprops</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">facecolor</span><span class="o">=</span><span class="s">&#39;black&#39;</span><span class="p">,</span> <span class="n">shrink</span><span class="o">=</span><span class="mf">0.05</span><span class="p">))</span> -<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s">&#39;&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">-.</span><span class="mi">1</span><span class="p">,</span> <span class="o">.</span><span class="mi">005</span><span class="p">),</span> <span class="p">(</span><span class="o">-.</span><span class="mi">5</span><span class="p">,</span> <span class="o">.</span><span class="mi">02</span><span class="p">),</span> - <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s">&#39;black&#39;</span><span class="p">,</span> <span class="s">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">})</span> -<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s">&#39;$\pm$ 1 std. dev. each day&#39;</span><span class="p">,</span> <span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="o">.</span><span class="mi">055</span><span class="p">),</span> <span class="p">(</span><span class="mf">2.5</span><span class="p">,</span> <span class="o">.</span><span class="mi">085</span><span class="p">),</span> - <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s">&#39;black&#39;</span><span class="p">,</span> <span class="s">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">})</span> -<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s">&#39;Min/Max each day&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">.</span><span class="mi">9</span><span class="p">,</span> <span class="o">-.</span><span class="mi">07</span><span class="p">),</span> <span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-.</span><span class="mi">1</span><span class="p">),</span> - <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s">&#39;black&#39;</span><span class="p">,</span> <span class="s">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">});</span> + <span class="n">arrowprops</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">facecolor</span><span class="o">=</span><span class="s1">&#39;black&#39;</span><span class="p">,</span> <span class="n">shrink</span><span class="o">=</span><span class="mf">0.05</span><span class="p">))</span> +<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">-.</span><span class="mi">1</span><span class="p">,</span> <span class="o">.</span><span class="mi">005</span><span class="p">),</span> <span class="p">(</span><span class="o">-.</span><span class="mi">5</span><span class="p">,</span> <span class="o">.</span><span class="mi">02</span><span class="p">),</span> + <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s1">&#39;black&#39;</span><span class="p">,</span> <span class="s1">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">})</span> +<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s1">&#39;$\pm$ 1 std. dev. each day&#39;</span><span class="p">,</span> <span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="o">.</span><span class="mi">055</span><span class="p">),</span> <span class="p">(</span><span class="mf">2.5</span><span class="p">,</span> <span class="o">.</span><span class="mi">085</span><span class="p">),</span> + <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s1">&#39;black&#39;</span><span class="p">,</span> <span class="s1">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">})</span> +<span class="n">plt</span><span class="o">.</span><span class="n">annotate</span><span class="p">(</span><span class="s1">&#39;Min/Max each day&#39;</span><span class="p">,</span> <span class="p">(</span><span class="o">.</span><span class="mi">9</span><span class="p">,</span> <span class="o">-.</span><span class="mi">07</span><span class="p">),</span> <span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-.</span><span class="mi">1</span><span class="p">),</span> + <span class="n">arrowprops</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;facecolor&#39;</span><span class="p">:</span> <span class="s1">&#39;black&#39;</span><span class="p">,</span> <span class="s1">&#39;shrink&#39;</span><span class="p">:</span> <span class="o">.</span><span class="mi">05</span><span class="p">});</span> </pre></div> </div> @@ -1870,24 +2137,24 @@ PlISLSIiIiLiIyXRIiIiIiI+UhItIiIiIuIjJdEiIiIiIj76P+bhd3LjvwIqAAAAAElFTkSuQmCC <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">FB</span><span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">18</span><span class="p">):],</span> <span class="n">FB_plot</span><span class="p">)</span> <span class="n">FB_truncated</span> <span class="o">=</span> <span class="n">FB</span><span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">18</span><span class="p">):</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">)]</span> -<span class="n">midpoint</span> <span class="o">=</span> <span class="n">FB_truncated</span><span class="p">[</span><span class="s">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">FB_truncated</span><span class="p">[</span><span class="s">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> -<span class="n">FB_plot</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">FB_truncated</span><span class="o">.</span><span class="n">index</span><span class="p">,</span> <span class="n">midpoint</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Midpoint&#39;</span><span class="p">)</span> +<span class="n">midpoint</span> <span class="o">=</span> <span class="n">FB_truncated</span><span class="p">[</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">FB_truncated</span><span class="p">[</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> +<span class="n">FB_plot</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">FB_truncated</span><span class="o">.</span><span class="n">index</span><span class="p">,</span> <span class="n">midpoint</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Midpoint&#39;</span><span class="p">)</span> <span class="n">FB_plot</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">27</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax_fb</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">FB_plot</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> -<span class="n">FB_plot</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s">&#39;FB Midpoint Plot&#39;</span><span class="p">)</span> +<span class="n">FB_plot</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;FB Midpoint Plot&#39;</span><span class="p">)</span> <span class="n">AAPL_plot</span> <span class="o">=</span> <span class="n">axarr</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="n">ohlc_dataframe</span><span class="p">(</span><span class="n">AAPL</span><span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">10</span><span class="p">):],</span> <span class="n">AAPL_plot</span><span class="p">)</span> <span class="n">AAPL_truncated</span> <span class="o">=</span> <span class="n">AAPL</span><span class="p">[</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">10</span><span class="p">):</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">)]</span> -<span class="n">midpoint</span> <span class="o">=</span> <span class="n">AAPL_truncated</span><span class="p">[</span><span class="s">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">AAPL_truncated</span><span class="p">[</span><span class="s">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> -<span class="n">AAPL_plot</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">AAPL_truncated</span><span class="o">.</span><span class="n">index</span><span class="p">,</span> <span class="n">midpoint</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Midpoint&#39;</span><span class="p">)</span> +<span class="n">midpoint</span> <span class="o">=</span> <span class="n">AAPL_truncated</span><span class="p">[</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">AAPL_truncated</span><span class="p">[</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> +<span class="n">AAPL_plot</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">AAPL_truncated</span><span class="o">.</span><span class="n">index</span><span class="p">,</span> <span class="n">midpoint</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Midpoint&#39;</span><span class="p">)</span> <span class="n">AAPL_plot</span><span class="o">.</span><span class="n">vlines</span><span class="p">(</span><span class="n">date2num</span><span class="p">(</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">26</span><span class="p">,</span> <span class="mi">12</span><span class="p">)),</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">0</span><span class="p">],</span> <span class="n">ax_aapl</span><span class="o">.</span><span class="n">get_ylim</span><span class="p">()[</span><span class="mi">1</span><span class="p">],</span> - <span class="n">color</span><span class="o">=</span><span class="s">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&#39;Earnings Release&#39;</span><span class="p">)</span> + <span class="n">color</span><span class="o">=</span><span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;Earnings Release&#39;</span><span class="p">)</span> <span class="n">AAPL_plot</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span> -<span class="n">AAPL_plot</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s">&#39;AAPL Midpoint Plot&#39;</span><span class="p">);</span> +<span class="n">AAPL_plot</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s1">&#39;AAPL Midpoint Plot&#39;</span><span class="p">);</span> </pre></div> </div> @@ -2366,74 +2633,74 @@ TkSuQmCC <div class="prompt input_prompt">In&nbsp;[7]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># Read in the events for each stock;</span> -<span class="c"># The file was created using the first code block in the Appendix</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># Read in the events for each stock;</span> +<span class="c1"># The file was created using the first code block in the Appendix</span> <span class="kn">import</span> <span class="nn">yaml</span> <span class="kn">from</span> <span class="nn">dateutil.parser</span> <span class="k">import</span> <span class="n">parse</span> <span class="kn">from</span> <span class="nn">progressbar</span> <span class="k">import</span> <span class="n">ProgressBar</span> -<span class="n">data_str</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> -<span class="c"># Need to remove invalid lines</span> -<span class="n">filtered</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s">&#39;{&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">x</span><span class="p">,</span> <span class="n">data_str</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">&#39;</span><span class="se">\n</span><span class="s">&#39;</span><span class="p">))</span> -<span class="n">earnings_data</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s">&#39;</span><span class="se">\n</span><span class="s">&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">filtered</span><span class="p">))</span> +<span class="n">data_str</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> +<span class="c1"># Need to remove invalid lines</span> +<span class="n">filtered</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s1">&#39;{&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">x</span><span class="p">,</span> <span class="n">data_str</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">))</span> +<span class="n">earnings_data</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">filtered</span><span class="p">))</span> -<span class="c"># Convert our earnings data into a list of (ticker, date) pairs</span> -<span class="c"># to make it easy to work with.</span> -<span class="c"># This is horribly inefficient, but should get us what we need</span> +<span class="c1"># Convert our earnings data into a list of (ticker, date) pairs</span> +<span class="c1"># to make it easy to work with.</span> +<span class="c1"># This is horribly inefficient, but should get us what we need</span> <span class="n">ticker_dates</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">date_list</span> <span class="ow">in</span> <span class="n">earnings_data</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> <span class="k">for</span> <span class="n">iso_str</span> <span class="ow">in</span> <span class="n">date_list</span><span class="p">:</span> <span class="n">ticker_dates</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">ticker</span><span class="p">,</span> <span class="n">parse</span><span class="p">(</span><span class="n">iso_str</span><span class="p">)))</span> <span class="k">def</span> <span class="nf">does_trend_down</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">event</span><span class="p">,</span> <span class="n">horizon</span><span class="p">):</span> - <span class="c"># Figure out if the `event` has a downtrend for</span> - <span class="c"># the `horizon` days preceding it</span> - <span class="c"># As an interpretation note: it is assumed that</span> - <span class="c"># the closing price of day `event` is the reference</span> - <span class="c"># point, and we want `horizon` days before that.</span> - <span class="c"># The price_data.hdf was created in the second appendix code block</span> + <span class="c1"># Figure out if the `event` has a downtrend for</span> + <span class="c1"># the `horizon` days preceding it</span> + <span class="c1"># As an interpretation note: it is assumed that</span> + <span class="c1"># the closing price of day `event` is the reference</span> + <span class="c1"># point, and we want `horizon` days before that.</span> + <span class="c1"># The price_data.hdf was created in the second appendix code block</span> <span class="k">try</span><span class="p">:</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s1">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> <span class="n">data</span> <span class="o">=</span> <span class="n">ticker_data</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">):</span><span class="n">event</span><span class="p">]</span> - <span class="n">midpoints</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">data</span><span class="p">[</span><span class="s">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> + <span class="n">midpoints</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> - <span class="c"># Shift dates one forward into the future and subtract</span> - <span class="c"># Effectively: do we trend down over all days?</span> + <span class="c1"># Shift dates one forward into the future and subtract</span> + <span class="c1"># Effectively: do we trend down over all days?</span> <span class="n">elems</span> <span class="o">=</span> <span class="n">midpoints</span> <span class="o">-</span> <span class="n">midpoints</span><span class="o">.</span><span class="n">shift</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">elems</span><span class="p">)</span><span class="o">-</span><span class="mi">1</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">elems</span><span class="o">.</span><span class="n">dropna</span><span class="p">()[</span><span class="n">elems</span> <span class="o">&lt;=</span> <span class="mi">0</span><span class="p">])</span> <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span> - <span class="c"># If the stock doesn&#39;t exist, it doesn&#39;t qualify as trending down</span> - <span class="c"># Mostly this is here to make sure the entire analysis doesn&#39;t </span> - <span class="c"># blow up if there were issues in data retrieval</span> - <span class="k">return</span> <span class="k">False</span> + <span class="c1"># If the stock doesn&#39;t exist, it doesn&#39;t qualify as trending down</span> + <span class="c1"># Mostly this is here to make sure the entire analysis doesn&#39;t </span> + <span class="c1"># blow up if there were issues in data retrieval</span> + <span class="k">return</span> <span class="kc">False</span> <span class="k">def</span> <span class="nf">study_trend</span><span class="p">(</span><span class="n">horizon</span><span class="p">,</span> <span class="n">trend_function</span><span class="p">):</span> <span class="n">five_day_events</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span> <span class="n">horizon</span><span class="o">*</span><span class="mi">2</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span> <span class="n">invalid_events</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">ProgressBar</span><span class="p">()(</span><span class="n">ticker_dates</span><span class="p">):</span> <span class="k">if</span> <span class="n">trend_function</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">event</span><span class="p">,</span> <span class="n">horizon</span><span class="p">):</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> - <span class="n">event_data</span> <span class="o">=</span> <span class="n">ticker_data</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">):</span><span class="n">event</span><span class="o">+</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">)][</span><span class="s">&#39;Close&#39;</span><span class="p">]</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s1">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> + <span class="n">event_data</span> <span class="o">=</span> <span class="n">ticker_data</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">):</span><span class="n">event</span><span class="o">+</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">)][</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span> <span class="k">try</span><span class="p">:</span> <span class="n">five_day_events</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">([</span><span class="n">five_day_events</span><span class="p">,</span> <span class="n">event_data</span><span class="p">])</span> <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span> - <span class="c"># Sometimes we don&#39;t get exactly the right number of values due to calendar</span> - <span class="c"># issues. I&#39;ve fixed most everything I can, and the few issues that are left</span> - <span class="c"># I assume don&#39;t systemically bias the results (i.e. data could be missing</span> - <span class="c"># because it doesn&#39;t exist, etc.). After running through, ~1% of events get</span> - <span class="c"># discarded this way</span> + <span class="c1"># Sometimes we don&#39;t get exactly the right number of values due to calendar</span> + <span class="c1"># issues. I&#39;ve fixed most everything I can, and the few issues that are left</span> + <span class="c1"># I assume don&#39;t systemically bias the results (i.e. data could be missing</span> + <span class="c1"># because it doesn&#39;t exist, etc.). After running through, ~1% of events get</span> + <span class="c1"># discarded this way</span> <span class="n">invalid_events</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">ticker</span><span class="p">,</span> <span class="n">event</span><span class="p">))</span> - <span class="c"># Remove our initial zero row</span> + <span class="c1"># Remove our initial zero row</span> <span class="n">five_day_events</span> <span class="o">=</span> <span class="n">five_day_events</span><span class="p">[</span><span class="mi">1</span><span class="p">:,:]</span> <span class="n">plot_study</span><span class="p">(</span><span class="n">five_day_events</span><span class="p">)</span> - <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&#39;Action over {} days: {} events&#39;</span> + <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s1">&#39;Action over {} days: {} events&#39;</span> <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">horizon</span><span class="p">,</span><span class="n">five_day_events</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> -<span class="c"># Start with a 5 day study</span> +<span class="c1"># Start with a 5 day study</span> <span class="n">study_trend</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">does_trend_down</span><span class="p">)</span> </pre></div> @@ -2913,7 +3180,7 @@ ADBuCBoAAAAAAMC4+f+DLBbJVWQG2gAAAABJRU5ErkJggg== <div class="prompt input_prompt">In&nbsp;[8]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># 8 day study next</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># 8 day study next</span> <span class="n">study_trend</span><span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="n">does_trend_down</span><span class="p">)</span> </pre></div> @@ -3392,7 +3659,7 @@ AAAAKMr/B9pAH9lY/+K6AAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[9]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># 3 day study after that</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># 3 day study after that</span> <span class="n">study_trend</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">does_trend_down</span><span class="p">)</span> </pre></div> @@ -3805,26 +4072,26 @@ STfmKmb+uQ4AAAAASUVORK5CYII= <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">does_trend_up</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">event</span><span class="p">,</span> <span class="n">horizon</span><span class="p">):</span> - <span class="c"># Figure out if the `event` has an uptrend for</span> - <span class="c"># the `horizon` days preceding it</span> - <span class="c"># As an interpretation note: it is assumed that</span> - <span class="c"># the closing price of day `event` is the reference</span> - <span class="c"># point, and we want `horizon` days before that.</span> - <span class="c"># The price_data.hdf was created in the second appendix code block</span> + <span class="c1"># Figure out if the `event` has an uptrend for</span> + <span class="c1"># the `horizon` days preceding it</span> + <span class="c1"># As an interpretation note: it is assumed that</span> + <span class="c1"># the closing price of day `event` is the reference</span> + <span class="c1"># point, and we want `horizon` days before that.</span> + <span class="c1"># The price_data.hdf was created in the second appendix code block</span> <span class="k">try</span><span class="p">:</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_hdf</span><span class="p">(</span><span class="s1">&#39;price_data.hdf&#39;</span><span class="p">,</span> <span class="n">ticker</span><span class="p">)</span> <span class="n">data</span> <span class="o">=</span> <span class="n">ticker_data</span><span class="p">[</span><span class="n">event</span><span class="o">-</span><span class="n">TradeDay</span><span class="p">(</span><span class="n">horizon</span><span class="p">):</span><span class="n">event</span><span class="p">]</span> - <span class="n">midpoints</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">data</span><span class="p">[</span><span class="s">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> + <span class="n">midpoints</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> <span class="o">+</span> <span class="n">data</span><span class="p">[</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span><span class="o">/</span><span class="mi">2</span> - <span class="c"># Shift dates one forward into the future and subtract</span> - <span class="c"># Effectively: do we trend down over all days?</span> + <span class="c1"># Shift dates one forward into the future and subtract</span> + <span class="c1"># Effectively: do we trend down over all days?</span> <span class="n">elems</span> <span class="o">=</span> <span class="n">midpoints</span> <span class="o">-</span> <span class="n">midpoints</span><span class="o">.</span><span class="n">shift</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">elems</span><span class="p">)</span><span class="o">-</span><span class="mi">1</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">elems</span><span class="o">.</span><span class="n">dropna</span><span class="p">()[</span><span class="n">elems</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">])</span> <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span> - <span class="c"># If the stock doesn&#39;t exist, it doesn&#39;t qualify as trending down</span> - <span class="c"># Mostly this is here to make sure the entire analysis doesn&#39;t </span> - <span class="c"># blow up if there were issues in data retrieval</span> - <span class="k">return</span> <span class="k">False</span> + <span class="c1"># If the stock doesn&#39;t exist, it doesn&#39;t qualify as trending down</span> + <span class="c1"># Mostly this is here to make sure the entire analysis doesn&#39;t </span> + <span class="c1"># blow up if there were issues in data retrieval</span> + <span class="k">return</span> <span class="kc">False</span> <span class="n">study_trend</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">does_trend_up</span><span class="p">)</span> </pre></div> @@ -5105,38 +5372,38 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="bp">self</span><span class="o">.</span><span class="n">dates</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">def</span> <span class="nf">handle_starttag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span> - <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;table&#39;</span><span class="p">:</span> + <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;table&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_dates</span> <span class="o">=</span> <span class="bp">True</span> <span class="k">def</span> <span class="nf">handle_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_dates</span><span class="p">:</span> - <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s">r&#39;\d+/\d+/\d+&#39;</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span> + <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">r&#39;\d+/\d+/\d+&#39;</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span> <span class="k">if</span> <span class="n">match</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">dates</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span> - <span class="c"># If a company reports before the bell, record the earnings date</span> - <span class="c"># being at midnight the day before. Ex: WMT reports 5/19/2016,</span> - <span class="c"># but we want the reference point to be the closing price on 5/18/2016</span> - <span class="k">if</span> <span class="s">&#39;After Close&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span> + <span class="c1"># If a company reports before the bell, record the earnings date</span> + <span class="c1"># being at midnight the day before. Ex: WMT reports 5/19/2016,</span> + <span class="c1"># but we want the reference point to be the closing price on 5/18/2016</span> + <span class="k">if</span> <span class="s1">&#39;After Close&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">earnings_offset</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> - <span class="k">elif</span> <span class="s">&#39;Before Open&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span> + <span class="k">elif</span> <span class="s1">&#39;Before Open&#39;</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">earnings_offset</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span> <span class="k">def</span> <span class="nf">handle_endtag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">):</span> - <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;table&#39;</span><span class="p">:</span> + <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;table&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_dates</span> <span class="o">=</span> <span class="bp">False</span> <span class="k">def</span> <span class="nf">earnings_releases</span><span class="p">(</span><span class="n">ticker</span><span class="p">):</span> - <span class="c">#print(&quot;Looking up ticker {}&quot;.format(ticker))</span> - <span class="n">user_agent</span> <span class="o">=</span> <span class="s">&#39;Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) &#39;</span>\ - <span class="s">&#39;Gecko/20100101 Firefox/46.0&#39;</span> - <span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s">&#39;user-agent&#39;</span><span class="p">:</span> <span class="n">user_agent</span><span class="p">}</span> - <span class="n">base_url</span> <span class="o">=</span> <span class="s">&#39;http://www.streetinsider.com/ec_earnings.php?q={}&#39;</span>\ + <span class="c1">#print(&quot;Looking up ticker {}&quot;.format(ticker))</span> + <span class="n">user_agent</span> <span class="o">=</span> <span class="s1">&#39;Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) &#39;</span>\ + <span class="s1">&#39;Gecko/20100101 Firefox/46.0&#39;</span> + <span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;user-agent&#39;</span><span class="p">:</span> <span class="n">user_agent</span><span class="p">}</span> + <span class="n">base_url</span> <span class="o">=</span> <span class="s1">&#39;http://www.streetinsider.com/ec_earnings.php?q={}&#39;</span>\ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ticker</span><span class="p">)</span> <span class="n">e</span> <span class="o">=</span> <span class="n">EarningsParser</span><span class="p">()</span> <span class="n">s</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span> <span class="n">a</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">adapters</span><span class="o">.</span><span class="n">HTTPAdapter</span><span class="p">(</span><span class="n">max_retries</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> - <span class="n">s</span><span class="o">.</span><span class="n">mount</span><span class="p">(</span><span class="s">&#39;http://&#39;</span><span class="p">,</span> <span class="n">a</span><span class="p">)</span> + <span class="n">s</span><span class="o">.</span><span class="n">mount</span><span class="p">(</span><span class="s1">&#39;http://&#39;</span><span class="p">,</span> <span class="n">a</span><span class="p">)</span> <span class="n">e</span><span class="o">.</span><span class="n">feed</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">s</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">base_url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span><span class="o">.</span><span class="n">content</span><span class="p">))</span> <span class="k">if</span> <span class="n">e</span><span class="o">.</span><span class="n">earnings_offset</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span> @@ -5144,18 +5411,18 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="n">past</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="o">&lt;</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(),</span> <span class="n">dates</span><span class="p">)</span> <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span><span class="o">.</span><span class="n">isoformat</span><span class="p">(),</span> <span class="n">past</span><span class="p">))</span> -<span class="c"># Use a Russell-3000 ETF tracker (ticker IWV) to get a list of holdings</span> -<span class="n">r3000</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s">&#39;https://www.ishares.com/us/products/239714/&#39;</span> - <span class="s">&#39;ishares-russell-3000-etf/1449138789749.ajax?&#39;</span> - <span class="s">&#39;fileType=csv&amp;fileName=IWV_holdings&amp;dataType=fund&#39;</span><span class="p">,</span> +<span class="c1"># Use a Russell-3000 ETF tracker (ticker IWV) to get a list of holdings</span> +<span class="n">r3000</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;https://www.ishares.com/us/products/239714/&#39;</span> + <span class="s1">&#39;ishares-russell-3000-etf/1449138789749.ajax?&#39;</span> + <span class="s1">&#39;fileType=csv&amp;fileName=IWV_holdings&amp;dataType=fund&#39;</span><span class="p">,</span> <span class="n">header</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span> -<span class="n">r3000_equities</span> <span class="o">=</span> <span class="n">r3000</span><span class="p">[(</span><span class="n">r3000</span><span class="p">[</span><span class="s">&#39;Exchange&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;NASDAQ&#39;</span><span class="p">)</span> <span class="o">|</span> - <span class="p">(</span><span class="n">r3000</span><span class="p">[</span><span class="s">&#39;Exchange&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;New York Stock Exchange Inc.&#39;</span><span class="p">)]</span> +<span class="n">r3000_equities</span> <span class="o">=</span> <span class="n">r3000</span><span class="p">[(</span><span class="n">r3000</span><span class="p">[</span><span class="s1">&#39;Exchange&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;NASDAQ&#39;</span><span class="p">)</span> <span class="o">|</span> + <span class="p">(</span><span class="n">r3000</span><span class="p">[</span><span class="s1">&#39;Exchange&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;New York Stock Exchange Inc.&#39;</span><span class="p">)]</span> -<span class="n">dates_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s">&#39;w&#39;</span><span class="p">)</span> +<span class="n">dates_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">with</span> <span class="n">futures</span><span class="o">.</span><span class="n">ThreadPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="mi">8</span><span class="p">)</span> <span class="k">as</span> <span class="n">pool</span><span class="p">:</span> - <span class="n">fs</span> <span class="o">=</span> <span class="p">{</span><span class="n">pool</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">earnings_releases</span><span class="p">,</span> <span class="n">r3000_equities</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">t</span><span class="p">][</span><span class="s">&#39;Ticker&#39;</span><span class="p">]):</span> <span class="n">t</span> + <span class="n">fs</span> <span class="o">=</span> <span class="p">{</span><span class="n">pool</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">earnings_releases</span><span class="p">,</span> <span class="n">r3000_equities</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">t</span><span class="p">][</span><span class="s1">&#39;Ticker&#39;</span><span class="p">]):</span> <span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">r3000_equities</span><span class="o">.</span><span class="n">index</span><span class="p">}</span> <span class="n">pbar</span> <span class="o">=</span> <span class="n">progressbar</span><span class="o">.</span><span class="n">ProgressBar</span><span class="p">(</span><span class="n">term_width</span><span class="o">=</span><span class="mi">80</span><span class="p">,</span> <span class="n">max_value</span><span class="o">=</span><span class="n">r3000_equities</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">max</span><span class="p">())</span> @@ -5163,7 +5430,7 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="o">.</span><span class="n">as_completed</span><span class="p">(</span><span class="n">fs</span><span class="p">):</span> <span class="n">i</span> <span class="o">=</span> <span class="n">fs</span><span class="p">[</span><span class="n">future</span><span class="p">]</span> <span class="n">pbar</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> - <span class="n">dates_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">yaml</span><span class="o">.</span><span class="n">dump</span><span class="p">({</span><span class="n">r3000_equities</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s">&#39;Ticker&#39;</span><span class="p">]:</span> + <span class="n">dates_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">yaml</span><span class="o">.</span><span class="n">dump</span><span class="p">({</span><span class="n">r3000_equities</span><span class="o">.</span><span class="n">ix</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">&#39;Ticker&#39;</span><span class="p">]:</span> <span class="n">future</span><span class="o">.</span><span class="n">result</span><span class="p">()}))</span> </pre></div> <p>Downloading stock price data needed for the event studies:</p> @@ -5176,31 +5443,31 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="kn">from</span> <span class="nn">progressbar</span> <span class="kn">import</span> <span class="n">ProgressBar</span> <span class="k">def</span> <span class="nf">fetch_ticker</span><span class="p">(</span><span class="n">ticker</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">):</span> - <span class="c"># Quandl is currently giving me issues with returning</span> - <span class="c"># the entire dataset and not slicing server-side.</span> - <span class="c"># So instead, we&#39;ll do it client-side!</span> - <span class="n">q_format</span> <span class="o">=</span> <span class="s">&#39;%Y-%m-</span><span class="si">%d</span><span class="s">&#39;</span> - <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&#39;YAHOO/&#39;</span> <span class="o">+</span> <span class="n">ticker</span><span class="p">,</span> + <span class="c1"># Quandl is currently giving me issues with returning</span> + <span class="c1"># the entire dataset and not slicing server-side.</span> + <span class="c1"># So instead, we&#39;ll do it client-side!</span> + <span class="n">q_format</span> <span class="o">=</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span> + <span class="n">ticker_data</span> <span class="o">=</span> <span class="n">quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;YAHOO/&#39;</span> <span class="o">+</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">start_date</span><span class="o">=</span><span class="n">start</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">q_format</span><span class="p">),</span> <span class="n">end_date</span><span class="o">=</span><span class="n">end</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">q_format</span><span class="p">),</span> <span class="n">authtoken</span><span class="o">=</span><span class="n">QUANDL_KEY</span><span class="p">)</span> <span class="k">return</span> <span class="n">ticker_data</span> -<span class="n">data_str</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> -<span class="c"># Need to remove invalid lines</span> -<span class="n">filtered</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s">&#39;{&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">x</span><span class="p">,</span> <span class="n">data_str</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">&#39;</span><span class="se">\n</span><span class="s">&#39;</span><span class="p">))</span> -<span class="n">earnings_data</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s">&#39;</span><span class="se">\n</span><span class="s">&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">filtered</span><span class="p">))</span> +<span class="n">data_str</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;earnings_dates.yaml&#39;</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> +<span class="c1"># Need to remove invalid lines</span> +<span class="n">filtered</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s1">&#39;{&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">x</span><span class="p">,</span> <span class="n">data_str</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">))</span> +<span class="n">earnings_data</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">filtered</span><span class="p">))</span> -<span class="c"># Get the first 1500 keys - split up into two statements</span> -<span class="c"># because of Quandl rate limits</span> +<span class="c1"># Get the first 1500 keys - split up into two statements</span> +<span class="c1"># because of Quandl rate limits</span> <span class="n">tickers</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">earnings_data</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> <span class="n">price_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">invalid_tickers</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">ticker</span> <span class="ow">in</span> <span class="n">ProgressBar</span><span class="p">()(</span><span class="n">tickers</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="mi">1500</span><span class="p">]):</span> <span class="k">try</span><span class="p">:</span> - <span class="c"># Replace &#39;.&#39; with &#39;-&#39; in name for some tickers</span> - <span class="n">fixed</span> <span class="o">=</span> <span class="n">ticker</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s">&#39;.&#39;</span><span class="p">,</span> <span class="s">&#39;-&#39;</span><span class="p">)</span> + <span class="c1"># Replace &#39;.&#39; with &#39;-&#39; in name for some tickers</span> + <span class="n">fixed</span> <span class="o">=</span> <span class="n">ticker</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;-&#39;</span><span class="p">)</span> <span class="n">event_strs</span> <span class="o">=</span> <span class="n">earnings_data</span><span class="p">[</span><span class="n">ticker</span><span class="p">]</span> <span class="n">events</span> <span class="o">=</span> <span class="p">[</span><span class="n">parse</span><span class="p">(</span><span class="n">event</span><span class="p">)</span> <span class="k">for</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">event_strs</span><span class="p">]</span> <span class="n">td</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span> @@ -5209,11 +5476,11 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="k">except</span> <span class="n">quandl</span><span class="o">.</span><span class="n">NotFoundError</span><span class="p">:</span> <span class="n">invalid_tickers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ticker</span><span class="p">)</span> -<span class="c"># Execute this after 10 minutes have passed</span> +<span class="c1"># Execute this after 10 minutes have passed</span> <span class="k">for</span> <span class="n">ticker</span> <span class="ow">in</span> <span class="n">ProgressBar</span><span class="p">()(</span><span class="n">tickers</span><span class="p">[</span><span class="mi">1500</span><span class="p">:]):</span> <span class="k">try</span><span class="p">:</span> - <span class="c"># Replace &#39;.&#39; with &#39;-&#39; in name for some tickers</span> - <span class="n">fixed</span> <span class="o">=</span> <span class="n">ticker</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s">&#39;.&#39;</span><span class="p">,</span> <span class="s">&#39;-&#39;</span><span class="p">)</span> + <span class="c1"># Replace &#39;.&#39; with &#39;-&#39; in name for some tickers</span> + <span class="n">fixed</span> <span class="o">=</span> <span class="n">ticker</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="s1">&#39;-&#39;</span><span class="p">)</span> <span class="n">event_strs</span> <span class="o">=</span> <span class="n">earnings_data</span><span class="p">[</span><span class="n">ticker</span><span class="p">]</span> <span class="n">events</span> <span class="o">=</span> <span class="p">[</span><span class="n">parse</span><span class="p">(</span><span class="n">event</span><span class="p">)</span> <span class="k">for</span> <span class="n">event</span> <span class="ow">in</span> <span class="n">event_strs</span><span class="p">]</span> <span class="n">td</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span> @@ -5222,7 +5489,7 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== <span class="k">except</span> <span class="n">quandl</span><span class="o">.</span><span class="n">NotFoundError</span><span class="p">:</span> <span class="n">invalid_tickers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ticker</span><span class="p">)</span> -<span class="n">prices_store</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">HDFStore</span><span class="p">(</span><span class="s">&#39;price_data.hdf&#39;</span><span class="p">)</span> +<span class="n">prices_store</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">HDFStore</span><span class="p">(</span><span class="s1">&#39;price_data.hdf&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">ticker</span><span class="p">,</span> <span class="n">prices</span> <span class="ow">in</span> <span class="n">price_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> <span class="n">prices_store</span><span class="p">[</span><span class="n">ticker</span><span class="p">]</span> <span class="o">=</span> <span class="n">prices</span> </pre></div> @@ -5235,7 +5502,7 @@ AAAAAGBWNBAAAAAAAMCsaCAAAAAAAIBZ0UAAAAAAAACz+n9WbX+0Yscg8AAAAABJRU5ErkJggg== MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>The Unfair Casino2016-05-15T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-05-15:the-unfair-casino.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>The Unfair Casino2016-05-15T00:00:00-04:002016-05-15T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-05-15:the-unfair-casino.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -5263,8 +5530,8 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <span class="n">fair_1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> <span class="n">fair_2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> -<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">fair_1</span> <span class="o">+</span> <span class="n">fair_2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;Fair Distribution&#39;</span><span class="p">);</span> +<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">fair_1</span> <span class="o">+</span> <span class="n">fair_2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s1">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;Fair Distribution&#39;</span><span class="p">);</span> </pre></div> </div> @@ -5475,8 +5742,8 @@ D_2 = \left\{ <span class="n">x1</span> <span class="o">=</span> <span class="n">unfair_die</span><span class="p">(</span><span class="n">d1</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> <span class="n">x2</span> <span class="o">=</span> <span class="n">unfair_die</span><span class="p">(</span><span class="n">d2</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> -<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">x1</span> <span class="o">+</span> <span class="n">x2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;$D_1$ biased low, $D_2$ biased high&#39;</span><span class="p">);</span> +<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">x1</span> <span class="o">+</span> <span class="n">x2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s1">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;$D_1$ biased low, $D_2$ biased high&#39;</span><span class="p">);</span> </pre></div> </div> @@ -5674,8 +5941,8 @@ D_2 = \left\{ <span class="n">x1</span> <span class="o">=</span> <span class="n">unfair_die</span><span class="p">(</span><span class="n">d1</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> <span class="n">x2</span> <span class="o">=</span> <span class="n">unfair_die</span><span class="p">(</span><span class="n">d2</span><span class="p">,</span> <span class="mi">10000</span><span class="p">)</span> -<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">x1</span> <span class="o">+</span> <span class="n">x2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&quot;$D_1$ and $D_2$ biased to 1 and 6&quot;</span><span class="p">);</span> +<span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">x1</span> <span class="o">+</span> <span class="n">x2</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s1">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s2">&quot;$D_1$ and $D_2$ biased to 1 and 6&quot;</span><span class="p">);</span> </pre></div> </div> @@ -5895,18 +6162,18 @@ and continue the next iteration.</li> <span class="kn">from</span> <span class="nn">numpy</span> <span class="k">import</span> <span class="n">polynomial</span> <span class="k">def</span> <span class="nf">density_coef</span><span class="p">(</span><span class="n">d1_density</span><span class="p">,</span> <span class="n">d2_density</span><span class="p">):</span> - <span class="c"># Calculating the probabilities of each outcome was taken</span> - <span class="c"># from this author: http://math.stackexchange.com/a/1710392/320784</span> + <span class="c1"># Calculating the probabilities of each outcome was taken</span> + <span class="c1"># from this author: http://math.stackexchange.com/a/1710392/320784</span> <span class="n">d1_p</span> <span class="o">=</span> <span class="n">polynomial</span><span class="o">.</span><span class="n">Polynomial</span><span class="p">(</span><span class="n">d1_density</span><span class="p">)</span> <span class="n">d2_p</span> <span class="o">=</span> <span class="n">polynomial</span><span class="o">.</span><span class="n">Polynomial</span><span class="p">(</span><span class="n">d2_density</span><span class="p">)</span> <span class="n">coefs</span> <span class="o">=</span> <span class="p">(</span><span class="n">d1_p</span> <span class="o">*</span> <span class="n">d2_p</span><span class="p">)</span><span class="o">.</span><span class="n">coef</span> <span class="k">return</span> <span class="n">coefs</span> <span class="k">def</span> <span class="nf">score</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">d1_density</span><span class="p">,</span> <span class="n">d2_density</span><span class="p">):</span> - <span class="c"># We&#39;ve now got the probabilities of each event, but we need</span> - <span class="c"># to shift the array a bit so we can use the x values to actually</span> - <span class="c"># index into it. This will allow us to do all the calculations</span> - <span class="c"># incredibly quickly</span> + <span class="c1"># We&#39;ve now got the probabilities of each event, but we need</span> + <span class="c1"># to shift the array a bit so we can use the x values to actually</span> + <span class="c1"># index into it. This will allow us to do all the calculations</span> + <span class="c1"># incredibly quickly</span> <span class="n">coefs</span> <span class="o">=</span> <span class="n">density_coef</span><span class="p">(</span><span class="n">d1_density</span><span class="p">,</span> <span class="n">d2_density</span><span class="p">)</span> <span class="n">coefs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">hstack</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">coefs</span><span class="p">))</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">coefs</span><span class="p">[</span><span class="n">x</span><span class="p">])</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> @@ -5933,23 +6200,23 @@ and continue the next iteration.</li> <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">permute</span><span class="p">(</span><span class="n">d1_density</span><span class="p">,</span> <span class="n">d2_density</span><span class="p">):</span> - <span class="c"># To ensure we have legitimate densities, we will randomly</span> - <span class="c"># increase one die face probability by `change`,</span> - <span class="c"># and decrease one by `change`.</span> - <span class="c"># This means there are something less than (1/`change`)^12 possibilities</span> - <span class="c"># we are trying to search over.</span> + <span class="c1"># To ensure we have legitimate densities, we will randomly</span> + <span class="c1"># increase one die face probability by `change`,</span> + <span class="c1"># and decrease one by `change`.</span> + <span class="c1"># This means there are something less than (1/`change`)^12 possibilities</span> + <span class="c1"># we are trying to search over.</span> <span class="n">change</span> <span class="o">=</span> <span class="o">.</span><span class="mi">01</span> <span class="n">d1_index1</span><span class="p">,</span> <span class="n">d1_index2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span> <span class="n">d2_index1</span><span class="p">,</span> <span class="n">d2_index2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span> - <span class="c"># Also make sure to copy. I&#39;ve had some weird aliasing issues</span> - <span class="c"># in the past that made everything blow up.</span> + <span class="c1"># Also make sure to copy. I&#39;ve had some weird aliasing issues</span> + <span class="c1"># in the past that made everything blow up.</span> <span class="n">new_d1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">d1_density</span><span class="p">))</span> <span class="n">new_d2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">d2_density</span><span class="p">))</span> - <span class="c"># While this doesn&#39;t account for the possibility that some</span> - <span class="c"># values go negative, in practice this never happens</span> + <span class="c1"># While this doesn&#39;t account for the possibility that some</span> + <span class="c1"># values go negative, in practice this never happens</span> <span class="n">new_d1</span><span class="p">[</span><span class="n">d1_index1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">change</span> <span class="n">new_d1</span><span class="p">[</span><span class="n">d1_index2</span><span class="p">]</span> <span class="o">-=</span> <span class="n">change</span> <span class="n">new_d2</span><span class="p">[</span><span class="n">d2_index1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">change</span> @@ -5982,24 +6249,24 @@ and continue the next iteration.</li> <span class="n">switch_failures</span> <span class="o">=</span> <span class="mi">0</span> <span class="n">iter_count</span> <span class="o">=</span> <span class="mi">0</span> - <span class="c"># Start with guessing fair dice</span> + <span class="c1"># Start with guessing fair dice</span> <span class="n">cur_d1</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="mi">1</span><span class="o">/</span><span class="mi">6</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> <span class="n">cur_d2</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="mi">1</span><span class="o">/</span><span class="mi">6</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> <span class="n">cur_score</span> <span class="o">=</span> <span class="n">score</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">cur_d1</span><span class="p">,</span> <span class="n">cur_d2</span><span class="p">)</span> - <span class="c"># Keep track of our best guesses - may not be</span> - <span class="c"># what we end with</span> + <span class="c1"># Keep track of our best guesses - may not be</span> + <span class="c1"># what we end with</span> <span class="n">max_score</span> <span class="o">=</span> <span class="n">cur_score</span> <span class="n">max_d1</span> <span class="o">=</span> <span class="n">cur_d1</span> <span class="n">max_d2</span> <span class="o">=</span> <span class="n">cur_d2</span> - <span class="c"># Optimization stops when we have failed to switch `conv_count`</span> - <span class="c"># times (presumably because we have a great guess), or we reach</span> - <span class="c"># the maximum number of iterations.</span> + <span class="c1"># Optimization stops when we have failed to switch `conv_count`</span> + <span class="c1"># times (presumably because we have a great guess), or we reach</span> + <span class="c1"># the maximum number of iterations.</span> <span class="k">while</span> <span class="n">switch_failures</span> <span class="o">&lt;</span> <span class="n">conv_count</span> <span class="ow">and</span> <span class="n">iter_count</span> <span class="o">&lt;</span> <span class="n">max_iter</span><span class="p">:</span> <span class="n">iter_count</span> <span class="o">+=</span> <span class="mi">1</span> <span class="k">if</span> <span class="n">iter_count</span> <span class="o">%</span> <span class="p">(</span><span class="n">max_iter</span> <span class="o">/</span> <span class="mi">10</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> - <span class="nb">print</span><span class="p">(</span><span class="s">&#39;Iteration: {}; Current score (higher is better): {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> + <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Iteration: {}; Current score (higher is better): {}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> <span class="n">iter_count</span><span class="p">,</span> <span class="n">cur_score</span><span class="p">))</span> <span class="n">new_d1</span><span class="p">,</span> <span class="n">new_d2</span> <span class="o">=</span> <span class="n">permute</span><span class="p">(</span><span class="n">cur_d1</span><span class="p">,</span> <span class="n">cur_d2</span><span class="p">)</span> @@ -6011,19 +6278,19 @@ and continue the next iteration.</li> <span class="n">max_d2</span> <span class="o">=</span> <span class="n">new_d2</span> <span class="k">if</span> <span class="n">new_score</span> <span class="o">&gt;</span> <span class="n">cur_score</span><span class="p">:</span> - <span class="c"># If the new permutation beats the old one,</span> - <span class="c"># automatically select it.</span> + <span class="c1"># If the new permutation beats the old one,</span> + <span class="c1"># automatically select it.</span> <span class="n">cur_score</span> <span class="o">=</span> <span class="n">new_score</span> <span class="n">cur_d1</span> <span class="o">=</span> <span class="n">new_d1</span> <span class="n">cur_d2</span> <span class="o">=</span> <span class="n">new_d2</span> <span class="n">switch_failures</span> <span class="o">=</span> <span class="mi">0</span> <span class="k">else</span><span class="p">:</span> - <span class="c"># We didn&#39;t beat the current score, but allow</span> - <span class="c"># for possibly switching anyways.</span> + <span class="c1"># We didn&#39;t beat the current score, but allow</span> + <span class="c1"># for possibly switching anyways.</span> <span class="n">accept_prob</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">new_score</span> <span class="o">-</span> <span class="n">cur_score</span><span class="p">)</span> <span class="n">coin_toss</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">()</span> <span class="k">if</span> <span class="n">coin_toss</span> <span class="o">&lt;</span> <span class="n">accept_prob</span><span class="p">:</span> - <span class="c"># We randomly switch to the new distribution</span> + <span class="c1"># We randomly switch to the new distribution</span> <span class="n">cur_score</span> <span class="o">=</span> <span class="n">new_score</span> <span class="n">cur_d1</span> <span class="o">=</span> <span class="n">new_d1</span> <span class="n">cur_d2</span> <span class="o">=</span> <span class="n">new_d2</span> @@ -6031,7 +6298,7 @@ and continue the next iteration.</li> <span class="k">else</span><span class="p">:</span> <span class="n">switch_failures</span> <span class="o">+=</span> <span class="mi">1</span> - <span class="c"># Return both our best guess, and the ending guess</span> + <span class="c1"># Return both our best guess, and the ending guess</span> <span class="k">return</span> <span class="n">max_d1</span><span class="p">,</span> <span class="n">max_d2</span><span class="p">,</span> <span class="n">cur_d1</span><span class="p">,</span> <span class="n">cur_d2</span> </pre></div> @@ -6065,20 +6332,20 @@ and continue the next iteration.</li> <span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">perf_counter</span><span class="p">()</span> <span class="n">max_d1</span><span class="p">,</span> <span class="n">max_d2</span><span class="p">,</span> <span class="n">final_d1</span><span class="p">,</span> <span class="n">final_d2</span> <span class="o">=</span> <span class="n">optimize</span><span class="p">(</span><span class="n">d1_vals</span> <span class="o">+</span> <span class="n">d2_vals</span><span class="p">)</span> <span class="n">end</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">perf_counter</span><span class="p">()</span> - <span class="nb">print</span><span class="p">(</span><span class="s">&quot;Simulated Annealing time: {:.02f}s&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">start</span><span class="p">))</span> + <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Simulated Annealing time: {:.02f}s&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">start</span><span class="p">))</span> <span class="n">coef_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">13</span><span class="p">)</span> <span class="o">-</span> <span class="o">.</span><span class="mi">5</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">221</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">bar</span><span class="p">(</span><span class="n">coef_range</span><span class="p">,</span> <span class="n">density_coef</span><span class="p">(</span><span class="n">d1_dist</span><span class="p">,</span> <span class="n">d2_dist</span><span class="p">),</span> <span class="n">width</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> - <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;True Distribution&#39;</span><span class="p">)</span> + <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;True Distribution&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">222</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">hist</span><span class="p">(</span><span class="n">d1_vals</span> <span class="o">+</span> <span class="n">d2_vals</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="mi">11</span><span class="p">)</span> - <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;Empirical Distribution&#39;</span><span class="p">)</span> + <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;Empirical Distribution&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">223</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">bar</span><span class="p">(</span><span class="n">coef_range</span><span class="p">,</span> <span class="n">density_coef</span><span class="p">(</span><span class="n">max_d1</span><span class="p">,</span> <span class="n">max_d2</span><span class="p">),</span> <span class="n">width</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> - <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s">&#39;Recovered Distribution&#39;</span><span class="p">)</span> + <span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s1">&#39;Recovered Distribution&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span> @@ -7963,7 +8230,7 @@ JDVg6JIkSWrA0CVJktSAoUuSJKkBQ5ckSVIDhi5JkqQG/gdFMjKH5D5DewAAAABJRU5ErkJggg== MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Tick Tock...2016-04-06T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-04-06:tick-tock.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Tick Tock...2016-04-06T00:00:00-04:002016-04-06T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-04-06:tick-tock.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -8014,7 +8281,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <div class="prompt input_prompt">In&nbsp;[2]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># Import all the OAuth secret information from a local file</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># Import all the OAuth secret information from a local file</span> <span class="kn">from</span> <span class="nn">secrets</span> <span class="k">import</span> <span class="n">CLIENT_SECRET</span><span class="p">,</span> <span class="n">CLIENT_ID</span><span class="p">,</span> <span class="n">CALLBACK_URL</span> </pre></div> @@ -8055,17 +8322,17 @@ Application: Thank you, now give me all the data.</code></pre> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="kn">import</span> <span class="nn">urllib</span> -<span class="n">FITBIT_URI</span> <span class="o">=</span> <span class="s">&#39;https://www.fitbit.com/oauth2/authorize&#39;</span> +<span class="n">FITBIT_URI</span> <span class="o">=</span> <span class="s1">&#39;https://www.fitbit.com/oauth2/authorize&#39;</span> <span class="n">params</span> <span class="o">=</span> <span class="p">{</span> - <span class="c"># If we need more than one scope, must be a CSV string</span> - <span class="s">&#39;scope&#39;</span><span class="p">:</span> <span class="s">&#39;heartrate&#39;</span><span class="p">,</span> - <span class="s">&#39;response_type&#39;</span><span class="p">:</span> <span class="s">&#39;token&#39;</span><span class="p">,</span> - <span class="s">&#39;expires_in&#39;</span><span class="p">:</span> <span class="mi">86400</span><span class="p">,</span> <span class="c"># 1 day</span> - <span class="s">&#39;redirect_uri&#39;</span><span class="p">:</span> <span class="n">CALLBACK_URL</span><span class="p">,</span> - <span class="s">&#39;client_id&#39;</span><span class="p">:</span> <span class="n">CLIENT_ID</span> + <span class="c1"># If we need more than one scope, must be a CSV string</span> + <span class="s1">&#39;scope&#39;</span><span class="p">:</span> <span class="s1">&#39;heartrate&#39;</span><span class="p">,</span> + <span class="s1">&#39;response_type&#39;</span><span class="p">:</span> <span class="s1">&#39;token&#39;</span><span class="p">,</span> + <span class="s1">&#39;expires_in&#39;</span><span class="p">:</span> <span class="mi">86400</span><span class="p">,</span> <span class="c1"># 1 day</span> + <span class="s1">&#39;redirect_uri&#39;</span><span class="p">:</span> <span class="n">CALLBACK_URL</span><span class="p">,</span> + <span class="s1">&#39;client_id&#39;</span><span class="p">:</span> <span class="n">CLIENT_ID</span> <span class="p">}</span> -<span class="n">request_url</span> <span class="o">=</span> <span class="n">FITBIT_URI</span> <span class="o">+</span> <span class="s">&#39;?&#39;</span> <span class="o">+</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlencode</span><span class="p">(</span><span class="n">params</span><span class="p">)</span> +<span class="n">request_url</span> <span class="o">=</span> <span class="n">FITBIT_URI</span> <span class="o">+</span> <span class="s1">&#39;?&#39;</span> <span class="o">+</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlencode</span><span class="p">(</span><span class="n">params</span><span class="p">)</span> </pre></div> </div> @@ -8088,10 +8355,10 @@ Application: Thank you, now give me all the data.</code></pre> <div class="prompt input_prompt">In&nbsp;[6]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># The `response_url` variable contains the full URL that</span> -<span class="c"># FitBit sent back to us, but most importantly,</span> -<span class="c"># contains the token we need for authorization.</span> -<span class="n">access_token</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">parse_qsl</span><span class="p">(</span><span class="n">response_url</span><span class="p">))[</span><span class="s">&#39;access_token&#39;</span><span class="p">]</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># The `response_url` variable contains the full URL that</span> +<span class="c1"># FitBit sent back to us, but most importantly,</span> +<span class="c1"># contains the token we need for authorization.</span> +<span class="n">access_token</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">parse_qsl</span><span class="p">(</span><span class="n">response_url</span><span class="p">))[</span><span class="s1">&#39;access_token&#39;</span><span class="p">]</span> </pre></div> </div> @@ -8120,41 +8387,41 @@ Application: Thank you, now give me all the data.</code></pre> <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span> <span class="n">session</span> <span class="o">=</span> <span class="n">OAuth2Session</span><span class="p">(</span><span class="n">token</span><span class="o">=</span><span class="p">{</span> - <span class="s">&#39;access_token&#39;</span><span class="p">:</span> <span class="n">access_token</span><span class="p">,</span> - <span class="s">&#39;token_type&#39;</span><span class="p">:</span> <span class="s">&#39;Bearer&#39;</span> + <span class="s1">&#39;access_token&#39;</span><span class="p">:</span> <span class="n">access_token</span><span class="p">,</span> + <span class="s1">&#39;token_type&#39;</span><span class="p">:</span> <span class="s1">&#39;Bearer&#39;</span> <span class="p">})</span> -<span class="n">format_str</span> <span class="o">=</span> <span class="s">&#39;%Y-%m-%d&#39;</span> +<span class="n">format_str</span> <span class="o">=</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">&#39;</span> <span class="n">start_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="n">end_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">31</span><span class="p">)</span> <span class="n">dr</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">date_range</span><span class="p">(</span><span class="n">start_date</span><span class="p">,</span> <span class="n">end_date</span><span class="p">)</span> -<span class="n">url</span> <span class="o">=</span> <span class="s">&#39;https://api.fitbit.com/1/user/-/activities/heart/date/{0}/1d/1min.json&#39;</span> +<span class="n">url</span> <span class="o">=</span> <span class="s1">&#39;https://api.fitbit.com/1/user/-/activities/heart/date/{0}/1d/1min.json&#39;</span> <span class="n">hr_responses</span> <span class="o">=</span> <span class="p">[</span><span class="n">session</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">format_str</span><span class="p">)))</span> <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">dr</span><span class="p">]</span> <span class="k">def</span> <span class="nf">record_to_df</span><span class="p">(</span><span class="n">record</span><span class="p">):</span> - <span class="k">if</span> <span class="s">&#39;activities-heart&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">record</span><span class="p">:</span> - <span class="k">return</span> <span class="k">None</span> - <span class="n">date_str</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s">&#39;activities-heart&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">][</span><span class="s">&#39;dateTime&#39;</span><span class="p">]</span> - <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s">&#39;activities-heart-intraday&#39;</span><span class="p">][</span><span class="s">&#39;dataset&#39;</span><span class="p">])</span> + <span class="k">if</span> <span class="s1">&#39;activities-heart&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">record</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">None</span> + <span class="n">date_str</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s1">&#39;activities-heart&#39;</span><span class="p">][</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;dateTime&#39;</span><span class="p">]</span> + <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s1">&#39;activities-heart-intraday&#39;</span><span class="p">][</span><span class="s1">&#39;dataset&#39;</span><span class="p">])</span> - <span class="n">df</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s">&#39;time&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span> - <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">date_str</span> <span class="o">+</span> <span class="s">&#39; &#39;</span> <span class="o">+</span> <span class="n">x</span><span class="p">,</span> <span class="s">&#39;%Y-%m-%d %H:%M:%S&#39;</span><span class="p">))</span> + <span class="n">df</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s1">&#39;time&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span> + <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">date_str</span> <span class="o">+</span> <span class="s1">&#39; &#39;</span> <span class="o">+</span> <span class="n">x</span><span class="p">,</span> <span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1"> %H:%M:%S&#39;</span><span class="p">))</span> <span class="k">return</span> <span class="n">df</span> <span class="n">hr_dataframes</span> <span class="o">=</span> <span class="p">[</span><span class="n">record_to_df</span><span class="p">(</span><span class="n">record</span><span class="o">.</span><span class="n">json</span><span class="p">())</span> <span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">hr_responses</span><span class="p">]</span> <span class="n">hr_df_concat</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">(</span><span class="n">hr_dataframes</span><span class="p">)</span> -<span class="c"># There are some minutes with missing data, so we need to correct that</span> +<span class="c1"># There are some minutes with missing data, so we need to correct that</span> <span class="n">full_daterange</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">date_range</span><span class="p">(</span><span class="n">hr_df_concat</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">hr_df_concat</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;min&#39;</span><span class="p">)</span> -<span class="n">hr_df_full</span> <span class="o">=</span> <span class="n">hr_df_concat</span><span class="o">.</span><span class="n">reindex</span><span class="p">(</span><span class="n">full_daterange</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s">&#39;nearest&#39;</span><span class="p">)</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;min&#39;</span><span class="p">)</span> +<span class="n">hr_df_full</span> <span class="o">=</span> <span class="n">hr_df_concat</span><span class="o">.</span><span class="n">reindex</span><span class="p">(</span><span class="n">full_daterange</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s1">&#39;nearest&#39;</span><span class="p">)</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Heartbeats from {} to {}: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Heartbeats from {} to {}: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> - <span class="n">hr_df_full</span><span class="p">[</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()))</span> + <span class="n">hr_df_full</span><span class="p">[</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()))</span> </pre></div> </div> @@ -8202,9 +8469,9 @@ hr_q \cdot n - hr_d \cdot (n-m) <div class="prompt input_prompt">In&nbsp;[8]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">quarterly_count</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">quarterly_count</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> <span class="n">leap_day_count</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">month</span> <span class="o">==</span> <span class="mi">2</span><span class="p">)</span> <span class="o">&amp;</span> - <span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">day</span> <span class="o">==</span> <span class="mi">29</span><span class="p">)][</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> + <span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">day</span> <span class="o">==</span> <span class="mi">29</span><span class="p">)][</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> <span class="n">num_quarters</span> <span class="o">=</span> <span class="mi">96</span> <span class="n">leap_days</span> <span class="o">=</span> <span class="mi">6</span> @@ -8249,17 +8516,17 @@ hr_q \cdot n - hr_d \cdot (n-m) <div class="prompt input_prompt">In&nbsp;[9]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">minute_mean</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> -<span class="c"># Don&#39;t you wish you knew?</span> -<span class="c"># birthday_minutes = ???</span> +<div class=" highlight hl-ipython3"><pre><span class="n">minute_mean</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> +<span class="c1"># Don&#39;t you wish you knew?</span> +<span class="c1"># birthday_minutes = ???</span> <span class="n">birthday_heartbeats</span> <span class="o">=</span> <span class="n">birthday_minutes</span> <span class="o">*</span> <span class="n">minute_mean</span> <span class="n">heartbeats_until_2016</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">birthday_heartbeats</span> <span class="o">+</span> <span class="n">jan_92_jan_16</span><span class="p">)</span> <span class="n">remaining_2016</span> <span class="o">=</span> <span class="n">total_heartbeats</span> <span class="o">-</span> <span class="n">heartbeats_until_2016</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Heartbeats so far: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">heartbeats_until_2016</span><span class="p">))</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Remaining heartbeats: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_2016</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Heartbeats so far: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">heartbeats_until_2016</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Remaining heartbeats: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_2016</span><span class="p">))</span> </pre></div> </div> @@ -8350,8 +8617,8 @@ hr_q \cdot n - hr_d \cdot (n - m) <div class=" highlight hl-ipython3"><pre><span class="n">four_year_periods</span> <span class="o">=</span> <span class="n">remaining_2016</span> <span class="o">//</span> <span class="n">heartbeats_4year</span> <span class="n">remaining_4y</span> <span class="o">=</span> <span class="n">remaining_2016</span> <span class="o">-</span> <span class="n">four_year_periods</span> <span class="o">*</span> <span class="n">heartbeats_4year</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Four year periods remaining: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">four_year_periods</span><span class="p">))</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Remaining heartbeats after 4 year periods: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_4y</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Four year periods remaining: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">four_year_periods</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Remaining heartbeats after 4 year periods: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_4y</span><span class="p">))</span> </pre></div> </div> @@ -8390,14 +8657,14 @@ Remaining heartbeats after 4 year periods: 48041640 <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="n">remaining_leap</span> <span class="o">=</span> <span class="n">remaining_4y</span> <span class="o">-</span> <span class="n">leap_day_count</span> -<span class="c"># Ignore leap day in the data set</span> +<span class="c1"># Ignore leap day in the data set</span> <span class="n">heartbeats_quarter</span> <span class="o">=</span> <span class="n">hr_df_full</span><span class="p">[(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">month</span> <span class="o">!=</span> <span class="mi">2</span><span class="p">)</span> <span class="o">&amp;</span> - <span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">day</span> <span class="o">!=</span> <span class="mi">29</span><span class="p">)][</span><span class="s">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> + <span class="p">(</span><span class="n">hr_df_full</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">day</span> <span class="o">!=</span> <span class="mi">29</span><span class="p">)][</span><span class="s1">&#39;value&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> <span class="n">quarters_left</span> <span class="o">=</span> <span class="n">remaining_leap</span> <span class="o">//</span> <span class="n">heartbeats_quarter</span> <span class="n">remaining_year</span> <span class="o">=</span> <span class="n">remaining_leap</span> <span class="o">-</span> <span class="n">quarters_left</span> <span class="o">*</span> <span class="n">heartbeats_quarter</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Quarters left starting 2068: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">quarters_left</span><span class="p">))</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Remaining heartbeats after that: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_year</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Quarters left starting 2068: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">quarters_left</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Remaining heartbeats after that: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">remaining_year</span><span class="p">))</span> </pre></div> </div> @@ -8483,7 +8750,7 @@ Remaining heartbeats after that: 4760716 MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Tweet Like Me2016-03-28T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-03-28:tweet-like-me.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Tweet Like Me2016-03-28T00:00:00-04:002016-03-28T00:00:00-04:00Bradlee Speicetag:bspeice.github.io,2016-03-28:tweet-like-me.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -8525,11 +8792,11 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span> -<span class="n">tweets</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s">&#39;tweets.csv&#39;</span><span class="p">)</span> +<span class="n">tweets</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;tweets.csv&#39;</span><span class="p">)</span> <span class="n">text</span> <span class="o">=</span> <span class="n">tweets</span><span class="o">.</span><span class="n">text</span> -<span class="c"># Don&#39;t include tweets in reply to or mentioning people</span> -<span class="n">replies</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span><span class="s">&#39;@&#39;</span><span class="p">)</span> +<span class="c1"># Don&#39;t include tweets in reply to or mentioning people</span> +<span class="n">replies</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span><span class="s1">&#39;@&#39;</span><span class="p">)</span> <span class="n">text_norep</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="o">~</span><span class="n">replies</span><span class="p">]</span> </pre></div> @@ -8584,10 +8851,10 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="kn">from</span> <span class="nn">functools</span> <span class="k">import</span> <span class="n">reduce</span> -<span class="c"># Get all possible words</span> +<span class="c1"># Get all possible words</span> <span class="n">all_words</span> <span class="o">=</span> <span class="n">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">+</span><span class="n">y</span><span class="p">,</span> <span class="n">tokens</span><span class="p">,</span> <span class="p">[])</span> <span class="n">unique_words</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">all_words</span><span class="p">)</span> -<span class="n">actual_words</span> <span class="o">=</span> <span class="nb">set</span><span class="p">([</span><span class="n">x</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s">&#39;.&#39;</span> <span class="k">else</span> <span class="k">None</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">unique_words</span><span class="p">])</span> +<span class="n">actual_words</span> <span class="o">=</span> <span class="nb">set</span><span class="p">([</span><span class="n">x</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;.&#39;</span> <span class="k">else</span> <span class="kc">None</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">unique_words</span><span class="p">])</span> <span class="n">word_dist</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">for</span> <span class="n">word</span> <span class="ow">in</span> <span class="nb">iter</span><span class="p">(</span><span class="n">actual_words</span><span class="p">):</span> @@ -8619,9 +8886,9 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['\$','\$']]}}); <div class=" highlight hl-ipython3"><pre><span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span> <span class="o">%</span><span class="k">matplotlib</span> inline -<span class="n">hashtags</span> <span class="o">=</span> <span class="n">text_norep</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s">&#39;#&#39;</span><span class="p">)</span> +<span class="n">hashtags</span> <span class="o">=</span> <span class="n">text_norep</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">&#39;#&#39;</span><span class="p">)</span> <span class="n">bins</span> <span class="o">=</span> <span class="n">hashtags</span><span class="o">.</span><span class="n">unique</span><span class="p">()</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> -<span class="n">hashtags</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="n">bins</span><span class="p">)</span> +<span class="n">hashtags</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">kind</span><span class="o">=</span><span class="s1">&#39;hist&#39;</span><span class="p">,</span> <span class="n">bins</span><span class="o">=</span><span class="n">bins</span><span class="p">)</span> </pre></div> </div> @@ -8793,12 +9060,12 @@ AElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[6]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">hashtags</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">all_words</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;#&#39;</span><span class="p">]</span> +<div class=" highlight hl-ipython3"><pre><span class="n">hashtags</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">all_words</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;#&#39;</span><span class="p">]</span> <span class="n">n_hashtags</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">hashtags</span><span class="p">)</span> -<span class="n">unique_hashtags</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">unique_words</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;#&#39;</span><span class="p">]))</span> -<span class="n">hashtag_dist</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s">&#39;hashtags&#39;</span><span class="p">:</span> <span class="n">unique_hashtags</span><span class="p">,</span> - <span class="s">&#39;prob&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">all_words</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="n">h</span><span class="p">)</span> <span class="o">/</span> <span class="n">n_hashtags</span> +<span class="n">unique_hashtags</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">unique_words</span> <span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;#&#39;</span><span class="p">]))</span> +<span class="n">hashtag_dist</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;hashtags&#39;</span><span class="p">:</span> <span class="n">unique_hashtags</span><span class="p">,</span> + <span class="s1">&#39;prob&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">all_words</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="n">h</span><span class="p">)</span> <span class="o">/</span> <span class="n">n_hashtags</span> <span class="k">for</span> <span class="n">h</span> <span class="ow">in</span> <span class="n">unique_hashtags</span><span class="p">]})</span> <span class="nb">len</span><span class="p">(</span><span class="n">hashtag_dist</span><span class="p">)</span> </pre></div> @@ -8856,7 +9123,7 @@ z: .17</code></pre> <span class="k">def</span> <span class="nf">multinom_sim</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">vals</span><span class="p">,</span> <span class="n">probs</span><span class="p">):</span> <span class="n">occurrences</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">multinomial</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">probs</span><span class="p">)</span> <span class="n">results</span> <span class="o">=</span> <span class="n">occurrences</span> <span class="o">*</span> <span class="n">vals</span> - <span class="k">return</span> <span class="s">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">results</span><span class="p">[</span><span class="n">results</span> <span class="o">!=</span> <span class="s">&#39;&#39;</span><span class="p">])</span> + <span class="k">return</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">results</span><span class="p">[</span><span class="n">results</span> <span class="o">!=</span> <span class="s1">&#39;&#39;</span><span class="p">])</span> <span class="k">def</span> <span class="nf">sim_n_hashtags</span><span class="p">(</span><span class="n">hashtag_freq</span><span class="p">):</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">poisson</span><span class="p">(</span><span class="n">hashtag_freq</span><span class="p">)</span> @@ -8866,7 +9133,7 @@ z: .17</code></pre> <span class="k">def</span> <span class="nf">sim_first_word</span><span class="p">(</span><span class="n">first_word_dist</span><span class="p">):</span> <span class="n">probs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">(</span><span class="n">first_word_dist</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> - <span class="k">return</span> <span class="n">multinom_sim</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">first_word_dist</span><span class="o">.</span><span class="n">reset_index</span><span class="p">()[</span><span class="s">&#39;index&#39;</span><span class="p">],</span> <span class="n">probs</span><span class="p">)</span> + <span class="k">return</span> <span class="n">multinom_sim</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">first_word_dist</span><span class="o">.</span><span class="n">reset_index</span><span class="p">()[</span><span class="s1">&#39;index&#39;</span><span class="p">],</span> <span class="n">probs</span><span class="p">)</span> <span class="k">def</span> <span class="nf">sim_next_word</span><span class="p">(</span><span class="n">current</span><span class="p">,</span> <span class="n">word_dist</span><span class="p">):</span> <span class="n">dist</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">word_dist</span><span class="p">[</span><span class="n">current</span><span class="p">])</span> @@ -8901,7 +9168,7 @@ z: .17</code></pre> <span class="n">fifth</span> <span class="o">=</span> <span class="n">sim_next_word</span><span class="p">(</span><span class="n">fourth</span><span class="p">,</span> <span class="n">word_dist</span><span class="p">)</span> <span class="n">hashtag</span> <span class="o">=</span> <span class="n">sim_hashtags</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">hashtag_dist</span><span class="p">)</span> -<span class="s">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">first</span><span class="p">,</span> <span class="n">second</span><span class="p">,</span> <span class="n">third</span><span class="p">,</span> <span class="n">fourth</span><span class="p">,</span> <span class="n">fifth</span><span class="p">,</span> <span class="n">hashtag</span><span class="p">))</span> +<span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">first</span><span class="p">,</span> <span class="n">second</span><span class="p">,</span> <span class="n">third</span><span class="p">,</span> <span class="n">fourth</span><span class="p">,</span> <span class="n">fifth</span><span class="p">,</span> <span class="n">hashtag</span><span class="p">))</span> </pre></div> </div> @@ -8950,13 +9217,13 @@ z: .17</code></pre> <span class="n">tweet</span> <span class="o">=</span> <span class="n">first</span> <span class="n">current</span> <span class="o">=</span> <span class="n">first</span> - <span class="k">while</span> <span class="n">chars_remaining</span> <span class="o">&gt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">tweet</span><span class="p">)</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="n">hashtags</span><span class="p">)</span> <span class="ow">and</span> <span class="n">current</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s">&#39;.&#39;</span> <span class="ow">and</span> <span class="n">current</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s">&#39;!&#39;</span><span class="p">:</span> + <span class="k">while</span> <span class="n">chars_remaining</span> <span class="o">&gt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">tweet</span><span class="p">)</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="n">hashtags</span><span class="p">)</span> <span class="ow">and</span> <span class="n">current</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;.&#39;</span> <span class="ow">and</span> <span class="n">current</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">&#39;!&#39;</span><span class="p">:</span> <span class="n">current</span> <span class="o">=</span> <span class="n">sim_next_word</span><span class="p">(</span><span class="n">current</span><span class="p">,</span> <span class="n">word_dist</span><span class="p">)</span> - <span class="n">tweet</span> <span class="o">+=</span> <span class="s">&#39; &#39;</span> <span class="o">+</span> <span class="n">current</span> + <span class="n">tweet</span> <span class="o">+=</span> <span class="s1">&#39; &#39;</span> <span class="o">+</span> <span class="n">current</span> <span class="n">tweet</span> <span class="o">=</span> <span class="n">tweet</span><span class="p">[:</span><span class="o">-</span><span class="mi">2</span><span class="p">]</span> <span class="o">+</span> <span class="n">tweet</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> - <span class="k">return</span> <span class="s">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">tweet</span><span class="p">,</span> <span class="n">hashtags</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> + <span class="k">return</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">tweet</span><span class="p">,</span> <span class="n">hashtags</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> </pre></div> </div> @@ -9058,7 +9325,7 @@ Aamazing conference when you married #DepartmentOfRedundancyDepartment Yep , but MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Predicting Santander Customer Happiness2016-03-05T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-03-05:predicting-santander-customer-happiness.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Predicting Santander Customer Happiness2016-03-05T00:00:00-05:002016-03-05T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-03-05:predicting-santander-customer-happiness.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -9080,13 +9347,13 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span> <span class="o">%</span><span class="k">matplotlib</span> inline -<span class="c"># Record how long it takes to run the notebook - I&#39;m curious.</span> +<span class="c1"># Record how long it takes to run the notebook - I&#39;m curious.</span> <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span> <span class="n">start</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> -<span class="n">dataset</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s">&#39;split_train.csv&#39;</span><span class="p">)</span> +<span class="n">dataset</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;split_train.csv&#39;</span><span class="p">)</span> <span class="n">dataset</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">ID</span> -<span class="n">X</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">drop</span><span class="p">([</span><span class="s">&#39;TARGET&#39;</span><span class="p">,</span> <span class="s">&#39;ID&#39;</span><span class="p">,</span> <span class="s">&#39;ID.1&#39;</span><span class="p">],</span> <span class="mi">1</span><span class="p">)</span> +<span class="n">X</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">drop</span><span class="p">([</span><span class="s1">&#39;TARGET&#39;</span><span class="p">,</span> <span class="s1">&#39;ID&#39;</span><span class="p">,</span> <span class="s1">&#39;ID.1&#39;</span><span class="p">],</span> <span class="mi">1</span><span class="p">)</span> <span class="n">y</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">TARGET</span> </pre></div> @@ -9216,13 +9483,13 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[5]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># First we need to `binarize` the data to 0-1; some of the labels are {0, 1},</span> -<span class="c"># some are {0, 3}, etc.</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># First we need to `binarize` the data to 0-1; some of the labels are {0, 1},</span> +<span class="c1"># some are {0, 3}, etc.</span> <span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="k">import</span> <span class="n">binarize</span> <span class="n">X_bin</span> <span class="o">=</span> <span class="n">binarize</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="n">b_class</span><span class="p">])</span> <span class="n">accuracy</span> <span class="o">=</span> <span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">X_bin</span><span class="p">[:,</span><span class="n">i</span><span class="p">]</span> <span class="o">==</span> <span class="n">y</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">b_class</span><span class="p">))]</span> -<span class="n">acc_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s">&quot;Accuracy&quot;</span><span class="p">:</span> <span class="n">accuracy</span><span class="p">},</span> <span class="n">index</span><span class="o">=</span><span class="n">b_class</span><span class="p">)</span> +<span class="n">acc_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;Accuracy&quot;</span><span class="p">:</span> <span class="n">accuracy</span><span class="p">},</span> <span class="n">index</span><span class="o">=</span><span class="n">b_class</span><span class="p">)</span> <span class="n">acc_df</span><span class="o">.</span><span class="n">describe</span><span class="p">()</span> </pre></div> @@ -9306,9 +9573,9 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="n">unsat</span> <span class="o">=</span> <span class="n">y</span><span class="p">[</span><span class="n">y</span> <span class="o">==</span> <span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">count</span><span class="p">()</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Satisfied customers: {}; Unsatisfied customers: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">y</span><span class="p">)</span> <span class="o">-</span> <span class="n">unsat</span><span class="p">,</span> <span class="n">unsat</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Satisfied customers: {}; Unsatisfied customers: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">y</span><span class="p">)</span> <span class="o">-</span> <span class="n">unsat</span><span class="p">,</span> <span class="n">unsat</span><span class="p">))</span> <span class="n">naive_guess</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span> <span class="o">==</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">y</span><span class="p">)))</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Naive guess accuracy: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Naive guess accuracy: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">))</span> </pre></div> </div> @@ -9369,9 +9636,9 @@ Naive guess accuracy: 0.9608561656706882 <span class="k">return</span> <span class="n">gnb</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_xform</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> <span class="n">dim_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">21</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">dim_range</span><span class="p">,</span> <span class="p">[</span><span class="n">evaluate_gnb</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="k">for</span> <span class="n">dim</span> <span class="ow">in</span> <span class="n">dim_range</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Gaussian NB Accuracy&quot;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s">&#39;k&#39;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Inverse Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s">&#39;k&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">dim_range</span><span class="p">,</span> <span class="p">[</span><span class="n">evaluate_gnb</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="k">for</span> <span class="n">dim</span> <span class="ow">in</span> <span class="n">dim_range</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Gaussian NB Accuracy&quot;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s1">&#39;k&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Inverse Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s1">&#39;k&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">();</span> </pre></div> @@ -9655,9 +9922,9 @@ rkJggg== <span class="k">return</span> <span class="n">gnb</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_xform</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> <span class="n">dim_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">21</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">dim_range</span><span class="p">,</span> <span class="p">[</span><span class="n">evaluate_gnb</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="k">for</span> <span class="n">dim</span> <span class="ow">in</span> <span class="n">dim_range</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Gaussian NB Accuracy&quot;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s">&#39;k&#39;</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s">&quot;Inverse Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s">&#39;k&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">dim_range</span><span class="p">,</span> <span class="p">[</span><span class="n">evaluate_gnb</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="k">for</span> <span class="n">dim</span> <span class="ow">in</span> <span class="n">dim_range</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Gaussian NB Accuracy&quot;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s1">&#39;k&#39;</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axhline</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">naive_guess</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">&quot;Inverse Naive Guess&quot;</span><span class="p">,</span> <span class="n">c</span><span class="o">=</span><span class="s1">&#39;k&#39;</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">12</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> <span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">();</span> </pre></div> @@ -9933,7 +10200,7 @@ rkJggg== <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="n">end</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Running time: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">start</span><span class="p">))</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Running time: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">start</span><span class="p">))</span> </pre></div> </div> @@ -9962,14 +10229,14 @@ rkJggg== <div class="text_cell_render border-box-sizing rendered_html"> <h1 id="Appendix">Appendix<a class="anchor-link" href="#Appendix">&#182;</a></h1><p>Code used to split the initial training data:</p> <div class="highlight"><pre><span class="kn">from</span> <span class="nn">sklearn.cross_validation</span> <span class="kn">import</span> <span class="n">train_test_split</span> -<span class="n">data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s">&#39;train.csv&#39;</span><span class="p">)</span> +<span class="n">data</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;train.csv&#39;</span><span class="p">)</span> <span class="n">data</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">ID</span> <span class="n">data_train</span><span class="p">,</span> <span class="n">data_validate</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span> <span class="n">data</span><span class="p">,</span> <span class="n">train_size</span><span class="o">=.</span><span class="mi">7</span><span class="p">)</span> -<span class="n">data_train</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s">&#39;split_train.csv&#39;</span><span class="p">)</span> -<span class="n">data_validate</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s">&#39;split_validate.csv&#39;</span><span class="p">)</span> +<span class="n">data_train</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s1">&#39;split_train.csv&#39;</span><span class="p">)</span> +<span class="n">data_validate</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s1">&#39;split_validate.csv&#39;</span><span class="p">)</span> </pre></div> </div> @@ -9980,7 +10247,7 @@ rkJggg== MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Profitability using the Investment Formula2016-02-26T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-02-26:profitability-using-the-investment-formula.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Profitability using the Investment Formula2016-02-26T00:00:00-05:002016-02-26T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-02-26:profitability-using-the-investment-formula.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -10017,7 +10284,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span> <span class="kn">from</span> <span class="nn">Quandl</span> <span class="k">import</span> <span class="n">get</span> <span class="k">as</span> <span class="n">qget</span> <span class="o">%</span><span class="k">matplotlib</span> inline -<span class="n">api_key</span> <span class="o">=</span> <span class="s">&#39;&#39;</span> +<span class="n">api_key</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span> <span class="n">profitability</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">p</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">m</span><span class="p">,</span> <span class="n">d</span><span class="p">:</span> <span class="mi">1000</span><span class="o">*</span><span class="n">p</span> <span class="o">/</span> <span class="p">(</span><span class="n">m</span> <span class="o">+</span> <span class="n">i</span><span class="o">*</span><span class="n">d</span><span class="p">)</span> @@ -10041,15 +10308,15 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <span class="n">expected_close</span> <span class="o">=</span> <span class="n">open_vals</span><span class="p">[</span><span class="n">day</span><span class="p">]</span> <span class="o">*</span> <span class="n">expected</span> <span class="n">todays_purchase</span> <span class="o">=</span> <span class="n">daily_investment</span><span class="p">(</span><span class="n">open_vals</span><span class="p">[</span><span class="n">day</span><span class="p">],</span> <span class="n">expected_close</span><span class="p">,</span> <span class="n">invested</span><span class="p">,</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="n">day</span><span class="p">])</span> <span class="n">invested</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">invested</span><span class="p">,</span> <span class="n">todays_purchase</span> <span class="o">+</span> <span class="n">bias</span><span class="p">)</span> - <span class="c"># expected_profit = expected_close * (invested / open_vals[0:len(invested)]).sum() - invested.sum()</span> + <span class="c1"># expected_profit = expected_close * (invested / open_vals[0:len(invested)]).sum() - invested.sum()</span> <span class="n">day</span> <span class="o">+=</span> <span class="mi">1</span> <span class="n">profitable</span> <span class="o">=</span> <span class="n">is_profitable</span><span class="p">(</span><span class="n">close_vals</span><span class="p">[</span><span class="n">day</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">invested</span><span class="p">,</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="nb">len</span><span class="p">(</span><span class="n">invested</span><span class="p">)])</span> \ <span class="ow">or</span> <span class="n">is_profitable</span><span class="p">(</span><span class="n">open_vals</span><span class="p">[</span><span class="n">day</span><span class="p">],</span> <span class="n">invested</span><span class="p">,</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="nb">len</span><span class="p">(</span><span class="n">invested</span><span class="p">)])</span> <span class="n">shares</span> <span class="o">=</span> <span class="p">(</span><span class="n">invested</span> <span class="o">/</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="nb">len</span><span class="p">(</span><span class="n">invested</span><span class="p">)])</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> - <span class="c"># Make sure we can&#39;t see into the future - we know either today&#39;s close or tomorrow&#39;s open</span> - <span class="c"># will be profitable, but we need to check which one.</span> + <span class="c1"># Make sure we can&#39;t see into the future - we know either today&#39;s close or tomorrow&#39;s open</span> + <span class="c1"># will be profitable, but we need to check which one.</span> <span class="k">if</span> <span class="n">is_profitable</span><span class="p">(</span><span class="n">close_vals</span><span class="p">[</span><span class="n">day</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">invested</span><span class="p">,</span> <span class="n">open_vals</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="nb">len</span><span class="p">(</span><span class="n">invested</span><span class="p">)]):</span> <span class="n">ending_price</span> <span class="o">=</span> <span class="n">close_vals</span><span class="p">[</span><span class="n">day</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> @@ -10062,21 +10329,21 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <span class="n">ticker_info</span> <span class="o">=</span> <span class="n">qget</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start</span><span class="p">,</span> <span class="n">api_key</span><span class="o">=</span><span class="n">api_key</span><span class="p">)</span> <span class="n">evaluation_times</span> <span class="o">=</span> <span class="n">ticker_info</span><span class="p">[:</span><span class="n">end</span><span class="p">]</span><span class="o">.</span><span class="n">index</span> - <span class="c"># Handle Google vs. YFinance data</span> - <span class="k">if</span> <span class="s">&quot;Adjusted Close&quot;</span> <span class="ow">in</span> <span class="n">ticker_info</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span> - <span class="n">close_column</span> <span class="o">=</span> <span class="s">&quot;Adjusted Close&quot;</span> + <span class="c1"># Handle Google vs. YFinance data</span> + <span class="k">if</span> <span class="s2">&quot;Adjusted Close&quot;</span> <span class="ow">in</span> <span class="n">ticker_info</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span> + <span class="n">close_column</span> <span class="o">=</span> <span class="s2">&quot;Adjusted Close&quot;</span> <span class="k">else</span><span class="p">:</span> - <span class="n">close_column</span> <span class="o">=</span> <span class="s">&quot;Close&quot;</span> + <span class="n">close_column</span> <span class="o">=</span> <span class="s2">&quot;Close&quot;</span> - <span class="n">sim</span> <span class="o">=</span> <span class="p">{</span><span class="n">d</span><span class="p">:</span> <span class="n">simulate_day</span><span class="p">(</span><span class="n">ticker_info</span><span class="p">[</span><span class="n">d</span><span class="p">:][</span><span class="s">&quot;Open&quot;</span><span class="p">],</span> <span class="n">ticker_info</span><span class="p">[</span><span class="n">d</span><span class="p">:][</span><span class="n">close_column</span><span class="p">],</span> + <span class="n">sim</span> <span class="o">=</span> <span class="p">{</span><span class="n">d</span><span class="p">:</span> <span class="n">simulate_day</span><span class="p">(</span><span class="n">ticker_info</span><span class="p">[</span><span class="n">d</span><span class="p">:][</span><span class="s2">&quot;Open&quot;</span><span class="p">],</span> <span class="n">ticker_info</span><span class="p">[</span><span class="n">d</span><span class="p">:][</span><span class="n">close_column</span><span class="p">],</span> <span class="mi">100</span><span class="p">,</span> <span class="mf">1.02</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span> <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">evaluation_times</span><span class="p">}</span> <span class="n">sim_series</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="n">sim</span><span class="p">)</span> <span class="n">result</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">()</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;profit&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;max&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">max</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;days&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;score&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">profitability</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">],</span> <span class="nb">max</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])))</span> - <span class="n">result</span><span class="p">[</span><span class="s">&quot;investments&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;profit&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;max&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">max</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;days&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;score&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">profitability</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">],</span> <span class="nb">max</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])))</span> + <span class="n">result</span><span class="p">[</span><span class="s2">&quot;investments&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sim_series</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="k">return</span> <span class="n">result</span> @@ -10111,7 +10378,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <div class="prompt input_prompt">In&nbsp;[7]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s">&quot;YAHOO/FB&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/AAPL&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/NFLX&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/GOOG&quot;</span><span class="p">])</span> +<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s2">&quot;YAHOO/FB&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/AAPL&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/NFLX&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/GOOG&quot;</span><span class="p">])</span> </pre></div> </div> @@ -10124,9 +10391,9 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}}); <div class="prompt input_prompt">In&nbsp;[8]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">8</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -10598,9 +10865,9 @@ kiSpNScYJEmSJElSa04wSJIkSZKk1pxgkCRJkiRJrf1/uiw9D8O0d6EAAAAASUVORK5CYII= <div class="prompt input_prompt">In&nbsp;[10]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">fang_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -12237,7 +12504,7 @@ EZIkSZIkSZIkSZIk6RIpIiRJkiRJkiRJkiRJ0iX+D91jLOlExGw8AAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[13]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s">&quot;YAHOO/X&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/CAT&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/NFLX&quot;</span><span class="p">,</span> <span class="s">&quot;YAHOO/GOOG&quot;</span><span class="p">])</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s2">&quot;YAHOO/X&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/CAT&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/NFLX&quot;</span><span class="p">,</span> <span class="s2">&quot;YAHOO/GOOG&quot;</span><span class="p">])</span> </pre></div> </div> @@ -12250,9 +12517,9 @@ EZIkSZIkSZIkSZIk6RIpIiRJkiRJkiRJkiRJ0iX+D91jLOlExGw8AAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[14]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">8</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -12738,9 +13005,9 @@ JEmSJEnSwGwwSJIkSZKkgdlgkCRJkiRJA7PBIEmSJEmSBvb/AfKAew4+oiojAAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[15]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cyclic_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -14523,7 +14790,7 @@ hUKhUCgUpor/A9AnO0PoIItTAAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[21]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s">&#39;YAHOO/REGN&#39;</span><span class="p">,</span> <span class="s">&#39;YAHOO/CELG&#39;</span><span class="p">,</span> <span class="s">&#39;GOOG/NASDAQ_BIB&#39;</span><span class="p">,</span> <span class="s">&#39;GOOG/NASDAQ_IBB&#39;</span><span class="p">])</span> +<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span> <span class="o">=</span> <span class="n">simulate_tickers</span><span class="p">([</span><span class="s1">&#39;YAHOO/REGN&#39;</span><span class="p">,</span> <span class="s1">&#39;YAHOO/CELG&#39;</span><span class="p">,</span> <span class="s1">&#39;GOOG/NASDAQ_BIB&#39;</span><span class="p">,</span> <span class="s1">&#39;GOOG/NASDAQ_IBB&#39;</span><span class="p">])</span> </pre></div> </div> @@ -14536,9 +14803,9 @@ hUKhUCgUpor/A9AnO0PoIItTAAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[22]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;days&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">hist</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">8</span><span class="p">);</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Distribution of Days Until Profitability&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -15089,9 +15356,9 @@ WKBTQVcCAAAAAElFTkSuQmCC <div class="prompt input_prompt">In&nbsp;[23]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> +<div class=" highlight hl-ipython3"><pre><span class="n">biotech_df</span><span class="o">.</span><span class="n">xs</span><span class="p">(</span><span class="s1">&#39;score&#39;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span> <span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">set_size_inches</span><span class="p">(</span><span class="mi">18</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span> -<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> +<span class="n">plt</span><span class="o">.</span><span class="n">gcf</span><span class="p">()</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">&quot;Profitability score over time&quot;</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="mi">18</span><span class="p">);</span> </pre></div> </div> @@ -16853,7 +17120,7 @@ JBKJRCJrRTQiRCKRSCQSiUQikUgkElkr/h9yZ17EmpnoFwAAAABJRU5ErkJggg== MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Guaranteed Money Maker2016-02-03T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-02-03:guaranteed-money-maker.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Guaranteed Money Maker2016-02-03T00:00:00-05:002016-02-03T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-02-03:guaranteed-money-maker.html<p> <div class="cell border-box-sizing text_cell rendered"> <div class="prompt input_prompt"> </div> @@ -17116,7 +17383,7 @@ Profit: 0.08304813163696423 MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Cloudy in Seattle2016-01-23T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-01-23:cloudy-in-seattle.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Cloudy in Seattle2016-01-23T00:00:00-05:002016-01-23T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-01-23:cloudy-in-seattle.html<p> <div class="cell border-box-sizing code_cell rendered"> <div class="input"> <div class="prompt input_prompt">In&nbsp;[1]:</div> @@ -17389,7 +17656,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[2]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">city_forecasts</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="s">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s">&#39;rb&#39;</span><span class="p">))</span> +<div class=" highlight hl-ipython3"><pre><span class="n">city_forecasts</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="s1">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">))</span> <span class="n">forecasts_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">city_forecasts</span><span class="p">)</span> </pre></div> @@ -17403,24 +17670,24 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[3]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cities</span> <span class="o">=</span> <span class="p">[</span><span class="s">&#39;binghamton&#39;</span><span class="p">,</span> <span class="s">&#39;cary&#39;</span><span class="p">,</span> <span class="s">&#39;nyc&#39;</span><span class="p">,</span> <span class="s">&#39;seattle&#39;</span><span class="p">]</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cities</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;binghamton&#39;</span><span class="p">,</span> <span class="s1">&#39;cary&#39;</span><span class="p">,</span> <span class="s1">&#39;nyc&#39;</span><span class="p">,</span> <span class="s1">&#39;seattle&#39;</span><span class="p">]</span> <span class="n">city_colors</span> <span class="o">=</span> <span class="p">{</span><span class="n">cities</span><span class="p">[</span><span class="n">i</span><span class="p">]:</span> <span class="n">Palette</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">4</span><span class="p">)}</span> <span class="k">def</span> <span class="nf">safe_cover</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s">&#39;cloudCover&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> - <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s">&#39;cloudCover&#39;</span><span class="p">]</span> + <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s1">&#39;cloudCover&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> + <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s1">&#39;cloudCover&#39;</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">monthly_avg_cloudcover</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">forecasts_df</span><span class="p">[</span><span class="n">city</span><span class="p">][</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">forecasts_df</span><span class="p">[</span><span class="n">city</span><span class="p">][</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">cloud_cover_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">cloud_cover_vals</span><span class="p">)))</span> - <span class="c"># Ignore an issue with nanmean having all NaN values. We&#39;ll discuss the data issues below.</span> + <span class="c1"># Ignore an issue with nanmean having all NaN values. We&#39;ll discuss the data issues below.</span> <span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span> - <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s">&#39;ignore&#39;</span><span class="p">)</span> + <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">nanmean</span><span class="p">(</span><span class="n">cloud_cover_vals</span><span class="p">),</span> <span class="n">cloud_cover_samples</span> </pre></div> @@ -17439,18 +17706,18 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="k">return</span> <span class="p">[</span><span class="n">monthly_avg_cloudcover</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">month</span><span class="p">)</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="n">months</span> <span class="o">=</span> <span class="p">[</span> - <span class="p">(</span><span class="s">&#39;July&#39;</span><span class="p">,</span> <span class="mi">7</span><span class="p">),</span> - <span class="p">(</span><span class="s">&#39;August&#39;</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> - <span class="p">(</span><span class="s">&#39;September&#39;</span><span class="p">,</span> <span class="mi">9</span><span class="p">),</span> - <span class="p">(</span><span class="s">&#39;October&#39;</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span> - <span class="p">(</span><span class="s">&#39;November&#39;</span><span class="p">,</span> <span class="mi">11</span><span class="p">)</span> + <span class="p">(</span><span class="s1">&#39;July&#39;</span><span class="p">,</span> <span class="mi">7</span><span class="p">),</span> + <span class="p">(</span><span class="s1">&#39;August&#39;</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> + <span class="p">(</span><span class="s1">&#39;September&#39;</span><span class="p">,</span> <span class="mi">9</span><span class="p">),</span> + <span class="p">(</span><span class="s1">&#39;October&#39;</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span> + <span class="p">(</span><span class="s1">&#39;November&#39;</span><span class="p">,</span> <span class="mi">11</span><span class="p">)</span> <span class="p">]</span> <span class="k">for</span> <span class="n">month</span><span class="p">,</span> <span class="n">month_id</span> <span class="ow">in</span> <span class="n">months</span><span class="p">:</span> <span class="n">month_averages</span> <span class="o">=</span> <span class="p">{</span><span class="n">city</span><span class="p">:</span> <span class="n">city_avg_cc</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">month_id</span><span class="p">)</span> <span class="k">for</span> <span class="n">city</span> <span class="ow">in</span> <span class="n">cities</span><span class="p">}</span> - <span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&quot;{} Average Cloud Cover&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">month</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">,</span> - <span class="n">y_axis_label</span><span class="o">=</span><span class="s">&#39;Cloud Cover Percentage&#39;</span><span class="p">)</span> + <span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s2">&quot;{} Average Cloud Cover&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">month</span><span class="p">),</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">,</span> + <span class="n">y_axis_label</span><span class="o">=</span><span class="s1">&#39;Cloud Cover Percentage&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">city</span> <span class="ow">in</span> <span class="n">cities</span><span class="p">:</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="p">[</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">month_averages</span><span class="p">[</span><span class="n">city</span><span class="p">]],</span> <span class="n">legend</span><span class="o">=</span><span class="n">city</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="n">city_colors</span><span class="p">[</span><span class="n">city</span><span class="p">])</span> @@ -17640,20 +17907,20 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">safe_precip</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s">&#39;precipProbability&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> - <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s">&#39;precipProbability&#39;</span><span class="p">]</span> + <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s1">&#39;precipProbability&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> + <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s1">&#39;precipProbability&#39;</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">monthly_avg_precip</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">precip_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">forecasts_df</span><span class="p">[</span><span class="n">city</span><span class="p">][</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">precip_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">forecasts_df</span><span class="p">[</span><span class="n">city</span><span class="p">][</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">precip_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">precip_vals</span><span class="p">)))</span> - <span class="c"># Ignore an issue with nanmean having all NaN values. We&#39;ll discuss the data issues below.</span> + <span class="c1"># Ignore an issue with nanmean having all NaN values. We&#39;ll discuss the data issues below.</span> <span class="k">with</span> <span class="n">warnings</span><span class="o">.</span><span class="n">catch_warnings</span><span class="p">():</span> - <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s">&#39;ignore&#39;</span><span class="p">)</span> + <span class="n">warnings</span><span class="o">.</span><span class="n">simplefilter</span><span class="p">(</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">nanmean</span><span class="p">(</span><span class="n">precip_vals</span><span class="p">),</span> <span class="n">precip_samples</span> <span class="k">def</span> <span class="nf">city_avg_precip</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> @@ -17661,9 +17928,9 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="k">for</span> <span class="n">month</span><span class="p">,</span> <span class="n">month_id</span> <span class="ow">in</span> <span class="n">months</span><span class="p">:</span> <span class="n">month_averages</span> <span class="o">=</span> <span class="p">{</span><span class="n">city</span><span class="p">:</span> <span class="n">city_avg_cc</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> <span class="n">month_id</span><span class="p">)</span> <span class="k">for</span> <span class="n">city</span> <span class="ow">in</span> <span class="n">cities</span><span class="p">}</span> - <span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&quot;{} Average Precipitation Chance&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">month</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">,</span> - <span class="n">y_axis_label</span><span class="o">=</span><span class="s">&#39;Precipitation Chance Percentage&#39;</span><span class="p">)</span> + <span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s2">&quot;{} Average Precipitation Chance&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">month</span><span class="p">),</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">,</span> + <span class="n">y_axis_label</span><span class="o">=</span><span class="s1">&#39;Precipitation Chance Percentage&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">city</span> <span class="ow">in</span> <span class="n">cities</span><span class="p">:</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="p">[</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">month_averages</span><span class="p">[</span><span class="n">city</span><span class="p">]],</span> <span class="n">legend</span><span class="o">=</span><span class="n">city</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="n">city_colors</span><span class="p">[</span><span class="n">city</span><span class="p">])</span> @@ -17852,7 +18119,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Complaining about the Weather2016-01-01T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-01-01:complaining-about-the-weather.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Complaining about the Weather2016-01-01T00:00:00-05:002016-01-01T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2016-01-01:complaining-about-the-weather.html<p> <div class="cell border-box-sizing code_cell rendered"> <div class="input"> <div class="prompt input_prompt">In&nbsp;[1]:</div> @@ -18118,7 +18385,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[2]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">city_forecasts</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="s">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s">&#39;rb&#39;</span><span class="p">))</span> +<div class=" highlight hl-ipython3"><pre><span class="n">city_forecasts</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="s1">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">))</span> <span class="n">forecast_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">city_forecasts</span><span class="p">)</span> </pre></div> @@ -18132,31 +18399,31 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[3]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="n">cary_forecast</span> <span class="o">=</span> <span class="n">forecast_df</span><span class="p">[</span><span class="s">&#39;cary&#39;</span><span class="p">]</span> +<div class=" highlight hl-ipython3"><pre><span class="n">cary_forecast</span> <span class="o">=</span> <span class="n">forecast_df</span><span class="p">[</span><span class="s1">&#39;cary&#39;</span><span class="p">]</span> <span class="n">years</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2016</span><span class="p">)</span> <span class="n">months</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="mi">7</span><span class="p">,</span> <span class="mi">12</span><span class="p">)</span> -<span class="n">months_str</span> <span class="o">=</span> <span class="p">[</span><span class="s">&#39;July&#39;</span><span class="p">,</span> <span class="s">&#39;August&#39;</span><span class="p">,</span> <span class="s">&#39;September&#39;</span><span class="p">,</span> <span class="s">&#39;October&#39;</span><span class="p">,</span> <span class="s">&#39;November&#39;</span><span class="p">]</span> +<span class="n">months_str</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;July&#39;</span><span class="p">,</span> <span class="s1">&#39;August&#39;</span><span class="p">,</span> <span class="s1">&#39;September&#39;</span><span class="p">,</span> <span class="s1">&#39;October&#39;</span><span class="p">,</span> <span class="s1">&#39;November&#39;</span><span class="p">]</span> <span class="k">def</span> <span class="nf">safe_cover</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s">&#39;cloudCover&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> - <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s">&#39;cloudCover&#39;</span><span class="p">]</span> + <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s1">&#39;cloudCover&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> + <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s1">&#39;cloudCover&#39;</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">monthly_avg_cloudcover</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">cloud_cover_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">cloud_cover_vals</span><span class="p">)))</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">nanmean</span><span class="p">(</span><span class="n">cloud_cover_vals</span><span class="p">),</span> <span class="n">cloud_cover_samples</span> <span class="n">monthly_cover_vals</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_avg_cloudcover</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Average Cloud Cover&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Average Cloud Cover&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">)</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_cover_vals</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> <span class="n">show</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> @@ -18223,8 +18490,8 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">monthly_cloudy_days</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">cloud_cover_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_cover</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">cloud_cover_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">cloud_cover_vals</span><span class="p">)))</span> <span class="n">cloudy_days</span> <span class="o">=</span> <span class="p">[</span><span class="n">cover</span> <span class="o">&gt;</span> <span class="o">.</span><span class="mi">5</span> <span class="k">for</span> <span class="n">cover</span> <span class="ow">in</span> <span class="n">cloud_cover_vals</span><span class="p">]</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">count_nonzero</span><span class="p">(</span><span class="n">cloudy_days</span><span class="p">),</span> <span class="n">cloud_cover_samples</span> @@ -18232,16 +18499,16 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">monthly_days_vals</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_cloudy_days</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> <span class="n">monthly_cover_samples</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_cloudy_days</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Cloudy Days&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Cloudy Days&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">)</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_days_vals</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> <span class="n">show</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Cloud Cover Samples&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Cloud Cover Samples&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">,</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">300</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_cover_samples</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> @@ -18337,24 +18604,24 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">safe_precip</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s">&#39;precipProbability&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> - <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s">&#39;precipProbability&#39;</span><span class="p">]</span> + <span class="k">if</span> <span class="n">frame</span> <span class="ow">and</span> <span class="s1">&#39;precipProbability&#39;</span> <span class="ow">in</span> <span class="n">frame</span><span class="p">:</span> + <span class="k">return</span> <span class="n">frame</span><span class="p">[</span><span class="s1">&#39;precipProbability&#39;</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">monthly_avg_precip</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">precip_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">precip_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">precip_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">precip_vals</span><span class="p">)))</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">nanmean</span><span class="p">(</span><span class="n">precip_vals</span><span class="p">),</span> <span class="n">precip_samples</span> <span class="n">monthly_avg_precip_vals</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_avg_precip</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Average Precipitation Chance&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Average Precipitation Chance&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">)</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_avg_precip_vals</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> <span class="n">show</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> @@ -18420,8 +18687,8 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">monthly_rainy_days</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">):</span> <span class="n">dates</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DatetimeIndex</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="n">month</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> - <span class="n">freq</span><span class="o">=</span><span class="s">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s">&#39;left&#39;</span><span class="p">)</span> - <span class="n">precip_prob_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> + <span class="n">freq</span><span class="o">=</span><span class="s1">&#39;D&#39;</span><span class="p">,</span> <span class="n">closed</span><span class="o">=</span><span class="s1">&#39;left&#39;</span><span class="p">)</span> + <span class="n">precip_prob_vals</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">safe_precip</span><span class="p">(</span><span class="n">cary_forecast</span><span class="p">[</span><span class="n">x</span><span class="p">][</span><span class="s1">&#39;currently&#39;</span><span class="p">]),</span> <span class="n">dates</span><span class="p">))</span> <span class="n">precip_prob_samples</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span><span class="p">,</span> <span class="n">precip_prob_vals</span><span class="p">)))</span> <span class="n">precip_days</span> <span class="o">=</span> <span class="p">[</span><span class="n">prob</span> <span class="o">&gt;</span> <span class="o">.</span><span class="mi">25</span> <span class="k">for</span> <span class="n">prob</span> <span class="ow">in</span> <span class="n">precip_prob_vals</span><span class="p">]</span> <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">count_nonzero</span><span class="p">(</span><span class="n">precip_days</span><span class="p">),</span> <span class="n">precip_prob_samples</span> @@ -18429,16 +18696,16 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">monthly_precip_days_vals</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_rainy_days</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> <span class="n">monthly_precip_samples</span> <span class="o">=</span> <span class="p">[[</span><span class="n">monthly_rainy_days</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">m</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">y</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">months</span><span class="p">]</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Rainy Days&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Rainy Days&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">)</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_precip_days_vals</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> <span class="n">show</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> -<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#39;Monthly Rainy Days Samples&#39;</span><span class="p">,</span> +<span class="n">f</span> <span class="o">=</span> <span class="n">figure</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s1">&#39;Monthly Rainy Days Samples&#39;</span><span class="p">,</span> <span class="n">x_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span> <span class="mi">2015</span><span class="p">),</span> - <span class="n">x_axis_label</span><span class="o">=</span><span class="s">&#39;Year&#39;</span><span class="p">,</span> + <span class="n">x_axis_label</span><span class="o">=</span><span class="s1">&#39;Year&#39;</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">300</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">months</span><span class="p">)):</span> <span class="n">f</span><span class="o">.</span><span class="n">line</span><span class="p">(</span><span class="n">years</span><span class="p">,</span> <span class="n">monthly_precip_samples</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">legend</span><span class="o">=</span><span class="n">months_str</span><span class="p">[</span><span class="n">x</span><span class="p">],</span> <span class="n">color</span><span class="o">=</span><span class="n">Palette</span><span class="p">[</span><span class="n">x</span><span class="p">])</span> @@ -18541,64 +18808,64 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="kn">import</span> <span class="nn">requests</span> <span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span> -<span class="c"># Coordinate data from http://itouchmap.com/latlong.html</span> +<span class="c1"># Coordinate data from http://itouchmap.com/latlong.html</span> <span class="n">cary_loc</span> <span class="o">=</span> <span class="p">(</span><span class="mf">35.79154</span><span class="p">,</span><span class="o">-</span><span class="mf">78.781117</span><span class="p">)</span> <span class="n">nyc_loc</span> <span class="o">=</span> <span class="p">(</span><span class="mf">40.78306</span><span class="p">,</span><span class="o">-</span><span class="mf">73.971249</span><span class="p">)</span> <span class="n">seattle_loc</span> <span class="o">=</span> <span class="p">(</span><span class="mf">47.60621</span><span class="p">,</span><span class="o">-</span><span class="mf">122.332071</span><span class="p">)</span> <span class="n">binghamton_loc</span> <span class="o">=</span> <span class="p">(</span><span class="mf">42.098687</span><span class="p">,</span><span class="o">-</span><span class="mf">75.917974</span><span class="p">)</span> <span class="n">cities</span> <span class="o">=</span> <span class="p">{</span> - <span class="s">&#39;cary&#39;</span><span class="p">:</span> <span class="n">cary_loc</span><span class="p">,</span> - <span class="s">&#39;nyc&#39;</span><span class="p">:</span> <span class="n">nyc_loc</span><span class="p">,</span> - <span class="s">&#39;seattle&#39;</span><span class="p">:</span> <span class="n">seattle_loc</span><span class="p">,</span> - <span class="s">&#39;binghamton&#39;</span><span class="p">:</span> <span class="n">binghamton_loc</span> + <span class="s1">&#39;cary&#39;</span><span class="p">:</span> <span class="n">cary_loc</span><span class="p">,</span> + <span class="s1">&#39;nyc&#39;</span><span class="p">:</span> <span class="n">nyc_loc</span><span class="p">,</span> + <span class="s1">&#39;seattle&#39;</span><span class="p">:</span> <span class="n">seattle_loc</span><span class="p">,</span> + <span class="s1">&#39;binghamton&#39;</span><span class="p">:</span> <span class="n">binghamton_loc</span> <span class="p">}</span> -<span class="n">apikey</span> <span class="o">=</span> <span class="s">&#39;&#39;</span> <span class="c"># My super-secret API Key</span> +<span class="n">apikey</span> <span class="o">=</span> <span class="s1">&#39;&#39;</span> <span class="c1"># My super-secret API Key</span> <span class="k">def</span> <span class="nf">get_forecast</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="n">date</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span> - <span class="n">forecast_base</span> <span class="o">=</span> <span class="s">&quot;https://api.forecast.io/forecast/&quot;</span> + <span class="n">forecast_base</span> <span class="o">=</span> <span class="s2">&quot;https://api.forecast.io/forecast/&quot;</span> <span class="k">if</span> <span class="n">date</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span> - <span class="n">url</span> <span class="o">=</span> <span class="n">forecast_base</span> <span class="o">+</span> <span class="n">apikey</span> <span class="o">+</span> <span class="s">&#39;/{},{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">)</span> + <span class="n">url</span> <span class="o">=</span> <span class="n">forecast_base</span> <span class="o">+</span> <span class="n">apikey</span> <span class="o">+</span> <span class="s1">&#39;/{},{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> <span class="n">epoch</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">())</span> - <span class="n">url</span> <span class="o">=</span> <span class="n">forecast_base</span> <span class="o">+</span> <span class="n">apikey</span> <span class="o">+</span> <span class="s">&#39;/{},{},{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="n">epoch</span><span class="p">)</span> + <span class="n">url</span> <span class="o">=</span> <span class="n">forecast_base</span> <span class="o">+</span> <span class="n">apikey</span> <span class="o">+</span> <span class="s1">&#39;/{},{},{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lat</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="n">epoch</span><span class="p">)</span> <span class="k">return</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> <span class="n">years</span> <span class="o">=</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1990</span><span class="p">,</span><span class="mi">2016</span><span class="p">)</span> -<span class="c"># For datetimes, the 12 is for getting the weather at noon.</span> -<span class="c"># We&#39;re doing this over midnight because we&#39;re more concerned</span> -<span class="c"># with what people see, and people don&#39;t typically see the weather</span> -<span class="c"># at midnight.</span> +<span class="c1"># For datetimes, the 12 is for getting the weather at noon.</span> +<span class="c1"># We&#39;re doing this over midnight because we&#39;re more concerned</span> +<span class="c1"># with what people see, and people don&#39;t typically see the weather</span> +<span class="c1"># at midnight.</span> <span class="n">dt_indices</span> <span class="o">=</span> <span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">date_range</span><span class="p">(</span><span class="n">start</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">12</span><span class="p">),</span> <span class="n">end</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="mi">11</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">12</span><span class="p">))</span> <span class="k">for</span> <span class="n">year</span> <span class="ow">in</span> <span class="n">years</span><span class="p">]</span> <span class="n">dt_merge</span> <span class="o">=</span> <span class="nb">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">union</span><span class="p">(</span><span class="n">y</span><span class="p">),</span> <span class="n">dt_indices</span><span class="p">)</span> -<span class="c"># Because we have to pay a little bit to use the API, we use for loops here</span> -<span class="c"># instead of a comprehension - if something breaks, we want to preserve the</span> -<span class="c"># data already retrieved</span> +<span class="c1"># Because we have to pay a little bit to use the API, we use for loops here</span> +<span class="c1"># instead of a comprehension - if something breaks, we want to preserve the</span> +<span class="c1"># data already retrieved</span> <span class="n">city_forecasts</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">for</span> <span class="n">city</span><span class="p">,</span> <span class="n">loc</span> <span class="ow">in</span> <span class="n">cities</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> - <span class="k">print</span><span class="p">(</span><span class="s">&quot;Retrieving data for {} starting at {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> - <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s">&quot;%I:%M:%S %p&quot;</span><span class="p">)))</span> + <span class="k">print</span><span class="p">(</span><span class="s2">&quot;Retrieving data for {} starting at {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">city</span><span class="p">,</span> + <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%I:%M:%S %p&quot;</span><span class="p">)))</span> <span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="n">dt_merge</span><span class="p">:</span> <span class="k">try</span><span class="p">:</span> <span class="n">city_forecasts</span><span class="p">[(</span><span class="n">city</span><span class="p">,</span> <span class="n">dt</span><span class="p">)]</span> <span class="o">=</span> <span class="n">get_forecast</span><span class="p">(</span><span class="o">*</span><span class="n">loc</span><span class="p">,</span> <span class="n">dt</span><span class="p">)</span> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="k">print</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="n">city_forecasts</span><span class="p">[(</span><span class="n">city</span><span class="p">,</span> <span class="n">dt</span><span class="p">)]</span> <span class="o">=</span> <span class="bp">None</span> -<span class="k">print</span><span class="p">(</span><span class="s">&quot;End forecast retrieval: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s">&quot;%I:%M:%S %p&quot;</span><span class="p">)))</span> +<span class="k">print</span><span class="p">(</span><span class="s2">&quot;End forecast retrieval: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%I:%M:%S %p&quot;</span><span class="p">)))</span> <span class="kn">import</span> <span class="nn">pickle</span> -<span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">city_forecasts</span><span class="p">,</span> <span class="nb">open</span><span class="p">(</span><span class="s">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s">&#39;wb&#39;</span><span class="p">))</span> +<span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">city_forecasts</span><span class="p">,</span> <span class="nb">open</span><span class="p">(</span><span class="s1">&#39;city_forecasts.p&#39;</span><span class="p">,</span> <span class="s1">&#39;wb&#39;</span><span class="p">))</span> -<span class="c">### Output:</span> -<span class="c"># Retrieving data for binghamton starting at 05:13:42 PM</span> -<span class="c"># Retrieving data for seattle starting at 05:30:51 PM</span> -<span class="c"># Retrieving data for nyc starting at 05:48:30 PM</span> -<span class="c"># Retrieving data for cary starting at 06:08:32 PM</span> -<span class="c"># End forecast retrieval: 06:25:21 PM</span> +<span class="c1">### Output:</span> +<span class="c1"># Retrieving data for binghamton starting at 05:13:42 PM</span> +<span class="c1"># Retrieving data for seattle starting at 05:30:51 PM</span> +<span class="c1"># Retrieving data for nyc starting at 05:48:30 PM</span> +<span class="c1"># Retrieving data for cary starting at 06:08:32 PM</span> +<span class="c1"># End forecast retrieval: 06:25:21 PM</span> </pre></div> </div> @@ -18608,7 +18875,7 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Testing Cramer2015-12-26T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-12-26:testing-cramer.html<p> +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Testing Cramer2015-12-26T00:00:00-05:002015-12-26T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-12-26:testing-cramer.html<p> <div class="cell border-box-sizing code_cell rendered"> <div class="input"> <div class="prompt input_prompt">In&nbsp;[1]:</div> @@ -18655,20 +18922,20 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">articles</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">def</span> <span class="nf">handle_starttag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span> - <span class="c">#if tag == &#39;div&#39; and (&quot;id&quot;, &quot;author_articles_wrapper&quot;) in attrs:</span> - <span class="c"># self.fetch_links = True</span> - <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;a&#39;</span> <span class="ow">and</span> <span class="p">(</span><span class="s">&#39;class&#39;</span><span class="p">,</span> <span class="s">&#39;dashboard_article_link&#39;</span><span class="p">)</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">:</span> - <span class="n">href</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;href&#39;</span><span class="p">,</span> <span class="n">attrs</span><span class="p">))[</span><span class="mi">0</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span> + <span class="c1">#if tag == &#39;div&#39; and (&quot;id&quot;, &quot;author_articles_wrapper&quot;) in attrs:</span> + <span class="c1"># self.fetch_links = True</span> + <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;a&#39;</span> <span class="ow">and</span> <span class="p">(</span><span class="s1">&#39;class&#39;</span><span class="p">,</span> <span class="s1">&#39;dashboard_article_link&#39;</span><span class="p">)</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">:</span> + <span class="n">href</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;href&#39;</span><span class="p">,</span> <span class="n">attrs</span><span class="p">))[</span><span class="mi">0</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span> <span class="bp">self</span><span class="o">.</span><span class="n">articles</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">href</span><span class="p">)</span> -<span class="n">base_url</span> <span class="o">=</span> <span class="s">&quot;http://seekingalpha.com/author/wall-street-breakfast/articles&quot;</span> -<span class="n">article_page_urls</span> <span class="o">=</span> <span class="p">[</span><span class="n">base_url</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="n">base_url</span> <span class="o">+</span> <span class="s">&#39;/{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">20</span><span class="p">)]</span> +<span class="n">base_url</span> <span class="o">=</span> <span class="s2">&quot;http://seekingalpha.com/author/wall-street-breakfast/articles&quot;</span> +<span class="n">article_page_urls</span> <span class="o">=</span> <span class="p">[</span><span class="n">base_url</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="n">base_url</span> <span class="o">+</span> <span class="s1">&#39;/{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">20</span><span class="p">)]</span> <span class="n">global_articles</span> <span class="o">=</span> <span class="p">[]</span> <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="n">article_page_urls</span><span class="p">:</span> - <span class="c"># We need to switch the user agent, as SA blocks the standard requests agent</span> + <span class="c1"># We need to switch the user agent, as SA blocks the standard requests agent</span> <span class="n">articles_html</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">page</span><span class="p">,</span> - <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="s">&quot;Wget/1.13.4&quot;</span><span class="p">})</span> + <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="s2">&quot;Wget/1.13.4&quot;</span><span class="p">})</span> <span class="n">parser</span> <span class="o">=</span> <span class="n">ArticleListParser</span><span class="p">()</span> <span class="n">parser</span><span class="o">.</span><span class="n">feed</span><span class="p">(</span><span class="n">articles_html</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="n">global_articles</span> <span class="o">+=</span> <span class="p">(</span><span class="n">parser</span><span class="o">.</span><span class="n">articles</span><span class="p">)</span> @@ -18685,24 +18952,24 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">class</span> <span class="nc">ArticleReturnParser</span><span class="p">(</span><span class="n">HTMLParser</span><span class="p">):</span> - <span class="s">&quot;Given an article, parse out the futures returns in it&quot;</span> + <span class="s2">&quot;Given an article, parse out the futures returns in it&quot;</span> - <span class="n">record_font_tags</span> <span class="o">=</span> <span class="k">False</span> - <span class="n">in_font_tag</span> <span class="o">=</span> <span class="k">False</span> + <span class="n">record_font_tags</span> <span class="o">=</span> <span class="kc">False</span> + <span class="n">in_font_tag</span> <span class="o">=</span> <span class="kc">False</span> <span class="n">counter</span> <span class="o">=</span> <span class="mi">0</span> - <span class="c"># data = {} # See __init__</span> + <span class="c1"># data = {} # See __init__</span> <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">def</span> <span class="nf">handle_starttag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span> - <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;span&#39;</span> <span class="ow">and</span> <span class="p">(</span><span class="s">&#39;itemprop&#39;</span><span class="p">,</span> <span class="s">&#39;datePublished&#39;</span><span class="p">)</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">:</span> - <span class="n">date_string</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s">&#39;content&#39;</span><span class="p">,</span> <span class="n">attrs</span><span class="p">))[</span><span class="mi">0</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span> + <span class="k">if</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;span&#39;</span> <span class="ow">and</span> <span class="p">(</span><span class="s1">&#39;itemprop&#39;</span><span class="p">,</span> <span class="s1">&#39;datePublished&#39;</span><span class="p">)</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">:</span> + <span class="n">date_string</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;content&#39;</span><span class="p">,</span> <span class="n">attrs</span><span class="p">))[</span><span class="mi">0</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span> <span class="n">date</span> <span class="o">=</span> <span class="n">dtparser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">date_string</span><span class="p">)</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;date&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">date</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;date&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">date</span> - <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span> <span class="o">=</span> <span class="n">tag</span> <span class="o">==</span> <span class="s">&#39;font&#39;</span> + <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span> <span class="o">=</span> <span class="n">tag</span> <span class="o">==</span> <span class="s1">&#39;font&#39;</span> <span class="k">def</span> <span class="nf">safe_float</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">string</span><span class="p">):</span> <span class="k">try</span><span class="p">:</span> @@ -18711,43 +18978,43 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">NaN</span> <span class="k">def</span> <span class="nf">handle_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">content</span><span class="p">):</span> - <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="ow">and</span> <span class="s">&quot;Futures at 6&quot;</span> <span class="ow">in</span> <span class="n">content</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="o">=</span> <span class="k">True</span> + <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="ow">and</span> <span class="s2">&quot;Futures at 6&quot;</span> <span class="ow">in</span> <span class="n">content</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="o">=</span> <span class="kc">True</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">record_font_tags</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span><span class="p">:</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;DOW&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;DOW&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;S&amp;P&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;S&amp;P&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;NASDAQ&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;NASDAQ&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">3</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;Crude&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;Crude&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">==</span> <span class="mi">4</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s">&#39;Gold&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;Gold&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">safe_float</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">counter</span> <span class="o">+=</span> <span class="mi">1</span> <span class="k">def</span> <span class="nf">handle_endtag</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tag</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span> <span class="o">=</span> <span class="k">False</span> + <span class="bp">self</span><span class="o">.</span><span class="n">in_font_tag</span> <span class="o">=</span> <span class="kc">False</span> <span class="k">def</span> <span class="nf">retrieve_data</span><span class="p">(</span><span class="n">url</span><span class="p">):</span> - <span class="n">sa</span> <span class="o">=</span> <span class="s">&quot;http://seekingalpha.com&quot;</span> + <span class="n">sa</span> <span class="o">=</span> <span class="s2">&quot;http://seekingalpha.com&quot;</span> <span class="n">article_html</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sa</span> <span class="o">+</span> <span class="n">url</span><span class="p">,</span> - <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="s">&quot;Wget/1.13.4&quot;</span><span class="p">})</span> + <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="s2">&quot;Wget/1.13.4&quot;</span><span class="p">})</span> <span class="n">parser</span> <span class="o">=</span> <span class="n">ArticleReturnParser</span><span class="p">()</span> <span class="n">parser</span><span class="o">.</span><span class="n">feed</span><span class="p">(</span><span class="n">article_html</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> - <span class="n">parser</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s">&quot;url&quot;</span><span class="p">:</span> <span class="n">url</span><span class="p">})</span> - <span class="n">parser</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s">&quot;text&quot;</span><span class="p">:</span> <span class="n">article_html</span><span class="o">.</span><span class="n">text</span><span class="p">})</span> + <span class="n">parser</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;url&quot;</span><span class="p">:</span> <span class="n">url</span><span class="p">})</span> + <span class="n">parser</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;text&quot;</span><span class="p">:</span> <span class="n">article_html</span><span class="o">.</span><span class="n">text</span><span class="p">})</span> <span class="k">return</span> <span class="n">parser</span><span class="o">.</span><span class="n">data</span> -<span class="c"># This copy **MUST** be in place. I&#39;m not sure why,</span> -<span class="c"># as you&#39;d think that the data being returned would already</span> -<span class="c"># represent a different memory location. Even so, it blows up</span> -<span class="c"># if you don&#39;t do this.</span> +<span class="c1"># This copy **MUST** be in place. I&#39;m not sure why,</span> +<span class="c1"># as you&#39;d think that the data being returned would already</span> +<span class="c1"># represent a different memory location. Even so, it blows up</span> +<span class="c1"># if you don&#39;t do this.</span> <span class="n">article_list</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">set</span><span class="p">(</span><span class="n">global_articles</span><span class="p">))</span> <span class="n">article_data</span> <span class="o">=</span> <span class="p">[</span><span class="n">copy</span><span class="p">(</span><span class="n">retrieve_data</span><span class="p">(</span><span class="n">url</span><span class="p">))</span> <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">article_list</span><span class="p">]</span> -<span class="c"># If there&#39;s an issue downloading the article, drop it.</span> +<span class="c1"># If there&#39;s an issue downloading the article, drop it.</span> <span class="n">article_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">article_data</span><span class="p">)</span><span class="o">.</span><span class="n">dropna</span><span class="p">()</span> </pre></div> @@ -18771,12 +19038,12 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="prompt input_prompt">In&nbsp;[4]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-ipython3"><pre><span class="c"># article_df is sorted by date, so we get the first row.</span> -<span class="n">start_date</span> <span class="o">=</span> <span class="n">article_df</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="n">by</span><span class="o">=</span><span class="s">&#39;date&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s">&#39;date&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> -<span class="n">SPY</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;GOOG/NYSE_SPY&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> -<span class="n">DJIA</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;GOOG/AMS_DIA&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> -<span class="n">RUSS</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;GOOG/AMEX_IWM&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> -<span class="n">NASDAQ</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&quot;GOOG/EPA_QQQ&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> +<div class=" highlight hl-ipython3"><pre><span class="c1"># article_df is sorted by date, so we get the first row.</span> +<span class="n">start_date</span> <span class="o">=</span> <span class="n">article_df</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="n">by</span><span class="o">=</span><span class="s1">&#39;date&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">&#39;date&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> +<span class="n">SPY</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;GOOG/NYSE_SPY&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> +<span class="n">DJIA</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;GOOG/AMS_DIA&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> +<span class="n">RUSS</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;GOOG/AMEX_IWM&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> +<span class="n">NASDAQ</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;GOOG/EPA_QQQ&quot;</span><span class="p">,</span> <span class="n">trim_start</span><span class="o">=</span><span class="n">start_date</span><span class="p">)</span> </pre></div> </div> @@ -18801,13 +19068,13 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">calculate_opening_ret</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="c"># I&#39;m not a huge fan of the appending for loop,</span> - <span class="c"># but it&#39;s a bit verbose for a comprehension</span> + <span class="c1"># I&#39;m not a huge fan of the appending for loop,</span> + <span class="c1"># but it&#39;s a bit verbose for a comprehension</span> <span class="n">data</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">frame</span><span class="p">)):</span> <span class="n">date</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">.</span><span class="n">name</span> - <span class="n">prior_close</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s">&#39;Close&#39;</span><span class="p">]</span> - <span class="n">open_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s">&#39;Open&#39;</span><span class="p">]</span> + <span class="n">prior_close</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span> + <span class="n">open_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span> <span class="n">data</span><span class="p">[</span><span class="n">date</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">open_val</span> <span class="o">-</span> <span class="n">prior_close</span><span class="p">)</span> <span class="o">/</span> <span class="n">prior_close</span> <span class="k">return</span> <span class="n">data</span> @@ -18818,8 +19085,8 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">NASDAQ_open_ret</span> <span class="o">=</span> <span class="n">calculate_opening_ret</span><span class="p">(</span><span class="n">NASDAQ</span><span class="p">)</span> <span class="k">def</span> <span class="nf">signs_match</span><span class="p">(</span><span class="n">list_1</span><span class="p">,</span> <span class="n">list_2</span><span class="p">):</span> - <span class="c"># This is a surprisingly difficult task - we have to match</span> - <span class="c"># up the dates in order to check if opening returns actually match</span> + <span class="c1"># This is a surprisingly difficult task - we have to match</span> + <span class="c1"># up the dates in order to check if opening returns actually match</span> <span class="n">index_dict_dt</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="o">.</span><span class="n">to_datetime</span><span class="p">():</span> <span class="n">list_2</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">list_2</span><span class="o">.</span><span class="n">keys</span><span class="p">()}</span> <span class="n">matches</span> <span class="o">=</span> <span class="p">[]</span> @@ -18835,26 +19102,26 @@ MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); <span class="n">matches</span> <span class="o">+=</span> <span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="n">matches</span> <span class="o">+=</span> <span class="p">[</span><span class="mi">0</span><span class="p">]</span> - <span class="c">#print(&quot;{}&quot;.format(list_2[index_dt]))</span> + <span class="c1">#print(&quot;{}&quot;.format(list_2[index_dt]))</span> <span class="k">return</span> <span class="n">matches</span> <span class="n">prediction_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">matches_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">count_dict</span> <span class="o">=</span> <span class="p">{}</span> -<span class="n">index_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s">&quot;SPY&quot;</span><span class="p">:</span> <span class="n">SPY_open_ret</span><span class="p">,</span> <span class="s">&quot;DJIA&quot;</span><span class="p">:</span> <span class="n">DJIA_open_ret</span><span class="p">,</span> <span class="s">&quot;RUSS&quot;</span><span class="p">:</span> <span class="n">RUSS_open_ret</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">:</span> <span class="n">NASDAQ_open_ret</span><span class="p">}</span> -<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="s">&quot;SPY&quot;</span><span class="p">,</span> <span class="s">&quot;DJIA&quot;</span><span class="p">,</span> <span class="s">&quot;RUSS&quot;</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">]</span> -<span class="n">futures</span> <span class="o">=</span> <span class="p">[</span><span class="s">&quot;Crude&quot;</span><span class="p">,</span> <span class="s">&quot;Gold&quot;</span><span class="p">,</span> <span class="s">&quot;DOW&quot;</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">,</span> <span class="s">&quot;S&amp;P&quot;</span><span class="p">]</span> +<span class="n">index_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;SPY&quot;</span><span class="p">:</span> <span class="n">SPY_open_ret</span><span class="p">,</span> <span class="s2">&quot;DJIA&quot;</span><span class="p">:</span> <span class="n">DJIA_open_ret</span><span class="p">,</span> <span class="s2">&quot;RUSS&quot;</span><span class="p">:</span> <span class="n">RUSS_open_ret</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">:</span> <span class="n">NASDAQ_open_ret</span><span class="p">}</span> +<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;SPY&quot;</span><span class="p">,</span> <span class="s2">&quot;DJIA&quot;</span><span class="p">,</span> <span class="s2">&quot;RUSS&quot;</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">]</span> +<span class="n">futures</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;Crude&quot;</span><span class="p">,</span> <span class="s2">&quot;Gold&quot;</span><span class="p">,</span> <span class="s2">&quot;DOW&quot;</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">,</span> <span class="s2">&quot;S&amp;P&quot;</span><span class="p">]</span> <span class="k">for</span> <span class="n">index</span> <span class="ow">in</span> <span class="n">indices</span><span class="p">:</span> - <span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">signs_match</span><span class="p">(</span><span class="n">article_df</span><span class="p">[[</span><span class="n">future</span><span class="p">,</span> <span class="s">&#39;date&#39;</span><span class="p">]],</span> + <span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">signs_match</span><span class="p">(</span><span class="n">article_df</span><span class="p">[[</span><span class="n">future</span><span class="p">,</span> <span class="s1">&#39;date&#39;</span><span class="p">]],</span> <span class="n">index_dict</span><span class="p">[</span><span class="n">index</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> <span class="n">count_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="n">future</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> <span class="n">prediction_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="n">future</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Articles Checked: &quot;</span><span class="p">)</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Articles Checked: &quot;</span><span class="p">)</span> <span class="nb">print</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">count_dict</span><span class="p">))</span> <span class="nb">print</span><span class="p">()</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Prediction Accuracy:&quot;</span><span class="p">)</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Prediction Accuracy:&quot;</span><span class="p">)</span> <span class="nb">print</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">prediction_dict</span><span class="p">))</span> </pre></div> @@ -18914,13 +19181,13 @@ S&amp;P 0.604478 0.597015 0.811808 0.848708 <div class="inner_cell"> <div class="input_area"> <div class=" highlight hl-ipython3"><pre><span class="k">def</span> <span class="nf">calculate_closing_ret</span><span class="p">(</span><span class="n">frame</span><span class="p">):</span> - <span class="c"># I&#39;m not a huge fan of the appending for loop,</span> - <span class="c"># but it&#39;s a bit verbose for a comprehension</span> + <span class="c1"># I&#39;m not a huge fan of the appending for loop,</span> + <span class="c1"># but it&#39;s a bit verbose for a comprehension</span> <span class="n">data</span> <span class="o">=</span> <span class="p">{}</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">frame</span><span class="p">)):</span> <span class="n">date</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">.</span><span class="n">name</span> - <span class="n">open_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s">&#39;Open&#39;</span><span class="p">]</span> - <span class="n">close_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s">&#39;Close&#39;</span><span class="p">]</span> + <span class="n">open_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">&#39;Open&#39;</span><span class="p">]</span> + <span class="n">close_val</span> <span class="o">=</span> <span class="n">frame</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">&#39;Close&#39;</span><span class="p">]</span> <span class="n">data</span><span class="p">[</span><span class="n">date</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">close_val</span> <span class="o">-</span> <span class="n">open_val</span><span class="p">)</span> <span class="o">/</span> <span class="n">open_val</span> <span class="k">return</span> <span class="n">data</span> @@ -18931,8 +19198,8 @@ S&amp;P 0.604478 0.597015 0.811808 0.848708 <span class="n">NASDAQ_close_ret</span> <span class="o">=</span> <span class="n">calculate_closing_ret</span><span class="p">(</span><span class="n">NASDAQ</span><span class="p">)</span> <span class="k">def</span> <span class="nf">signs_match</span><span class="p">(</span><span class="n">list_1</span><span class="p">,</span> <span class="n">list_2</span><span class="p">):</span> - <span class="c"># This is a surprisingly difficult task - we have to match</span> - <span class="c"># up the dates in order to check if opening returns actually match</span> + <span class="c1"># This is a surprisingly difficult task - we have to match</span> + <span class="c1"># up the dates in order to check if opening returns actually match</span> <span class="n">index_dict_dt</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="o">.</span><span class="n">to_datetime</span><span class="p">():</span> <span class="n">list_2</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">list_2</span><span class="o">.</span><span class="n">keys</span><span class="p">()}</span> <span class="n">matches</span> <span class="o">=</span> <span class="p">[]</span> @@ -18948,28 +19215,28 @@ S&amp;P 0.604478 0.597015 0.811808 0.848708 <span class="n">matches</span> <span class="o">+=</span> <span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">else</span><span class="p">:</span> <span class="n">matches</span> <span class="o">+=</span> <span class="p">[</span><span class="mi">0</span><span class="p">]</span> - <span class="c">#print(&quot;{}&quot;.format(list_2[index_dt]))</span> + <span class="c1">#print(&quot;{}&quot;.format(list_2[index_dt]))</span> <span class="k">return</span> <span class="n">matches</span> <span class="n">matches_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">count_dict</span> <span class="o">=</span> <span class="p">{}</span> <span class="n">prediction_dict</span> <span class="o">=</span> <span class="p">{}</span> -<span class="n">index_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s">&quot;SPY&quot;</span><span class="p">:</span> <span class="n">SPY_close_ret</span><span class="p">,</span> <span class="s">&quot;DJIA&quot;</span><span class="p">:</span> <span class="n">DJIA_close_ret</span><span class="p">,</span> - <span class="s">&quot;RUSS&quot;</span><span class="p">:</span> <span class="n">RUSS_close_ret</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">:</span> <span class="n">NASDAQ_close_ret</span><span class="p">}</span> -<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="s">&quot;SPY&quot;</span><span class="p">,</span> <span class="s">&quot;DJIA&quot;</span><span class="p">,</span> <span class="s">&quot;RUSS&quot;</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">]</span> -<span class="n">futures</span> <span class="o">=</span> <span class="p">[</span><span class="s">&quot;Crude&quot;</span><span class="p">,</span> <span class="s">&quot;Gold&quot;</span><span class="p">,</span> <span class="s">&quot;DOW&quot;</span><span class="p">,</span> <span class="s">&quot;NASDAQ&quot;</span><span class="p">,</span> <span class="s">&quot;S&amp;P&quot;</span><span class="p">]</span> +<span class="n">index_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;SPY&quot;</span><span class="p">:</span> <span class="n">SPY_close_ret</span><span class="p">,</span> <span class="s2">&quot;DJIA&quot;</span><span class="p">:</span> <span class="n">DJIA_close_ret</span><span class="p">,</span> + <span class="s2">&quot;RUSS&quot;</span><span class="p">:</span> <span class="n">RUSS_close_ret</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">:</span> <span class="n">NASDAQ_close_ret</span><span class="p">}</span> +<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;SPY&quot;</span><span class="p">,</span> <span class="s2">&quot;DJIA&quot;</span><span class="p">,</span> <span class="s2">&quot;RUSS&quot;</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">]</span> +<span class="n">futures</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;Crude&quot;</span><span class="p">,</span> <span class="s2">&quot;Gold&quot;</span><span class="p">,</span> <span class="s2">&quot;DOW&quot;</span><span class="p">,</span> <span class="s2">&quot;NASDAQ&quot;</span><span class="p">,</span> <span class="s2">&quot;S&amp;P&quot;</span><span class="p">]</span> <span class="k">for</span> <span class="n">index</span> <span class="ow">in</span> <span class="n">indices</span><span class="p">:</span> - <span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">signs_match</span><span class="p">(</span><span class="n">article_df</span><span class="p">[[</span><span class="n">future</span><span class="p">,</span> <span class="s">&#39;date&#39;</span><span class="p">]],</span> + <span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">signs_match</span><span class="p">(</span><span class="n">article_df</span><span class="p">[[</span><span class="n">future</span><span class="p">,</span> <span class="s1">&#39;date&#39;</span><span class="p">]],</span> <span class="n">index_dict</span><span class="p">[</span><span class="n">index</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> <span class="n">count_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="n">future</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> <span class="n">prediction_dict</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">future</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">matches_dict</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="n">future</span><span class="p">])</span> <span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">futures</span><span class="p">}</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Articles Checked:&quot;</span><span class="p">)</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Articles Checked:&quot;</span><span class="p">)</span> <span class="nb">print</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">count_dict</span><span class="p">))</span> <span class="nb">print</span><span class="p">()</span> -<span class="nb">print</span><span class="p">(</span><span class="s">&quot;Prediction Accuracy:&quot;</span><span class="p">)</span> +<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Prediction Accuracy:&quot;</span><span class="p">)</span> <span class="nb">print</span><span class="p">(</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">prediction_dict</span><span class="p">))</span> </pre></div> @@ -19031,7 +19298,7 @@ S&amp;P 0.585821 0.608209 0.535055 0.535055 MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Autocallable Bonds2015-11-27T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-11-27:autocallable-bonds.html<p>My only non-core class this semester has been in Structure Products. We've been surveying a wide variety of products, and the final project was to pick one to report on. +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Autocallable Bonds2015-11-27T00:00:00-05:002015-11-27T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-11-27:autocallable-bonds.html<p>My only non-core class this semester has been in Structure Products. We've been surveying a wide variety of products, and the final project was to pick one to report on. Because these are all very similar, we decided to demonstrate all 3 products at once.</p> <p>What follows below is a notebook demonstrating the usage of <a href="http://julialang.com">Julia</a> for Monte-Carlo simulation of some exotic products.</p> <p> @@ -19079,7 +19346,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at <div class=" highlight hl-julia"><pre><span class="n">S0</span> <span class="o">=</span> <span class="mf">102.2</span> <span class="n">nominal</span> <span class="o">=</span> <span class="mi">100</span> <span class="n">q</span> <span class="o">=</span> <span class="mf">2.84</span> <span class="o">/</span> <span class="mi">100</span> -<span class="err">σ</span> <span class="o">=</span> <span class="mf">15.37</span> <span class="o">/</span> <span class="mi">100</span> +<span class="n">σ</span> <span class="o">=</span> <span class="mf">15.37</span> <span class="o">/</span> <span class="mi">100</span> <span class="n">term</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">.</span><span class="mi">49</span><span class="p">,</span> <span class="o">.</span><span class="mi">9</span><span class="p">,</span> <span class="mf">1.21</span><span class="p">,</span> <span class="mf">1.45</span><span class="p">,</span> <span class="mf">1.69</span><span class="p">]</span> <span class="o">/</span> <span class="mi">100</span> <span class="o">+</span> <span class="mi">1</span> <span class="c">###</span> @@ -19132,7 +19399,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at <div class="prompt input_prompt">In&nbsp;[3]:</div> <div class="inner_cell"> <div class="input_area"> -<div class=" highlight hl-julia"><pre><span class="n">simulate_gbm</span> <span class="o">=</span> <span class="n">function</span><span class="p">(</span><span class="n">S0</span><span class="p">,</span> <span class="err">μ</span><span class="p">,</span> <span class="err">σ</span><span class="p">,</span> <span class="n">T</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> +<div class=" highlight hl-julia"><pre><span class="n">simulate_gbm</span> <span class="o">=</span> <span class="n">function</span><span class="p">(</span><span class="n">S0</span><span class="p">,</span> <span class="n">μ</span><span class="p">,</span> <span class="n">σ</span><span class="p">,</span> <span class="n">T</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> <span class="c"># Set the initial state</span> <span class="n">m</span> <span class="o">=</span> <span class="n">length</span><span class="p">(</span><span class="n">S0</span><span class="p">)</span> <span class="n">t</span> <span class="o">=</span> <span class="n">T</span> <span class="o">/</span> <span class="n">n</span> @@ -19141,7 +19408,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at <span class="c"># Build out all states</span> <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:(</span><span class="n">n</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> - <span class="n">motion</span><span class="p">[:,</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">motion</span><span class="p">[:,</span><span class="n">i</span><span class="p">]</span> <span class="o">.*</span> <span class="n">exp</span><span class="p">((</span><span class="err">μ</span> <span class="o">-</span> <span class="err">σ</span><span class="o">^</span><span class="mi">2</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span><span class="o">*</span><span class="n">t</span><span class="p">)</span> <span class="o">.*</span> <span class="n">exp</span><span class="p">(</span><span class="n">sqrt</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="o">*</span> <span class="err">σ</span> <span class="o">.*</span> <span class="n">randn</span><span class="p">(</span><span class="n">m</span><span class="p">))</span> + <span class="n">motion</span><span class="p">[:,</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">motion</span><span class="p">[:,</span><span class="n">i</span><span class="p">]</span> <span class="o">.*</span> <span class="n">exp</span><span class="p">((</span><span class="n">μ</span> <span class="o">-</span> <span class="n">σ</span><span class="o">^</span><span class="mi">2</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span><span class="o">*</span><span class="n">t</span><span class="p">)</span> <span class="o">.*</span> <span class="n">exp</span><span class="p">(</span><span class="n">sqrt</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="o">*</span> <span class="n">σ</span> <span class="o">.*</span> <span class="n">randn</span><span class="p">(</span><span class="n">m</span><span class="p">))</span> <span class="k">end</span> <span class="k">return</span> <span class="n">motion</span> @@ -19205,7 +19472,7 @@ Because these are all very similar, we decided to demonstrate all 3 products at <div class="input_area"> <div class=" highlight hl-julia"><pre><span class="n">initial</span> <span class="o">=</span> <span class="n">ones</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span> <span class="o">*</span> <span class="n">S0</span> <span class="c"># Using μ=0, T=.25 for now, we&#39;ll use the proper values later</span> -<span class="n">motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">initial</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="err">σ</span><span class="p">,</span> <span class="o">.</span><span class="mi">25</span><span class="p">,</span> <span class="mi">200</span><span class="p">)</span> +<span class="n">motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">initial</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">σ</span><span class="p">,</span> <span class="o">.</span><span class="mi">25</span><span class="p">,</span> <span class="mi">200</span><span class="p">)</span> <span class="n">display_motion</span><span class="p">(</span><span class="n">motion</span><span class="p">,</span> <span class="o">.</span><span class="mi">25</span><span class="p">)</span> </pre></div> @@ -20888,8 +21155,8 @@ fig.select("#fig-3a6dd25ad25c4037a166889ee51bb151-element-20") <div class=" highlight hl-julia"><pre><span class="n">full_motion</span> <span class="o">=</span> <span class="n">ones</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span> <span class="o">*</span> <span class="n">S0</span> <span class="n">full_term</span> <span class="o">=</span> <span class="n">vcat</span><span class="p">(</span><span class="n">term</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">forward_yield</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:</span><span class="n">T</span> - <span class="err">μ</span> <span class="o">=</span> <span class="p">(</span><span class="n">full_term</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">q</span><span class="p">)</span> - <span class="n">year_motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">full_motion</span><span class="p">[:,</span><span class="k">end</span><span class="p">],</span> <span class="err">μ</span><span class="p">,</span> <span class="err">σ</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> + <span class="n">μ</span> <span class="o">=</span> <span class="p">(</span><span class="n">full_term</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">q</span><span class="p">)</span> + <span class="n">year_motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">full_motion</span><span class="p">[:,</span><span class="k">end</span><span class="p">],</span> <span class="n">μ</span><span class="p">,</span> <span class="n">σ</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> <span class="n">full_motion</span> <span class="o">=</span> <span class="n">hcat</span><span class="p">(</span><span class="n">full_motion</span><span class="p">,</span> <span class="n">year_motion</span><span class="p">)</span> <span class="k">end</span> @@ -22452,8 +22719,8 @@ fig.select("#fig-0378e04b897742b597befd2e8e1c169e-element-20") <span class="c"># And an S0 to kick things off.</span> <span class="n">final_motion</span> <span class="o">=</span> <span class="n">ones</span><span class="p">(</span><span class="n">m</span><span class="p">)</span> <span class="o">*</span> <span class="n">S0</span> <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:</span><span class="n">T</span> - <span class="err">μ</span> <span class="o">=</span> <span class="p">(</span><span class="n">forward</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">q</span><span class="p">)</span> - <span class="n">year_motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">final_motion</span><span class="p">[:,</span><span class="k">end</span><span class="p">],</span> <span class="err">μ</span><span class="p">,</span> <span class="err">σ</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> + <span class="n">μ</span> <span class="o">=</span> <span class="p">(</span><span class="n">forward</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">q</span><span class="p">)</span> + <span class="n">year_motion</span> <span class="o">=</span> <span class="n">simulate_gbm</span><span class="p">(</span><span class="n">final_motion</span><span class="p">[:,</span><span class="k">end</span><span class="p">],</span> <span class="n">μ</span><span class="p">,</span> <span class="n">σ</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span> <span class="n">final_motion</span> <span class="o">=</span> <span class="n">hcat</span><span class="p">(</span><span class="n">final_motion</span><span class="p">,</span> <span class="n">year_motion</span><span class="p">)</span> <span class="k">end</span> <span class="k">return</span> <span class="n">final_motion</span> @@ -22521,7 +22788,7 @@ fig.select("#fig-0378e04b897742b597befd2e8e1c169e-element-20") <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:</span><span class="n">t</span> <span class="n">price</span> <span class="o">=</span> <span class="n">year_prices</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> - <span class="k">if</span> <span class="n">price</span> <span class="err">≥</span> <span class="n">call_barrier</span> + <span class="k">if</span> <span class="n">price</span> <span class="n">≥</span> <span class="n">call_barrier</span> <span class="k">return</span> <span class="p">(</span><span class="n">nominal</span> <span class="o">+</span> <span class="n">coupon</span><span class="o">*</span><span class="n">i</span><span class="p">)</span> <span class="o">*</span> <span class="n">exp</span><span class="p">((</span><span class="n">prod</span><span class="p">(</span><span class="n">forward_structure</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="k">end</span><span class="p">])</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">t</span><span class="o">-</span><span class="n">i</span><span class="p">))</span> <span class="k">end</span> <span class="k">end</span> @@ -22631,9 +22898,9 @@ Present value of Athena note: $95.00, notional: $100.00</pre> <span class="k">for</span> <span class="n">i</span><span class="o">=</span><span class="mi">1</span><span class="p">:</span><span class="n">t</span> <span class="n">price</span> <span class="o">=</span> <span class="n">year_prices</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> - <span class="k">if</span> <span class="n">price</span> <span class="err">≥</span> <span class="n">call_barrier</span> + <span class="k">if</span> <span class="n">price</span> <span class="n">≥</span> <span class="n">call_barrier</span> <span class="k">return</span> <span class="p">(</span><span class="n">nominal</span> <span class="o">+</span> <span class="n">coupon</span> <span class="o">+</span> <span class="n">total_coupons</span><span class="p">)</span><span class="o">*</span><span class="n">exp</span><span class="p">((</span><span class="n">prod</span><span class="p">(</span><span class="n">forward_structure</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="k">end</span><span class="p">])</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">t</span><span class="o">-</span><span class="n">i</span><span class="p">))</span> - <span class="k">elseif</span> <span class="n">price</span> <span class="err">≥</span> <span class="n">coupon_barrier</span> + <span class="k">elseif</span> <span class="n">price</span> <span class="n">≥</span> <span class="n">coupon_barrier</span> <span class="n">total_coupons</span> <span class="o">=</span> <span class="n">total_coupons</span> <span class="o">*</span> <span class="n">exp</span><span class="p">(</span><span class="n">forward_structure</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="n">coupon</span> <span class="k">else</span> <span class="n">total_coupons</span> <span class="o">*=</span> <span class="n">exp</span><span class="p">(</span><span class="n">forward_structure</span><span class="p">[</span><span class="n">i</span><span class="p">]</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span> @@ -22810,7 +23077,7 @@ Present value of Phoenix with memory note: $100.09</pre> MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\(','\)']]}}); </script> -<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Welcome, and an algorithm2015-12-05T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-11-19:welcome-and-an-algorithm.html<p>Hello! Glad to meet you. I'm currently a student at Columbia University +<script async src='https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_CHTML'></script>Welcome, and an algorithm2015-12-05T00:00:00-05:002015-12-05T00:00:00-05:00Bradlee Speicetag:bspeice.github.io,2015-11-19:welcome-and-an-algorithm.html<p>Hello! Glad to meet you. I'm currently a student at Columbia University studying Financial Engineering, and want to give an overview of the projects I'm working on!</p> <p>To start things off, Columbia has been hosting a trading competition that @@ -22885,12 +23152,12 @@ to make some tweaks over the coming weeks, and do another forward test in Januar <span class="kn">import</span> <span class="nn">Quandl</span> <span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span> -<span class="n">tickers</span> <span class="o">=</span> <span class="p">[</span><span class="s">&#39;XOM&#39;</span><span class="p">,</span> <span class="s">&#39;CVX&#39;</span><span class="p">,</span> <span class="s">&#39;CLB&#39;</span><span class="p">,</span> <span class="s">&#39;OXY&#39;</span><span class="p">,</span> <span class="s">&#39;SLB&#39;</span><span class="p">]</span> -<span class="n">market_ticker</span> <span class="o">=</span> <span class="s">&#39;GOOG/NYSE_VOO&#39;</span> +<span class="n">tickers</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;XOM&#39;</span><span class="p">,</span> <span class="s1">&#39;CVX&#39;</span><span class="p">,</span> <span class="s1">&#39;CLB&#39;</span><span class="p">,</span> <span class="s1">&#39;OXY&#39;</span><span class="p">,</span> <span class="s1">&#39;SLB&#39;</span><span class="p">]</span> +<span class="n">market_ticker</span> <span class="o">=</span> <span class="s1">&#39;GOOG/NYSE_VOO&#39;</span> <span class="n">lookback</span> <span class="o">=</span> <span class="mi">30</span> -<span class="n">d_col</span> <span class="o">=</span> <span class="s">&#39;Close&#39;</span> +<span class="n">d_col</span> <span class="o">=</span> <span class="s1">&#39;Close&#39;</span> -<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="n">tick</span><span class="p">:</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">&#39;YAHOO/{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tick</span><span class="p">))[</span><span class="o">-</span><span class="n">lookback</span><span class="p">:]</span> <span class="k">for</span> <span class="n">tick</span> <span class="ow">in</span> <span class="n">tickers</span><span class="p">}</span> +<span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="n">tick</span><span class="p">:</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;YAHOO/{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tick</span><span class="p">))[</span><span class="o">-</span><span class="n">lookback</span><span class="p">:]</span> <span class="k">for</span> <span class="n">tick</span> <span class="ow">in</span> <span class="n">tickers</span><span class="p">}</span> <span class="n">market</span> <span class="o">=</span> <span class="n">Quandl</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">market_ticker</span><span class="p">)</span> </pre></div> @@ -23064,22 +23331,22 @@ s.t.\ \ & \vec{1} \omega = 1\\ <div class=" highlight hl-ipython3"><pre><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> <span class="kn">from</span> <span class="nn">scipy.optimize</span> <span class="k">import</span> <span class="n">minimize</span> -<span class="c">#sharpe_limit = .1</span> +<span class="c1">#sharpe_limit = .1</span> <span class="n">drawdown_limit</span> <span class="o">=</span> <span class="o">.</span><span class="mi">05</span> <span class="n">leverage</span> <span class="o">=</span> <span class="mi">250</span> -<span class="c"># Use the map so we can guarantee we maintain the correct order</span> -<span class="c"># sharpe_a = np.array(list(map(lambda tick: sharpes[tick], tickers))) * -1 # So we can write as upper-bound</span> +<span class="c1"># Use the map so we can guarantee we maintain the correct order</span> +<span class="c1"># sharpe_a = np.array(list(map(lambda tick: sharpes[tick], tickers))) * -1 # So we can write as upper-bound</span> <span class="n">dd_a</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">tick</span><span class="p">:</span> <span class="n">drawdowns</span><span class="p">[</span><span class="n">tick</span><span class="p">],</span> <span class="n">tickers</span><span class="p">)))</span> -<span class="n">returns_a</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">tick</span><span class="p">:</span> <span class="n">returns</span><span class="p">[</span><span class="n">tick</span><span class="p">]</span><span class="o">.</span><span class="n">mean</span><span class="p">(),</span> <span class="n">tickers</span><span class="p">)))</span> <span class="c"># Because minimizing</span> +<span class="n">returns_a</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">tick</span><span class="p">:</span> <span class="n">returns</span><span class="p">[</span><span class="n">tick</span><span class="p">]</span><span class="o">.</span><span class="n">mean</span><span class="p">(),</span> <span class="n">tickers</span><span class="p">)))</span> <span class="c1"># Because minimizing</span> <span class="n">meets_sharpe</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">*</span> <span class="n">sharpe_a</span><span class="p">)</span> <span class="o">-</span> <span class="n">sharpe_limit</span> <span class="k">def</span> <span class="nf">meets_dd</span><span class="p">(</span><span class="n">x</span><span class="p">):</span> <span class="n">portfolio</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">))</span> <span class="k">if</span> <span class="n">portfolio</span> <span class="o">&lt;</span> <span class="o">.</span><span class="mi">1</span><span class="p">:</span> - <span class="c"># If there are no stocks in the portfolio,</span> - <span class="c"># we can accidentally induce division by 0,</span> - <span class="c"># or division by something small enough to cause infinity</span> + <span class="c1"># If there are no stocks in the portfolio,</span> + <span class="c1"># we can accidentally induce division by 0,</span> + <span class="c1"># or division by something small enough to cause infinity</span> <span class="k">return</span> <span class="mi">0</span> <span class="k">return</span> <span class="n">drawdown_limit</span> <span class="o">-</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">*</span> <span class="n">dd_a</span><span class="p">)</span> <span class="o">/</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">))</span> @@ -23089,41 +23356,41 @@ s.t.\ \ & \vec{1} \omega = 1\\ <span class="k">def</span> <span class="nf">within_leverage</span><span class="p">(</span><span class="n">x</span><span class="p">):</span> <span class="k">return</span> <span class="n">leverage</span> <span class="o">-</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">x</span><span class="p">))</span> -<span class="n">objective</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">sum</span><span class="p">(</span><span class="n">x</span> <span class="o">*</span> <span class="n">returns_a</span><span class="p">)</span> <span class="o">*</span> <span class="o">-</span><span class="mi">1</span> <span class="c"># Because we&#39;re minimizing</span> -<span class="n">bounds</span> <span class="o">=</span> <span class="p">((</span><span class="k">None</span><span class="p">,</span> <span class="k">None</span><span class="p">),)</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">tickers</span><span class="p">)</span> +<span class="n">objective</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="nb">sum</span><span class="p">(</span><span class="n">x</span> <span class="o">*</span> <span class="n">returns_a</span><span class="p">)</span> <span class="o">*</span> <span class="o">-</span><span class="mi">1</span> <span class="c1"># Because we&#39;re minimizing</span> +<span class="n">bounds</span> <span class="o">=</span> <span class="p">((</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">),)</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">tickers</span><span class="p">)</span> <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">tickers</span><span class="p">))</span> <span class="n">constraints</span> <span class="o">=</span> <span class="p">[</span> <span class="p">{</span> - <span class="s">&#39;type&#39;</span><span class="p">:</span> <span class="s">&#39;eq&#39;</span><span class="p">,</span> - <span class="s">&#39;fun&#39;</span><span class="p">:</span> <span class="n">is_portfolio</span> + <span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;eq&#39;</span><span class="p">,</span> + <span class="s1">&#39;fun&#39;</span><span class="p">:</span> <span class="n">is_portfolio</span> <span class="p">},</span> <span class="p">{</span> - <span class="s">&#39;type&#39;</span><span class="p">:</span> <span class="s">&#39;ineq&#39;</span><span class="p">,</span> - <span class="s">&#39;fun&#39;</span><span class="p">:</span> <span class="n">within_leverage</span> - <span class="c">#}, {</span> - <span class="c"># &#39;type&#39;: &#39;ineq&#39;,</span> - <span class="c"># &#39;fun&#39;: meets_sharpe</span> + <span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;ineq&#39;</span><span class="p">,</span> + <span class="s1">&#39;fun&#39;</span><span class="p">:</span> <span class="n">within_leverage</span> + <span class="c1">#}, {</span> + <span class="c1"># &#39;type&#39;: &#39;ineq&#39;,</span> + <span class="c1"># &#39;fun&#39;: meets_sharpe</span> <span class="p">},</span> <span class="p">{</span> - <span class="s">&#39;type&#39;</span><span class="p">:</span> <span class="s">&#39;ineq&#39;</span><span class="p">,</span> - <span class="s">&#39;fun&#39;</span><span class="p">:</span> <span class="n">meets_dd</span> + <span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;ineq&#39;</span><span class="p">,</span> + <span class="s1">&#39;fun&#39;</span><span class="p">:</span> <span class="n">meets_dd</span> <span class="p">}</span> <span class="p">]</span> <span class="n">optimal</span> <span class="o">=</span> <span class="n">minimize</span><span class="p">(</span><span class="n">objective</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">bounds</span><span class="o">=</span><span class="n">bounds</span><span class="p">,</span> <span class="n">constraints</span><span class="o">=</span><span class="n">constraints</span><span class="p">,</span> - <span class="n">options</span><span class="o">=</span><span class="p">{</span><span class="s">&#39;maxiter&#39;</span><span class="p">:</span> <span class="mi">500</span><span class="p">})</span> + <span class="n">options</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;maxiter&#39;</span><span class="p">:</span> <span class="mi">500</span><span class="p">})</span> -<span class="c"># Optimization time!</span> +<span class="c1"># Optimization time!</span> <span class="n">display</span><span class="p">(</span><span class="n">optimal</span><span class="o">.</span><span class="n">message</span><span class="p">)</span> -<span class="n">display</span><span class="p">(</span><span class="s">&quot;Holdings: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">tickers</span><span class="p">,</span> <span class="n">optimal</span><span class="o">.</span><span class="n">x</span><span class="p">))))</span> +<span class="n">display</span><span class="p">(</span><span class="s2">&quot;Holdings: {}&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">tickers</span><span class="p">,</span> <span class="n">optimal</span><span class="o">.</span><span class="n">x</span><span class="p">))))</span> -<span class="n">expected_return</span> <span class="o">=</span> <span class="n">optimal</span><span class="o">.</span><span class="n">fun</span> <span class="o">*</span> <span class="o">-</span><span class="mi">100</span> <span class="c"># multiply by -100 to scale, and compensate for minimizing</span> -<span class="n">display</span><span class="p">(</span><span class="s">&quot;Expected Return: {:.3f}%&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">expected_return</span><span class="p">))</span> +<span class="n">expected_return</span> <span class="o">=</span> <span class="n">optimal</span><span class="o">.</span><span class="n">fun</span> <span class="o">*</span> <span class="o">-</span><span class="mi">100</span> <span class="c1"># multiply by -100 to scale, and compensate for minimizing</span> +<span class="n">display</span><span class="p">(</span><span class="s2">&quot;Expected Return: {:.3f}%&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">expected_return</span><span class="p">))</span> <span class="n">expected_drawdown</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">optimal</span><span class="o">.</span><span class="n">x</span><span class="p">)</span> <span class="o">*</span> <span class="n">dd_a</span><span class="p">)</span> <span class="o">/</span> <span class="nb">sum</span><span class="p">(</span><span class="nb">abs</span><span class="p">(</span><span class="n">optimal</span><span class="o">.</span><span class="n">x</span><span class="p">))</span> <span class="o">*</span> <span class="mi">100</span> -<span class="n">display</span><span class="p">(</span><span class="s">&quot;Expected Max Drawdown: {0:.2f}%&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">expected_drawdown</span><span class="p">))</span> +<span class="n">display</span><span class="p">(</span><span class="s2">&quot;Expected Max Drawdown: {0:.2f}%&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">expected_drawdown</span><span class="p">))</span> -<span class="c"># TODO: Calculate expected Sharpe</span> +<span class="c1"># TODO: Calculate expected Sharpe</span> </pre></div> </div> diff --git a/index.html b/index.html index 4cf8857..f092a70 100644 --- a/index.html +++ b/index.html @@ -83,6 +83,8 @@

+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
Wed 08 June 2016
Event Studies and Earnings Releases
Sun 15 May 2016
@@ -101,8 +103,6 @@
Cloudy in Seattle
Fri 01 January 2016
Complaining about the Weather
-
Sat 26 December 2015
-
Testing Cramer
diff --git a/index2.html b/index2.html index ca596ec..57b0c98 100644 --- a/index2.html +++ b/index2.html @@ -83,6 +83,8 @@

- page 2

+
Sat 26 December 2015
+
Testing Cramer
Fri 27 November 2015
Autocallable Bonds
Thu 19 November 2015
diff --git a/predicting-santander-customer-happiness.html b/predicting-santander-customer-happiness.html index 871635f..0f8b0c7 100644 --- a/predicting-santander-customer-happiness.html +++ b/predicting-santander-customer-happiness.html @@ -108,13 +108,13 @@ import matplotlib.pyplot as plt %matplotlib inline -# Record how long it takes to run the notebook - I'm curious. +# Record how long it takes to run the notebook - I'm curious. from datetime import datetime start = datetime.now() -dataset = pd.read_csv('split_train.csv') +dataset = pd.read_csv('split_train.csv') dataset.index = dataset.ID -X = dataset.drop(['TARGET', 'ID', 'ID.1'], 1) +X = dataset.drop(['TARGET', 'ID', 'ID.1'], 1) y = dataset.TARGET
@@ -244,13 +244,13 @@
In [5]:
-
# First we need to `binarize` the data to 0-1; some of the labels are {0, 1},
-# some are {0, 3}, etc.
+
# First we need to `binarize` the data to 0-1; some of the labels are {0, 1},
+# some are {0, 3}, etc.
 from sklearn.preprocessing import binarize
 X_bin = binarize(X[b_class])
 
 accuracy = [np.mean(X_bin[:,i] == y) for i in range(0, len(b_class))]
-acc_df = pd.DataFrame({"Accuracy": accuracy}, index=b_class)
+acc_df = pd.DataFrame({"Accuracy": accuracy}, index=b_class)
 acc_df.describe()
 
@@ -334,9 +334,9 @@
unsat = y[y == 1].count()
-print("Satisfied customers: {}; Unsatisfied customers: {}".format(len(y) - unsat, unsat))
+print("Satisfied customers: {}; Unsatisfied customers: {}".format(len(y) - unsat, unsat))
 naive_guess = np.mean(y == np.zeros(len(y)))
-print("Naive guess accuracy: {}".format(naive_guess))
+print("Naive guess accuracy: {}".format(naive_guess))
 
@@ -397,9 +397,9 @@ Naive guess accuracy: 0.9608561656706882 return gnb.score(X_xform, y) dim_range = np.arange(1, 21) -plt.plot(dim_range, [evaluate_gnb(dim) for dim in dim_range], label="Gaussian NB Accuracy") -plt.axhline(naive_guess, label="Naive Guess", c='k') -plt.axhline(1 - naive_guess, label="Inverse Naive Guess", c='k') +plt.plot(dim_range, [evaluate_gnb(dim) for dim in dim_range], label="Gaussian NB Accuracy") +plt.axhline(naive_guess, label="Naive Guess", c='k') +plt.axhline(1 - naive_guess, label="Inverse Naive Guess", c='k') plt.gcf().set_size_inches(12, 6) plt.legend();
@@ -683,9 +683,9 @@ rkJggg== return gnb.score(X_xform, y) dim_range = np.arange(1, 21) -plt.plot(dim_range, [evaluate_gnb(dim) for dim in dim_range], label="Gaussian NB Accuracy") -plt.axhline(naive_guess, label="Naive Guess", c='k') -plt.axhline(1 - naive_guess, label="Inverse Naive Guess", c='k') +plt.plot(dim_range, [evaluate_gnb(dim) for dim in dim_range], label="Gaussian NB Accuracy") +plt.axhline(naive_guess, label="Naive Guess", c='k') +plt.axhline(1 - naive_guess, label="Inverse Naive Guess", c='k') plt.gcf().set_size_inches(12, 6) plt.legend();
@@ -961,7 +961,7 @@ rkJggg==
end = datetime.now()
-print("Running time: {}".format(end - start))
+print("Running time: {}".format(end - start))
 
@@ -990,14 +990,14 @@ rkJggg==

Appendix

Code used to split the initial training data:

from sklearn.cross_validation import train_test_split
-data = pd.read_csv('train.csv')
+data = pd.read_csv('train.csv')
 data.index = data.ID
 
 data_train, data_validate = train_test_split(
     data, train_size=.7)
 
-data_train.to_csv('split_train.csv')
-data_validate.to_csv('split_validate.csv')
+data_train.to_csv('split_train.csv')
+data_validate.to_csv('split_validate.csv')
 
diff --git a/profitability-using-the-investment-formula.html b/profitability-using-the-investment-formula.html index e665456..66d695a 100644 --- a/profitability-using-the-investment-formula.html +++ b/profitability-using-the-investment-formula.html @@ -123,7 +123,7 @@ import matplotlib.pyplot as plt from Quandl import get as qget %matplotlib inline -api_key = '' +api_key = '' profitability = lambda p, i, m, d: 1000*p / (m + i*d) @@ -147,15 +147,15 @@ expected_close = open_vals[day] * expected todays_purchase = daily_investment(open_vals[day], expected_close, invested, open_vals[0:day]) invested = np.append(invested, todays_purchase + bias) - # expected_profit = expected_close * (invested / open_vals[0:len(invested)]).sum() - invested.sum() + # expected_profit = expected_close * (invested / open_vals[0:len(invested)]).sum() - invested.sum() day += 1 profitable = is_profitable(close_vals[day-1], invested, open_vals[0:len(invested)]) \ or is_profitable(open_vals[day], invested, open_vals[0:len(invested)]) shares = (invested / open_vals[0:len(invested)]).sum() - # Make sure we can't see into the future - we know either today's close or tomorrow's open - # will be profitable, but we need to check which one. + # Make sure we can't see into the future - we know either today's close or tomorrow's open + # will be profitable, but we need to check which one. if is_profitable(close_vals[day-1], invested, open_vals[0:len(invested)]): ending_price = close_vals[day-1] else: @@ -168,21 +168,21 @@ ticker_info = qget(name, trim_start=start, api_key=api_key) evaluation_times = ticker_info[:end].index - # Handle Google vs. YFinance data - if "Adjusted Close" in ticker_info.columns: - close_column = "Adjusted Close" + # Handle Google vs. YFinance data + if "Adjusted Close" in ticker_info.columns: + close_column = "Adjusted Close" else: - close_column = "Close" + close_column = "Close" - sim = {d: simulate_day(ticker_info[d:]["Open"], ticker_info[d:][close_column], + sim = {d: simulate_day(ticker_info[d:]["Open"], ticker_info[d:][close_column], 100, 1.02, 10) for d in evaluation_times} sim_series = pd.Series(sim) result = pd.DataFrame() - result["profit"] = sim_series.apply(lambda x: x[1]) - result["max"] = sim_series.apply(lambda x: max(x[0])) - result["days"] = sim_series.apply(lambda x: len(x[0])) - result["score"] = sim_series.apply(lambda x: profitability(x[1], x[0][0], max(x[0]), len(x[0]))) - result["investments"] = sim_series.apply(lambda x: x[0]) + result["profit"] = sim_series.apply(lambda x: x[1]) + result["max"] = sim_series.apply(lambda x: max(x[0])) + result["days"] = sim_series.apply(lambda x: len(x[0])) + result["score"] = sim_series.apply(lambda x: profitability(x[1], x[0][0], max(x[0]), len(x[0]))) + result["investments"] = sim_series.apply(lambda x: x[0]) return result @@ -217,7 +217,7 @@
In [7]:
-
fang_df = simulate_tickers(["YAHOO/FB", "YAHOO/AAPL", "YAHOO/NFLX", "YAHOO/GOOG"])
+
fang_df = simulate_tickers(["YAHOO/FB", "YAHOO/AAPL", "YAHOO/NFLX", "YAHOO/GOOG"])
 
@@ -230,9 +230,9 @@
In [8]:
-
fang_df.xs('days', axis=1, level=1).hist()
+
fang_df.xs('days', axis=1, level=1).hist()
 plt.gcf().set_size_inches(18, 8);
-plt.gcf().suptitle("Distribution of Days Until Profitability", fontsize=18);
+plt.gcf().suptitle("Distribution of Days Until Profitability", fontsize=18);
 
@@ -704,9 +704,9 @@ kiSpNScYJEmSJElSa04wSJIkSZKk1pxgkCRJkiRJrf1/uiw9D8O0d6EAAAAASUVORK5CYII=
In [10]:
-
fang_df.xs('score', axis=1, level=1).plot()
+
fang_df.xs('score', axis=1, level=1).plot()
 plt.gcf().set_size_inches(18, 6)
-plt.gcf().suptitle("Profitability score over time", fontsize=18);
+plt.gcf().suptitle("Profitability score over time", fontsize=18);
 
@@ -2343,7 +2343,7 @@ EZIkSZIkSZIkSZIk6RIpIiRJkiRJkiRJkiRJ0iX+D91jLOlExGw8AAAAAElFTkSuQmCC
In [13]:
-
cyclic_df = simulate_tickers(["YAHOO/X", "YAHOO/CAT", "YAHOO/NFLX", "YAHOO/GOOG"])
+
cyclic_df = simulate_tickers(["YAHOO/X", "YAHOO/CAT", "YAHOO/NFLX", "YAHOO/GOOG"])
 
@@ -2356,9 +2356,9 @@ EZIkSZIkSZIkSZIk6RIpIiRJkiRJkiRJkiRJ0iX+D91jLOlExGw8AAAAAElFTkSuQmCC
In [14]:
-
cyclic_df.xs('days', axis=1, level=1).hist()
+
cyclic_df.xs('days', axis=1, level=1).hist()
 plt.gcf().set_size_inches(18, 8);
-plt.gcf().suptitle("Distribution of Days Until Profitability", fontsize=18);
+plt.gcf().suptitle("Distribution of Days Until Profitability", fontsize=18);
 
@@ -2844,9 +2844,9 @@ JEmSJEnSwGwwSJIkSZKkgdlgkCRJkiRJA7PBIEmSJEmSBvb/AfKAew4+oiojAAAAAElFTkSuQmCC
In [15]:
-
cyclic_df.xs('score', axis=1, level=1).plot()
+
cyclic_df.xs('score', axis=1, level=1).plot()
 plt.gcf().set_size_inches(18, 6)
-plt.gcf().suptitle("Profitability score over time", fontsize=18);
+plt.gcf().suptitle("Profitability score over time", fontsize=18);
 
@@ -4629,7 +4629,7 @@ hUKhUCgUpor/A9AnO0PoIItTAAAAAElFTkSuQmCC
In [21]:
-
biotech_df = simulate_tickers(['YAHOO/REGN', 'YAHOO/CELG', 'GOOG/NASDAQ_BIB', 'GOOG/NASDAQ_IBB'])
+
biotech_df = simulate_tickers(['YAHOO/REGN', 'YAHOO/CELG', 'GOOG/NASDAQ_BIB', 'GOOG/NASDAQ_IBB'])
 
@@ -4642,9 +4642,9 @@ hUKhUCgUpor/A9AnO0PoIItTAAAAAElFTkSuQmCC
In [22]:
-
biotech_df.xs('days', axis=1, level=1).hist()
+
biotech_df.xs('days', axis=1, level=1).hist()
 plt.gcf().set_size_inches(18, 8);
-plt.gcf().suptitle("Distribution of Days Until Profitability", fontsize=18);
+plt.gcf().suptitle("Distribution of Days Until Profitability", fontsize=18);
 
@@ -5195,9 +5195,9 @@ WKBTQVcCAAAAAElFTkSuQmCC
In [23]:
-
biotech_df.xs('score', axis=1, level=1).plot()
+
biotech_df.xs('score', axis=1, level=1).plot()
 plt.gcf().set_size_inches(18, 6)
-plt.gcf().suptitle("Profitability score over time", fontsize=18);
+plt.gcf().suptitle("Profitability score over time", fontsize=18);
 
diff --git a/tag/nutone.html b/tag/nutone.html new file mode 100644 index 0000000..c8f4282 --- /dev/null +++ b/tag/nutone.html @@ -0,0 +1,123 @@ + + + + + + + + + + + nutone - Bradlee Speice + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+ + +
+
+ + + +
+
+
+
+

: #nutone

+
+

#nutone

+
+
+
+
+ + +
+ + + + +
+
+

nutone

+
+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
+
+
+
+ + + + + + + \ No newline at end of file diff --git a/tag/rust.html b/tag/rust.html new file mode 100644 index 0000000..b91efbc --- /dev/null +++ b/tag/rust.html @@ -0,0 +1,123 @@ + + + + + + + + + + + Rust - Bradlee Speice + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+ + +
+
+ + + +
+
+
+
+

: #Rust

+
+

#Rust

+
+
+
+
+ + +
+ + + + +
+
+

Rust

+
+
Sat 22 October 2016
+
A Rustic Re-Podcasting Server (Part 1)
+
+
+
+ + + + + + + \ No newline at end of file diff --git a/tags.html b/tags.html index 85f91f3..022a29b 100644 --- a/tags.html +++ b/tags.html @@ -115,10 +115,14 @@
1 article
monte carlo
1 article
+
nutone
+
1 article
probability
1 article
python
1 article
+
Rust
+
1 article
simulated annealing
1 article
simulation
diff --git a/testing-cramer.html b/testing-cramer.html index 91d489d..f1e75db 100644 --- a/testing-cramer.html +++ b/testing-cramer.html @@ -4,7 +4,7 @@ - + @@ -133,20 +133,20 @@ articles = [] def handle_starttag(self, tag, attrs): - #if tag == 'div' and ("id", "author_articles_wrapper") in attrs: - # self.fetch_links = True - if tag == 'a' and ('class', 'dashboard_article_link') in attrs: - href = list(filter(lambda x: x[0] == 'href', attrs))[0][1] + #if tag == 'div' and ("id", "author_articles_wrapper") in attrs: + # self.fetch_links = True + if tag == 'a' and ('class', 'dashboard_article_link') in attrs: + href = list(filter(lambda x: x[0] == 'href', attrs))[0][1] self.articles.append(href) -base_url = "http://seekingalpha.com/author/wall-street-breakfast/articles" -article_page_urls = [base_url] + [base_url + '/{}'.format(i) for i in range(2, 20)] +base_url = "http://seekingalpha.com/author/wall-street-breakfast/articles" +article_page_urls = [base_url] + [base_url + '/{}'.format(i) for i in range(2, 20)] global_articles = [] for page in article_page_urls: - # We need to switch the user agent, as SA blocks the standard requests agent + # We need to switch the user agent, as SA blocks the standard requests agent articles_html = requests.get(page, - headers={"User-Agent": "Wget/1.13.4"}) + headers={"User-Agent": "Wget/1.13.4"}) parser = ArticleListParser() parser.feed(articles_html.text) global_articles += (parser.articles) @@ -163,24 +163,24 @@
class ArticleReturnParser(HTMLParser):
-    "Given an article, parse out the futures returns in it"
+    "Given an article, parse out the futures returns in it"
     
-    record_font_tags = False
-    in_font_tag = False
+    record_font_tags = False
+    in_font_tag = False
     counter = 0
-    # data = {} # See __init__
+    # data = {} # See __init__
     
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.data = {}
     
     def handle_starttag(self, tag, attrs):
-        if tag == 'span' and ('itemprop', 'datePublished') in attrs:
-            date_string = list(filter(lambda x: x[0] == 'content', attrs))[0][1]
+        if tag == 'span' and ('itemprop', 'datePublished') in attrs:
+            date_string = list(filter(lambda x: x[0] == 'content', attrs))[0][1]
             date = dtparser.parse(date_string)
-            self.data['date'] = date
+            self.data['date'] = date
             
-        self.in_font_tag = tag == 'font'
+        self.in_font_tag = tag == 'font'
         
     def safe_float(self, string):
         try:
@@ -189,43 +189,43 @@
             return np.NaN
             
     def handle_data(self, content):
-        if not self.record_font_tags and "Futures at 6" in content:
-            self.record_font_tags = True
+        if not self.record_font_tags and "Futures at 6" in content:
+            self.record_font_tags = True
             
         if self.record_font_tags and self.in_font_tag:
             if self.counter == 0:
-                self.data['DOW'] = self.safe_float(content)
+                self.data['DOW'] = self.safe_float(content)
             elif self.counter == 1:
-                self.data['S&P'] = self.safe_float(content)
+                self.data['S&P'] = self.safe_float(content)
             elif self.counter == 2:
-                self.data['NASDAQ'] = self.safe_float(content)
+                self.data['NASDAQ'] = self.safe_float(content)
             elif self.counter == 3:
-                self.data['Crude'] = self.safe_float(content)
+                self.data['Crude'] = self.safe_float(content)
             elif self.counter == 4:
-                self.data['Gold'] = self.safe_float(content)
+                self.data['Gold'] = self.safe_float(content)
             
             self.counter += 1
             
     def handle_endtag(self, tag):
-        self.in_font_tag = False
+        self.in_font_tag = False
 
 def retrieve_data(url):
-    sa = "http://seekingalpha.com"
+    sa = "http://seekingalpha.com"
     article_html = requests.get(sa + url,
-                               headers={"User-Agent": "Wget/1.13.4"})
+                               headers={"User-Agent": "Wget/1.13.4"})
     parser = ArticleReturnParser()
     parser.feed(article_html.text)
-    parser.data.update({"url": url})
-    parser.data.update({"text": article_html.text})
+    parser.data.update({"url": url})
+    parser.data.update({"text": article_html.text})
     return parser.data
 
-# This copy **MUST** be in place. I'm not sure why,
-# as you'd think that the data being returned would already
-# represent a different memory location. Even so, it blows up
-# if you don't do this.
+# This copy **MUST** be in place. I'm not sure why,
+# as you'd think that the data being returned would already
+# represent a different memory location. Even so, it blows up
+# if you don't do this.
 article_list = list(set(global_articles))
 article_data = [copy(retrieve_data(url)) for url in article_list]
-# If there's an issue downloading the article, drop it.
+# If there's an issue downloading the article, drop it.
 article_df = pd.DataFrame.from_dict(article_data).dropna()
 
@@ -249,12 +249,12 @@
In [4]:
-
# article_df is sorted by date, so we get the first row.
-start_date = article_df.sort_values(by='date').iloc[0]['date'] - relativedelta(days=1)
-SPY = Quandl.get("GOOG/NYSE_SPY", trim_start=start_date)
-DJIA = Quandl.get("GOOG/AMS_DIA", trim_start=start_date)
-RUSS = Quandl.get("GOOG/AMEX_IWM", trim_start=start_date)
-NASDAQ = Quandl.get("GOOG/EPA_QQQ", trim_start=start_date)
+
# article_df is sorted by date, so we get the first row.
+start_date = article_df.sort_values(by='date').iloc[0]['date'] - relativedelta(days=1)
+SPY = Quandl.get("GOOG/NYSE_SPY", trim_start=start_date)
+DJIA = Quandl.get("GOOG/AMS_DIA", trim_start=start_date)
+RUSS = Quandl.get("GOOG/AMEX_IWM", trim_start=start_date)
+NASDAQ = Quandl.get("GOOG/EPA_QQQ", trim_start=start_date)
 
@@ -279,13 +279,13 @@
def calculate_opening_ret(frame):
-    # I'm not a huge fan of the appending for loop,
-    # but it's a bit verbose for a comprehension
+    # I'm not a huge fan of the appending for loop,
+    # but it's a bit verbose for a comprehension
     data = {}
     for i in range(1, len(frame)):
         date = frame.iloc[i].name
-        prior_close = frame.iloc[i-1]['Close']
-        open_val = frame.iloc[i]['Open']
+        prior_close = frame.iloc[i-1]['Close']
+        open_val = frame.iloc[i]['Open']
         data[date] = (open_val - prior_close) / prior_close
         
     return data
@@ -296,8 +296,8 @@
 NASDAQ_open_ret = calculate_opening_ret(NASDAQ)
 
 def signs_match(list_1, list_2):
-    # This is a surprisingly difficult task - we have to match
-    # up the dates in order to check if opening returns actually match
+    # This is a surprisingly difficult task - we have to match
+    # up the dates in order to check if opening returns actually match
     index_dict_dt = {key.to_datetime(): list_2[key] for key in list_2.keys()}
     
     matches = []
@@ -313,26 +313,26 @@
                     matches += [1]
             else:
                 matches += [0]
-            #print("{}".format(list_2[index_dt]))
+            #print("{}".format(list_2[index_dt]))
     return matches
     
     
 prediction_dict = {}
 matches_dict = {}
 count_dict = {}
-index_dict = {"SPY": SPY_open_ret, "DJIA": DJIA_open_ret, "RUSS": RUSS_open_ret, "NASDAQ": NASDAQ_open_ret}
-indices = ["SPY", "DJIA", "RUSS", "NASDAQ"]
-futures = ["Crude", "Gold", "DOW", "NASDAQ", "S&P"]
+index_dict = {"SPY": SPY_open_ret, "DJIA": DJIA_open_ret, "RUSS": RUSS_open_ret, "NASDAQ": NASDAQ_open_ret}
+indices = ["SPY", "DJIA", "RUSS", "NASDAQ"]
+futures = ["Crude", "Gold", "DOW", "NASDAQ", "S&P"]
 for index in indices:
-    matches_dict[index] = {future: signs_match(article_df[[future, 'date']],
+    matches_dict[index] = {future: signs_match(article_df[[future, 'date']],
                                                index_dict[index]) for future in futures}
     count_dict[index] = {future: len(matches_dict[index][future]) for future in futures}
     prediction_dict[index] = {future: np.mean(matches_dict[index][future])
                               for future in futures}
-print("Articles Checked: ")
+print("Articles Checked: ")
 print(pd.DataFrame.from_dict(count_dict))
 print()
-print("Prediction Accuracy:")
+print("Prediction Accuracy:")
 print(pd.DataFrame.from_dict(prediction_dict))
 
@@ -392,13 +392,13 @@ S&P 0.604478 0.597015 0.811808 0.848708
def calculate_closing_ret(frame):
-    # I'm not a huge fan of the appending for loop,
-    # but it's a bit verbose for a comprehension
+    # I'm not a huge fan of the appending for loop,
+    # but it's a bit verbose for a comprehension
     data = {}
     for i in range(0, len(frame)):
         date = frame.iloc[i].name
-        open_val = frame.iloc[i]['Open']
-        close_val = frame.iloc[i]['Close']
+        open_val = frame.iloc[i]['Open']
+        close_val = frame.iloc[i]['Close']
         data[date] = (close_val - open_val) / open_val
         
     return data
@@ -409,8 +409,8 @@ S&P     0.604478  0.597015  0.811808  0.848708
 NASDAQ_close_ret = calculate_closing_ret(NASDAQ)
 
 def signs_match(list_1, list_2):
-    # This is a surprisingly difficult task - we have to match
-    # up the dates in order to check if opening returns actually match
+    # This is a surprisingly difficult task - we have to match
+    # up the dates in order to check if opening returns actually match
     index_dict_dt = {key.to_datetime(): list_2[key] for key in list_2.keys()}
     
     matches = []
@@ -426,28 +426,28 @@ S&P     0.604478  0.597015  0.811808  0.848708
                     matches += [1]
             else:
                 matches += [0]
-            #print("{}".format(list_2[index_dt]))
+            #print("{}".format(list_2[index_dt]))
     return matches
     
     
 matches_dict = {}
 count_dict = {}
 prediction_dict = {}
-index_dict = {"SPY": SPY_close_ret, "DJIA": DJIA_close_ret,
-              "RUSS": RUSS_close_ret, "NASDAQ": NASDAQ_close_ret}
-indices = ["SPY", "DJIA", "RUSS", "NASDAQ"]
-futures = ["Crude", "Gold", "DOW", "NASDAQ", "S&P"]
+index_dict = {"SPY": SPY_close_ret, "DJIA": DJIA_close_ret,
+              "RUSS": RUSS_close_ret, "NASDAQ": NASDAQ_close_ret}
+indices = ["SPY", "DJIA", "RUSS", "NASDAQ"]
+futures = ["Crude", "Gold", "DOW", "NASDAQ", "S&P"]
 for index in indices:
-    matches_dict[index] = {future: signs_match(article_df[[future, 'date']],
+    matches_dict[index] = {future: signs_match(article_df[[future, 'date']],
                                                index_dict[index]) for future in futures}
     count_dict[index] = {future: len(matches_dict[index][future]) for future in futures}
     prediction_dict[index] = {future: np.mean(matches_dict[index][future])
                               for future in futures}
     
-print("Articles Checked:")
+print("Articles Checked:")
 print(pd.DataFrame.from_dict(count_dict))
 print()
-print("Prediction Accuracy:")
+print("Prediction Accuracy:")
 print(pd.DataFrame.from_dict(prediction_dict))
 
diff --git a/the-unfair-casino.html b/the-unfair-casino.html index 5540ca1..bfbc7f7 100644 --- a/the-unfair-casino.html +++ b/the-unfair-casino.html @@ -4,7 +4,7 @@ - + @@ -114,8 +114,8 @@ fair_1 = np.random.randint(1, 7, 10000) fair_2 = np.random.randint(1, 7, 10000) -pd.Series(fair_1 + fair_2).plot(kind='hist', bins=11); -plt.title('Fair Distribution'); +pd.Series(fair_1 + fair_2).plot(kind='hist', bins=11); +plt.title('Fair Distribution');
@@ -326,8 +326,8 @@ D_2 = \left\{ x1 = unfair_die(d1, 10000) x2 = unfair_die(d2, 10000) -pd.Series(x1 + x2).plot(kind='hist', bins=11); -plt.title('$D_1$ biased low, $D_2$ biased high'); +pd.Series(x1 + x2).plot(kind='hist', bins=11); +plt.title('$D_1$ biased low, $D_2$ biased high');
@@ -525,8 +525,8 @@ D_2 = \left\{ x1 = unfair_die(d1, 10000) x2 = unfair_die(d2, 10000) -pd.Series(x1 + x2).plot(kind='hist', bins=11) -plt.title("$D_1$ and $D_2$ biased to 1 and 6"); +pd.Series(x1 + x2).plot(kind='hist', bins=11) +plt.title("$D_1$ and $D_2$ biased to 1 and 6");
@@ -746,18 +746,18 @@ and continue the next iteration. from numpy import polynomial def density_coef(d1_density, d2_density): - # Calculating the probabilities of each outcome was taken - # from this author: http://math.stackexchange.com/a/1710392/320784 + # Calculating the probabilities of each outcome was taken + # from this author: http://math.stackexchange.com/a/1710392/320784 d1_p = polynomial.Polynomial(d1_density) d2_p = polynomial.Polynomial(d2_density) coefs = (d1_p * d2_p).coef return coefs def score(x, d1_density, d2_density): - # We've now got the probabilities of each event, but we need - # to shift the array a bit so we can use the x values to actually - # index into it. This will allow us to do all the calculations - # incredibly quickly + # We've now got the probabilities of each event, but we need + # to shift the array a bit so we can use the x values to actually + # index into it. This will allow us to do all the calculations + # incredibly quickly coefs = density_coef(d1_density, d2_density) coefs = np.hstack((0, 0, coefs)) return np.log(coefs[x]).sum() @@ -784,23 +784,23 @@ and continue the next iteration.
def permute(d1_density, d2_density):
-    # To ensure we have legitimate densities, we will randomly
-    # increase one die face probability by `change`,
-    # and decrease one by `change`.
-    # This means there are something less than (1/`change`)^12 possibilities
-    # we are trying to search over.
+    # To ensure we have legitimate densities, we will randomly
+    # increase one die face probability by `change`,
+    # and decrease one by `change`.
+    # This means there are something less than (1/`change`)^12 possibilities
+    # we are trying to search over.
     change = .01
     
     d1_index1, d1_index2 = np.random.randint(0, 6, 2)
     d2_index1, d2_index2 = np.random.randint(0, 6, 2)
     
-    # Also make sure to copy. I've had some weird aliasing issues
-    # in the past that made everything blow up.
+    # Also make sure to copy. I've had some weird aliasing issues
+    # in the past that made everything blow up.
     new_d1 = np.float64(np.copy(d1_density))
     new_d2 = np.float64(np.copy(d2_density))
     
-    # While this doesn't account for the possibility that some
-    # values go negative, in practice this never happens
+    # While this doesn't account for the possibility that some
+    # values go negative, in practice this never happens
     new_d1[d1_index1] += change
     new_d1[d1_index2] -= change
     new_d2[d2_index1] += change
@@ -833,24 +833,24 @@ and continue the next iteration.
     switch_failures = 0
     iter_count = 0
     
-    # Start with guessing fair dice
+    # Start with guessing fair dice
     cur_d1 = np.repeat(1/6, 6)
     cur_d2 = np.repeat(1/6, 6)
     cur_score = score(data, cur_d1, cur_d2)
     
-    # Keep track of our best guesses - may not be
-    # what we end with
+    # Keep track of our best guesses - may not be
+    # what we end with
     max_score = cur_score
     max_d1 = cur_d1
     max_d2 = cur_d2
     
-    # Optimization stops when we have failed to switch `conv_count`
-    # times (presumably because we have a great guess), or we reach
-    # the maximum number of iterations.
+    # Optimization stops when we have failed to switch `conv_count`
+    # times (presumably because we have a great guess), or we reach
+    # the maximum number of iterations.
     while switch_failures < conv_count and iter_count < max_iter:
         iter_count += 1
         if iter_count % (max_iter / 10) == 0:
-            print('Iteration: {}; Current score (higher is better): {}'.format(
+            print('Iteration: {}; Current score (higher is better): {}'.format(
                     iter_count, cur_score))
                 
         new_d1, new_d2 = permute(cur_d1, cur_d2)
@@ -862,19 +862,19 @@ and continue the next iteration.
             max_d2 = new_d2
             
         if new_score > cur_score:
-            # If the new permutation beats the old one,
-            # automatically select it.
+            # If the new permutation beats the old one,
+            # automatically select it.
             cur_score = new_score
             cur_d1 = new_d1
             cur_d2 = new_d2
             switch_failures = 0
         else:
-            # We didn't beat the current score, but allow
-            # for possibly switching anyways.
+            # We didn't beat the current score, but allow
+            # for possibly switching anyways.
             accept_prob = np.exp(new_score - cur_score)
             coin_toss = np.random.rand()
             if coin_toss < accept_prob:
-                # We randomly switch to the new distribution
+                # We randomly switch to the new distribution
                 cur_score = new_score
                 cur_d1 = new_d1
                 cur_d2 = new_d2
@@ -882,7 +882,7 @@ and continue the next iteration.
             else:
                 switch_failures += 1
             
-    # Return both our best guess, and the ending guess
+    # Return both our best guess, and the ending guess
     return max_d1, max_d2, cur_d1, cur_d2
 
@@ -916,20 +916,20 @@ and continue the next iteration. start = time.perf_counter() max_d1, max_d2, final_d1, final_d2 = optimize(d1_vals + d2_vals) end = time.perf_counter() - print("Simulated Annealing time: {:.02f}s".format(end - start)) + print("Simulated Annealing time: {:.02f}s".format(end - start)) coef_range = np.arange(2, 13) - .5 plt.subplot(221) plt.bar(coef_range, density_coef(d1_dist, d2_dist), width=1) - plt.title('True Distribution') + plt.title('True Distribution') plt.subplot(222) plt.hist(d1_vals + d2_vals, bins=11) - plt.title('Empirical Distribution') + plt.title('Empirical Distribution') plt.subplot(223) plt.bar(coef_range, density_coef(max_d1, max_d2), width=1) - plt.title('Recovered Distribution') + plt.title('Recovered Distribution') plt.gcf().set_size_inches(10, 10) diff --git a/tick-tock.html b/tick-tock.html index b0c971b..c3cc173 100644 --- a/tick-tock.html +++ b/tick-tock.html @@ -137,7 +137,7 @@
In [2]:
-
# Import all the OAuth secret information from a local file
+
# Import all the OAuth secret information from a local file
 from secrets import CLIENT_SECRET, CLIENT_ID, CALLBACK_URL
 
@@ -178,17 +178,17 @@ Application: Thank you, now give me all the data.
import urllib
 
-FITBIT_URI = 'https://www.fitbit.com/oauth2/authorize'
+FITBIT_URI = 'https://www.fitbit.com/oauth2/authorize'
 params = {
-    # If we need more than one scope, must be a CSV string
-    'scope': 'heartrate',
-    'response_type': 'token',
-    'expires_in': 86400, # 1 day
-    'redirect_uri': CALLBACK_URL,
-    'client_id': CLIENT_ID
+    # If we need more than one scope, must be a CSV string
+    'scope': 'heartrate',
+    'response_type': 'token',
+    'expires_in': 86400, # 1 day
+    'redirect_uri': CALLBACK_URL,
+    'client_id': CLIENT_ID
 }
 
-request_url = FITBIT_URI + '?' + urllib.parse.urlencode(params)
+request_url = FITBIT_URI + '?' + urllib.parse.urlencode(params)
 
@@ -211,10 +211,10 @@ Application: Thank you, now give me all the data.
In [6]:
-
# The `response_url` variable contains the full URL that
-# FitBit sent back to us, but most importantly,
-# contains the token we need for authorization.
-access_token = dict(urllib.parse.parse_qsl(response_url))['access_token']
+
# The `response_url` variable contains the full URL that
+# FitBit sent back to us, but most importantly,
+# contains the token we need for authorization.
+access_token = dict(urllib.parse.parse_qsl(response_url))['access_token']
 
@@ -243,41 +243,41 @@ Application: Thank you, now give me all the data.
from datetime import datetime session = OAuth2Session(token={ - 'access_token': access_token, - 'token_type': 'Bearer' + 'access_token': access_token, + 'token_type': 'Bearer' }) -format_str = '%Y-%m-%d' +format_str = '%Y-%m-%d' start_date = datetime(2016, 1, 1) end_date = datetime(2016, 3, 31) dr = pd.date_range(start_date, end_date) -url = 'https://api.fitbit.com/1/user/-/activities/heart/date/{0}/1d/1min.json' +url = 'https://api.fitbit.com/1/user/-/activities/heart/date/{0}/1d/1min.json' hr_responses = [session.get(url.format(d.strftime(format_str))) for d in dr] def record_to_df(record): - if 'activities-heart' not in record: - return None - date_str = record['activities-heart'][0]['dateTime'] - df = pd.DataFrame(record['activities-heart-intraday']['dataset']) + if 'activities-heart' not in record: + return None + date_str = record['activities-heart'][0]['dateTime'] + df = pd.DataFrame(record['activities-heart-intraday']['dataset']) - df.index = df['time'].apply( - lambda x: datetime.strptime(date_str + ' ' + x, '%Y-%m-%d %H:%M:%S')) + df.index = df['time'].apply( + lambda x: datetime.strptime(date_str + ' ' + x, '%Y-%m-%d %H:%M:%S')) return df hr_dataframes = [record_to_df(record.json()) for record in hr_responses] hr_df_concat = pd.concat(hr_dataframes) -# There are some minutes with missing data, so we need to correct that +# There are some minutes with missing data, so we need to correct that full_daterange = pd.date_range(hr_df_concat.index[0], hr_df_concat.index[-1], - freq='min') -hr_df_full = hr_df_concat.reindex(full_daterange, method='nearest') + freq='min') +hr_df_full = hr_df_concat.reindex(full_daterange, method='nearest') -print("Heartbeats from {} to {}: {}".format(hr_df_full.index[0], +print("Heartbeats from {} to {}: {}".format(hr_df_full.index[0], hr_df_full.index[-1], - hr_df_full['value'].sum())) + hr_df_full['value'].sum()))
@@ -325,9 +325,9 @@ hr_q \cdot n - hr_d \cdot (n-m)
In [8]:
-
quarterly_count = hr_df_full['value'].sum()
+
quarterly_count = hr_df_full['value'].sum()
 leap_day_count = hr_df_full[(hr_df_full.index.month == 2) &
-                            (hr_df_full.index.day == 29)]['value'].sum()
+                            (hr_df_full.index.day == 29)]['value'].sum()
 num_quarters = 96
 leap_days = 6
 
@@ -372,17 +372,17 @@ hr_q \cdot n - hr_d \cdot (n-m)
 
In [9]:
-
minute_mean = hr_df_full['value'].mean()
-# Don't you wish you knew?
-# birthday_minutes = ???
+
minute_mean = hr_df_full['value'].mean()
+# Don't you wish you knew?
+# birthday_minutes = ???
 
 birthday_heartbeats = birthday_minutes * minute_mean
 
 heartbeats_until_2016 = int(birthday_heartbeats + jan_92_jan_16)
 remaining_2016 = total_heartbeats - heartbeats_until_2016
 
-print("Heartbeats so far: {}".format(heartbeats_until_2016))
-print("Remaining heartbeats: {}".format(remaining_2016))
+print("Heartbeats so far: {}".format(heartbeats_until_2016))
+print("Remaining heartbeats: {}".format(remaining_2016))
 
@@ -473,8 +473,8 @@ hr_q \cdot n - hr_d \cdot (n - m)
four_year_periods = remaining_2016 // heartbeats_4year
 remaining_4y = remaining_2016 - four_year_periods * heartbeats_4year
 
-print("Four year periods remaining: {}".format(four_year_periods))
-print("Remaining heartbeats after 4 year periods: {}".format(remaining_4y))
+print("Four year periods remaining: {}".format(four_year_periods))
+print("Remaining heartbeats after 4 year periods: {}".format(remaining_4y))
 
@@ -513,14 +513,14 @@ Remaining heartbeats after 4 year periods: 48041640
remaining_leap = remaining_4y - leap_day_count
-# Ignore leap day in the data set
+# Ignore leap day in the data set
 heartbeats_quarter = hr_df_full[(hr_df_full.index.month != 2) &
-                                (hr_df_full.index.day != 29)]['value'].sum()
+                                (hr_df_full.index.day != 29)]['value'].sum()
 quarters_left = remaining_leap // heartbeats_quarter
 remaining_year = remaining_leap - quarters_left * heartbeats_quarter
 
-print("Quarters left starting 2068: {}".format(quarters_left))
-print("Remaining heartbeats after that: {}".format(remaining_year))
+print("Quarters left starting 2068: {}".format(quarters_left))
+print("Remaining heartbeats after that: {}".format(remaining_year))
 
diff --git a/tweet-like-me.html b/tweet-like-me.html index 05f9e34..f9fda20 100644 --- a/tweet-like-me.html +++ b/tweet-like-me.html @@ -4,7 +4,7 @@ - + @@ -128,11 +128,11 @@
import pandas as pd
 
-tweets = pd.read_csv('tweets.csv')
+tweets = pd.read_csv('tweets.csv')
 text = tweets.text
 
-# Don't include tweets in reply to or mentioning people
-replies = text.str.contains('@')
+# Don't include tweets in reply to or mentioning people
+replies = text.str.contains('@')
 text_norep = text.loc[~replies]
 
@@ -187,10 +187,10 @@
from functools import reduce
 
-# Get all possible words
+# Get all possible words
 all_words = reduce(lambda x, y: x+y, tokens, [])
 unique_words = set(all_words)
-actual_words = set([x if x[0] != '.' else None for x in unique_words])
+actual_words = set([x if x[0] != '.' else None for x in unique_words])
 
 word_dist = {}
 for word in iter(actual_words):
@@ -222,9 +222,9 @@
 
import matplotlib.pyplot as plt
 %matplotlib inline
 
-hashtags = text_norep.str.count('#')
+hashtags = text_norep.str.count('#')
 bins = hashtags.unique().max()
-hashtags.plot(kind='hist', bins=bins)
+hashtags.plot(kind='hist', bins=bins)
 
@@ -396,12 +396,12 @@ AElFTkSuQmCC
In [6]:
-
hashtags = [x for x in all_words if x[0] == '#']
+
hashtags = [x for x in all_words if x[0] == '#']
 n_hashtags = len(hashtags)
 
-unique_hashtags = list(set([x for x in unique_words if x[0] == '#']))
-hashtag_dist = pd.DataFrame({'hashtags': unique_hashtags,
-                             'prob': [all_words.count(h) / n_hashtags
+unique_hashtags = list(set([x for x in unique_words if x[0] == '#']))
+hashtag_dist = pd.DataFrame({'hashtags': unique_hashtags,
+                             'prob': [all_words.count(h) / n_hashtags
                                       for h in unique_hashtags]})
 len(hashtag_dist)
 
@@ -459,7 +459,7 @@ z: .17
def multinom_sim(n, vals, probs): occurrences = np.random.multinomial(n, probs) results = occurrences * vals - return ' '.join(results[results != '']) + return ' '.join(results[results != '']) def sim_n_hashtags(hashtag_freq): return np.random.poisson(hashtag_freq) @@ -469,7 +469,7 @@ z: .17
def sim_first_word(first_word_dist): probs = np.float64(first_word_dist.values) - return multinom_sim(1, first_word_dist.reset_index()['index'], probs) + return multinom_sim(1, first_word_dist.reset_index()['index'], probs) def sim_next_word(current, word_dist): dist = pd.Series(word_dist[current]) @@ -504,7 +504,7 @@ z: .17
fifth = sim_next_word(fourth, word_dist) hashtag = sim_hashtags(1, hashtag_dist) -' '.join((first, second, third, fourth, fifth, hashtag)) +' '.join((first, second, third, fourth, fifth, hashtag))
@@ -553,13 +553,13 @@ z: .17 tweet = first current = first - while chars_remaining > len(tweet) + len(hashtags) and current[0] != '.' and current[0] != '!': + while chars_remaining > len(tweet) + len(hashtags) and current[0] != '.' and current[0] != '!': current = sim_next_word(current, word_dist) - tweet += ' ' + current + tweet += ' ' + current tweet = tweet[:-2] + tweet[-1] - return ' '.join((tweet, hashtags)).strip() + return ' '.join((tweet, hashtags)).strip()
diff --git a/welcome-and-an-algorithm.html b/welcome-and-an-algorithm.html index ac5ec5a..5e25490 100644 --- a/welcome-and-an-algorithm.html +++ b/welcome-and-an-algorithm.html @@ -4,7 +4,7 @@ - + @@ -161,12 +161,12 @@ to make some tweaks over the coming weeks, and do another forward test in Januar import Quandl from datetime import datetime, timedelta -tickers = ['XOM', 'CVX', 'CLB', 'OXY', 'SLB'] -market_ticker = 'GOOG/NYSE_VOO' +tickers = ['XOM', 'CVX', 'CLB', 'OXY', 'SLB'] +market_ticker = 'GOOG/NYSE_VOO' lookback = 30 -d_col = 'Close' +d_col = 'Close' -data = {tick: Quandl.get('YAHOO/{}'.format(tick))[-lookback:] for tick in tickers} +data = {tick: Quandl.get('YAHOO/{}'.format(tick))[-lookback:] for tick in tickers} market = Quandl.get(market_ticker)
@@ -340,22 +340,22 @@ s.t.\ \ & \vec{1} \omega = 1\\
import numpy as np
 from scipy.optimize import minimize
 
-#sharpe_limit = .1
+#sharpe_limit = .1
 drawdown_limit = .05
 leverage = 250
 
-# Use the map so we can guarantee we maintain the correct order
-# sharpe_a = np.array(list(map(lambda tick: sharpes[tick], tickers))) * -1 # So we can write as upper-bound
+# Use the map so we can guarantee we maintain the correct order
+# sharpe_a = np.array(list(map(lambda tick: sharpes[tick], tickers))) * -1 # So we can write as upper-bound
 dd_a = np.array(list(map(lambda tick: drawdowns[tick], tickers)))
-returns_a = np.array(list(map(lambda tick: returns[tick].mean(), tickers))) # Because minimizing
+returns_a = np.array(list(map(lambda tick: returns[tick].mean(), tickers))) # Because minimizing
 
 meets_sharpe = lambda x: sum(abs(x) * sharpe_a) - sharpe_limit
 def meets_dd(x):
     portfolio = sum(abs(x))
     if portfolio < .1:
-        # If there are no stocks in the portfolio,
-        # we can accidentally induce division by 0,
-        # or division by something small enough to cause infinity
+        # If there are no stocks in the portfolio,
+        # we can accidentally induce division by 0,
+        # or division by something small enough to cause infinity
         return 0
     
     return drawdown_limit - sum(abs(x) * dd_a) / sum(abs(x))
@@ -365,41 +365,41 @@ s.t.\ \ & \vec{1} \omega = 1\\
 def within_leverage(x):
     return leverage - sum(abs(x))
 
-objective = lambda x: sum(x * returns_a) * -1 # Because we're minimizing
-bounds = ((None, None),) * len(tickers)
+objective = lambda x: sum(x * returns_a) * -1 # Because we're minimizing
+bounds = ((None, None),) * len(tickers)
 x = np.zeros(len(tickers))
 
 constraints = [
     {
-        'type': 'eq',
-        'fun': is_portfolio
+        'type': 'eq',
+        'fun': is_portfolio
     }, {
-        'type': 'ineq',
-        'fun': within_leverage
-    #}, {
-    #    'type': 'ineq',
-    #    'fun': meets_sharpe
+        'type': 'ineq',
+        'fun': within_leverage
+    #}, {
+    #    'type': 'ineq',
+    #    'fun': meets_sharpe
     }, {
-        'type': 'ineq',
-        'fun': meets_dd
+        'type': 'ineq',
+        'fun': meets_dd
     }
 ]
 
 optimal = minimize(objective, x, bounds=bounds, constraints=constraints,
-                  options={'maxiter': 500})
+                  options={'maxiter': 500})
 
-# Optimization time!
+# Optimization time!
 display(optimal.message)
 
-display("Holdings: {}".format(list(zip(tickers, optimal.x))))
+display("Holdings: {}".format(list(zip(tickers, optimal.x))))
 
-expected_return = optimal.fun * -100  # multiply by -100 to scale, and compensate for minimizing
-display("Expected Return: {:.3f}%".format(expected_return))
+expected_return = optimal.fun * -100  # multiply by -100 to scale, and compensate for minimizing
+display("Expected Return: {:.3f}%".format(expected_return))
 
 expected_drawdown = sum(abs(optimal.x) * dd_a) / sum(abs(optimal.x)) * 100
-display("Expected Max Drawdown: {0:.2f}%".format(expected_drawdown))
+display("Expected Max Drawdown: {0:.2f}%".format(expected_drawdown))
 
-# TODO: Calculate expected Sharpe
+# TODO: Calculate expected Sharpe