From fcc280b96cf1cfb6207052192a6221affa5be4fd Mon Sep 17 00:00:00 2001 From: bspeice Date: Tue, 17 May 2016 10:40:23 -0400 Subject: [PATCH] Refactoring work to make this a real program --- .idea/misc.xml | 18 +- example_conf.yaml | 17 +- repod.iml | 4 +- repod/conf_parser.py | 30 --- repod/server.py | 10 - {repod => src}/__init__.py | 0 src/conf_parser.py | 38 ++++ {repod/modules => src/podcasters}/__init__.py | 0 repod/podcast.py => src/podcasters/base.py | 37 ++-- .../modules => src/podcasters}/bassdrive.py | 185 +++++++++--------- src/server.py | 38 ++++ src/tests/__init__.py | 0 src/tests/test_build_configurator.py | 12 ++ 13 files changed, 215 insertions(+), 174 deletions(-) delete mode 100644 repod/conf_parser.py delete mode 100644 repod/server.py rename {repod => src}/__init__.py (100%) create mode 100644 src/conf_parser.py rename {repod/modules => src/podcasters}/__init__.py (100%) rename repod/podcast.py => src/podcasters/base.py (93%) rename {repod/modules => src/podcasters}/bassdrive.py (90%) create mode 100644 src/server.py create mode 100644 src/tests/__init__.py create mode 100644 src/tests/test_build_configurator.py diff --git a/.idea/misc.xml b/.idea/misc.xml index c777e63..66f7a4c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -13,23 +13,7 @@ - + - - - - - Python 3.5.1 (C:\Users\Bradlee Speice\Anaconda3\python.exe) - - - - - - - \ No newline at end of file diff --git a/example_conf.yaml b/example_conf.yaml index bc9a3de..b520c75 100644 --- a/example_conf.yaml +++ b/example_conf.yaml @@ -1,12 +1,15 @@ -# Format: (all args are passed to __init__ as kwargs +# Format (all args are passed to __init__ as kwargs: # # : # class: # args: # key: value -subfactory-show: - package: bassdrive - class: BassdriveFeed - args: - url: http://archives.bassdrivearchive.com/1%20-%20Monday/Subfactory%20Show%20-%20DJ%20Spim/ - logo: http://www.bassdrive.com/img/radio_schedule_entries/image/original/subfactory-web-add-56.jpg +server: + port: 10000 + +podcasts: + subfactory-show: + class: podcasters.BassdriveFeed + args: + url: http://archives.bassdrivearchive.com/1%20-%20Monday/Subfactory%20Show%20-%20DJ%20Spim/ + logo: http://www.bassdrive.com/img/radio_schedule_entries/image/original/subfactory-web-add-56.jpg diff --git a/repod.iml b/repod.iml index ad3c0a3..d3ce09f 100644 --- a/repod.iml +++ b/repod.iml @@ -2,7 +2,9 @@ - + + + diff --git a/repod/conf_parser.py b/repod/conf_parser.py deleted file mode 100644 index 93fc399..0000000 --- a/repod/conf_parser.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Given a configuration file, set up everything needed to kick -off the server. -""" -from importlib import import_module -import yaml -from pyramid.config import Configurator -from os.path import expanduser, join - -# Needed for import_module call -# noinspection PyUnresolvedReferences -import modules - - -def build_configuration(conf=None) -> Configurator: - if conf is None: - conf = join(expanduser('~'), '.repodrc') - - with open(conf) as conf_file: - conf_dict = yaml.load(conf_file) - server_conf = Configurator() - for mountpoint, feed in conf_dict.items(): - feed_package = import_module('modules.' + feed['package']) - feed_class = getattr(feed_package, feed['class']) - feed_instance = feed_class(**feed['args']) - - server_conf.add_route(mountpoint, '/' + mountpoint + '/') - server_conf.add_view(feed_instance.view, route_name=mountpoint) - - return server_conf diff --git a/repod/server.py b/repod/server.py deleted file mode 100644 index baa2c74..0000000 --- a/repod/server.py +++ /dev/null @@ -1,10 +0,0 @@ -from wsgiref.simple_server import make_server -from conf_parser import build_configuration - -def start_server(): - app = build_configuration().make_wsgi_app() - server = make_server('0.0.0.0', 8080, app) - server.serve_forever() - -if __name__ == '__main__': - start_server() diff --git a/repod/__init__.py b/src/__init__.py similarity index 100% rename from repod/__init__.py rename to src/__init__.py diff --git a/src/conf_parser.py b/src/conf_parser.py new file mode 100644 index 0000000..5f75322 --- /dev/null +++ b/src/conf_parser.py @@ -0,0 +1,38 @@ +""" +Given a configuration file, set up everything needed to kick +off the server. +""" +from importlib import import_module + +import yaml +from pyramid.config import Configurator + + +def build_configurator(podcasts: dict) -> Configurator: + server_conf = Configurator() + for mountpoint, feed in podcasts: + package, class_name = feed['class'].rsplit('.', 1) + feed_package = import_module(package) + feed_class = getattr(feed_package, class_name) + feed_instance = feed_class(**feed['args']) + + server_conf.add_route(mountpoint, '/' + mountpoint + '/') + server_conf.add_view(feed_instance.view, route_name=mountpoint) + + +def build_configuration_text(file_str: str) -> (dict, Configurator): + conf_dict = yaml.load(file_str) + + server_opts = conf_dict.get('server', None) + podcasts = build_configurator(conf_dict['podcasts']) + return server_opts, podcasts + + +def build_configuration(file_name) -> (dict, Configurator): + try: + with open(file_name) as conf_file: + return build_configuration_text(conf_file.read()) + except FileNotFoundError: + print("Could not locate configuration file " + + "(does {} exist?)".format(file_name)) + raise diff --git a/repod/modules/__init__.py b/src/podcasters/__init__.py similarity index 100% rename from repod/modules/__init__.py rename to src/podcasters/__init__.py diff --git a/repod/podcast.py b/src/podcasters/base.py similarity index 93% rename from repod/podcast.py rename to src/podcasters/base.py index e5da250..041b9d5 100644 --- a/repod/podcast.py +++ b/src/podcasters/base.py @@ -1,18 +1,19 @@ -""" -Base skeleton for what needs to be implemented by a podcast provider -""" -from feedgen.feed import FeedGenerator -from pyramid.response import Response - - -class BasePodcast(): - - def build_feed(self) -> FeedGenerator: - "Return a list of all episodes, in descending date order" - pass - - def view(self, request): - fg = self.build_feed() - response = Response(fg.rss_str(pretty=True)) - response.content_type = 'application/rss+xml' - return response +""" +Base skeleton for what needs to be implemented by a podcast provider +""" +from feedgen.feed import FeedGenerator +from pyramid.response import Response + + +class BasePodcast(): + + def build_feed(self) -> FeedGenerator: + "Return a list of all episodes, in descending date order" + pass + + # noinspection PyUnusedLocal + def view(self, request): + fg = self.build_feed() + response = Response(fg.rss_str(pretty=True)) + response.content_type = 'application/rss+xml' + return response diff --git a/repod/modules/bassdrive.py b/src/podcasters/bassdrive.py similarity index 90% rename from repod/modules/bassdrive.py rename to src/podcasters/bassdrive.py index 6bfd95d..2672b0c 100644 --- a/repod/modules/bassdrive.py +++ b/src/podcasters/bassdrive.py @@ -1,91 +1,94 @@ -""" -Podcast provider for the Bassdrive Archives -""" -from html.parser import HTMLParser -from urllib.parse import unquote - -import requests -from feedgen.feed import FeedGenerator - -from podcast import BasePodcast -from datetime import datetime -from pytz import UTC - - -class BassdriveParser(HTMLParser): - record_link_text = False - link_url = '' - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.links = [] - - def handle_starttag(self, tag, attrs): - href = '' - for attr, val in attrs: - if attr == 'href': - href = val - - if tag == 'a' and href.find('mp3') != -1: - self.record_link_text = True - self.link_url = href - - def handle_data(self, data): - if self.record_link_text: - self.links.append((data, self.link_url)) - self.record_link_text = False - - def get_links(self): - # Reverse to sort in descending date order - return self.links - - def clear_links(self): - self.links = [] - - -class BassdriveFeed(BasePodcast): - def __init__(self, *args, **kwargs): - self.url = kwargs['url'] - self.logo = kwargs['logo'] - # Get the title and DJ while handling trailing slash - url_pretty = unquote(self.url) - elems = filter(lambda x: x, url_pretty.split('/')) - self.title, self.dj = list(elems)[-1].split(' - ') - - def build_feed(self): - "Build the feed given our existing URL" - # Get all the episodes - page_content = str(requests.get(self.url).content) - parser = BassdriveParser() - parser.feed(page_content) - links = parser.get_links() - - # And turn them into something usable - fg = FeedGenerator() - #fg.load_extension('podcast') - fg.id(self.url) - fg.title(self.title) - fg.description(self.title) - fg.author({'name': self.dj}) - fg.language('en') - fg.link({'href': self.url, 'rel': 'alternate'}) - fg.logo(self.logo) - - for link in links: - fe = fg.add_entry() - fe.author({'name': self.dj}) - fe.title(link[0]) - fe.description(link[0]) - fe.enclosure(self.url + link[1], 0, 'audio/mpeg') - - # Bassdrive always uses date strings of - # [yyyy.mm.dd] with 0 padding, so that - # makes our lives easy - date_start = link[0].find('[') - date_str = link[0][date_start:date_start+12] - published = datetime.strptime(date_str, '[%Y.%m.%d]') - fe.pubdate(UTC.localize(published)) - fe.guid((link[0])) - - parser.clear_links() - return fg +""" +Podcast provider for the Bassdrive Archives +""" +from datetime import datetime +from html.parser import HTMLParser +from urllib.parse import unquote + +import requests +from feedgen.feed import FeedGenerator +from pytz import UTC + +from podcasters.base import BasePodcast + + +class BassdriveParser(HTMLParser): + def error(self, message): + return super().error(message) + + record_link_text = False + link_url = '' + + def __init__(self, *args, **kwargs): + # noinspection PyArgumentList + super().__init__(*args, **kwargs) + self.links = [] + + def handle_starttag(self, tag, attrs): + href = '' + for attr, val in attrs: + if attr == 'href': + href = val + + if tag == 'a' and href.find('mp3') != -1: + self.record_link_text = True + self.link_url = href + + def handle_data(self, data): + if self.record_link_text: + self.links.append((data, self.link_url)) + self.record_link_text = False + + def get_links(self): + # Reverse to sort in descending date order + return self.links + + def clear_links(self): + self.links = [] + + +class BassdriveFeed(BasePodcast): + def __init__(self, *args, **kwargs): + self.url = kwargs['url'] + self.logo = kwargs['logo'] + # Get the title and DJ while handling trailing slash + url_pretty = unquote(self.url) + elems = filter(lambda x: x, url_pretty.split('/')) + self.title, self.dj = list(elems)[-1].split(' - ') + + def build_feed(self): + "Build the feed given our existing URL" + # Get all the episodes + page_content = str(requests.get(self.url).content) + parser = BassdriveParser() + parser.feed(page_content) + links = parser.get_links() + + # And turn them into something usable + fg = FeedGenerator() + fg.id(self.url) + fg.title(self.title) + fg.description(self.title) + fg.author({'name': self.dj}) + fg.language('en') + fg.link({'href': self.url, 'rel': 'alternate'}) + fg.logo(self.logo) + + for link in links: + fe = fg.add_entry() + fe.author({'name': self.dj}) + fe.title(link[0]) + fe.description(link[0]) + fe.enclosure(self.url + link[1], 0, 'audio/mpeg') + + # Bassdrive always uses date strings of + # [yyyy.mm.dd] with 0 padding on days and months, + # so that makes our lives easy + date_start = link[0].find('[') + date_str = link[0][date_start:date_start+12] + published = datetime.strptime(date_str, '[%Y.%m.%d]') + fe.pubdate(UTC.localize(published)) + fe.guid((link[0])) + + parser.clear_links() + return fg diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..377cf7a --- /dev/null +++ b/src/server.py @@ -0,0 +1,38 @@ +import argparse +from wsgiref.simple_server import make_server +from conf_parser import build_configuration +from os.path import expanduser, join + + +# noinspection PyUnresolvedReferences +def start_server(cmd_args: dict): + try: + server_conf, configurator = build_configuration(cmd_args.configuration) + app = configurator.make_wsgi_app() + server = make_server(cmd_args.host, cmd_args.port, app) + + server.serve_forever() + except FileNotFoundError: + print("Unable to find configuration file. Does {} exist?" + .format(cmd_args.configuration)) + except AttributeError: + print("Unable to parse configuration file. Is {} a valid YML file?" + .format(cmd_args.configuration)) + except KeyError: + print('Unable to parse configuration file. Is there a `podcasts`' + 'section?') + + +if __name__ == '__main__': + default_rc = join(expanduser('~'), '.repodrc') + parser = argparse.ArgumentParser() + parser.add_argument('--verbose', help='Run server in verbose mode') + parser.add_argument('--port', type=int, default=10000, + help='Port to use when starting the server') + parser.add_argument('--host', type=str, default='0.0.0.0', + help='Host address to start the server') + parser.add_argument('--configuration', type=str, default=default_rc, + help='Configuration file to start the server') + + args = parser.parse_args() + start_server(args) diff --git a/src/tests/__init__.py b/src/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tests/test_build_configurator.py b/src/tests/test_build_configurator.py new file mode 100644 index 0000000..ca8e45e --- /dev/null +++ b/src/tests/test_build_configurator.py @@ -0,0 +1,12 @@ +from unittest import TestCase +import conf_parser + + +class TestBuild_configurator(TestCase): + def test_build_configurator(self): + try: + # noinspection PyTypeChecker + conf_parser.build_configurator(None) + self.fail("Must have dictionary to set up configurator") + except TypeError: + pass \ No newline at end of file