mirror-github-bspeice/Melodia
1
0
Fork 0
mirror of https://github.com/bspeice/Melodia synced 2025-11-04 02:10:42 -05:00
Code Issues Releases Wiki Activity

Create a models package, add template file suffix

Browse Source 
Also, add AngularJS
This commit is contained in:
Bradlee Speice
2013-05-09 17:16:43 -04:00
parent 17960be548
commit 6e767fb3fb
13 changed files with 887 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
archiver/models/__init__.py Normal file
View File

@ -0,0 +1,5 @@
# Create your models here.
from archive import Archive
from song import Song
from playlist import Playlist
from feed import Feed

188
archiver/models/archive.py Normal file
View File

@ -0,0 +1,188 @@
from django.db import models
"""
This is the archive model for the archiving backend of Melodia.
It's purpose is to control the high-level functionality of managing
multiple archives of music. It is different from a playlist both conceptually
and practically - an archive describes a group of files, while a playlist
describes a group of songs.
In this way, you back up archives of music - you don't back up the songs in a
playlist. Additionally, you may want to re-organize your music to use a
cleaner directory structure - a playlist doesn't care about this.
"""
class Archive (models.Model):
class Meta:
app_label = 'archiver'
"""
The archive model itself, and all functions used to interact with it.
The archive is built up from a grouping of songs, and the functions
that are used to interact with many songs at a single time. The archive
for example allows you to re-organize a specific set of music files into
a cleaner directory structure.
The archive is given a folder to use as its root directory - it finds all
music files under there, and takes control of them from there.
"""
import datetime
name = models.CharField(max_length = 64)
#Note that we're not using FilePathField since this is actually a folder
root_folder = models.CharField(max_length = 255)
#We've removed the reference to "songs" - instead define it as a ForeignKey,
#and do lookups via song_set
#Backup settings
backup_location = models.CharField(max_length = 255, default = None, null = True)
backup_frequency = models.IntegerField(default = 604800) #1 week in seconds
last_backup = models.DateTimeField(default = datetime.datetime.now) #Note that this by default will be the time the archive was instantiated
def _scan_filesystem(self):
"Scan the archive's root filesystem and add any new songs without adding metadata, delete songs that exist no more"
#This method is implemented since the other scan methods all need to use the same code
#DRY FTW
import re, os, itertools
from django.core.exceptions import ObjectDoesNotExist
from Melodia.melodia_settings import SUPPORTED_AUDIO_EXTENSIONS
from Melodia.melodia_settings import HASH_FUNCTION as hash
_regex = '|'.join(( '.*' + ext + '$' for ext in SUPPORTED_AUDIO_EXTENSIONS))
regex = re.compile(_regex, re.IGNORECASE)
#It's hackish, but far fewer transactions to delete everything first, and add it all back.
#If we get interrupted, just re-run it.
song_set.all().delete()
#Add new songs
for dirname, dirnames, filenames in os.walk(self.root_folder):
#For each filename that is supported
for filename in itertools.ifilter(lambda filename: re.match(regex, filename), filenames):
rel_url = os.path.join(dirname, filename)
full_url = os.path.abspath(rel_url)
new_song = Song(url = full_url)
new_song.save()
song_set.add(new_song)
def _update_song_metadata(self, use_echonest = False, progress_callback = lambda x, y: None):
"""Scan every song in this archive (database only) and make sure all songs are correct
The progress_callback function is called with the current song being operated on first, and the total songs second."""
#This method operates only on the songs that are in the database - if you need to make
#sure that new songs are added, use the _scan_filesystem() method in addition
total_songs = song_set.count()
for index, song in enumerate(song_set.all()):
song.populate_metadata(use_echonest = use_echonest)
song.save()
progress_callback(index + 1, total_songs)
def _needs_backup(self):
"Check if the current archive is due for a backup"
import datetime
prev_backup_time = self.last_backup
current_time = datetime.datetime.now()
delta = current_time - prev_backup_time
if delta > datetime.timedelta(seconds = self.backup_frequency):
return True
else:
return False
def quick_scan(self):
"Scan this archive's root folder and make sure that all songs are in the database."
#This is a quick scan - only validate whether or not songs should exist in the database
self._scan_filesystem()
def scan(self):
"Scan this archive's root folder and make sure any local metadata are correct."
#This is a longer scan - validate whether songs should exist, and use local data to update
#the database
self._scan_filesystem()
self._update_song_metadata()
def deep_scan(self):
"Scan this archive's root folder and make sure that all songs are in the database, and use EchoNest to update metadata as necessary"
#This is a very long scan - validate whether songs should exist, and use Echonest to make sure
#that metadata is as accurate as possible.
self._scan_filesystem()
self._update_song_metadata(use_echonest = True)
def run_backup(self, force_backup = False):
"Backup the current archive"
if force_backup or self._needs_backup():
import subprocess
subprocess.call(['rsync', '-av', self.root_folder, self.backup_location])
def reorganize(self, format_string, progress_function = lambda w, x, y, z: None, dry_run = False):
"""Reorganize a music archive using a specified format string.
Recognized escape characters:
%a - Artist Name %A - Album Name
%d - Disc Number %e - Number of discs
%f - Current Filename (with extension) %g - Current Filename (no extension)
%n - Track Number %o - Number of tracks on disc
%y - Album year
Note that all organization takes place relative to the archive's root folder.
The progress_function is called with the current song number as its first argument, total songs as its second,
current song URL as the third argument, and new URL as the fourth.
"""
import os, shutil, errno
total_songs = song_set.count()
for index, song in enumerate(song_set.all()):
_current_filename = os.path.basename(song.url)
_current_filename_no_extension = os.path.splitext(_current_filename)[0]
_release_year = song.release_date.year
new_location = format_string.replace("%a", song.artist)\
.replace("%A", song.album)\
.replace("%d", str(song.disc_number))\
.replace("%e", str(song.disc_total))\
.replace("%f", _current_filename)\
.replace("%g", _current_filename_no_extension)\
.replace("%n", str(song.track_number))\
.replace("%o", str(song.track_total))\
.replace("%y", str(_release_year))
new_url = os.path.join(self.root_folder, new_location)
progress_function(index + 1, total_songs, song.url, new_url)
if not dry_run:
new_folder = os.path.dirname(new_url)
try:
#`mkdir -p` functionality
if not os.path.isdir(new_folder):
os.makedirs(new_folder)
#Safely copy the file - don't 'move' it, but do a full 'copy' 'rm'
#This way if the process is ever interrupted, we have an unaltered copy
#of the file.
shutil.copyfile(song.url, new_url)
shutil.copystat(song.url, new_url)
#Notify the database about the new URL
old_url = song.url
song.url = new_url
song.save()
#Actually remove the file since all references to the original location have been removed
os.remove(old_url)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(new_folder):
#This is safe to skip - makedirs() is complaining about a folder already existing
pass
else: raise
except IOError as exc:
#shutil error - likely that folders weren't specified correctly
raise

85
archiver/models/feed.py Normal file
View File

@ -0,0 +1,85 @@
from django.db import models
import datetime, time
import feedparser
from archive import Archive
"""
The "Feed" model describes a podcast feed using any of RSS, Atom, etc.
Backend handling is processed by 'feedparser', we just download all the podcast files,
control how many are stored, etc. The feed is intended to belong to an archive -
this way the feed is backed up automatically (and we don't have the podcast spewing
files everywhere).
It is important to note - the "max_episodes" field regulates how many episodes are
stored and backed up. A value < 1 indicates storing all episodes.
"""
_audio_type_mime_types = [
u'audio/mpeg'
]
_audio_type_mime_types_string = "\n".join(_audio_type_mime_types)
class Feed(models.Model):
class Meta:
app_label = 'archiver'
url = models.URLField()
name = models.CharField(max_length = 64)
max_episodes = models.IntegerField(default = 0) # Default store everything
current_episodes = models.IntegerField(default = 0)
last_episode = models.DateTimeField(default = datetime.datetime(1970, 1, 1))
parent_archive = models.ForeignKey(Archive)
def _get_episode_time(episode):
"""
Get a datetime.datetime object of a podcast episode's published time.
Expects a specific element from feed_object.entries.
"""
t = time.mktime(episode.published_parsed)
return datetime.datetime.fromtimestamp(t)
def _calculate_new_episodes(feed_object):
"""
Calculate how many new episodes there are of a podcast (and consequently
how many we need to remove).
"""
num_episodes = 0
#feed_object.entries starts at the most recent
for episode in feed_object.entries:
if _get_episode_time(episode) > last_episode:
num_episodes += 1
#Don't set ourselves up to download any more than max_episodes
if num_episodes > max_episodes and max_episodes > 0:
return num_episodes
return num_episodes
def _download_podcast(feed_object, num_episodes = -1):
"""
Update this podcast with episodes from the server copy. The feed_object is a reference to a
feedparser object so we don't have to redownload a feed multiple times.
"""
num_episodes = _calculate_new_episodes()
#feedparser-specific way of building the list
new_episodes = feed_object.entries[:num_episodes]
for episode in new_episodes:
episode_audio_links = [link for link in episodes['links']
if link['type'] in _audio_type_mime_types_string]
print episode_audio_links
def sync_podcast(dry_run = False, forbid_delete = False):
"""
Update the podcast with episodes from the server copy. If dry_run, don't actually download episodes,
but show what changes would have been made (implies forbid_delete). If forbid_delete, download all new
episodes, ignoring the max_episodes count.
"""
pass

56
archiver/models/listfield.py Normal file
View File

@ -0,0 +1,56 @@
from django.db import models
import re, itertools
class IntegerListField(models.TextField):
class Meta:
app_label = 'archiver'
"""
Store a list of integers in a database string.
Format is:
[<int_1>, <int_2>, <int_3>, ... , <int_n>]
"""
description = "Field type for storing lists of integers."
__metaclass__ = models.SubfieldBase
def __init__(self, *args, **kwargs):
super(IntegerListField, self).__init__(*args, **kwargs)
#Convert database to python
def to_python(self, value):
if isinstance(value, list):
return value
#Process a database string
#Validation first
if len(value) <= 0:
return []
if value[0] != '[' or value[-1] != ']':
raise ValidationError("Invalid input to parse a list of integers!")
#Note that any non-digit string is a valid separator
_csv_regex = "[0-9]"
csv_regex = re.compile(_csv_regex)
#Synonymous to:
#string_list = filter(None, csv_regex.findall(value))
string_list = itertools.ifilter(None, csv_regex.findall(value))
value_list = [int(i) for i in string_list]
return value_list
#Convert python to database
def get_prep_value(self, value):
if not isinstance(value, list):
raise ValidationError("Invalid list given to put in database!")
separator_string = ", "
list_elements = separator_string.join(map(str, value))
return "[" + list_elements + "]"

225
archiver/models/playlist.py Normal file
View File

@ -0,0 +1,225 @@
from django.db import models
from django.core.exceptions import ObjectDoesNotExist
from song import Song
from listfield import IntegerListField
import re
from warnings import warn
"""
Playlist model
Each playlist is a high-level ordering of songs. There really isn't much to a playlist - just its name, and the songs inside it.
However, we need to have a way to guarantee song order, in addition to re-ordering. A ManyToMany field can't do this.
As such, a custom IntegerListField is implemented - it takes a python list of ints, converts it to a text field in the DB,
and then back to a python list. This way, we can guarantee order, and have a song appear multiple times.
The IntegerListField itself uses the ID of each song as the int in a list. For example, a list of:
[1, 3, 5, 17]
Means that the playlist is made up of four songs. The order of the playlist is the song with index 1, 3, 5, and 17.
Additionally, the ManyToMany field is included to make sure we don't use the global Songs manager - it just seems hackish.
"""
class Playlist(models.Model):
class Meta:
app_label = 'archiver'
"""
The Playlist class defines the playlist model and its operations.
Currently supported are add, move, and remove operations, as well as exporting to
multiple formats.
"""
name = models.CharField(max_length = 255)
song_list = IntegerListField()
#This is a bit of a backup field, since technically the song PK's are in
#the song_order field. However, it might be useful to just get a reference
#to the songs in this playlist. Additionally, it's kind of hackish to reference
#the global Song manager, rather than something that is specific for this playlist.
songs = models.ManyToManyField(Song)
def _populate_songs(self):
"""
Make sure that the 'songs' relation is up-to-date.
"""
#This operation works by getting the ID's for all songs currently in the playlist,
#calculates what we need to add, what we need to remove, and then does it.
#As much work as is possible is done on the python side to avoid the DB at all costs.
current_song_ids = [song.id for song in self.songs.all()]
current_song_ids_set = set(current_song_ids)
new_song_ids_set = set(self.song_list)
remove_set = current_song_ids_set.difference(new_song_ids_set)
add_set = new_song_ids_set.difference(current_song_ids_set)
for song_id in remove_set:
song = self.songs.get(id = song_id)
song.remove()
for song_id in add_set:
song = Song.objects.get(id = song_id) #Using the global Songs manager is unavoidable for this one
self.songs.add(song)
def insert(self, position, new_song):
"""
Insert a new song into the playlist at a specific position.
"""
if not isinstance(new_song, Song):
#Not given a song reference, raise an error
raise ValidationError("Not given a song reference to insert.")
self.songs.add(new_song)
self.song_list.insert(position, new_song.id)
self._populate_songs()
def append(self, new_song):
"""
Add a new song to the end of the playlist.
"""
if not isinstance(new_song, Song):
#Not given a song reference, raise an error
raise ValidationError("Not given a song reference to insert.")
self.songs.add(new_song)
self.song_list.append(new_song.id)
self._populate_songs()
def move(self, original_position, new_position):
"""
Move a song from one position to another
"""
if original_position == new_position:
return
song_id = self.song_list[original_position]
#This is actually a bit more complicated than it first appears, since the index we're moving to may actually change
#when we remove an item.
if new_position < original_position:
del self.song_list[original_position]
self.song_list.insert(new_position, song_id)
else:
del self.song_list[original_position]
self.song_list.insert(new_position - 1, song_id) #Account for the list indices shifting down.
self._populate_songs()
def remove(self, position):
if position > len(self.song_list):
return False
del self.song_list[position]
self._populate_songs()
def export(self, playlist_type = "m3u"):
"""
Export this internal playlist to a file format.
Supported formats:
-pls
-m3u
Return value is a string containing the playlist -
you can write it to file as you deem necessary.
"""
if playlist_type == "pls":
#Playlist header
playlist_string = "[playlist]"
#Playlist body
for index, song_id in enumerate(self.song_list):
song = self.songs.get(id = song_id)
playlist_string += "File" + str(index + 1) + "=" + song.url + "\n"
playlist_string += "Title" + str(index + 1) + "=" + song.title + "\n"
playlist_string += "Length" + str(index + 1) + "=" + str(song.duration) + "\n"
#Playlist footer
playlist_string += "NumberOfEntries=" + str(len(self.song_list))
playlist_string += "Version=2"
return playlist_string
elif playlist_type == "m3u":
#Playlist header
playlist_string += "#EXTM3U" + "\n"
#Playlist body
for song_id in self.song_list:
song = self.songs.get(id = song_id)
playlist_string += "#EXTINF:" + str(song.duration) + "," + song.artist + " - " + song.title + "\n"
playlist_string += song.url + "\n\n"
return playlist_string
def _import(self, playlist_string = None):
"""
Import and convert a playlist into native DB format.
This function will return true if the playlist format was recognized, false otherwise.
It will return true even if there are errors processing individual songs in the playlist.
As a side note - the _import() name is used since python doesn't let
you name a function import().
"""
#TODO: Code playlist importing
self.song_list = []
if not playlist_string:
#Make sure we have a string to operate on.
return False
#Figure out what format we're in
if playlist_string[0:7] == "#EXTM3U":
#Import m3u format playlist
print playlist_string
#Expected format is "#EXTINF:" followed by the song url on the next line.
line_iterator = playlist_string.split("\n").__iter__()
#In case we end iteration early
try:
for line in line_iterator:
if line[0:8] == "#EXTINF:":
song_url = line_iterator.next() #Consume the next line
try:
song = Song.objects.get(url = song_url)
self.append(song)
except ObjectDoesNotExist:
#The URL of our song could not be found
warn("The playlist entry: " + song_url + " could not be found, and has not been added to your playlist.")
continue
#Silently end processing
except StopIteration:
pass
return True
if playlist_string[0:10] == "[playlist]":
#Import pls format playlist
#This one is a bit simpler - we're just looking for lines that start with "File="
pls_regex = re.compile("^File=", re.IGNORECASE)
for file_line in pls_regex.match(pls_regex, playlist_string):
song_url = file_line[5:]
try:
song = Song.objects.get(url = song_url)
self.append(song)
except ObjectDoesNotExist:
#The URL of our song could not be found
warn("The playlist entry: " + song_url + " could not be found, and has not been added to your playlist.")
continue
return True
#If we got here, the playlist format wasn't recognized.
return False

162
archiver/models/song.py Normal file
View File

@ -0,0 +1,162 @@
from django.db import models
from Melodia import melodia_settings
from archive import Archive
import datetime
import os.path
"""
The Song model
Each instance of a Song represents a single music file.
This database model is used for storing the metadata information about a song,
and helps in doing sorting etc.
"""
_default_string = "_UNAVAILABLE_"
_default_date = datetime.datetime.now
_default_int = -1
_default_rating = 0
_default_rating_bad = 1
_default_rating_ok = 2
_default_rating_decent = 3
_default_rating_good = 4
_default_rating_excellent = 5
_default_rating_choices = (
(_default_rating, 'Default'),
(_default_rating_bad, 'Bad'),
(_default_rating_ok, 'OK'),
(_default_rating_decent, 'Decent'),
(_default_rating_good, 'Good'),
(_default_rating_excellent, 'Excellent'),
)
class Song (models.Model):
class Meta:
app_label = 'archiver'
"""
This class defines the fields and functions related to controlling
individual music files.
Note that the Playlist model depends on this model's PK being an int.
"""
#Standard song metadata
title = models.CharField(max_length = 64, default = _default_string)
artist = models.CharField(max_length = 64, default = _default_string)
album = models.CharField(max_length = 64, default = _default_string)
year = models.IntegerField(default = _default_string)
genre = models.CharField(max_length = 64, default = _default_string)
bpm = models.IntegerField(default = _default_int)
disc_number = models.IntegerField(default = _default_int)
disc_total = models.IntegerField(default = _default_int)
track_number = models.IntegerField(default = _default_int)
track_total = models.IntegerField(default = _default_int)
comment = models.CharField(default = _default_string, max_length=512)
#File metadata
bit_rate = models.IntegerField(default = _default_int)
duration = models.FloatField(default = _default_int)
add_date = models.DateField(default = _default_date)
echonest_song_id = models.CharField(max_length = 64, default = _default_string)
url = models.CharField(max_length = 255)
file_hash = melodia_settings.HASH_RESULT_DB_TYPE
file_size = models.IntegerField(default = _default_int)
#Melodia metadata
play_count = models.IntegerField(default = _default_int)
skip_count = models.IntegerField(default = _default_int)
rating = models.IntegerField(default = _default_int, choices = _default_rating_choices)
#Link back to the archive this comes from
parent_archive = models.ForeignKey(Archive)
#Set a static reference to the rating options
RATING_DEFAULT = _default_rating
RATING_BAD = _default_rating_bad
RATING_OK = _default_rating_ok
RATING_DECENT = _default_rating_decent
RATING_GOOD = _default_rating_good
RATING_EXCELLENT = _default_rating_excellent
def _get_full_url(self):
"Combine this song's URL with the URL of its parent"
return os.path.join(parent_archive.root_folder, self.url)
def _file_not_changed(self):
"Make sure the hash for this file is valid - return True if it has not changed."
#Overload the hash function with whatever Melodia as a whole is using
from Melodia.melodia_settings import HASH_FUNCTION as hash
#Check if there's a hash entry - if there is, the song may not have changed,
#and we can go ahead and return
if self.file_hash != None:
song_file = open(self._get_full_url, 'rb')
current_file_hash = hash(song_file.read())
if current_file_hash == self.file_hash:
#The song data hasn't changed at all, we don't need to do anything
return True
return False
def _grab_file_info(self):
"Populate file-based metadata about this song."
import os
#Overload the hash function with whatever Melodia as a whole is using
from Melodia.melodia_settings import HASH_FUNCTION as hash
file_handle = open(self._get_full_url, 'rb')
self.file_hash = hash(file_handle.read())
self.file_size = os.stat(self._get_full_url).st_size
def _grab_metadata_echonest(self):
"Populate this song's metadata using EchoNest"
pass
def _grab_metadata_local(self):
"Populate this song's metadata using what is locally available"
#Use mutagen to get the song metadata
import mutagen
try:
#Use mutagen to scan local metadata - don't update anything else (i.e. play_count)
track = mutagen.File(self._get_full_url)
track_easy = mutagen.File(self._get_full_url, easy=True)
self.title = track_easy['title'][0] or _default_string
self.artist = track_easy['artist'][0] or _default_string
self.album_artist = track_easy['albumartist'][0] or _default_string
self.album = track_easy['album'][0] or _default_string
self.year = int(track_easy['date'][0][0:4]) or _default_int
self.genre = track_easy["genre"][0] or _default_string
self.disc_number = int(track_easy['discnumber'][0].split('/')[0]) or _default_int
self.disc_total = int(track_easy['discnumber'][0].split('/')[-1]) or _default_int
self.track_number = int(track_easy['track_number'][0].split('/')[0]) or _default_int
self.track_total = int(track_easy['track_number'][0].split('/')[-1]) or _default_int
self.comment = track_easy['comment'][0] or _default_string
self.bit_rate = track.info.bitrate or _default_int
self.duration = track.info.length or _default_int
except:
#Couldn't grab the local data
return False
def populate_metadata(self, use_echonest = False):
"Populate the metadata of this song (only if file hash has changed), and save the result."
if self._file_not_changed():
return
#If we've gotten to here, we do actually need to fully update the metadata
if use_echonest:
self._grab_echonest()
else:
self._grab_metadata_local()
def convert(self, output_location, output_format, progress_func = lambda x, y: None):
"Convert a song to a new format."
pass #Need to get pydub code in place