Delete stale files after move to models package

2024-12-25 08:08:13 -05:00 · 2013-05-09 18:03:08 -04:00 · 2013-05-09 18:03:08 -04:00 · 33eed75bf0
commit 33eed75bf0
parent 6e767fb3fb
6 changed files with 0 additions and 5374 deletions
--- a/archiver/archive.py
+++ b/archiver/archive.py
@ -1,185 +0,0 @@
 from django.db import models
 """
 This is the archive model for the archiving backend of Melodia.
 It's purpose is to control the high-level functionality of managing
 multiple archives of music. It is different from a playlist both conceptually
 and practically - an archive describes a group of files, while a playlist
 describes a group of songs.
 In this way, you back up archives of music - you don't back up the songs in a
 playlist. Additionally, you may want to re-organize your music to use a
 cleaner directory structure - a playlist doesn't care about this.
 """
 class Archive (models.Model):
 	import datetime
 	"""
 	The archive model itself, and all functions used to interact with it.
 	The archive is built up from a grouping of songs, and the functions
 	that are used to interact with many songs at a single time. The archive
 	for example allows you to re-organize a specific set of music files into
 	a cleaner directory structure.
 	The archive is given a folder to use as its root directory - it finds all
 	music files under there, and takes control of them from there.
 	"""
 	name        = models.CharField(max_length = 64)
 	#Note that we're not using FilePathField since this is actually a folder
 	root_folder = models.CharField(max_length = 255)
 	#We've removed the reference to "songs" - instead define it as a ForeignKey,
 	#and do lookups via song_set
 	#Backup settings
 	backup_location  = models.CharField(max_length = 255, default = "/dev/null")
 	backup_frequency = models.IntegerField(default = 604800) #1 week in seconds
 	last_backup      = models.DateTimeField(default = datetime.datetime.now()) #Note that this by default will be the time the archive was instantiated
 	def _scan_filesystem(self):
 		"Scan the archive's root filesystem and add any new songs without adding metadata, delete songs that exist no more"
 		#This method is implemented since the other scan methods all need to use the same code
 		#DRY FTW
 		import re, os, itertools
 		from django.core.exceptions import ObjectDoesNotExist
 		from Melodia.melodia_settings import SUPPORTED_AUDIO_EXTENSIONS
 		from Melodia.melodia_settings import HASH_FUNCTION as hash
 		_regex = '|'.join(( '.*' + ext + '$' for ext in SUPPORTED_AUDIO_EXTENSIONS))
 		regex  = re.compile(_regex, re.IGNORECASE)
 		#It's hackish, but far fewer transactions to delete everything first, and add it all back.
 		#If we get interrupted, just re-run it.
 		song_set.all().delete()
 		#Add new songs
 		for dirname, dirnames, filenames in os.walk(self.root_folder):
 			#For each filename that is supported
 			for filename in itertools.ifilter(lambda filename: re.match(regex, filename), filenames):
 				rel_url = os.path.join(dirname, filename)
 				full_url = os.path.abspath(rel_url)
 				new_song = Song(url = full_url)
 				new_song.save()
 				song_set.add(new_song)
 	def _update_song_metadata(self, use_echonest = False, progress_callback = lambda x, y: None):
 		"""Scan every song in this archive (database only) and make sure all songs are correct
 		The progress_callback function is called with the current song being operated on first, and the total songs second."""
 		#This method operates only on the songs that are in the database - if you need to make
 		#sure that new songs are added, use the _scan_filesystem() method in addition
 		total_songs  = song_set.count()
 		for index, song in enumerate(song_set.all()):
 			song.populate_metadata(use_echonest = use_echonest)
 			song.save()
 			progress_callback(index + 1, total_songs)
 	def _needs_backup(self):
 		"Check if the current archive is due for a backup"
 		import datetime
 		prev_backup_time = self.last_backup
 		current_time     = datetime.datetime.now()
 		delta = current_time - prev_backup_time
 		if delta > datetime.timedelta(seconds = self.backup_frequency):
 			return True
 		else:
 			return False
 	def quick_scan(self):
 		"Scan this archive's root folder and make sure that	all songs are in the database."
 		#This is a quick scan - only validate whether or not songs should exist in the database
 		self._scan_filesystem()
 	def scan(self):
 		"Scan this archive's root folder and make sure any local metadata are correct."
 		#This is a longer scan - validate whether songs should exist, and use local data to update
 		#the database
 		self._scan_filesystem()
 		self._update_song_metadata()
 	def deep_scan(self):
 		"Scan this archive's root folder and make sure that	all songs are in the database, and use EchoNest to update metadata as necessary"
 		#This is a very long scan - validate whether songs should exist, and use Echonest to make sure
 		#that metadata is as accurate as possible.
 		self._scan_filesystem()
 		self._update_song_metadata(use_echonest = True)
 	def run_backup(self, force_backup = False):
 		"Backup the current archive"
 		if force_backup or self._needs_backup():
 			import subprocess
 			subprocess.call(['rsync', '-av', self.root_folder, self.backup_location])
 	def reorganize(self, format_string, progress_function = lambda w, x, y, z: None, dry_run = False):
 		"""Reorganize a music archive using a specified format string.
 		Recognized escape characters:
 		%a - Artist Name                       %A - Album Name
 		%d - Disc Number                       %e - Number of discs
 		%f - Current Filename (with extension) %g - Current Filename (no extension)
 		%n - Track Number                      %o - Number of tracks on disc
 		%y - Album year
 		Note that all organization takes place relative to the archive's root folder.
 		The progress_function is called with the current song number as its first argument, total songs as its second,
 		current song URL as the third argument, and new URL as the fourth.
 		"""
 		import os, shutil, errno
 		total_songs = song_set.count()
 		for index, song in enumerate(song_set.all()):
 			_current_filename              = os.path.basename(song.url)
 			_current_filename_no_extension = os.path.splitext(_current_filename)[0]
 			_release_year = song.release_date.year
 			new_location = format_string.replace("%a", song.artist)\
 										.replace("%A", song.album)\
 										.replace("%d", str(song.disc_number))\
 										.replace("%e", str(song.disc_total))\
 										.replace("%f", _current_filename)\
 										.replace("%g", _current_filename_no_extension)\
 										.replace("%n", str(song.track_number))\
 										.replace("%o", str(song.track_total))\
 										.replace("%y", str(_release_year))
 			new_url = os.path.join(self.root_folder, new_location)
 			progress_function(index + 1, total_songs, song.url, new_url)
 			if not dry_run:
 				new_folder = os.path.dirname(new_url)
 				try:
 					#`mkdir -p` functionality
 					if not os.path.isdir(new_folder):
 						os.makedirs(new_folder)
 					#Safely copy the file - don't 'move' it, but do a full 'copy' 'rm'
 					#This way if the process is ever interrupted, we have an unaltered copy
 					#of the file.
 					shutil.copyfile(song.url, new_url)
 					shutil.copystat(song.url, new_url)
 					#Notify the database about the new URL
 					old_url  = song.url
 					song.url = new_url
 					song.save()
 					#Actually remove the file since all references to the original location have been removed
 					os.remove(old_url)
 				except OSError as exc:
 					if exc.errno == errno.EEXIST and os.path.isdir(new_folder):
 						#This is safe to skip - makedirs() is complaining about a folder already existing
 						pass
 					else: raise
 				except IOError as exc:
 					#shutil error - likely that folders weren't specified correctly
 					raise
--- a/archiver/listfield.py
+++ b/archiver/listfield.py
@ -1,53 +0,0 @@
 from django.db import models
 import re, itertools
 class IntegerListField(models.TextField):
 	"""
 	Store a list of integers in a database string.
 	Format is: 
 	[<int_1>, <int_2>, <int_3>, ... , <int_n>]
 	"""
 	description = "Field type for storing lists of integers."
 	__metaclass__ = models.SubfieldBase
 	def __init__(self, *args, **kwargs):
 		super(IntegerListField, self).__init__(*args, **kwargs)
 	#Convert database to python
 	def to_python(self, value):
 		if isinstance(value, list):
 			return value
 		#Process a database string
 		#Validation first
 		if len(value) <= 0:
 			return []
 		if value[0] != '[' or value[-1] != ']':
 			raise ValidationError("Invalid input to parse a list of integers!")
 		#Note that any non-digit string is a valid separator
 		_csv_regex = "[0-9]"
 		csv_regex  = re.compile(_csv_regex)
 		#Synonymous to:
 		#string_list = filter(None, csv_regex.findall(value))
 		string_list  = itertools.ifilter(None, csv_regex.findall(value))
 		value_list   = [int(i) for i in string_list]
 		return value_list
 	#Convert python to database
 	def get_prep_value(self, value):
 		if not isinstance(value, list):
 			raise ValidationError("Invalid list given to put in database!")
 		separator_string = ", "
 		list_elements = separator_string.join(map(str, value))
 		return "[" + list_elements + "]"
--- a/archiver/models.py
+++ b/archiver/models.py
@ -1,6 +0,0 @@
 from django.db import models
 # Create your models here.
 from archive import Archive
 from song import Song
 from playlist import Playlist
--- a/archiver/playlist.py
+++ b/archiver/playlist.py
@ -1,228 +0,0 @@
 from django.db import models
 from django.core.exceptions import ObjectDoesNotExist
 from song import Song
 from listfield import IntegerListField
 import re
 from warnings import warn
 """
 Playlist model
 Each playlist is a high-level ordering of songs. There really isn't much to a playlist - just its name, and the songs inside it.
 However, we need to have a way to guarantee song order, in addition to re-ordering. A ManyToMany field can't do this.
 As such, a custom IntegerListField is implemented - it takes a python list of ints, converts it to a text field in the DB,
 and then back to a python list. This way, we can guarantee order, and have a song appear multiple times.
 The IntegerListField itself uses the ID of each song as the int in a list. For example, a list of:
 	[1, 3, 5, 17]
 Means that the playlist is made up of four songs. The order of the playlist is the song with index 1, 3, 5, and 17.
 Additionally, the ManyToMany field is included to make sure we don't use the global Songs manager - it just seems hackish.
 """
 class Playlist (models.Model):
 	"""
 	The Playlist class defines the playlist model and its operations.
 	Currently supported are add, move, and remove operations, as well as exporting to
 	multiple formats.
 	"""
 	name       = models.CharField(max_length = 255)
 	song_list = IntegerListField()
 	#This is a bit of a backup field, since technically the song PK's are in
 	#the song_order field. However, it might be useful to just get a reference
 	#to the songs in this playlist. Additionally, it's kind of hackish to reference
 	#the global Song manager, rather than something that is specific for this playlist.
 	songs      = models.ManyToManyField(Song)
 	def _populate_songs(self):
 		"""
 		Make sure that the 'songs' relation is up-to-date.
 		"""
 		#This operation works by getting the ID's for all songs currently in the playlist,
 		#calculates what we need to add, what we need to remove, and then does it.
 		#As much work as is possible is done on the python side to avoid the DB at all costs.
 		current_song_ids     = [song.id for song in self.songs.all()]
 		current_song_ids_set = set(current_song_ids)
 		new_song_ids_set = set(self.song_list)
 		remove_set = current_song_ids_set.difference(new_song_ids_set)
 		add_set    = new_song_ids_set.difference(current_song_ids_set)
 		for song_id in remove_set:
 			song = self.songs.get(id = song_id)
 			song.remove()
 		for song_id in add_set:
 			song = Song.objects.get(id = song_id) #Using the global Songs manager is unavoidable for this one
 			self.songs.add(song)
 	def insert(self, position, new_song):
 		"""
 		Insert a new song into the playlist at a specific position.
 		"""
 		if not isinstance(new_song, Song):
 			#Not given a song reference, raise an error
 			raise ValidationError("Not given a song reference to insert.")
 		self.songs.add(new_song)
 		self.song_list.insert(position, new_song.id)
 		self._populate_songs()
 	def append(self, new_song):
 		"""
 		Add a new song to the end of the playlist.
 		"""
 		if not isinstance(new_song, Song):
 			#Not given a song reference, raise an error
 			raise ValidationError("Not given a song reference to insert.")
 		self.songs.add(new_song)
 		self.song_list.append(new_song.id)
 		self._populate_songs()
 	def move(self, original_position, new_position):
 		"""
 		Move a song from one position to another
 		"""
 		if original_position == new_position:
 			return
 		song_id = self.song_list[original_position]
 		#This is actually a bit more complicated than it first appears, since the index we're moving to may actually change
 		#when we remove an item.
 		if new_position < original_position:
 			del self.song_list[original_position]
 			self.song_list.insert(new_position, song_id)
 		else:
 			del self.song_list[original_position]
 			self.song_list.insert(new_position - 1, song_id) #Account for the list indices shifting down.
 		self._populate_songs()
 	def remove(self, position):
 		if position > len(self.song_list):
 			return False
 		del self.song_list[position]
 		self._populate_songs()
 	def export(self, playlist_type = "m3u"):
 		"""
 		Export this internal playlist to a file format.
 		Supported formats:
 			-pls
 			-m3u
 		Return value is a string containing the playlist -
 		you can write it to file as you deem necessary.
 		"""
 		#Default m3u playlist type, support others as I build them.
 		playlist_string = ""
 		if playlist_type == "pls":
 			#Playlist header
 			playlist_string += "[playlist]"
 			#Playlist body
 			for index, song_id in enumerate(self.song_list):
 				song = self.songs.get(id = song_id)
 				playlist_string += "File" + str(index + 1) + "=" + song.url + "\n"
 				playlist_string += "Title" + str(index + 1) + "=" + song.title + "\n"
 				playlist_string += "Length" + str(index + 1) + "=" + str(song.duration) + "\n"
 			#Playlist footer
 			playlist_string += "NumberOfEntries=" + str(len(self.song_list))
 			playlist_string += "Version=2"
 		else:
 			#Export m3u, default option if nothing else is recognized
 			#Playlist header
 			playlist_string += "#EXTM3U" + "\n"
 			#Playlist body
 			for song_id in self.song_list:
 				song = self.songs.get(id = song_id)
 				playlist_string += "#EXTINF:" + str(song.duration) + "," + song.artist + " - " + song.title + "\n"
 				playlist_string += song.url + "\n\n"
 			#Playlist footer
 		return playlist_string
 	def _import(self, playlist_string = None):
 		"""
 		Import and convert a playlist into native DB format.
 		This function will return true if the playlist format was recognized, false otherwise.
 		It will return true even if there are errors processing individual songs in the playlist.
 		As a side note - the _import() name is used since python doesn't let
 		you name a function import().
 		"""
 		#TODO: Code playlist importing
 		self.song_list = []
 		if not playlist_string:
 			#Make sure we have a string to operate on.
 			return False
 		#Figure out what format we're in
 		if playlist_string[0:7] == "#EXTM3U":
 			#Import m3u format playlist
 			print playlist_string
 			#Expected format is "#EXTINF:" followed by the song url on the next line.
 			line_iterator = playlist_string.split("\n").__iter__()
 			#In case we end iteration early
 			try:
 				for line in line_iterator:
 					if line[0:8] == "#EXTINF:":
 						song_url = line_iterator.next() #Consume the next line
 						try:
 							song = Song.objects.get(url = song_url)
 							self.append(song)
 						except ObjectDoesNotExist:
 							#The URL of our song could not be found
 							warn("The playlist entry: " + song_url + " could not be found, and has not been added to your playlist.")
 							continue
 			#Silently end processing
 			except StopIteration:
 				pass
 			return True
 		if playlist_string[0:10] == "[playlist]":
 			#Import pls format playlist
 			#This one is a bit simpler - we're just looking for lines that start with "File="
 			pls_regex = re.compile("^File=", re.IGNORECASE)
 			for file_line in pls_regex.match(pls_regex, playlist_string):
 				song_url = file_line[5:]
 				try:
 					song = Song.objects.get(url = song_url)
 					self.append(song)
 				except ObjectDoesNotExist:
 					#The URL of our song could not be found
 					warn("The playlist entry: " + song_url + " could not be found, and has not been added to your playlist.")
 					continue
 			return True
 		#If we got here, the playlist format wasn't recognized.
 		return False
--- a/archiver/song.py
+++ b/archiver/song.py
@ -1,160 +0,0 @@
 from django.db import models
 from Melodia import melodia_settings
 from archive import Archive
 import datetime
 import os.path
 """
 The Song model
 Each instance of a Song represents a single music file.
 This database model is used for storing the metadata information about a song,
 and helps in doing sorting etc.
 """
 _default_string = "_UNAVAILABLE_"
 _default_date   = datetime.datetime.now
 _default_int    = -1
 _default_rating = 0
 _default_rating_bad       = 1
 _default_rating_ok        = 2
 _default_rating_decent    = 3
 _default_rating_good      = 4
 _default_rating_excellent = 5
 _default_rating_choices = (
 		(_default_rating, 'Default'),
 		(_default_rating_bad, 'Bad'),
 		(_default_rating_ok, 'OK'),
 		(_default_rating_decent, 'Decent'),
 		(_default_rating_good, 'Good'),
 		(_default_rating_excellent, 'Excellent'),
 		)
 class Song (models.Model):
 	"""
 	This class defines the fields and functions related to controlling
 	individual music files.
 	Note that the Playlist model depends on this model's PK being an int.
 	"""
 	#Standard song metadata
 	title        = models.CharField(max_length = 64, default = _default_string)
 	artist       = models.CharField(max_length = 64, default = _default_string)
 	album        = models.CharField(max_length = 64, default = _default_string)
 	year         = models.IntegerField(default = _default_string)
 	genre        = models.CharField(max_length = 64, default = _default_string)
 	bpm          = models.IntegerField(default = _default_int)
 	disc_number  = models.IntegerField(default = _default_int)
 	disc_total   = models.IntegerField(default = _default_int)
 	track_number = models.IntegerField(default = _default_int)
 	track_total  = models.IntegerField(default = _default_int)
 	comment      = models.CharField(default = _default_string, max_length=512)
 	#File metadata
 	bit_rate         = models.IntegerField(default = _default_int)
 	duration         = models.FloatField(default = _default_int)
 	add_date         = models.DateField(default = _default_date)
 	echonest_song_id = models.CharField(max_length = 64, default = _default_string)
 	url              = models.CharField(max_length = 255)
 	file_hash        = melodia_settings.HASH_RESULT_DB_TYPE
 	file_size        = models.IntegerField(default = _default_int)
 	#Melodia metadata
 	play_count = models.IntegerField(default = _default_int)
 	skip_count = models.IntegerField(default = _default_int)
 	rating     = models.IntegerField(default = _default_int, choices = _default_rating_choices)
 	#Link back to the archive this comes from
 	parent_archive = models.ForeignKey(Archive)
 	#Set a static reference to the rating options
 	RATING_DEFAULT   = _default_rating
 	RATING_BAD       = _default_rating_bad
 	RATING_OK        = _default_rating_ok
 	RATING_DECENT    = _default_rating_decent
 	RATING_GOOD      = _default_rating_good
 	RATING_EXCELLENT = _default_rating_excellent
 	def _get_full_url(self):
 		"Combine this song's URL with the URL of its parent"
 		return os.path.join(parent_archive.root_folder, self.url)
 	def _file_not_changed(self):
 		"Make sure the hash for this file is valid - return True if it has not changed."
 		#Overload the hash function with whatever Melodia as a whole is using
 		from Melodia.melodia_settings import HASH_FUNCTION as hash
 		#Check if there's a hash entry - if there is, the song may not have changed,
 		#and we can go ahead and return
 		if self.file_hash != None:
 			song_file = open(self._get_full_url, 'rb')
 			current_file_hash = hash(song_file.read())
 			if current_file_hash == self.file_hash:
 				#The song data hasn't changed at all, we don't need to do anything
 				return True
 		return False
 	def _grab_file_info(self):
 		"Populate file-based metadata about this song."
 		import os
 		#Overload the hash function with whatever Melodia as a whole is using
 		from Melodia.melodia_settings import HASH_FUNCTION as hash
 		file_handle = open(self._get_full_url, 'rb')
 		self.file_hash = hash(file_handle.read())
 		self.file_size = os.stat(self._get_full_url).st_size
 	def _grab_metadata_echonest(self):
 		"Populate this song's metadata using EchoNest"
 		pass
 	def _grab_metadata_local(self):
 		"Populate this song's metadata using what is locally available"
 		#Use mutagen to get the song metadata
 		import mutagen
 		try:
 			#Use mutagen to scan local metadata - don't update anything else (i.e. play_count)
 			track             = mutagen.File(self._get_full_url)
 			track_easy        = mutagen.File(self._get_full_url, easy=True)
 			self.title        = track_easy['title'][0]  or _default_string
 			self.artist       = track_easy['artist'][0] or _default_string
 			self.album_artist = track_easy['albumartist'][0] or _default_string
 			self.album        = track_easy['album'][0]  or _default_string
 			self.year         = int(track_easy['date'][0][0:4]) or _default_int
 			self.genre        = track_easy["genre"][0] or _default_string
 			self.disc_number  = int(track_easy['discnumber'][0].split('/')[0]) or _default_int
 			self.disc_total   = int(track_easy['discnumber'][0].split('/')[-1]) or _default_int
 			self.track_number = int(track_easy['track_number'][0].split('/')[0]) or _default_int
 			self.track_total  = int(track_easy['track_number'][0].split('/')[-1])  or _default_int
 			self.comment      = track_easy['comment'][0] or _default_string
 			self.bit_rate         = track.info.bitrate or _default_int
 			self.duration         = track.info.length or _default_int
 		except:
 			#Couldn't grab the local data
 			return False
 	def populate_metadata(self, use_echonest = False):
 		"Populate the metadata of this song (only if file hash has changed), and save the result."
 		if self._file_not_changed():
 			return
 		#If we've gotten to here, we do actually need to fully update the metadata
 		if use_echonest:
 			self._grab_echonest()
 		else:
 			self._grab_metadata_local()
 	def convert(self, output_location, output_format, progress_func = lambda x, y: None):
 		"Convert a song to a new format."
 		pass #Need to get pydub code in place
--- a/web/static/css/bootstrap.css
+++ b/web/static/css/bootstrap.css