Sunday, November 22, 2009

Time Traveler... a sort of Time machine system

This is not a nice script, but it does the job for me in the way I need it to work. I share it with you in the case it can be useful to you.
I did it to store Time Traveler data on a separate disk, it is why I don't use a maximum backuped data size, and instead a free space to keep before starting the backup.
All backups are stored in folders starting with the time_t value, then the date in human readable format.
It is used in a server, so it sends the result to an email address. You can use it with cron for example.

The script is in python, so just copy it in the file timetraveler.py, change source and destination folder and email address at the top of the file, then chmod 700 timetraveler.py, then run it. Be sure you have python and rsync installed. Us only a linux compatible file system as backup folder (ext2, ext3, ext4 or other... not fat32 or ntfs, since I don't know if it will work)

Here is the script.
#!/usr/bin/env python
# coding: utf-8
#  TimeTraveler (c) 2009 Walter  http://computersstoneage.blogspot.com/
#
#
#TODO:
#     Répér dans les logs la sortie des commandes:
#            rm
#            rsync
#
import commands
import datetime
import time
import array
import string
import os


freeSpaceLimit = 50*1024*1024; #required disk space to start the backup in Ko. here it is 50Go.
path = "/mnt/timemachine/"  #folder where to store backups
source = "/mnt/data/"  #rértoire à auvegarder
mail = "/usr/bin/mail" #mail software
email = "timemachine"    #mail address to send the result of the backup


def disk_get_free(path):
    res = commands.getoutput('df -k '+path).splitlines()
    return int(res[1].split()[3])
  
def get_size(val):
    if val < (1024):
        val = str(val)+ ' KB'           
    elif  val < 1048576: #(1024*1024):
        val = str(float(int(val/102.4))/10.0)+ ' MB'       
    elif val < 1073741824 :#(1024*1024*1024):
        val = str(float(int(val/(104857.6)))/10.0)+ ' GB'   
    else:
        val = str(float(int(val/(107374182.4)))/10.0)+ ' TB'
    return val




date = str(datetime.datetime.now())
date = date.split()[0]
total = int(time.time())
#print(str(total)+"_"+date)

dest = path+str(total)+"_"+date

log = "STARTING TIMEMACHINE: "+str(datetime.datetime.now())+"\n"
log=log+"Backup folder: "+path+"\n"
log=log+"Source folder: "+source+"\n"
#lastInt = 0
#lastName = date+"_"+str(total)




#methode 1: je purge en fonction de l'espace disque
#print disk_get_free(path)
freeSpace = disk_get_free(path)
log = log+"Starting disk space recovery:\n"
log = log+"\tActual disk size: "+get_size(freeSpace)+"\n"
log=log+"\tDisk space to keep: "+get_size(freeSpaceLimit)+"\n"
if(freeSpace >= freeSpaceLimit):
    log=log+"\tNo need to free disk space.\n"
empty = False #cela me permet de savoir si j'ai encore des fichiers àffacer
while freeSpace < freeSpaceLimit and empty==False:
    files = os.listdir(path)
    files.sort()
    empty = True
    if len(files)>0:
         tab = files[0].split('_')
         if(len(tab) > 1) and str(tab[0]).isdigit():
            command = "rm -rf "+path+files[0]
            #print(command)
            log = log+commands.getoutput( command )
            freeSpaceNew = disk_get_free(path)
            log=log+"\tRemoving "+files[0]+": "+command+" recovered "+get_size(freeSpaceNew - freeSpace)+" => "+get_size(freeSpaceNew)+" free.\n"
            freeSpace = freeSpaceNew
            empty = False
   
if(empty==True):
    log=log+"ERROR: Can't recover enough space from previous backup. Current free space: "+get_size(freeSpace)+".\nABORTING BACKUP.\n"
    #print(log)
    p = os.popen(mail+" -s  \"Time machine "+str(datetime.datetime.now())+" - ERROR\" "+email, 'w')
    p.write(log)
    exitcode = p.close()
    exit()
log=log+"Disk space recovery finished.\n"



files = os.listdir(path)
files.sort()
found = False
i=0;
lastBackup = ""
size = len(files)
while found==False and i
    #print("i:"+str(i)+" size:"+str(size))
    tab = files[size-i-1].split('_')
    if(len(tab) > 1) and str(tab[0]).isdigit():
        found = True
        lastBackup = files[size-i-1]
    i = i+1
   
   
#lastBackup = "path"+files[len(files)-1]
#La, j'ai plus qu'a faire mon backup
command = ""
if lastBackup != "":
    log=log+"Found previous backup to increment: "+ lastBackup +"\n";
    command = "/usr/bin/rsync -a --safe-links  --stats --link-dest="+path+lastBackup+" "+source+" "+dest
    log=log+"Starting incremental backup:\n"+command+"\n"
else:
    log=log+"No previous backup to increment found.\n";
    command = "/usr/bin/rsync -a --safe-links --stats "+source+" "+dest
    log=log+"Starting complete backup:\n"+command+"\n"

log = log+commands.getoutput( command )
log=log+"\nBACKUP FINISHED!"
#print("Commad: "+command)



log = log+"Time machine finished: "+str(datetime.datetime.now())+"\n"
p = os.popen(mail+" -s  \"Time machine "+str(datetime.datetime.now())+" - OK\" "+email, 'w')
p.write(log)
exitcode = p.close()


 That's it

Sunday, November 15, 2009

Frets on Fire and FoFix Song Cover Art and/or Album Cover Art Downloader

Maybe you are a fan of Frest On Fire or FoFix (personnaly, I prefer Fofix, since it has much more capabilities and less bugs).
Maybe you have some songs to play, and you would like to have cover art for your song list. To help you retrieving them automatically, I created this short script.
It is a very dirty script, and is not able to get all cover art. However It seems to make the job pretty well
For linux users, this script is usefull to, since it changes filenames starting with a upper case to lower case in songs folders.

WARNING: BE VERY CAREFUL. THIS SCRIPT IS JUST AN EXAMPLE OF WHAT IT IS POSSIBLE TO DO. THIS SCRIPT CONNECT TO AMAZON TO FIND THE COVER ART. YOU HAVE TO CHECK WITH AMAZON FIRST TO KNOW IF YOU CAN USE IT AS IS. I'M NOT RESPONSIBLE FOR ANY THING THIS SCRIPT CAN DO. USE IT AT YOUR OWN RISK AND AT YOUR OWN RESPONSIBILITY. CHECK IT DOESN'T DO THINGS WHICH MAY BE FORBIDDEN BEFORE USING IT.

Maybe you can modify this script to use another search engine.
This script needs ImageMagick program called "convert" to work.
It is a python script. To use it, create a file called coverart.py in your "songs" folder, then copy/paste the script below, and save the file. Then, using a terminal, go in your song folder and execute: python coverart.py
The script will check for songs in sub-folders recursively and download album cover art for each song.

Here is the bad and dirty script:


#!/usr/bin/env python
# coding: utf-8


##########################################################
#                                                        #
# Copyright Walter http://computersstoneage.blogspot.com #
#                                                        #
# This program is free software; you can redistribute    #
# it and/or modify it under the terms of the GNU General #
# Public License as published by the Free Software       #
# Foundation; either version 2 of the License, or (at    #
# your option) any later version.                        #
##########################################################


import urllib
import shutil
import re, os
import commands

def download(url):
    webFile = urllib.urlopen(url)
    a = webFile.read()
    start = a.find('._SL')

    a = a[start-70:start]
    start = a.find('http://')
    img =  a[start:]+"._SL500_AA240_.jpg"
    #print img
    return img

def getinfo(path, cover):
    artist = ""
    song = ""
    try:   
        f = open(path+"song.ini", "r")
        tmp = f.readlines(2000)
        f.close()
        for line in tmp:
            if (line.strip().startswith('name') or line.strip().startswith('Name')):               
                song = line.strip().split("=")[1].strip()
                a = song.find('(')
                if a > 0:
                    song = song[:a]
            if (line.strip().startswith('artist') or line.strip().startswith('Artist')):
                artist = line.strip().split("=")[1].strip()
                a = artist.find('(')
                if a > 0:
                    artist = artist[:a]
        song = song.replace(' ','+');
        artist = artist.replace(' ','+');
       
        return 'http://www.amazon.com/s/ref=nb_ss?url=search-alias%3Dpopular&field-keywords=%22'+artist+'%22+%22'+song+'%22&x=0&y=0'
    except:
        print "Failed to open "+path+"song.ini"
        #return ""
        exit
       
   
def AddCoverArt(path):
    url = getinfo(path,True)
    if len(url) < 10:
        return
    img = download(url)
    if ((len(img) < 20) or (len(img)>1000)):
        print("Nothing found with cover for: "+path+" "+url);
        return
    img=urllib.unquote_plus(img)
    print("Loading: "+img)
    try:
        webFile = urllib.urlopen(img)
        localFile = open(path+"album.tmp.png", 'w')
        localFile.write(webFile.read())
        localFile.close()
    except:
        print "Can't get: "+img
        return ""


def ScanDir(g):
        files = os.listdir(g)
        for f in files:
                if(os.path.isdir(g+f)):
                        print('Entering: '+g+f+'/')
                        ScanDir(g+f+'/')
        if(os.path.exists(g+'Song.ogg')):
                print( 'mv '+g+'Song.ogg '+g+'song.ogg')
                os.rename(g+'Song.ogg', g+'song.ogg')
        if(os.path.exists(g+'Song.ini')):
                print( 'mv '+g+'Song.ini '+g+'song.ini')
                os.rename(g+'Song.ini', g+'song.ini')
        if(os.path.exists(g+'Notes.mid')):
                print( 'mv '+g+'Notes.mid '+g+'notes.mid')
                os.rename(g+'Notes.mid', g+'notes.mid')
        if(os.path.exists(g+'Rhythm.ogg')):
                print( 'mv '+g+'Rhythm.ogg '+g+'rhythm.ogg')
                os.rename(g+'Rhythm.ogg', g+'rhythm.ogg')
        if(os.path.exists(g+'Guitar.ogg')):
                print( 'mv '+g+'Guitar.ogg '+g+'guitar.ogg')
                os.rename(g+'Guitar.ogg', g+'guitar.ogg')
        if(os.path.exists(g+'guitar.ogg')) and (not os.path.exists(g+'song.ogg')):
                shutil.copy2(g+'guitar.ogg', g+'song.ogg')
                print( 'cp '+g+'guitar.ogg '+g+'song.ogg')
    if(os.path.exists(g+'Album.png')):
        os.rename(g+'Album.png', g+'album.png')
                print( 'mv '+g+'Album.png '+g+'album.png')
    if(os.path.exists(g+'song.ini'))and (not os.path.exists(g+'album.png')):
        AddCoverArt(g)
    if(os.path.exists(g+'album.tmp.png')):
        #os.rename(g+'album.png', g+'album.tmp.png')
        cmd = 'convert "'+g+'album.tmp.png" "'+g+'album.png"'
        print('Convertion: '+cmd)
        commands.getstatusoutput(cmd)
        os.unlink(g+'album.tmp.png')
       
ScanDir('./')



That's it...