Tag Archive for python

Python log setup

def setupLogger(self, log_dir):
                if not isdir(log_dir):
                        os.mkdir(log_dir)

                # create logger
                log_file = join(log_dir, time.strftime('%Y_%m_%d') + '.log')
                logger = logging.getLogger(log_file)
                logger.setLevel(logging.INFO)

                # create console handler and set level to debug
                ch = logging.FileHandler(log_file)
                ch.setLevel(logging.INFO)

                # create formatter
                formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")

                # add formatter to ch
                ch.setFormatter(formatter)

                # add ch to logger
                logger.addHandler(ch)

                return logger

source

Hello World Python

print 'Hello World
'

source

Send free sms through way2sms (India)

#! /usr/bin/python
#----------------------------------------------------------------------
#
# Author:      Laszlo Nagy
#
# Copyright:   (c) 2005 by Szoftver Messias Bt.
# Licence:     BSD style
#
#
#----------------------------------------------------------------------
# WAY2SMS SMS Sender
#----------------------------------------------------------------------
#
# Author:      Kasi Viswanadh Yakkala
#
# Usage:       way2sms <YourMessage> <PhoneNumber>
#
#----------------------------------------------------------------------

import os
import md5
import urllib
import urllib2
import mimetypes
#from gzip import GzipFile
import cStringIO
from cPickle import loads,dumps
import cookielib
import random
import sys
import base64


class MozillaCacher(object):
    """A dictionary like object, that can cache results on a storage device."""
    def __init__(self,cachedir='.cache'):
        self.cachedir = cachedir
        if not os.path.isdir(cachedir):
            os.mkdir(cachedir)
    def name2fname(self,name):
        return os.path.join(self.cachedir,name)
    def __getitem__(self,name):
        if not isinstance(name,str):
            raise TypeError()
        fname = self.name2fname(name)
        if os.path.isfile(fname):
            return file(fname,'rb').read()
        else:
            raise IndexError()
    def __setitem__(self,name,value):
        if not isinstance(name,str):
            raise TypeError()
        fname = self.name2fname(name)
        if os.path.isfile(fname):
            os.unlink(fname)
        f = file(fname,'wb+')
        try:
            f.write(value)
        finally:
            f.close()
    def __delitem__(self,name):
        if not isinstance(name,str):
            raise TypeError()
        fname = self.name2fname(name)
        if os.path.isfile(fname):
            os.unlink(fname)
    def __iter__(self):
        raise NotImplementedError()
    def has_key(self,name):
        return os.path.isfile(self.name2fname(name))

class MozillaEmulator(object):
    def __init__(self,cacher={},trycount=0):
        """Create a new MozillaEmulator object.

        @param cacher: A dictionary like object, that can cache search results on a storage device.
            You can use a simple dictionary here, but it is not recommended.
            You can also put None here to disable caching completely.
        @param trycount: The download() method will retry the operation if it fails. You can specify -1 for infinite retrying.
                A value of 0 means no retrying. A value of 1 means one retry. etc."""
        self.cacher = cacher
        self.cookies = cookielib.CookieJar()
        self.debug = False
        self.trycount = trycount
    def _hash(self,data):
        h = md5.new()
        h.update(data)
        return h.hexdigest()

    def build_opener(self,url,postdata=None,extraheaders={},forbid_redirect=False):
        txheaders = {
            'Accept':'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
            'Accept-Language':'en,hu;q=0.8,en-us;q=0.5,hu-hu;q=0.3',
#            'Accept-Encoding': 'gzip, deflate',
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
#            'Keep-Alive': '300',
#            'Connection': 'keep-alive',
#            'Cache-Control': 'max-age=0',
        }
        for key,value in extraheaders.iteritems():
            txheaders[key] = value
        req = urllib2.Request(url, postdata, txheaders)
        self.cookies.add_cookie_header(req)
        if forbid_redirect:
            redirector = HTTPNoRedirector()
        else:
            redirector = urllib2.HTTPRedirectHandler()

        http_handler = urllib2.HTTPHandler(debuglevel=self.debug)
        https_handler = urllib2.HTTPSHandler(debuglevel=self.debug)

        u = urllib2.build_opener(http_handler,https_handler,urllib2.HTTPCookieProcessor(self.cookies),redirector)
        u.addheaders = [('User-Agent','Mozilla/5.0 (Windows; U; Windows NT 5.1; hu-HU; rv:1.7.8) Gecko/20050511 Firefox/1.0.4')]
        if not postdata is None:
            req.add_data(postdata)
        return (req,u)

    def download(self,url,postdata=None,extraheaders={},forbid_redirect=False,
            trycount=None,fd=None,onprogress=None,only_head=False):
        """Download an URL with GET or POST methods.

        @param postdata: It can be a string that will be POST-ed to the URL.
            When None is given, the method will be GET instead.
        @param extraheaders: You can add/modify HTTP headers with a dict here.
        @param forbid_redirect: Set this flag if you do not want to handle
            HTTP 301 and 302 redirects.
        @param trycount: Specify the maximum number of retries here.
            0 means no retry on error. Using -1 means infinite retring.
            None means the default value (that is self.trycount).
        @param fd: You can pass a file descriptor here. In this case,
            the data will be written into the file. Please note that
            when you save the raw data into a file then it won't be cached.
        @param onprogress: A function that has two parameters:
            the size of the resource and the downloaded size. This will be
            called for each 1KB chunk. (If the HTTP header does not contain
            the content-length field, then the size parameter will be zero!)
        @param only_head: Create the openerdirector and return it. In other
            words, this will not retrieve any content except HTTP headers.

        @return: The raw HTML page data, unless fd was specified. When fd
            was given, the return value is undefined.
        """
        if trycount is None:
            trycount = self.trycount
        cnt = 0
        while True:
            try:
                key = self._hash(url)
                if (self.cacher is None) or (not self.cacher.has_key(key)):
                    req,u = self.build_opener(url,postdata,extraheaders,forbid_redirect)
                    openerdirector = u.open(req)
                    if self.debug:
                        print req.get_method(),url
                        print openerdirector.code,openerdirector.msg
                        print openerdirector.headers
                    self.cookies.extract_cookies(openerdirector,req)
                    if only_head:
                        return openerdirector
                    if openerdirector.headers.has_key('content-length'):
                        length = long(openerdirector.headers['content-length'])
                    else:
                        length = 0
                    dlength = 0
                    if fd:
                        while True:
                            data = openerdirector.read(1024)
                            dlength += len(data)
                            fd.write(data)
                            if onprogress:
                                onprogress(length,dlength)
                            if not data:
                                break
                    else:
                        data = ''
                        while True:
                            newdata = openerdirector.read(1024)
                            dlength += len(newdata)
                            data += newdata
                            if onprogress:
                                onprogress(length,dlength)
                            if not newdata:
                                break
                        #data = openerdirector.read()
                        if not (self.cacher is None):
                            self.cacher[key] = data
                else:
                    data = self.cacher[key]
                #try:
                #    d2= GzipFile(fileobj=cStringIO.StringIO(data)).read()
                #    data = d2
                #except IOError:
                #    pass
                return data
            except urllib2.URLError:
                cnt += 1
                if (trycount > -1) and (trycount < cnt):
                    raise
                # Retry <img src='http://www.snippetsmania.com/wp-includes/images/smilies/icon_smile.gif' alt=':-)' class='wp-smiley' /> 
                if self.debug:
                    print "MozillaEmulator: urllib2.URLError, retryting ",cnt


    def post_multipart(self,url,fields, files, forbid_redirect=True):
        """Post fields and files to an http host as multipart/form-data.
        fields is a sequence of (name, value) elements for regular form fields.
        files is a sequence of (name, filename, value) elements for data to be uploaded as files
        Return the server's response page.
        """
        content_type, post_data = encode_multipart_formdata(fields, files)
        result = self.download(url,post_data,{
            'Content-Type': content_type,
            'Content-Length': str(len(post_data))
        },forbid_redirect=forbid_redirect
        )
        return result


class HTTPNoRedirector(urllib2.HTTPRedirectHandler):
    """This is a custom http redirect handler that FORBIDS redirection."""
    def http_error_302(self, req, fp, code, msg, headers):
        e = urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
        if e.code in (301,302):
            if 'location' in headers:
                newurl = headers.getheaders('location')[0]
            elif 'uri' in headers:
                newurl = headers.getheaders('uri')[0]
            e.newurl = newurl
        raise e


            
def encode_multipart_formdata(fields, files):
    """
    fields is a sequence of (name, value) elements for regular form fields.
    files is a sequence of (name, filename, value) elements for data to be uploaded as files
    Return (content_type, body) ready for httplib.HTTP instance
    """
    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
    CRLF = ''
    L = []
    for (key, value) in fields:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"' % key)
        L.append('')
        L.append(value)
    for (key, filename, value) in files:
        L.append('--' + BOUNDARY)
        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
        L.append('Content-Type: %s' % get_content_type(filename))
        L.append('')
        L.append(value)
    L.append('--' + BOUNDARY + '--')
    L.append('')
    body = CRLF.join(L)
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    return content_type, body

def get_content_type(filename):
    return mimetypes.guess_type(filename)[0] or 'application/octet-stream'    

def smsg(sms_message="Default Message from way2sms script",phone_number=""):
    dl = MozillaEmulator()           
    """ Make sure that we get cookies from the server before logging in """
    frontpage = dl.download("<a href="http://wwwm.way2sms.com/content/index.html",{},{},False)" >http://wwwm.way2sms.com/content/index.html",{},{},False)</a>
    """ Log in to Way2sms server """
    post_data = "login=Login&username=yourusername&password=yourpassword"
    page = dl.download("<a href="http://wwwm.way2sms.com/auth.cl",post_data,{'Referer':'http://wwwm.way2sms.com/content/index.html'},False)" >http://wwwm.way2sms.com/auth.cl",post_data,{'Referer':'http://wwwm.way2sms.com/content/index.html'},False)</a>

    if "Problem while Authentication.. Please try again" in page:
        print "Cant login"
    elif "Hello" in page:
        print "Login Successful"
    post_data = "custid=undefine&Action=custfrom100000&HiddenAction=instantsms&login=&pass=&MobNo="+phone_number+"&textArea="+sms_message
    page = dl.download("<a href="http://wwwm.way2sms.com/FirstServletsms?custid=",post_data,{'Referer':'http://wwwm.way2sms.com/jsp/InstantSMS.jsp?val=0'},False)" >http://wwwm.way2sms.com/FirstServletsms?custid=",post_data,{'Referer':'http://wwwm.way2sms.com/jsp/InstantSMS.jsp?val=0'},False)</a>
    if "Message has been submitted successfully" in page:
        print "SMS sent successfully"
    logoutpage = dl.download("<a href="http://wwwm.way2sms.com/chat/logout.cl",{},{'Referer':'http://www2.way2sms.com//jsp/Main.jsp?id=41825F6E1C22654DE17497102AB859C5.d702'},False)" >http://wwwm.way2sms.com/chat/logout.cl",{},{'Referer':'http://www2.way2sms.com//jsp/Main.jsp?id=41825F6E1C22654DE17497102AB859C5.d702'},False)</a>
    logoutpage = dl.download("<a href="http://wwwm.way2sms.com/Signut","ysignout=yahoo",{'Referer':'http://www2.way2sms.com//jsp/Main.jsp?id=41825F6E1C22654DE17497102AB859C5.d702'},False)" >http://wwwm.way2sms.com/Signut","ysignout=yahoo",{'Referer':'http://www2.way2sms.com//jsp/Main.jsp?id=41825F6E1C22654DE17497102AB859C5.d702'},False)</a>
    logoutpage = dl.download("<a href="http://wwwm.way2sms.com/LogOut","folder=inbox",{'Referer':'http://www2.way2sms.com//jsp/Main.jsp?id=41825F6E1C22654DE17497102AB859C5.d702'},False)" >http://wwwm.way2sms.com/LogOut","folder=inbox",{'Referer':'http://www2.way2sms.com//jsp/Main.jsp?id=41825F6E1C22654DE17497102AB859C5.d702'},False)</a>
    logoutpage = dl.download("<a href="http://wwwm.way2sms.com/jsp/logout.jsp","",{'Referer':'http://www2.way2sms.com//jsp/Main.jsp?id=41825F6E1C22654DE17497102AB859C5.d702'},False)" >http://wwwm.way2sms.com/jsp/logout.jsp","",{'Referer':'http://www2.way2sms.com//jsp/Main.jsp?id=41825F6E1C22654DE17497102AB859C5.d702'},False)</a>
    if 'successfully logged out' in logoutpage:
        print "Successfully logged out"



# HOW TO USE
""" Main Function """
if __name__ == "__main__":
    if len(sys.argv) < 2:
        sms_message="Default Message from way2sms script"
        phone_number="phone_number"
    elif len(sys.argv) == 2:
        sms_message=sys.argv[1]
        phone_number="phone_number"
    else:
        sms_message=sys.argv[1]
        phone_number=sys.argv[2]
    sms_message.lstrip('')
    sms_message.rstrip('')
    smsg(sms_message, phone_number)

source

List all sequences (recursively) in a dir

import os, sys
#from sets import Set
path = 'Z:/job/TOTEM/SHOTS/AZ003/Frames/render/character/'
files = os.listdir(path)

finalSeqList = []

def padFrame(frame,pad):
    return '0' * (pad - len(str(frame))) + str(frame)

def seqLS (dirPath):
    files = os.listdir(dirPath)
    for file in files:
        try:
            prefix, frame, suffix = file.split('.')
    
            # build a dictionary of the sequences as {name: frames, suffix}
            #
            # eg beauty.01.tif ... beauty.99.tif  will convert to
            # { beauty : [01,02,...,98,99], tif }
    
            try:
                result[prefix][0].append(frame)
            except KeyError:
                # we have a new file sequence, so create a new key:value pair
                result[prefix] = [[frame],suffix]   
        except ValueError:
            # the file isn't in a sequence, add a dummy key:value pair
            result[file] = file
    
    
    for prefix in result:
        if result[prefix] != prefix:
            frames = result[prefix][0]
            frames.sort()
    
            # find gaps in sequence
            startFrame = int(frames[0])
            endFrame = int(frames[-1])
            pad = len(frames[0])
            idealRange = set(range(startFrame,endFrame))
            realFrames = set([int(x) for x in frames])
            # sets can't be sorted, so cast to a list here
            missingFrames = list(idealRange - realFrames)
            missingFrames.sort()
    
            #calculate fancy ranges
            subRanges = []
            for gap in missingFrames:
                if startFrame != gap:
                    rangeStart = padFrame(startFrame,pad)
                    rangeEnd  = padFrame(gap-1,pad)
                    subRanges.append('-'.join([rangeStart, rangeEnd]))
                startFrame = gap+1
                
            subRanges.append('-'.join([padFrame(startFrame,pad), padFrame(endFrame,pad) ]))
            frameRanges = ','.join(subRanges)
            frameRanges = '[%s]' % (frameRanges)
            suffix = result[prefix][1]
            sortedList.append('.'.join([prefix, frameRanges ,suffix]))
            print ('	' + '.'.join([prefix, frameRanges ,suffix]))
        else: sortedList.append(prefix)
   
    
if __name__ == '__main__':
    if len(sys.argv) > 1:
        path = sys.argv[1]
        
    for root, dirs, files in os.walk(path):
        for curDir in dirs:
            fulldir = os.path.join(root, curDir)
            print 'Scanning : %s' % (fulldir)
            result = {}
            sortedList = []
            seqLS(fulldir)

source

adding elements of two lists

summed = [sum(pair) for pair in zip(listA, listB)]

or

from operator import add 
summed = map(add,listA,listB) 

or

summed = map(sum,zip(listA,listB))

source

python’s random

from random import *


for x in range(10):
	print random()
	

print '============================='

z = ['ciao', 'piccolo', 'bambino', 'sono', 'qua', 'io', 'in verita']

for x in z:
	# picks a random el from a list
	print choice(z)
	

print '============================='

# "Two ways to randomly attach elements"


for x in range(10):
	shuffle(z)  # shuffle modifies the element in place so you have to call it beforehand
	print " ".join(z)
	

# an alternative method
print '============================='

for x in range(10):
	print " ".join(sample(z, len(z)))

source

python Singleton

class Singleton(object):
    def __new__(cls, *p, **k):
        if not '_the_instance' in cls.__dict__:
            cls._the_instance = object.__new__(cls)
        return cls._the_instance

source

Hidden Markov Model

import numpy as np
import pymc
import pdb

def unconditionalProbability(Ptrans):
   """Compute the unconditional probability for the states of a
   Markov chain."""

   m = Ptrans.shape[0]

   P = np.column_stack((Ptrans, 1. - Ptrans.sum(axis=1)))

   I = np.eye(m)
   U = np.ones((m, m))
   u = np.ones(m)

   return np.linalg.solve((I - P + U).T, u)

data = np.loadtxt('test_data.txt',
                 dtype=np.dtype([('state', np.uint8),
                                 ('emission', np.float)]),
                 delimiter=',',
                 skiprows=1)

# Two state model for simplicity.
N_states = 2
N_chain = len(data)

# Transition probability stochastic
theta = np.ones(N_states) + 1.

def Ptrans_logp(value, theta):
   logp = 0.
   for i in range(value.shape[0]):
       logp = logp + pymc.dirichlet_like(value[i], theta)
   return logp

def Ptrans_random(theta):
   return pymc.rdirichlet(theta, size=len(theta))

Ptrans = pymc.Stochastic(logp=Ptrans_logp,
                        doc='Transition matrix',
                        name='Ptrans',
                        parents={'theta': theta},
                        random=Ptrans_random)

#Hidden states stochastic
def states_logp(value, Ptrans=Ptrans):
    
    if sum(value>1):
        return -np.inf

    P = np.column_stack((Ptrans, 1. - Ptrans.sum(axis=1)))

    Pinit = unconditionalProbability(Ptrans)

    logp = pymc.categorical_like(value[0], Pinit)

    for i in range(1, len(value)):
       try:
          logp = logp + pymc.categorical_like(value[i], P[value[i-1]])
       except:
          pdb.set_trace()

    return logp

def states_random(Ptrans=Ptrans, N_chain=N_chain):
   P = np.column_stack((Ptrans, 1. - Ptrans.sum(axis=1)))

   Pinit = unconditionalProbability(Ptrans)

   states = np.empty(N_chain, dtype=np.uint8)

   states[0] = pymc.rcategorical(Pinit)

   for i in range(1, N_chain):
       states[i] = pymc.rcategorical(P[states[i-1]])

   return states

states = pymc.Stochastic(logp=states_logp,
                        doc='Hidden states',
                        name='states',
                        parents={'Ptrans': Ptrans},
                        random=states_random,
                        dtype=np.uint8)

# Gaussian emission parameters
mu = pymc.Normal('mu', 0., 1.e-6, value=np.random.randn(N_states))
sigma = pymc.Uniform('sigma', 0., 100.,
value=np.random.rand(N_states))

y = pymc.Normal('y', mu[states], 1./sigma[states]**2,
value=data['emission'], observed=True)

source

Batch iPod Video Encoder

__author__ = "OtonVM"
__date__ = "$Jan 15, 2010 2:18:00 PM$"

import sys
import os
import subprocess
import os.path

if sys.platform == 'win32':
    source = r"C:UsersOtonDesktopHandBrake"
    if os.path.exists(r"C:UsersOtonDesktopHB Out") == False:
        os.mkdir(r"C:UsersOtonDesktopHB Out")
    destination = r"C:UsersOtonDesktopHB Out"
    encoder = sys.path[0] + r'	oolshandbrakecliHandBrakeCLI.exe'
    audioFormat = "faac"

if sys.platform == 'darwin':
    source = r"/Users/Oton/Desktop/Encode"
    if os.path.exists(r"/Users/Oton/Desktop/HB Out") == False:
        os.mkdir(r"/Users/Oton/Desktop/HB Out")
    destination = r"/Users/Oton/Desktop/HB Out"
    encoder = sys.path[0] + r'/tools/handbrakecli/HandBrakeCLI'
    audioFormat = "ca_aac"

def encode(input, output):
    for root, dirs, files in os.walk(input):
        for file in files:
            (base, ext) = os.path.splitext(file)
            if ext.lower() == ".avi" or ext.lower() == ".mkv" or ext.lower() == ".mp4" or ext.lower() == ".m4v" or ext.lower() == ".mov":
                newfile = os.path.join(output, base + ".m4v")
                infile = os.path.join(root, file)
                if os.path.exists(newfile):
                    print('Skipping file: ' + infile)
                else:
                    encode = encoder + ' --encoder x264 --quality 30.0 --audio 1 --aencoder ' + audioFormat + ' --arate 44.1 --mixdown dpl2 --ab 96 --drc 2.0 --format mp4 --ipod-atom --maxHeight 240 --keep-display-aspect --modulus 16 --vfr --x264opts level=30:bframes=0:mbtree=1:weightp=0:cabac=0:ref=1:vbv-maxrate=768:vbv-bufsize=2000:analyse=all:8x8dct=0:me=umh:no-fast-pskip=1:no-dct-decimate=1:filter=-1,-1:qpmin=16:merange=128:subme=9:mixed-refs=0:trellis=0 --verbose 2' + ' -i "' + infile + '" -o "' + newfile + '"'
                    subprocess.call(encode)
                    os.remove(infile)

if __name__ == "__main__":
    encode(source, destination)

source

python web with bottle and session (beaker)

#!/usr/bin/python
# -*- coding: utf-8 -*-
import bottle as web
from beaker.middleware import SessionMiddleware

@web.route('/')
def index():
    session=web.request.environ["beaker.session"]
    if "cpt" in session:
        session["cpt"]+=1
    else:
        session["cpt"]=1
    return "cpt:"+str(session["cpt"])

if __name__=="__main__":

    session_opts = {
        "session.type": "file",
        'session.cookie_expires': True,
        'session.auto': True,
        'session.data_dir': "cache",
    }

    app = web.default_app()
    
    app = SessionMiddleware(app, session_opts)

    web.run(app=app,reloader=True)

source