Export emails from Outlook

"""
Exports email address from MS Outlook from recent eemails
"""

from win32com.client import Dispatch, constants
import os.path

class Exporter(object):

    app = None
    emails = []
    
    def __init__ (self):

 self.app = Dispatch("Outlook.Application")
        if self.app == None:
            raise Exception, "Unable to create an Outlook application object"

    def export_emails(self, foldername, outfile="email.csv", maxcount=1000):

        emails = [] 
 ns = self.app.GetNamespace("MAPI")
        folders = ns.Folders.Item(1).Folders #Usually the users folders
        folder = folders.Item(foldername)
        cnt = len(folder.Items)
        if cnt > maxcount:
            cnt = maxcount
        for i in range(1, cnt + 1):
            item = folder.Items.Item(i)
            print "Processing item %d" % i
            self.export_from_field(item, "To")
            self.export_from_field(item, "CC")

        self.emails.sort()
        emailfile = open(outfile, 'w')
        for email in self.emails:
            emailfile.write(email + "\n")
        emailfile.close()

    def export_from_field(self, item, fieldname, maxcount = 10):
        if not hasattr(item, fieldname):
            return
        ms = getattr(item, fieldname).split(';')
        if len(ms) > maxcount:
            return
        for m in ms:
            parts = m.split(',')
            if len(parts) <> 2:
                continue
            email = "%s_%s@intuit.com" % (parts[1].strip(), parts[0].strip())
            if not email in self.emails:
                self.emails.append(email)
     
    def __del__ (self):

        self.app = None
        self.emails = None

def main ():
    export = Exporter()
    export.export_emails("InBox", "emails.csv", 1000)
    
if __name__ == "__main__":
    main();

Web page smoke test script

#!/usr/local/bin python

"""
This script is used to check urls with a pattern. The configuration file has following format

    protocol_name|url|additional_args

example:
    simpleGET|http://www.google.com/|Google
    simpleGET|http://www.aol.com/|WhatWhat

Example output:
    http://www.google/ -> True
    http://www.aol.com/ -> False

"""

import os.path, re, getopt, sys, urllib2, urllib

verbose = False
FETCHER = 'fetcher'
MATCHER = 'matcher'

def say(s):
    if verbose:
        print s

class UrlChecker:

    def __init__(self, urls, protocols):
        self.urls = urls
        self.protocols = protocols

    def check(self):
        for rurl in self.urls:
            self._check_url(rurl)
            
    def _check_url(self, rurl):
        protocol, url, args = rurl.split('|')
        say("Check URL %s with protocol %s. Args = %s" % (url, protocol, args))
        rslt = self.protocols[protocol][FETCHER](url, args)
        self.protocols[protocol][MATCHER](rslt, url, args)

def load_urls(urlfile):
    urls = []
    f = open(urlfile, 'r')
    try:
        for line in f:
            say("Load one raw url %s" % (line.strip()))
            urls.append(line.strip())
    finally:
        f.close()
    return urls

def add_simpleGET(protocols):

    def simple_get(url, ignore):
        r = urllib2.urlopen(url)
        rslt = r.read()
        r.close()
        return rslt

    def str_match(body, url, pattern):
        print "%s -> %s" % (url, body.find(pattern) != -1)

    protocols['simpleGET'] = {FETCHER : simple_get, MATCHER : str_match}

def init_protocols():
    protocols = {};
    add_simpleGET(protocols)
    return protocols

if __name__ == '__main__':

    def usage():
        print "Usage:"
        print "python check_url.py -f <url_file> -v"

    try:                                
        opts, args = getopt.getopt(sys.argv[1:], "f:v")
    except getopt.GetoptError:
        usage()
        sys.exit(2)                     

    urls = None
    protocols = init_protocols()
    for opt, arg in opts:
        if opt in ("-v"):
            verbose = True
        elif opt in ("-f"):
            urls = load_urls(arg)
    if urls is None:
        usage()
        sys.exit(1)

    checker = UrlChecker(urls, protocols)
    checker.check()

Email forwarding script

#!/usr/local/bin python

"""
This script is used to forward email
"""

import os, sys, poplib, getopt, getpass, email, smtplib

from_email = '???'
to_email = '???'
pop3_server_url = '???'
pop3_server_port = 110
pop3_user = '???'
pop3_pass = '???'
delete_after = True
subject_filter = '????'

def match_filter(subject, sfilter):
    return subject.find(sfilter) == 0

def forward_mail(femail, temail, mail):

    for (header, value) in mail.items():
        if not header in ('Content-Type', 'Content-Transfer-Encoding', 'Subject', 'Mime-Version'):
            del mail[header]

    mail["From"] = femail
    mail["To"] = temail
    for (header, value) in mail.items():
        print "%s: %s" % (header, value)

    server = smtplib.SMTP('localhost')
    server.set_debuglevel(0)
    server.sendmail(femail, temail, mail.as_string())
    server.quit()    

def process_mail(pop3_mail):
    
    mail = email.message_from_string(pop3_mail)
    subject = mail.get("Subject")
    if match_filter(subject, subject_filter):
        print "Forward email: %s" % (subject)
        forward_mail(from_email, to_email, mail)
        return True
    else:
        print "Skip email: %s" % (subject)
        return False

if __name__ == '__main__':

    def usage():

        print "Usag:"
        print "python forward_mail.py -s <pop3_server> -p <pop3_port> -u <user> -c <credential> -f <from_email> -t <to_email>"

    try:                                
        opts, args = getopt.getopt(sys.argv[1:], "s:p:u:c:f:t:")
    except getopt.GetoptError:
        usage()
        sys.exit(2)                     

    for opt, arg in opts:
        if opt in ("-s"):
            pop3_server_url = arg
        elif opt in ("-p"):
            pop3_server_port = int(arg)
        elif opt in ("-u"):
            pop3_user = arg
        elif opt in ("-c"):
            pop3_pass = arg
        elif opt in ("-f"):
            from_email = arg
        elif opt in ("-t"):
            to_email = arg

    if pop3_user is None:
        pop3_user = getpass.getUser()
    if pop3_pass is None:
        pop3_pass = getpass.getpass()

    pop3 = poplib.POP3(pop3_server_url, pop3_server_port)
    pop3.user(pop3_user)
    pop3.pass_(pop3_pass)
    msg_count = len(pop3.list()[1])
    print "Total messages in Inbox: %d" % (msg_count)

    for i in range(msg_count):
        resp, text, octets = pop3.retr(i+1)
        if process_mail('\n'.join(text)) and delete_after:
            print "Forwarded email is deleted"
            pop3.dele(i+1)
    pop3.quit()

Search text pattern in log files

#!/usr/local/bin python

"""
This script is used to analyze log file
"""

import os.path, time, re, getopt, sys, fnmatch

verbose = False

def say(s):
   if verbose:
       print s

def searchPattern(args, dirname, filenames):

   file_pattern, text_pattern, count = args
   say("Processing directory %s now" % (dirname))
   say("File pattern: %s" % (file_pattern))
   say("Text pattern: %s" % (text_pattern))
   say("Count only: %s" % (count))

   counts = {}
   def incMatchCount(matched):
       counts[matched] = counts.get(matched, 0) + 1
  
   compiled_pattern = re.compile(text_pattern)
   for filename in fnmatch.filter(filenames, file_pattern):
       say("Processing file %s now" % (filename))
       log = open(os.path.join(dirname, filename), 'r')
       for line in log:
           m = re.match(compiled_pattern, line)
           if m:
               if count:
                   incMatchCount(m.group(1))
               else:
                   print ';;'.join(m.groups())
       if count:
           for matched in counts.keys():
               print " -> ".join([matched, str(counts[matched])])

if __name__ == '__main__':

   def usage():
       print "Usage:"
       print "python find_pattern.py -d <root_dir> -f <file_pattern> -p <search_pattern> -c -v"

   try:                               
       opts, args = getopt.getopt(sys.argv[1:], "d:f:p:cv")
   except getopt.GetoptError:
       usage()
       sys.exit(2)                    

   count = False
   root_dir = '.'
   file_pattern = '*.log'
   text_pattern = '(.+)'
   for opt, arg in opts:
       if opt in ("-v"):
           verbose = True
       elif opt in ("-c"):
           count = True
       elif opt in ("-d"):
           root_dir = arg
       elif opt in ("-f"):
           file_pattern = arg
       elif opt in ("-p"):
           text_pattern = arg

   os.path.walk(root_dir, searchPattern, (file_pattern, text_pattern, count))

Python utility for generating certificate

Here is a Python script that generates a certificate with various formats, including JKS and PFX. It also requires JDK and OpenSSL being installed.

-----------------------------

#!/usr/local/bin python

"""
Create a certificate with Python.
"""

import urllib, sys, getopt, os, shutil
from M2Crypto import SSL, httpslib
from M2Crypto import RSA, X509, EVP, m2, Rand, Err

keystorepass = 'secret'

def passphrase_callback(v):
  return keystorepass

def generateRSAKey():
  return RSA.gen_key(1024, m2.RSA_F4)

def makePKey(key):
  pkey = EVP.PKey()
  pkey.assign_rsa(key)
  return pkey

def makeRequest(pkey, server_dns):
  req = X509.Request()
  # Seems to default to 0, but we can now set it as well, so just API test
  req.set_version(req.get_version())
  req.set_pubkey(pkey)
  name = X509.X509_Name()
  name.CN = server_dns
  name.OU = 'My Unit'
  name.O = 'My Company'
  name.L = 'My City'
  name.ST = 'My State'
  name.C = 'US'
  req.set_subject_name(name)
  ext1 = X509.new_extension('Comment', 'Auto Generated')
  extstack = X509.X509_Extension_Stack()
  extstack.push(ext1)
  req.add_extensions(extstack)
  req.sign(pkey, 'md5')
  return req

def sendRequest(crtreq):
  # send to a web service to sign the certificate
  return crtresp

def extractCert(crtresp):

  crt = ''
  # extract crt from the response
  return crt

def createJKS(fqdn):

  java_home = os.environ['JAVA_HOME']
  if java_home is None:
      raise 'JAVA_HOME needs to be set.'

  # an empty Java keystore         
  jks_template = 'template.jks'
  jksfile = ''.join([fqdn, '.jks'])
  certfile = ''.join([fqdn, '.crt'])
  keyfile = ''.join([fqdn, '.der'])
  shutil.copyfile(jks_template, jksfile)

  os.spawnl(os.P_WAIT, ''.join([java_home, '/bin/java.exe']), 'java', '-cp', '.', 'KeyStoreImport', jksfile, keystorepass, fqdn, certfile, keyfile, keystorepass)

def createPFX(fqdn):

  openssl_home = "c:/tools/openssl"
  if openssl_home is None:
      raise 'openssl needs to be installed.'

  certfile = ''.join([fqdn, '.crt'])
  keyfile = ''.join([fqdn, '.key'])
  pfxfile = ''.join([fqdn, '.pfx'])
  inpass = ":".join(['pass', keystorepass])
  outpass = ":".join(['pass', keystorepass])

  os.spawnl(os.P_WAIT, ''.join([openssl_home, '/bin/openssl.exe']), 'openssl', 'pkcs12', '-export', '-inkey', keyfile, \
            '-in', certfile, '-out', pfxfile, '-passin', inpass, '-passout', outpass)

def moveFiles(fqdn):

  if os.path.isdir(fqdn):
      os.rmdir(fqdn)
  os.mkdir(fqdn)
  shutil.move(''.join([fqdn, '.key']), fqdn)
  shutil.move(''.join([fqdn, '.der']), fqdn)
  shutil.move(''.join([fqdn, '.crt']), fqdn)
  shutil.move(''.join([fqdn, '.jks']), fqdn)
  shutil.move(''.join([fqdn, '.pfx']), fqdn)

def makeCert(fqdn):

  print '####### Generate RSA Key #######'
  rsa = generateRSAKey()
  rsa.save_key(''.join([fqdn, '.key']), cipher='aes_256_cbc', callback=passphrase_callback)
  rsa.save_key_der(''.join([fqdn, '.der']))

  print '####### Generate Pub/Pri Keys #######'
  pkey = makePKey(rsa)

  print '####### Generate Certificate Request #######'
  req = makeRequest(pkey, fqdn)

  print '####### Generate Certificate Request PEM #######'
  crtreq = req.as_pem()
  print '####### Certificate Request #######'
  print crtreq
  print '####### Certificate Request #######', '\n'

  crtresp = sendRequest(crtreq, fqdn)
  # print '####### Certificate Response #######'
  # print crtresp
  # print '####### Certificate Response #######', '\n'
  crtfile = open(''.join([fqdn, '.crt']), 'w')
  crtfile.write(crtresp)
  crtfile.close()

  crtfile = open(''.join([fqdn, '.crt']), 'r')
  crt = extractCert(crtfile)
  print '####### Certificate #######'
  print crt
  print '####### Certificate #######', '\n'
  crtfile.close()

  crtfile = open(''.join([fqdn, '.crt']), 'w')
  crtfile.write(crt)
  crtfile.close()

  print '####### Generate JKS #######'
  createJKS(fqdn)

  print '####### Generate PFX #######'
  createPFX(fqdn)

  print '####### Move Files #######'
  moveFiles(fqdn)

  print "Certificate generated"

def usage():

  print "Usage:"
  print "python gencrt.py -n <fqdn>"

if __name__ == '__main__':

  try:                             
      opts, args = getopt.getopt(sys.argv[1:], "hn:", ["help", "fqdn="])
  except getopt.GetoptError:
      usage()
      sys.exit(2)                  

  fqdn = None
  for opt, arg in opts:
      if opt in ("-h", "--help"):
          usage()  
          sys.exit()
      elif opt in ("-n", "--fqdn"):
          fqdn = arg

  if fqdn == None:
      usage()  
      sys.exit(2)
       
  makeCert(fqdn)</fqdn>

A Python script to sync. with Perforce

I have been using Fitnesse for a while. But I am not happy with the file system based backup. There is no easy way to setup a SCM back end for the Fitnesse. So I end up have to write a Python script to periodically sync. my local page with a Perforce depot. It works quite convenient.

Mapping xml to Python objects

I need to process XML in my python code and prefer to deal with the data in the pythonic way. From a wide options available, I choose to play with 3 of them.

* generateDS
* gnosis.xml.objectify
* amara

Because I am a fan of XmlBeans, it seems that generateDS should be my favorite choice. However, when I tried to run the code-generating script on my schema file, it encountered some recursive problem and complained:

.... parent.collectElementNames(elementNames) File "C:\Python\Lib\site-packages\generateDS.py", line 478, in collectElementNames parent.collectElementNames(elementNames) File "C:\Python\Lib\site-packages\generateDS.py", line 475, in collectElementNames base = self.getBase() RuntimeError: maximum recursion depth exceeded

The same schema file works fine with XMLBeans however.

gnosis.xml.objectify and amara are similar in that they don't require a schema file and will just turn any well-formed xml file into python objects. I had no problem parse the xml files with both modules. However, the gnosis.xml.objectify module seems not aware of the namespace prefix. Given following xml element:

<ns:book xmlns:ns="http://mycompany/bookproject">..

It generates an object called ns_book. It does not recognize : as a namespace prefix and just blindly translates it to _. I tried to search for a solution, but could not find anything.

The amara module, however, correct translates the object name to book and also keeps the namespace information. It seems that I will use this module at this time.

Successfully installed Twisted

I need to install the latest Twisted with Python 2.4. Unfortunately I don't have Visual Studio on my machine. Is there a way to do it. I find this excellent instruction on how to install Python extensions without VS. It works as advertised. I am happily running Twisted now!

Python web service

In my previous post, I have discussed using Perl (SoapLite) as client to consume a web service implemented using Java(Apache Axis). I am curious to find how it will work out for other lanuages such as Python and Ruby. Here is my try for Python.

The tool I use is ZSI which stands for Zolera SOAP Infrastructure. It provides a tool wsdl2py that can generate Python stub code from a wsdl file. Run the tool on the wsdl file described in my previous post, I get two Python modules: MessageRouter_services_types.py and MessageRouter_services.py. With the help of these two modules, consuming a web service is straight forward. Here is the code:

def sendTo(destination, payload):

 loc = MessageRoutingServiceLocator()
 kw = { 'tracefile' : sys.stdout }
 portType = loc.getMessageRouterPortType(**kw)

 dest = ns1.DestinationType_Def()
 dest._protocol = "http"
 dest._category = "post"
 dest._server = "127.0.0.1"
 dest._port = 80
 dest._target = "/router"

 msg = ns1.MessageBodyType_Def()
 msg._content = "payload"

 req = RouteRequestWrapper()
 req._destination = dest
 req._request = msg

 try:
  response = portType.RouteMessage(req)._response._content
  print "Response message is:\n %s" % response
 except Exception, e:
  response = repr(e)
  print "Failed: ", e
  raise e