from __future__ import generators # This module is part of the spambayes project, which is Copyright 2002-3 # The Python Software Foundation and is covered by the Python Software # Foundation license. # Simple Python library for Outlook Express mailboxes handling # Based on C++ work by Arne Schloh __author__ = "Romain Guy" __credits__ = "All the SpamBayes folk" import binascii import os import struct import msgs import StringIO import sys from time import gmtime, strftime try: import win32api import win32con from win32com.shell import shell, shellcon except ImportError: # Not win32, or win32all not installed. # Some functions will not work, but some will. win32api = win32con = shell = shellcon = None ########################################################################### ## DBX FILE HEADER ########################################################################### class dbxFileHeader: """ Each Outlook Express DBX file has a file header. This header defines many properties, only a few of which interest us. The only properties which are required are defined by indexes. The indexes are static attributes of the class and their names begin with "fh". You can access their values through the method getEntry(). """ HEADER_SIZE = 0x24bc # total header size HEADER_ENTRIES = HEADER_SIZE >> 2 # total of entries in the header MAGIC_NUMBER = 0xfe12adcfL # specific to DBX files OFFLINE = 0x26fe9d30L # specific to offline.dbx FOLDERS = 0x6f74fdc6L # specific to folders.dbx POP3UIDL = 0x6f74fdc7L # specific to pop3uidl.dbx # various entries indexes FH_FILE_INFO_LENGTH = 0x07 # file info length FH_FIRST_FOLDER_LIST_NODE = 0x1b # pointer to the first folder list node FH_LAST_FOLDER_LIST_NODE = 0x1c # pointer to the last folder list node FH_MESSAGE_CONDITIONS_PTR = 0x22 # pointer to the message conditions object FH_FOLDER_CONDITIONS_PTR = 0x23 # pointer to the folder conditions object FH_ENTRIES = 0x31 # entries in tree FH_TREE_ROOT_NODE_PTR = 0x39 # pointer to the root node of a tree FILE_HEADER_ENTRIES = \ [ ( 0x07, "file info length" ), ( 0x09, "pointer to the last variable segment" ), ( 0x0a, "length of a variable segment" ), ( 0x0b, "used space of the last variable segment" ), ( 0x0c, "pointer to the last tree segment" ), ( 0x0d, "length of a tree segment" ), ( 0x0e, "used space of the last tree segment" ), ( 0x0f, "pointer to the last message segment" ), ( 0x10, "length of a message segment" ), ( 0x11, "used space of the last message segment" ), ( 0x12, "root pointer to the deleted message list" ), ( 0x13, "root pointer to the deleted tree list" ), ( 0x15, "used space in the middle sector of the file" ), ( 0x16, "reusable space in the middle sector of the file" ), ( 0x17, "index of the last entry in the tree" ), ( 0x1b, "pointer to the first folder list node" ), ( 0x1c, "pointer to the last folder list node" ), ( 0x1f, "used space of the file" ), ( 0x22, "pointer to the message conditions object" ), ( 0x23, "pointer to the folder conditions object" ), ( 0x31, "entries in the tree" ), ( 0x32, "entries in the 2.nd tree" ), ( 0x33, "entries in the 3.rd tree" ), ( 0x39, "pointer to the root node of the tree" ), ( 0x3a, "pointer to the root node of the 2.nd tree" ), ( 0x3b, "pointer to the root node of the 3.rd tree" ), ( 0x9f, "used space for indexed info objects" ), ( 0xa0, "used space for conditions objects" ), ( 0xa2, "used space for folder list objects" ), ( 0xa3, "used space for tree objects" ), ( 0xa4, "used space for message objects" )] def __init__(self, dbxStream): """Initialize the DBX header by reading it directly from the passed stream.""" dbxStream.seek(0) self.dbxBuffer = dbxStream.read(dbxFileHeader.HEADER_SIZE) def isMessages(self): """Return true iff the DBX is a messages DBX.""" return not (self.isFolders() or self.isPOP3UIDL() or self.isOffline()) def isFolders(self): """Return true if the DBX is the folders DBX.""" return self.getEntry(1) == dbxFileHeader.FOLDERS def isPOP3UIDL(self): """Return true if the DBX is the POP3UIDL DBX.""" return self.getEntry(1) == dbxFileHeader.POP3UIDL def isOffline(self): """Return true if the DBX is the offline DBX.""" return self.getEntry(1) == dbxFileHeader.OFFLINE def isValid(self): """Return true if the DBX is a valid DBX file.""" return self.getEntry(0) == dbxFileHeader.MAGIC_NUMBER def getHeaderBuffer(self): """Return the bytes buffer containing the whole header.""" return self.dbxBuffer def getEntry(self, dbxEntry): """Return the n-th entry as a long integer.""" return struct.unpack("L", self.dbxBuffer[dbxEntry * 4:(dbxEntry * 4) + 4])[0] def getEntryAsHexStr(self, dbxEntry): """Return the n-th entry as an hexadecimal string. (Little endian encoding!)""" return '0x' + \ binascii.hexlify(self.dbxBuffer[dbxEntry * 4:(dbxEntry * 4) + 4]) ########################################################################### ## DBX FILE INFO ########################################################################### class dbxFileInfo: """ Following the DBX header there is DBX info. This part gives the name of the folder described by the current DBX. """ MESSAGE_FILE_INFO = 0x618 def __init__(self, dbxStream, dbxLength): """Reads the DBX info part from a DBX stream.""" dbxStream.seek(dbxFileHeader.HEADER_SIZE) self.dbxLength = dbxLength self.dbxBuffer = dbxStream.read(dbxLength) def isFoldersInfo(self): """Return true if the info belongs to folders.dbx.""" return self.dbxLength != dbxFileInfo.MESSAGE_FILE_INFO def getFolderName(self): """Returns the folder name.""" if not self.isFoldersInfo(): name = [c for c in self.dbxBuffer[0x105:0x210] if ord(c) != 0] return "".join(name) else: return None def getCreationTime(self): """Not implemented yet.""" if self.isFoldersInfo(): return "Not implemented yet" else: return None ########################################################################### ## DBX TREE ########################################################################### class dbxTree: """Stands for the tree which stores the messages in a given folder.""" TREE_NODE_SIZE = 0x27c # size of a tree node def __init__(self, dbxStream, dbxAddress, dbxValues): """Reads the addresses of the stored messages.""" self.dbxValues = [i for i in range(dbxValues)] # XXX : silly fix ! if dbxAddress > 0: self.__readValues(dbxStream, 0, dbxAddress, 0, dbxValues) def __readValues(self, dbxStream, dbxParent, dbxAddress, dbxPosition, dbxValues): dbxStream.seek(dbxAddress) dbxBuffer = dbxStream.read(dbxTree.TREE_NODE_SIZE) count = 0 entries = ((self.getEntry(dbxBuffer, 4) >> 8) & 0xff) if self.getEntry(dbxBuffer, 2) != 0: self.__readValues(dbxStream, dbxAddress, self.getEntry(dbxBuffer, 2), dbxPosition, self.getEntry(dbxBuffer, 5)) count += self.getEntry(dbxBuffer, 5) for i in range(entries): pos = 6 + i * 3 if self.getEntry(dbxBuffer, pos) != 0: count += 1 value = dbxPosition + count self.dbxValues[value - 1] = self.getEntry(dbxBuffer, pos) if self.getEntry(dbxBuffer, pos + 1) != 0: self.__readValues(dbxStream, dbxAddress, self.getEntry(dbxBuffer, pos + 1), dbxPosition + count, self.getEntry(dbxBuffer, pos + 2)) count += self.getEntry(dbxBuffer, pos + 2) def getEntry(self, dbxBuffer, dbxEntry): """Return the n-th entry as a long integer.""" return struct.unpack("L", dbxBuffer[dbxEntry * 4:(dbxEntry * 4) + 4])[0] def getValue(self, dbxIndex): """Return the address of the n-th message.""" return self.dbxValues[dbxIndex] ########################################################################### ## DBX INDEXED INFO ########################################################################### class dbxIndexedInfo: """ Messages and folders mailboxes contain the "message info" and "folders info" entities. These entities are indexed info sequences. This is their base class. """ MAX_INDEX = 0x20 # max index DT_NONE = 0 # data type none def __init__(self, dbxStream, dbxAddress): """Reads the indexed infos from the passed stream.""" self.dbxBodyLength = 0L self.dbxObjectLength = 0L self.dbxEntries = 0L self.dbxCounter = 0L self.dbxBuffer = [] self.dbxIndexes = 0L self.dbxBegin = [0L for i in range(dbxIndexedInfo.MAX_INDEX)] self.dbxLength = [i for i in self.dbxBegin] self.dbxAddress = dbxAddress self.__readIndexedInfo(dbxStream) def __readIndexedInfo(self, dbxStream): dbxStream.seek(self.dbxAddress) temp = dbxStream.read(12) self.dbxBodyLength = self.__getEntry(temp, 1) self.dbxObjectLength = self.__getEntry(temp, 2) & 0xffff self.dbxEntries = (self.__getEntry(temp, 2) >> 16) & 0xff self.dbxCounter = (self.__getEntry(temp, 1) >> 24) & 0xff self.dbxBuffer = dbxStream.read(self.dbxBodyLength) # bytes array isIndirect = bool(0) # boolean lastIndirect = 0 data = self.dbxEntries << 2 # index within dbxBuffer for i in range(self.dbxEntries): value = self.__getEntry(self.dbxBuffer, i) isDirect = value & 0x80 index = value & 0x7f value >>= 8 if isDirect: self.__setIndex(index, (i << 2) + 1, 3) else: self.__setIndex(index, data + value) if isIndirect: self.__setEnd(lastIndirect, data + value) isIndirect = bool(1) lastIndirect = index self.dbxIndexes |= 1 << index if isIndirect: self.__setEnd(lastIndirect, self.dbxBodyLength) def __setIndex(self, dbxIndex, dbxBegin, dbxLength = 0): if dbxIndex < dbxIndexedInfo.MAX_INDEX: self.dbxBegin[dbxIndex] = dbxBegin self.dbxLength[dbxIndex] = dbxLength def __setEnd(self, dbxIndex, dbxEnd): if dbxIndex < dbxIndexedInfo.MAX_INDEX: self.dbxLength[dbxIndex] = dbxEnd - self.dbxBegin[dbxIndex] def __getEntry(self, dbxBuffer, dbxEntry): return struct.unpack("L", dbxBuffer[dbxEntry * 4:(dbxEntry * 4) + 4])[0] def getIndexText(self, dbxIndex): """Returns the description of the given indexed field.""" return "" def getIndexDataType(self, dbxIndex): """Returns the data type of the given index.""" return DT_NONE def getValue(self, dbxIndex): """Returns a tuple : (index in buffer of the info, length of the info).""" return (self.dbxBegin[dbxIndex], self.dbxLength[dbxIndex]) def getValueAsLong(self, dbxIndex): """Returns the indexed info as a long value.""" data, length = self.getValue(dbxIndex) value = 0 if data: value = struct.unpack("L", self.dbxBuffer[data:data + 4])[0] if length < 4: value &= (1 << (length << 3)) - 1 return value def getString(self, dbxIndex): """Returns the indexed info as a string value.""" index = self.dbxBegin[dbxIndex] end = index for c in self.dbxBuffer[index:]: if ord(c) == 0: break end += 1 return self.dbxBuffer[index:end] def getAddress(self): return self.dbxAddress def getBodyLength(self): return self.dbxBodyLength def getEntries(self): return self.dbxEntries def getCounter(self): return self.dbxCounter def getIndexes(self): return self.dbxIndexes def isIndexed(self, dbxIndex): return self.dbxIndexes & (1 << dbxIndex) ########################################################################### ## DBX MESSAGE INFO ########################################################################### class dbxMessageInfo(dbxIndexedInfo): """ The message info structure inherits from the index info one. It just defines extra constants which allow to access pertinent info. """ MI_INDEX = 0x0 # index of the message MI_FLAGS = 0x1 # the message flags MI_MESSAGE_ADDRESS = 0x4 # the address of the message MI_SUBJECT = 0x8 # the subject of the message # label of each indexed info INDEX_LABEL = \ [ "message index" , "flags" , "time message created/send" , "body lines" , "message address" , "original subject" , "time message saved" , "message id" , "subject" , "sender eMail address and name" , "answered to message id" , "server/newsgroup/message number", "server" , "sender name" , "sender eMail address" , "id 0f" , "message priority" , "message text length" , "time message created/received", "receiver name" , "receiver eMail address" , "id 15" , "id 16" , "id 17" , "id 18" , "id 19" , "OE account name" , "OE account registry key" , "message text structure" , "id 1d" , "id 1e" , "id 1f" ] DT_NONE = 0 # index is none DT_INT4 = 1 # index is a long integer (32 bits) DT_STRING = 2 # index is a string DT_DATE_TIME = 3 # index is date/time DT_DATA = 4 # index is data # the data type of each index INDEX_DATA_TYPE = \ [ DT_INT4 , DT_INT4 , DT_DATE_TIME, DT_INT4 , DT_INT4 , DT_STRING, DT_DATE_TIME, DT_STRING, DT_STRING, DT_STRING, DT_STRING , DT_STRING, DT_STRING, DT_STRING, DT_STRING , DT_NONE , DT_INT4 , DT_INT4 , DT_DATE_TIME, DT_STRING, DT_STRING, DT_NONE , DT_INT4 , DT_NONE , DT_INT4 , DT_INT4 , DT_STRING , DT_STRING, DT_DATA , DT_NONE , DT_NONE , DT_NONE ] def getIndexText(self, dbxIndex): return dbxMessageInfo.INDEX_LABEL[dbxIndex] def getIndexDataType(self, dbxIndex): return dbxMessageInfo.INDEX_DATA_TYPE[dbxIndex] ########################################################################### ## DBX MESSAGE ########################################################################### class dbxMessage: def __init__(self, dbxStream, dbxAddress): self.dbxAddress = dbxAddress self.dbxText = "" self.dbxLength = 0L self.__readMessageText(dbxStream) def __getEntry(self, dbxBuffer, dbxEntry): if len(dbxBuffer) < (dbxEntry * 4) + 4: return None return struct.unpack("L", dbxBuffer[dbxEntry * 4:(dbxEntry * 4) + 4])[0] def __readMessageText(self, dbxStream): address = self.dbxAddress header = "" while (address): dbxStream.seek(address) header = dbxStream.read(16) self.dbxLength += self.__getEntry(header, 2) address = self.__getEntry(header, 3) pos = "" address = self.dbxAddress while (address): dbxStream.seek(address) header = dbxStream.read(16) pos += dbxStream.read(self.__getEntry(header, 2)) address = self.__getEntry(header, 3) self.dbxText = pos def getText(self): return self.dbxText # This started its SpamBayes life as a private method of the UserInterface # class, but is really a general purpose (Outlook Express) function. def convertToMbox(content): """Check if the given buffer is in a non-mbox format, and convert it into mbox format if so. If it's already an mbox, return it unchanged. """ dbxStream = StringIO.StringIO(content) header = dbxFileHeader(dbxStream) if header.isValid() and header.isMessages(): file_info_len = dbxFileHeader.FH_FILE_INFO_LENGTH fh_entries = dbxFileHeader.FH_ENTRIES fh_ptr = dbxFileHeader.FH_TREE_ROOT_NODE_PTR info = dbxFileInfo(dbxStream, header.getEntry(file_info_len)) entries = header.getEntry(fh_entries) address = header.getEntry(fh_ptr) if address and entries: tree = dbxTree(dbxStream, address, entries) dbxBuffer = "" for i in range(entries): address = tree.getValue(i) messageInfo = dbxMessageInfo(dbxStream, address) if messageInfo.isIndexed(dbxMessageInfo.MI_MESSAGE_ADDRESS): address = dbxMessageInfo.MI_MESSAGE_ADDRESS messageAddress = messageInfo.getValueAsLong(address) message = dbxMessage(dbxStream, messageAddress) # This fakes up a from header to conform to mbox # standards. It would be better to extract this # data from the message itself, as this will # result in incorrect tokens. dbxBuffer += "From spambayes@spambayes.org %s\n%s" \ % (strftime("%a %b %d %H:%M:%S MET %Y", gmtime()), message.getText()) content = dbxBuffer dbxStream.close() return content def OEIdentityKeys(): """Return the OE identity keys. Tested with Outlook Express 6.0 with Windows XP.""" if sys.platform != "win32": # AFAIK, there is only a Win32 OE, and a Mac OE. # The Mac OE should be easy enough, but I don't know # where the dbx files are stored (I presume they are in the # same format). raise NotImplementedError if win32api is None: # Delayed import error from top. raise ImportError("win32all not installed") reg = win32api.RegOpenKeyEx(win32con.HKEY_USERS, "") user_index = 0 while True: # Loop through all the users try: user_name = "%s\\Identities" % \ (win32api.RegEnumKey(reg, user_index),) except win32api.error: break user_index += 1 try: user_key = win32api.RegOpenKeyEx(win32con.HKEY_USERS, user_name) except win32api.error: # Not this one continue identity_index = 0 while True: # Loop through all the identities try: identity_name = win32api.RegEnumKey(user_key, identity_index) except win32api.error: break identity_index += 1 subkey_name = "%s\\%s\\%s" % (user_name, identity_name, "Software\\Microsoft\\Outlook " \ "Express\\5.0") try: subkey = win32api.RegOpenKeyEx(win32con.HKEY_USERS, subkey_name, 0, win32con.KEY_READ) except win32api.error: # Not this user continue yield subkey def OEStoreRoot(): """Return the path to the Outlook Express Store Root. Tested with Outlook Express 6.0 with Windows XP.""" # Run through the identity keys, using the first that # works. raw = "" for identity in OEIdentityKeys(): try: raw = win32api.RegQueryValueEx(identity, "Store Root") except win32api.error: pass else: break # I can't find a shellcon to that is the same as %UserProfile%, # so extract it from CSIDL_LOCAL_APPDATA UserDirectory = shell.SHGetFolderPath \ (0, shellcon.CSIDL_LOCAL_APPDATA, 0, 0) parts = UserDirectory.split(os.sep) UserProfile = os.sep.join(parts[:-2]) raw = raw[0].replace("%UserProfile%", UserProfile) return raw def OEAccountKeys(permission = None): """Return registry keys for each of the OE mail accounts, along with information about what type of mail account it is.""" if permission is None: # Can't do this in the parameter, because then it requires # win32con to be available for the module to be imported. permission = win32con.KEY_READ | win32con.KEY_SET_VALUE possible_root_keys = [] # This appears to be the place for OE6 and WinXP # (So I'm guessing also for NT4) if sys.getwindowsversion()[0] >= 4: possible_root_keys = ["Software\\Microsoft\\" \ "Internet Account Manager\\Accounts"] else: # This appears to be the place for OE6 and Win98 # (So I'm guessing also for Win95) possible_root_keys = oe_mailbox.OEIdentityKeys() for key in possible_root_keys: reg = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER, key) account_index = 0 while True: # Loop through all the accounts account = {} try: subkey_name = "%s\\%s" % \ (key, win32api.RegEnumKey(reg, account_index)) except win32api.error: break account_index += 1 index = 0 subkey = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER, subkey_name, 0, permission) while True: # Loop through all the keys so that we can determine # what type of account this is. try: name, value, typ = win32api.RegEnumValue(subkey, index) except win32api.error: break account[name] = (value, typ) index += 1 # Yield, as appropriate. if account.has_key("POP3 Server"): yield("POP3", subkey, account) elif account.has_key("IMAP Server"): yield("IMAP4", subkey, account) def OEIsInstalled(): """Return True if Outlook Express appears to be installed, and in use (I think if sys.platform == "win32" would say if it was installed at all).""" # Our heuristic is that there is at least one mail account setup. if len(list(OEAccountKeys)) > 0: return True return False ## For use by the test tools. class OEMsg(msgs.Msg): def __init__(self, guts, id): self.tag = id self.guts = guts # The iterator yields a stream of Msg objects, taken from a list of # dbx files. class OEMsgStream(msgs.MsgStream): def __init__(self, tag, dbxes, keep=None): msgs.MsgStream.__init__(self, tag, dbxes, keep) def produce(self): if self.keep is None: for dbx in self.directories: folder = convertToMbox(file(dbx)) all = folder.split("\nFrom ") # XXX Is this right? count = 0 for msg in all: id = "%s::%s" % (dbx, count) count += 1 yield OEMsg(msg, id) return # We only want part of the msgs. Shuffle each directory list, but # in such a way that we'll get the same result each time this is # called on the same directory list. for directory in self.directories: folder = convertToMbox(file(dbx)) all = folder.split("\nFrom ") # XXX Is this right? random.seed(hash(max(all)) ^ SEED) # reproducible across calls random.shuffle(all) del all[self.keep:] all.sort() # for consistency with MsgStream count = 0 for msg in all: id = "%s::%s" % (dbx, count) count += 1 yield OEMsg(msg, id) class OEHamStream(msgs.HamStream): def __init__(self, tag, dbxes, train=0): msgs.HamStream.__init__(self, tag, dbxes, train) class OESpamStream(msgs.SpamStream): def __init__(self, tag, dbxes, train=0): msgs.SpamStream.__init__(self, tag, dbxes, train) ########################################################################### ## TEST DRIVER ########################################################################### def test(): import sys import getopt try: opts, args = getopt.getopt(sys.argv[1:], 'hp') except getopt.error, msg: print >>sys.stderr, str(msg) + '\n\n' + __doc__ sys.exit() print_message = False for opt, arg in opts: if opt == '-h': print >>sys.stderr, __doc__ sys.exit() elif opt == '-p': print_message = True if args: MAILBOX_DIR = args[0] else: MAILBOX_DIR = OEStoreRoot() files = [os.path.join(MAILBOX_DIR, file) for file in \ os.listdir(MAILBOX_DIR) if os.path.splitext(file)[1] == '.dbx'] for file in files: try: print print file dbx = open(file, "rb", 0) header = dbxFileHeader(dbx) print "IS VALID DBX :", header.isValid() if header.isMessages(): info = dbxFileInfo(dbx, header.getEntry(dbxFileHeader.FH_FILE_INFO_LENGTH)) print "MAILBOX NAME :", info.getFolderName() print "CREATION TIME :", info.getCreationTime() entries = header.getEntry(dbxFileHeader.FH_ENTRIES) address = header.getEntry(dbxFileHeader.FH_TREE_ROOT_NODE_PTR) if address and entries: tree = dbxTree(dbx, address, entries) for i in range(entries): address = tree.getValue(i) messageInfo = dbxMessageInfo(dbx, address) if messageInfo.isIndexed(dbxMessageInfo.MI_MESSAGE_ADDRESS): messageAddress = messageInfo.getValueAsLong(dbxMessageInfo.MI_MESSAGE_ADDRESS) message = dbxMessage(dbx, messageAddress) if print_message: print print "Message :", messageInfo.getString(dbxMessageInfo.MI_SUBJECT) print "=" * (len(messageInfo.getString(dbxMessageInfo.MI_SUBJECT)) + 9) print print message.getText() except Exception, (strerror): print strerror dbx.close() if __name__ == '__main__': test()