from BisonGen import Errors def Generate(parser): lexer = Lexer() lexer.generate(parser.lexer) return lexer class Lexer: def __init__(self): self.patterns = {} self.action_index = {} self.actions = [] self.states = {} self.charsets = [] return def generate(self, lexer): import Parser, Compiler compiler = Compiler.Compiler() action_map = {} for scope, patterns in lexer.patterns.items(): (exclusive, number) = lexer.states[scope] self.patterns[number] = [] self.action_index[number] = [] for pattern in patterns: parser = Parser.Parser(pattern.expression, lexer.defines) try: ast = parser.parse() except SyntaxError, e: raise Errors.GeneralError(str(e), (pattern.lineno, pattern.filename)) pattern.compiled = compiler.compile(ast) self.patterns[number].append(pattern) index = action_map.setdefault(pattern, len(action_map)) self.action_index[number].append(index) # Translate the map into a sorted list of actions self._build_actions(action_map) self._build_states(lexer.states) self._build_charsets(compiler.bitmaps) self._build_blockmaps(compiler.headers) return def _build_actions(self, actions): self.actions = [None]*len(actions) for name, index in actions.items(): self.actions[index] = name return def _build_states(self, states): for name, (exclusive, number) in states.items(): self.states[name] = number return def _build_charsets(self, charsets): self.charsets = [None]*len(charsets) for charset, index in charsets.items(): mask = 1 byte = 0 bitmap = [] for bit in charset: if bit: byte += mask mask += mask if mask > 255: bitmap.append(byte) mask = 1 byte = 0 self.charsets[index] = bitmap return def _build_blockmaps(self, blockmaps): self.blockmaps = [None]*len(blockmaps) for block, index in blockmaps.items(): self.blockmaps[index] = block return