############################################################################## # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE. # ############################################################################## """ Code generator for TALInterpreter intermediate code. """ import re import cgi import TALDefs from TALDefs import NAME_RE, TAL_VERSION from TALDefs import I18NError, METALError, TALError from TALDefs import parseSubstitution from TranslationContext import TranslationContext, DEFAULT_DOMAIN I18N_REPLACE = 1 I18N_CONTENT = 2 I18N_EXPRESSION = 3 class TALGenerator: inMacroUse = 0 inMacroDef = 0 source_file = None def __init__(self, expressionCompiler=None, xml=1, source_file=None): if not expressionCompiler: from DummyEngine import DummyEngine expressionCompiler = DummyEngine() self.expressionCompiler = expressionCompiler self.CompilerError = expressionCompiler.getCompilerError() # This holds the emitted opcodes representing the input self.program = [] # The program stack for when we need to do some sub-evaluation for an # intermediate result. E.g. in an i18n:name tag for which the # contents describe the ${name} value. self.stack = [] # Another stack of postponed actions. Elements on this stack are a # dictionary; key/values contain useful information that # emitEndElement needs to finish its calculations self.todoStack = [] self.macros = {} self.slots = {} self.slotStack = [] self.xml = xml self.emit("version", TAL_VERSION) self.emit("mode", xml and "xml" or "html") if source_file is not None: self.source_file = source_file self.emit("setSourceFile", source_file) self.i18nContext = TranslationContext() def getCode(self): assert not self.stack assert not self.todoStack return self.optimize(self.program), self.macros def optimize(self, program): output = [] collect = [] rawseen = cursor = 0 if self.xml: endsep = "/>" else: endsep = " />" for cursor in xrange(len(program)+1): try: item = program[cursor] except IndexError: item = (None, None) opcode = item[0] if opcode == "rawtext": collect.append(item[1]) continue if opcode == "endTag": collect.append("" % item[1]) continue if opcode == "startTag": if self.optimizeStartTag(collect, item[1], item[2], ">"): continue if opcode == "startEndTag": if self.optimizeStartTag(collect, item[1], item[2], endsep): continue if opcode in ("beginScope", "endScope"): # Push *Scope instructions in front of any text instructions; # this allows text instructions separated only by *Scope # instructions to be joined together. output.append(self.optimizeArgsList(item)) continue if opcode == 'noop': # This is a spacer for end tags in the face of i18n:name # attributes. We can't let the optimizer collect immediately # following end tags into the same rawtextOffset. opcode = None pass text = "".join(collect) if text: i = text.rfind("\n") if i >= 0: i = len(text) - (i + 1) output.append(("rawtextColumn", (text, i))) else: output.append(("rawtextOffset", (text, len(text)))) if opcode != None: output.append(self.optimizeArgsList(item)) rawseen = cursor+1 collect = [] return self.optimizeCommonTriple(output) def optimizeArgsList(self, item): if len(item) == 2: return item else: return item[0], tuple(item[1:]) # These codes are used to indicate what sort of special actions # are needed for each special attribute. (Simple attributes don't # get action codes.) # # The special actions (which are modal) are handled by # TALInterpreter.attrAction() and .attrAction_tal(). # # Each attribute is represented by a tuple: # # (name, value) -- a simple name/value pair, with # no special processing # # (name, value, action, *extra) -- attribute with special # processing needs, action is a # code that indicates which # branch to take, and *extra # contains additional, # action-specific information # needed by the processing # def optimizeStartTag(self, collect, name, attrlist, end): # return true if the tag can be converted to plain text if not attrlist: collect.append("<%s%s" % (name, end)) return 1 opt = 1 new = ["<" + name] for i in range(len(attrlist)): item = attrlist[i] if len(item) > 2: opt = 0 name, value, action = item[:3] attrlist[i] = (name, value, action) + item[3:] else: if item[1] is None: s = item[0] else: s = '%s="%s"' % (item[0], TALDefs.attrEscape(item[1])) attrlist[i] = item[0], s new.append(" " + s) # if no non-optimizable attributes were found, convert to plain text if opt: new.append(end) collect.extend(new) return opt def optimizeCommonTriple(self, program): if len(program) < 3: return program output = program[:2] prev2, prev1 = output for item in program[2:]: if ( item[0] == "beginScope" and prev1[0] == "setPosition" and prev2[0] == "rawtextColumn"): position = output.pop()[1] text, column = output.pop()[1] prev1 = None, None closeprev = 0 if output and output[-1][0] == "endScope": closeprev = 1 output.pop() item = ("rawtextBeginScope", (text, column, position, closeprev, item[1])) output.append(item) prev2 = prev1 prev1 = item return output def todoPush(self, todo): self.todoStack.append(todo) def todoPop(self): return self.todoStack.pop() def compileExpression(self, expr): try: return self.expressionCompiler.compile(expr) except self.CompilerError, err: raise TALError('%s in expression %s' % (err.args[0], `expr`), self.position) def pushProgram(self): self.stack.append(self.program) self.program = [] def popProgram(self): program = self.program self.program = self.stack.pop() return self.optimize(program) def pushSlots(self): self.slotStack.append(self.slots) self.slots = {} def popSlots(self): slots = self.slots self.slots = self.slotStack.pop() return slots def emit(self, *instruction): self.program.append(instruction) def emitStartTag(self, name, attrlist, isend=0): if isend: opcode = "startEndTag" else: opcode = "startTag" self.emit(opcode, name, attrlist) def emitEndTag(self, name): if self.xml and self.program and self.program[-1][0] == "startTag": # Minimize empty element self.program[-1] = ("startEndTag",) + self.program[-1][1:] else: self.emit("endTag", name) def emitOptTag(self, name, optTag, isend): program = self.popProgram() #block start = self.popProgram() #start tag if (isend or not program) and self.xml: # Minimize empty element start[-1] = ("startEndTag",) + start[-1][1:] isend = 1 cexpr = optTag[0] if cexpr: cexpr = self.compileExpression(optTag[0]) self.emit("optTag", name, cexpr, optTag[1], isend, start, program) def emitRawText(self, text): self.emit("rawtext", text) def emitText(self, text): self.emitRawText(cgi.escape(text)) def emitDefines(self, defines): for part in TALDefs.splitParts(defines): m = re.match( r"(?s)\s*(?:(global|local)\s+)?(%s)\s+(.*)\Z" % NAME_RE, part) if not m: raise TALError("invalid define syntax: " + `part`, self.position) scope, name, expr = m.group(1, 2, 3) scope = scope or "local" cexpr = self.compileExpression(expr) if scope == "local": self.emit("setLocal", name, cexpr) else: self.emit("setGlobal", name, cexpr) def emitOnError(self, name, onError, TALtag, isend): block = self.popProgram() key, expr = parseSubstitution(onError) cexpr = self.compileExpression(expr) if key == "text": self.emit("insertText", cexpr, []) else: assert key == "structure" self.emit("insertStructure", cexpr, {}, []) if TALtag: self.emitOptTag(name, (None, 1), isend) else: self.emitEndTag(name) handler = self.popProgram() self.emit("onError", block, handler) def emitCondition(self, expr): cexpr = self.compileExpression(expr) program = self.popProgram() self.emit("condition", cexpr, program) def emitRepeat(self, arg): m = re.match("(?s)\s*(%s)\s+(.*)\Z" % NAME_RE, arg) if not m: raise TALError("invalid repeat syntax: " + `arg`, self.position) name, expr = m.group(1, 2) cexpr = self.compileExpression(expr) program = self.popProgram() self.emit("loop", name, cexpr, program) def emitSubstitution(self, arg, attrDict={}): key, expr = parseSubstitution(arg) cexpr = self.compileExpression(expr) program = self.popProgram() if key == "text": self.emit("insertText", cexpr, program) else: assert key == "structure" self.emit("insertStructure", cexpr, attrDict, program) def emitI18nVariable(self, varname, action, expression): # Used for i18n:name attributes. arg is extra information describing # how the contents of the variable should get filled in, and it will # either be a 1-tuple or a 2-tuple. If arg[0] is None, then the # i18n:name value is taken implicitly from the contents of the tag, # e.g. "I live in the USA". In this # case, arg[1] is the opcode sub-program describing the contents of # the tag. # # When arg[0] is not None, it contains the tal expression used to # calculate the contents of the variable, e.g. # "I live in " key = cexpr = None program = self.popProgram() if action == I18N_REPLACE: # This is a tag with an i18n:name and a tal:replace (implicit or # explicit). Get rid of the first and last elements of the # program, which are the start and end tag opcodes of the tag. program = program[1:-1] elif action == I18N_CONTENT: # This is a tag with an i18n:name and a tal:content # (explicit-only). Keep the first and last elements of the # program, so we keep the start and end tag output. pass else: assert action == I18N_EXPRESSION key, expr = parseSubstitution(expression) cexpr = self.compileExpression(expr) # XXX Would key be anything but 'text' or None? assert key in ('text', None) self.emit('i18nVariable', varname, program, cexpr) def emitTranslation(self, msgid, i18ndata): program = self.popProgram() if i18ndata is None: self.emit('insertTranslation', msgid, program) else: key, expr = parseSubstitution(i18ndata) cexpr = self.compileExpression(expr) assert key == 'text' self.emit('insertTranslation', msgid, program, cexpr) def emitDefineMacro(self, macroName): program = self.popProgram() macroName = macroName.strip() if self.macros.has_key(macroName): raise METALError("duplicate macro definition: %s" % `macroName`, self.position) if not re.match('%s$' % NAME_RE, macroName): raise METALError("invalid macro name: %s" % `macroName`, self.position) self.macros[macroName] = program self.inMacroDef = self.inMacroDef - 1 self.emit("defineMacro", macroName, program) def emitUseMacro(self, expr): cexpr = self.compileExpression(expr) program = self.popProgram() self.inMacroUse = 0 self.emit("useMacro", expr, cexpr, self.popSlots(), program) def emitDefineSlot(self, slotName): program = self.popProgram() slotName = slotName.strip() if not re.match('%s$' % NAME_RE, slotName): raise METALError("invalid slot name: %s" % `slotName`, self.position) self.emit("defineSlot", slotName, program) def emitFillSlot(self, slotName): program = self.popProgram() slotName = slotName.strip() if self.slots.has_key(slotName): raise METALError("duplicate fill-slot name: %s" % `slotName`, self.position) if not re.match('%s$' % NAME_RE, slotName): raise METALError("invalid slot name: %s" % `slotName`, self.position) self.slots[slotName] = program self.inMacroUse = 1 self.emit("fillSlot", slotName, program) def unEmitWhitespace(self): collect = [] i = len(self.program) - 1 while i >= 0: item = self.program[i] if item[0] != "rawtext": break text = item[1] if not re.match(r"\A\s*\Z", text): break collect.append(text) i = i-1 del self.program[i+1:] if i >= 0 and self.program[i][0] == "rawtext": text = self.program[i][1] m = re.search(r"\s+\Z", text) if m: self.program[i] = ("rawtext", text[:m.start()]) collect.append(m.group()) collect.reverse() return "".join(collect) def unEmitNewlineWhitespace(self): collect = [] i = len(self.program) while i > 0: i = i-1 item = self.program[i] if item[0] != "rawtext": break text = item[1] if re.match(r"\A[ \t]*\Z", text): collect.append(text) continue m = re.match(r"(?s)^(.*)(\n[ \t]*)\Z", text) if not m: break text, rest = m.group(1, 2) collect.reverse() rest = rest + "".join(collect) del self.program[i:] if text: self.emit("rawtext", text) return rest return None def replaceAttrs(self, attrlist, repldict): # Each entry in attrlist starts like (name, value). # Result is (name, value, action, expr, xlat) if there is a # tal:attributes entry for that attribute. Additional attrs # defined only by tal:attributes are added here. # # (name, value, action, expr, xlat) if not repldict: return attrlist newlist = [] for item in attrlist: key = item[0] if repldict.has_key(key): expr, xlat = repldict[key] item = item[:2] + ("replace", expr, xlat) del repldict[key] newlist.append(item) # Add dynamic-only attributes for key, (expr, xlat) in repldict.items(): newlist.append((key, None, "insert", expr, xlat)) return newlist def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict, position=(None, None), isend=0): if not taldict and not metaldict and not i18ndict: # Handle the simple, common case self.emitStartTag(name, attrlist, isend) self.todoPush({}) if isend: self.emitEndElement(name, isend) return self.position = position for key, value in taldict.items(): if key not in TALDefs.KNOWN_TAL_ATTRIBUTES: raise TALError("bad TAL attribute: " + `key`, position) if not (value or key == 'omit-tag'): raise TALError("missing value for TAL attribute: " + `key`, position) for key, value in metaldict.items(): if key not in TALDefs.KNOWN_METAL_ATTRIBUTES: raise METALError("bad METAL attribute: " + `key`, position) if not value: raise TALError("missing value for METAL attribute: " + `key`, position) for key, value in i18ndict.items(): if key not in TALDefs.KNOWN_I18N_ATTRIBUTES: raise I18NError("bad i18n attribute: " + `key`, position) if not value and key in ("attributes", "data", "id"): raise I18NError("missing value for i18n attribute: " + `key`, position) todo = {} defineMacro = metaldict.get("define-macro") useMacro = metaldict.get("use-macro") defineSlot = metaldict.get("define-slot") fillSlot = metaldict.get("fill-slot") define = taldict.get("define") condition = taldict.get("condition") repeat = taldict.get("repeat") content = taldict.get("content") replace = taldict.get("replace") attrsubst = taldict.get("attributes") onError = taldict.get("on-error") omitTag = taldict.get("omit-tag") TALtag = taldict.get("tal tag") i18nattrs = i18ndict.get("attributes") # Preserve empty string if implicit msgids are used. We'll generate # code with the msgid='' and calculate the right implicit msgid during # interpretation phase. msgid = i18ndict.get("translate") varname = i18ndict.get('name') i18ndata = i18ndict.get('data') if i18ndata and not msgid: raise I18NError("i18n:data must be accompanied by i18n:translate", position) if len(metaldict) > 1 and (defineMacro or useMacro): raise METALError("define-macro and use-macro cannot be used " "together or with define-slot or fill-slot", position) if replace: if content: raise TALError( "tal:content and tal:replace are mutually exclusive", position) if msgid is not None: raise I18NError( "i18n:translate and tal:replace are mutually exclusive", position) repeatWhitespace = None if repeat: # Hack to include preceding whitespace in the loop program repeatWhitespace = self.unEmitNewlineWhitespace() if position != (None, None): # XXX at some point we should insist on a non-trivial position self.emit("setPosition", position) if self.inMacroUse: if fillSlot: self.pushProgram() if self.source_file is not None: self.emit("setSourceFile", self.source_file) todo["fillSlot"] = fillSlot self.inMacroUse = 0 else: if fillSlot: raise METALError("fill-slot must be within a use-macro", position) if not self.inMacroUse: if defineMacro: self.pushProgram() self.emit("version", TAL_VERSION) self.emit("mode", self.xml and "xml" or "html") if self.source_file is not None: self.emit("setSourceFile", self.source_file) todo["defineMacro"] = defineMacro self.inMacroDef = self.inMacroDef + 1 if useMacro: self.pushSlots() self.pushProgram() todo["useMacro"] = useMacro self.inMacroUse = 1 if defineSlot: if not self.inMacroDef: raise METALError( "define-slot must be within a define-macro", position) self.pushProgram() todo["defineSlot"] = defineSlot if defineSlot or i18ndict: domain = i18ndict.get("domain") or self.i18nContext.domain source = i18ndict.get("source") or self.i18nContext.source target = i18ndict.get("target") or self.i18nContext.target if ( domain != DEFAULT_DOMAIN or source is not None or target is not None): self.i18nContext = TranslationContext(self.i18nContext, domain=domain, source=source, target=target) self.emit("beginI18nContext", {"domain": domain, "source": source, "target": target}) todo["i18ncontext"] = 1 if taldict or i18ndict: dict = {} for item in attrlist: key, value = item[:2] dict[key] = value self.emit("beginScope", dict) todo["scope"] = 1 if onError: self.pushProgram() # handler if TALtag: self.pushProgram() # start self.emitStartTag(name, list(attrlist)) # Must copy attrlist! if TALtag: self.pushProgram() # start self.pushProgram() # block todo["onError"] = onError if define: self.emitDefines(define) todo["define"] = define if condition: self.pushProgram() todo["condition"] = condition if repeat: todo["repeat"] = repeat self.pushProgram() if repeatWhitespace: self.emitText(repeatWhitespace) if content: todo["content"] = content if replace: # tal:replace w/ i18n:name has slightly different semantics. What # we're actually replacing then is the contents of the ${name} # placeholder. if varname: todo['i18nvar'] = (varname, replace) else: todo["replace"] = replace self.pushProgram() # i18n:name w/o tal:replace uses the content as the interpolation # dictionary values elif varname: todo['i18nvar'] = (varname, None) self.pushProgram() if msgid is not None: todo['msgid'] = msgid if i18ndata: todo['i18ndata'] = i18ndata optTag = omitTag is not None or TALtag if optTag: todo["optional tag"] = omitTag, TALtag self.pushProgram() if attrsubst or i18nattrs: if attrsubst: repldict = TALDefs.parseAttributeReplacements(attrsubst) else: repldict = {} if i18nattrs: i18nattrs = i18nattrs.split() else: i18nattrs = () # Convert repldict's name-->expr mapping to a # name-->(compiled_expr, translate) mapping for key, value in repldict.items(): repldict[key] = self.compileExpression(value), key in i18nattrs for key in i18nattrs: if not repldict.has_key(key): repldict[key] = None, 1 else: repldict = {} if replace: todo["repldict"] = repldict repldict = {} self.emitStartTag(name, self.replaceAttrs(attrlist, repldict), isend) if optTag: self.pushProgram() if content: self.pushProgram() if msgid is not None: self.pushProgram() if todo and position != (None, None): todo["position"] = position self.todoPush(todo) if isend: self.emitEndElement(name, isend) def emitEndElement(self, name, isend=0, implied=0): todo = self.todoPop() if not todo: # Shortcut if not isend: self.emitEndTag(name) return self.position = position = todo.get("position", (None, None)) defineMacro = todo.get("defineMacro") useMacro = todo.get("useMacro") defineSlot = todo.get("defineSlot") fillSlot = todo.get("fillSlot") repeat = todo.get("repeat") content = todo.get("content") replace = todo.get("replace") condition = todo.get("condition") onError = todo.get("onError") define = todo.get("define") repldict = todo.get("repldict", {}) scope = todo.get("scope") optTag = todo.get("optional tag") msgid = todo.get('msgid') i18ncontext = todo.get("i18ncontext") varname = todo.get('i18nvar') i18ndata = todo.get('i18ndata') if implied > 0: if defineMacro or useMacro or defineSlot or fillSlot: exc = METALError what = "METAL" else: exc = TALError what = "TAL" raise exc("%s attributes on <%s> require explicit " % (what, name, name), position) # If there's no tal:content or tal:replace in the tag with the # i18n:name, tal:replace is the default. i18nNameAction = I18N_REPLACE if content: if varname: i18nNameAction = I18N_CONTENT self.emitSubstitution(content, {}) # If we're looking at an implicit msgid, emit the insertTranslation # opcode now, so that the end tag doesn't become part of the implicit # msgid. If we're looking at an explicit msgid, it's better to emit # the opcode after the i18nVariable opcode so we can better handle # tags with both of them in them (and in the latter case, the contents # would be thrown away for msgid purposes). if msgid is not None and not varname: self.emitTranslation(msgid, i18ndata) if optTag: self.emitOptTag(name, optTag, isend) elif not isend: # If we're processing the end tag for a tag that contained # i18n:name, we need to make sure that optimize() won't collect # immediately following end tags into the same rawtextOffset, so # put a spacer here that the optimizer will recognize. if varname: self.emit('noop') self.emitEndTag(name) # If i18n:name appeared in the same tag as tal:replace then we're # going to do the substitution a little bit differently. The results # of the expression go into the i18n substitution dictionary. if replace: self.emitSubstitution(replace, repldict) elif varname: if varname[1] is not None: i18nNameAction = I18N_EXPRESSION # o varname[0] is the variable name # o i18nNameAction is either # - I18N_REPLACE for implicit tal:replace # - I18N_CONTENT for tal:content # - I18N_EXPRESSION for explicit tal:replace # o varname[1] will be None for the first two actions and the # replacement tal expression for the third action. self.emitI18nVariable(varname[0], i18nNameAction, varname[1]) # Do not test for "msgid is not None", i.e. we only want to test for # explicit msgids here. See comment above. if msgid is not None and varname: self.emitTranslation(msgid, i18ndata) if repeat: self.emitRepeat(repeat) if condition: self.emitCondition(condition) if onError: self.emitOnError(name, onError, optTag and optTag[1], isend) if scope: self.emit("endScope") if i18ncontext: self.emit("endI18nContext") assert self.i18nContext.parent is not None self.i18nContext = self.i18nContext.parent if defineSlot: self.emitDefineSlot(defineSlot) if fillSlot: self.emitFillSlot(fillSlot) if useMacro: self.emitUseMacro(useMacro) if defineMacro: self.emitDefineMacro(defineMacro) def test(): t = TALGenerator() t.pushProgram() t.emit("bar") p = t.popProgram() t.emit("foo", p) if __name__ == "__main__": test()