import threading
import re, sys, types
from itertools import izip, count, chain
from decorator import decorator
from contextlib import contextmanager

from redbeans import latex
from redbeans.formats import BaseFormat, Format
from redbeans.creole import tokenize
from redbeans.parser import Parser
from redbeans.tokens import *

from . import custom, db, dependencies, cache, structure
from .flavors import FLAVORS, FORMATS, Reference
from .benchmark import benchmarking

class WikiException(Exception):
    pass

class NoContentException(WikiException):
    pass

class ElementNotFound(WikiException):
    pass

class PropNotFound(WikiException):
    pass

class moverride(object):
    def __init__(self, kw):
        for v in kw.values():
            assert isinstance(v, unicode), kw
        for k in kw:
            assert not k[0].isupper() and k[0] != '*', kw
        self.kw = kw
    def __enter__(self):
        self.old_moverrides = dict(eval_state.moverrides)
        eval_state.moverrides.update(self.kw)
    def __exit__(self, exc_type, exc_val, exc_tb):
        eval_state.moverrides = self.old_moverrides

class eoverride(object):
    def __init__(self, kw):
        self.kw = kw
        for k in kw:
            assert k[0].islower(), kw
            assert hasattr(kw[k], 'ename') or hasattr(kw[k], 'element') or kw[k] is None, kw
    def __enter__(self):
        self.old_eoverrides = dict(eval_state.eoverrides)
        eval_state.eoverrides.update(self.kw)
    def __exit__(self, exc_type, exc_val, exc_tb):
        eval_state.eoverrides = self.old_eoverrides
class push_current(eoverride):
    def __init__(self, curr):
        eover = {u'current': curr}
        if hasattr(curr, 'owner') and curr.owner is not None:
            # Reference
            eover[u'owner'] = curr.owner
        eoverride.__init__(self, eover)
    def __enter__(self):
        eval_state.current_stack.append(self.kw[u'current'])
        eoverride.__enter__(self)
    def __exit__(self, exc_type, exc_val, exc_tb):
        eoverride.__exit__(self, exc_type, exc_val, exc_tb)
        assert eval_state.current_stack[-1] == self.kw[u'current']
        eval_state.current_stack.pop()
        
class poverride(object):
    def __init__(self, kw):
        for k in kw:
            assert k.startswith('*'), kw
        self.kw = kw        
    def __enter__(self):
        self.old_poverrides = dict(eval_state.poverrides)
        eval_state.poverrides.update(self.kw)
    def __exit__(self, exc_type, exc_val, exc_tb):
        eval_state.poverrides = self.old_poverrides

class context(object):
    def __init__(self, kw):
        self.kw = kw        
    def __enter__(self):
        self.old_context = dict(eval_state.context)
        eval_state.context.update(self.kw)
    def __exit__(self, exc_type, exc_val, exc_tb):
        eval_state.context = self.old_context

# Use a weird singleton becuse None and False could be valid return values.
NO_MEMO = (None,)

def get_memo_or_push(key):
    ret, restr = cache.get_memo(key + get_context_tag(), (NO_MEMO, None))
    found = False
    if ret is not NO_MEMO:
        # Check restr
        for typ, name, val in restr:
            if typ == 'eoverride':
                if eval_state.eoverrides.get(name, None) != val:
                    break
            elif typ == 'moverride':
                if eval_state.moverrides.get(name, None) != val:
                    break
            elif typ == 'poverride':
                if eval_state.poverrides.get(name, None) != val:
                    break
        else:
            found = True
    if found:
        return ret
    else:
        eval_state.memoable_stack.append(set())
        return NO_MEMO

def set_memo_and_pop(key, val):
    memoable = eval_state.memoable_stack.pop()
    restr = set()
    for typ, name in memoable:
        if typ == 'eoverride':
            restr.add(('eoverride', name,
                       eval_state.eoverrides.get(name, None)))
        elif typ == 'poverride':
            restr.add(('poverride', name,
                       eval_state.poverrides.get(name, None)))
        elif typ == 'moverride':
            restr.add(('moverride', name,
                       eval_state.moverrides.get(name, None)))
        else:
            assert False, memoable
    cache.set_memo(key + get_context_tag(), (val, restr))
    eval_state.memoable_stack[-1].update(memoable)

def recursive_get(elm, prop_name, *args, **restrictions):
    incl_elm = True
    with_owner = False
    if len(args) > 0:
        if len(args) > 1:
            assert len(args) == 2, args
            with_owner = args[1]
        filter = args[0]
        # prune: include the first thing that fails the restrictions,
        #        but don't recurse into that thing.
        # exclude: exclude anything that fails the restrictions.
        # exclude!: exclude, but don't include elm itself.
        # invert: include only everything that would be excluded by exclude,
        #         but would've been included without restrictions.
        assert filter in ('prune', 'invert', 'exclude', 'exclude!')
        if filter.startswith('exclude'):
            if '!' in filter:
                incl_elm = False
            filter = 'exclude'
    else:
        filter = 'prune'
    ret = to_python(elm, prop_name)
    new = [([elm], list(ret), False)]
    if filter != 'prune':
        # We only add things once they pass filters
        ret = []
    if filter != 'invert' and incl_elm:
        ret = [elm] + ret
    if with_owner:
        ret = [(e, with_owner) for e in ret]
    while len(new) > 0:
        newnew = []
        # "pruned" indicates that this element would have been pruned
        for path, l, pruned in new:
            for n in l:
                if n in path:
                    raise WikiException(
                        "Infinite %s loop in recursive_get: %s!"
                        % (prop_name, '->'.join(path+[n])))
                passed_restr = all((to_python(n, r)
                                    if n.has_propval(r) else None)
                                   == restrictions[r]
                                   for r in (unicode(sr, 'utf-8')
                                             for sr in restrictions))
                if passed_restr or filter == 'invert':
                    if (filter == 'exclude' or
                        (filter == 'invert' and (not passed_restr or pruned))):
                        # We only add it after it meets the restrictions
                        if with_owner:
                            if n.has_propval(u'subowned') and to_python(n, u'subowned'):
                                ret.append((n, path[-1]))
                            else:
                                ret.append((n, with_owner))
                        else:
                            ret.append(n)
                    if n.has_propval(prop_name):
                        adding = to_python(n, prop_name)
                        if filter == 'prune':
                            assert not with_owner
                            ret += adding
                    newnew.append((path+[n], adding,
                                   not passed_restr or pruned))
        new = newnew
    return ret

class RefExtractingFormat(BaseFormat):
    def __init__(self):
        self.refs = []
        self.link_toks = None
    def text(self, text):
        if self.link_toks is not None:
            self.link_toks.append(Text(text))
        return u''
    def error(self, text):
        #???
        return u''
    def start(self, t, arg=None):
        if t == LINK:
            self.link_toks = []
        elif self.link_toks is not None:
            self.link_toks.append(Start(t, arg))
        return u''
    def end(self, t, arg=None):
        if t == LINK:
            if 'ename' in arg:
                assert isinstance(arg['ename'], unicode), repr(arg)
                self.refs.append((arg, self.link_toks))
                #assert False, self.link_toks
            self.link_toks = None
        elif self.link_toks is not None:
            self.link_toks.append(End(t, arg))
        return u''
    def entity(self, t, arg=None):
        if self.link_toks is not None:
            self.link_toks.append(Entity(t, arg))
        return u''
def ref_link_func(h, sty):
    #print >>sys.stderr, "r_l_f", h, sty
    if sty == LINK and not h.startswith('/') and '://' not in h:
        args = None
        if '/' in h:
            args = h.split('/')
            h = args.pop(0)
        if '.' not in h and ' ' not in h:
            elm = get_element(h)
            if elm is not None:
                ename = elm.ename
            else:
                ename = h
            return u'', dict(ename=ename, args=args,
                             owner=get_overridden_element(u'owner'))
    return u'', {}

def get_reference_enames_with_labels(text_or_elm, propname=None, propval=None):
    if propval is not None:
        # Could be memoized
        key = 'g_r_e_w_l:%s.%s' % (propval.element.ename, propval.propname)
        ret = get_memo_or_push(key)
        if ret is not NO_MEMO:
            return ret
    
    if propname is not None:
        assert not structure.get_flavor(propname).binary
        pv = text_or_elm.get_propval(propname)
        text = unicode(pv.value, 'utf-8')
        eval_state.dependencies.addDep(pv)
    else:
        assert isinstance(text_or_elm, unicode)
        text = text_or_elm
    old_mentions = eval_state.mentions
    old_errors = eval_state.errors
    eval_state.errors = []
    ref = RefExtractingFormat()

    with context({'in_get_refs': True}):
        eval_state.mainparser.parse(text, format=ref, link_func=ref_link_func)
    eval_state.mentions = old_mentions
    #assert not eval_state.errors, (foo, eval_state.errors)
    new_errors = eval_state.errors
    eval_state.errors = old_errors
    #print >>sys.stderr, "Refs:", ref.refs

    if propval is not None:
        set_memo_and_pop(key, (ref.refs, new_errors))
    
    return ref.refs, new_errors

def get_reference_enames(text_or_elm, propname=None, propval=None):
    with_labels, errors = get_reference_enames_with_labels(text_or_elm,
                                                           propname,
                                                           propval)
    return [d for d,l in with_labels], errors

def get_references(text_or_elm, propname=None, propval=None):
    with_args, errors = get_reference_enames(text_or_elm, propname, propval)
    return [structure.get_element(d['ename']) for d in with_args]

def get_mention_enames(text_or_elm, propname=None):
    return get_eval_state_delta('mentions', text_or_elm, propname)

def get_mentions(text_or_elm, propname=None):
    """Return all elements linked to or named in this wikitext or propval."""
    return [structure.get_element(e)
            for e in get_mention_enames(text_or_elm, propname)]

def has_errors(elm, propname):
    """Return true if any errors are encountered evaluating this propval.

    This probably only works properly if the logged in user is omniscient; we
    could catch the non-wiki exceptions that can arise, but we don't."""
    try:
        return len(get_eval_state_delta('errors', elm, propname)) > 0
    except WikiException:
        return True

def has_ancestor(element, ancestor):
    if hasattr(element, 'element'):
        element = element.element
    return ancestor.is_ancestor_of(element)
def is_a(element, ancestor):
    if hasattr(element, 'element'):
        element = element.element
    return ancestor == element or ancestor.is_ancestor_of(element)

def safe_getattr(object, name, *args):
    assert '__' not in name
    return getattr(object, name, *args)

# We do this for the exception handling; if builtin hasattr() gets any
# exception, it silently returns False.
def has_prop(obj, name):
    assert isinstance(obj, attrelm), obj
    try:
        getattr(obj, name)
    except PropNotFound:
        return False
    return True

def baz_eval_wrap(arg, toPython):
    if hasattr(arg, 'element'):
        return attrelm(arg.element, toPython, arg.args, arg.owner)
    elif hasattr(arg, 'ename'):
        return attrelm(arg, toPython)
    elif isinstance(arg, list):
        return [baz_eval_wrap(a, toPython) for a in arg]
    elif isinstance(arg, dict):
        return dict((baz_eval_wrap(k, toPython),
                     baz_eval_wrap(arg[k], toPython)) for k in arg)
    else:
        return arg
def baz_eval_wrap_func(func, toPython):
    @decorator
    def helper(func, *args, **kw):
        args = [baz_eval_unwrap(a) for a in args]
        kw = dict((k,baz_eval_unwrap(v)) for k,v in kw.items())
        return baz_eval_wrap(func(*args, **kw), toPython)
    return helper(func)
def baz_eval_unwrap(ret):
    if isinstance(ret, attrelm):
        return ret.unwrap__()
    elif isinstance(ret, list):
        return [baz_eval_unwrap(r) for r in ret]
    elif isinstance(ret, dict):
        return dict((baz_eval_unwrap(k), baz_eval_unwrap(ret[k])) for k in ret)
    elif isinstance(ret, str):
        return unicode(ret, 'utf-8')
    else:
        return ret

def get_context(key):
    return eval_state.context.get(key, False)
def get_context_tag():
    tag = ''
    for k in eval_state.context:
        val = eval_state.context[k]
        assert val in (True, False), eval_state.context
        if val:
            tag += '+' + k
    return tag

BAZ_EVAL_GLOBALS = dict((toPython, {"__builtins__": {},
                              "repr": repr,
                              "str": str,
                              "True": True,
                              "False": False,
                              "None": None,
                              "dict": baz_eval_wrap_func(
    lambda *args,**kw: dict(*args, **kw), toPython),
                              "any": any,
                              "all": all, 
                              "len": len,
                              "hasattr": has_prop,
                              "getattr": safe_getattr,
                              "list_props":
                              baz_eval_wrap_func(lambda e: list(e), toPython),
                              "recursive_get":
                              baz_eval_wrap_func(recursive_get, toPython),
                              "has_ancestor":
                              baz_eval_wrap_func(has_ancestor, toPython),
                              "is_a": baz_eval_wrap_func(is_a, toPython),
                              "get_references":
                              baz_eval_wrap_func(get_references, toPython),
                              "get_mentions":
                              baz_eval_wrap_func(get_mentions, toPython),
                              "has_errors":
                              baz_eval_wrap_func(has_errors, toPython),
                              "get_context": get_context
                              })
                        for toPython in (False, True))

def baz_eval(expr, toPython=True):
    if len(expr.strip()) == 0:
        return u''
    if '__' in expr:
        # Protect against hackery
        raise WikiException("The string '__' is not allowed to appear in expressions!")
    else:
        expr = expr.replace('\n', ' ')
        key = 'baz_eval%s:%s' % (toPython, expr)
        ret = get_memo_or_push(key)
        if ret is not NO_MEMO:
            return ret
        #print >>sys.stderr, 'baz_eval', expr
        
        elemlocs = _elemdict(toPython=toPython)
        try:
            eval_globals = dict(BAZ_EVAL_GLOBALS[toPython])
            eval_globals.update(
                {"defined": (lambda s: s in elemlocs),
                 "get": (lambda s,d=None: elemlocs.get(s,d))})
            #print >>sys.stderr, expr
            ret = eval(expr, eval_globals, elemlocs)
            ret = baz_eval_unwrap(ret)
        except AssertionError:
            raise
        except Exception,e:
            raise #!!!
            if (isinstance(e, NameError)
                and re.search("^global name '(\w+)' is not defined$",
                              e.args[0])):
                extra = " (If you're using a generator expression, try a list comprehension.)"
            else:
                extra = ''
            raise WikiException(repr(e)+extra)
        else:
            set_memo_and_pop(key, ret)
            return ret


def full_eval(expr):
    """Returns a generator for tokens representing the given baz_eval expr."""
    ret = baz_eval(expr, toPython=False)
    if isinstance(ret, basestring):
        return [Text(ret)]
    else:
        return ret

def unicode_eval(expr):
    ret = baz_eval(expr)
    if hasattr(ret, '__call__'):
        gen = ret(None, None)
        ret = tokens_text(gen)
    else:
        ret = unicode(ret)
    return ret

def string_tokenize(s):
    return FLAVORS['string'].tokenize(s, eval_state.mainparser)

def tokens_text(toks):
    ret = u''
    for t in toks:
        if t.op == ENTITY and t.style == MACRO:
            assert t.arg[1] is None, t
            ret += unicode_eval(t.arg[0])
        else:
            assert t.op == TEXT, t
            ret += t.arg
    return ret

def render_allow_override(element, prop_name, default=None,
                          eoverrides={}, moverrides={}):
    # Could be an moverride.
    eval_state.memoable_stack[-1].add(('moverride', prop_name))
    for v in moverrides.values():
        assert isinstance(v, unicode)
    if prop_name in eval_state.moverrides:
        return eval_state.moverrides[prop_name]
    else:
        prop_name = get_propname(prop_name)
        eval_state.dependencies.addPropvalDep(element, prop_name)
        if element.has_propval(prop_name):
            return render_propval(element, prop_name, eoverrides=eoverrides,
                                  moverrides=moverrides)
        else:
            return default

def render_propval(element, prop_name, eoverrides={}, moverrides={},
                   content=None):
    """Render the propval as plain text, tracking dependencies."""
    for v in moverrides.values():
        assert isinstance(v, unicode)
    for k in eoverrides:
        assert k[0].islower(), eoverrides
        assert hasattr(eoverrides[k], 'ename'), eoverrides
    for k in moverrides:
        assert not k[0].isupper() and k[0] != '*', moverrides
    # TODO(xavid): theoretically this should actively be position-independent
    parser = Parser(FORMATS['txt'], macro_func, link_func, makeRestricted)
    old_eoverrides = dict(eval_state.eoverrides)
    old_moverrides = dict(eval_state.moverrides)
    old_parser = eval_state.mainparser
    eval_state.eoverrides.update(eoverrides)
    eval_state.moverrides.update(moverrides)
    eval_state.mainparser = parser
    ret = parser.render(recurse(element, prop_name, content=content))
    eval_state.eoverrides = old_eoverrides
    eval_state.moverrides = old_moverrides
    eval_state.mainparser = old_parser
    return ret

KW_ARG_PAT = re.compile(r'^(\w+)[=]([^=].*)$')

def parse_macro_args(arglist):
    """Parse an argument list into a map of names to values.

    name will be an integer for positional parameters and a string for
    keyword parameters."""
    nextname = 1
    retmap = {}
    for word in arglist:
        m = KW_ARG_PAT.search(word)
        if m:
            retmap[m.group(1)] = m.group(2)
        else:
            retmap[nextname] = word
            nextname += 1
    return retmap
            

def _list_descendants(e):
    assert e.ename is not None
    eval_state.dependencies.addChildrenDep(e)
    # -! make more efficient, possibly move to structure
    everyone = e.get_descendants()
    for dude in everyone:
        eval_state.dependencies.addChildrenDep(dude)
    return everyone

def safesplit_tags(toks, splitters):
    retlist = [([], None)]
    macro_level = 0
    for t in toks:
        if (macro_level == 0 and t.op == ENTITY and t.style == MACRO
            and t.arg[0] in splitters):
            retlist.append(([], t.arg))
        else:
            if t.style == MACRO:
                if t.op == START:
                    macro_level += 1
                elif t.op == END:
                    macro_level -= 1
            retlist[-1][0].append(t)
    return retlist
def safesplit(toks, splitters):
    return [p[0] for p in safesplit_tags(toks, splitters)]

def get_overridden_element(ename):
    # Only allow overridden elements that start with a lowercase letter.
    # TODO(xavid): we can probably clean up lots of stuff given this new
    #              simplifying assumption.
    assert isinstance(ename, unicode), repr(ename)
    if hasattr(eval_state, 'dependencies') and ename[0].islower():
        # parent can't be overridden, for optimization reasons.
        if ename == custom.PARENT_ELEMENT:
            eval_state.dependencies.addParentDep(eval_state.me)
            return eval_state.me.get_parent()
        else:
            eval_state.memoable_stack[-1].add(('eoverride', ename))
            if ename in eval_state.eoverrides:
                return eval_state.eoverrides[ename]
            elif ename == custom.ME_ELEMENT:
                return eval_state.me
    return None
def get_element(ename):
    e = get_overridden_element(ename)
    if e is not None:
        return e
    else:
        elm = structure.get_element(ename)
        if elm is None:
            if hasattr(eval_state, 'dependencies'):
                eval_state.dependencies.addExistsDep(ename)
            return None
        return elm

def get_current_element():
    """Like get_element(u'leaf'), but allows explicit overriding by setting
    current."""
    #print >>sys.stderr, "g_c_e", eval_state.eoverrides, get_element(custom.LEAF_ELEMENT)
    eval_state.memoable_stack[-1].add(('eoverride', u'current'))
    if u'current' in eval_state.eoverrides:
        ret = eval_state.eoverrides[u'current']
        if ret is not None:
            return ret
    return get_element(custom.LEAF_ELEMENT)

def get_current_for_prop(propname):
    for curr in reversed(eval_state.current_stack):
        if curr.has_propval(propname):
            return curr
    return None

def get_propname(pname):
    if pname[0] != '*' and pname != u'this':
        return pname
    if pname != u'this':
        eval_state.memoable_stack[-1].add(('poverride', pname))
    if pname in eval_state.poverrides:
        return eval_state.poverrides[pname]
    else:
        raise WikiException("Undefined propname '%s'!" % pname)

def get_moverride(mname):
    if mname in eval_state.moverrides:
        return eval_state.moverrides[mname]
    else:
        return None

class Arg(object):
    def __init__(self, raw):
        self.raw = raw
    def __call__(self, args, content):
        yield Text(unicode_eval(self.raw))

def to_python(ename_or_e, pname):
    pname = get_propname(pname)
    if hasattr(ename_or_e, 'ename'):
        memo_ename = ename_or_e.ename
    elif hasattr(ename_or_e, 'element'):
        # It's a Reference, we need to include the args.
        memo_ename = u"%s/%s" % (ename_or_e.element.ename,
                                '/'.join(ename_or_e.args))
    else:
        memo_ename = get_overridden_element(ename_or_e)
        if memo_ename is None:
            memo_ename = ename_or_e
    key = 'to_python:%s.%s' % (memo_ename, pname)
    ret = get_memo_or_push(key)
    if ret is not NO_MEMO:
        return ret

    ls = list(recurse(ename_or_e, pname, to_python=True))
    #print >>sys.stderr, "to_python", ls
    assert len(ls) == 1

    set_memo_and_pop(key, ls[0])
    
    return ls[0]
def recurse(ename, pname, to_python=False, content=None, arglist=[]):
    #print >>sys.stderr, "recurse", ename, pname
    assert ename is not None
    if hasattr(ename, 'ename'):
        e = ename
        ename = e.ename
        assert hasattr(e, 'ename'), e
    elif hasattr(ename, 'element'):
        e = ename
        ename = e.element.ename
    else:
        e = get_element(ename)
        if e is None:
            raise ElementNotFound(ename)

    pname = get_propname(pname)
    assert pname is not None
    assert e is not None

    extra_mover = {}
    extra_eover = {}
    orig_e = e
    if hasattr(e, 'element'):
        # This is a flavors.Reference.
        for i in xrange(len(e.args)):
            extra_mover[unicode(i+1)] = e.args[i]
        if e.owner:
            extra_eover = {u'owner': e.owner}
        e = e.element
    assert hasattr(e, 'ename'), e

    if pname == custom.ELEMENT_NAME_PROP:
        # TODO(xavid): Figure out what sort of dependency this should introduce
        if to_python:
            yield e.ename
        else:
            yield Text(e.ename)
    else:
        if hasattr(eval_state, 'dependencies'):
            assert hasattr(e, 'get_propval'), e
            eval_state.dependencies.addPropvalDep(e, pname)

        if pname == custom.NAME_PROP:
            if ename not in eval_state.mentions:
                eval_state.mentions.append(ename)

        oldme = eval_state.me
        oldeover = dict(eval_state.eoverrides)
        oldmover = dict(eval_state.moverrides)
        oldpover = dict(eval_state.poverrides)
        old_current = list(eval_state.current_stack)
        added_current = False
        try:
            eval_state.me = e
            eval_state.poverrides['this'] = pname
            # Maybe update leaf
            if ename != custom.PARENT_ELEMENT:
                eval_state.eoverrides[custom.LEAF_ELEMENT] = orig_e
                eval_state.eoverrides[custom.CURRENT_ELEMENT] = orig_e
                eval_state.current_stack.append(orig_e)
                added_current = True
            # Add macro params
            if content is not None:
                content = list(content)
                def render_content(arglist, cont):
                    newleaf = eval_state.eoverrides[custom.LEAF_ELEMENT]
                    eval_state.eoverrides[custom.LEAF_ELEMENT] = oldeover[
                        custom.LEAF_ELEMENT]
                    for t in content:
                        yield t
                    eval_state.eoverrides[custom.LEAF_ELEMENT] = newleaf
                eval_state.moverrides[u'content'] = render_content
            elif ename != custom.PARENT_ELEMENT:
                eval_state.moverrides[u'content'] = None
            if ename != custom.PARENT_ELEMENT or arglist:
                eval_state.moverrides[u'args'] = u' '.join(arglist)
                if arglist:
                    for var,arg in parse_macro_args(arglist).items():
                        eval_state.moverrides[
                            u'arg'+unicode(var)] = Arg(arg)

            flavor = structure.get_flavor(pname)

            if flavor.owned:
                eval_state.eoverrides[u'owner'] = orig_e
            
            eval_state.moverrides.update(extra_mover)
            eval_state.eoverrides.update(extra_eover)

            pv = e.get_propval(pname)
            if pv is None:
                if pname == custom.SUBSTITUTION_PROP:
                    if to_python:
                        yield tokens_text(macros.contexteval())
                    else:
                        for t in macros.contexteval():
                            yield t
                    return
                else:
                    raise PropNotFound(
                        "Element ##%s## (called as ##%s## by ##%s##) has no property ##%s##!"
                        % (e.ename if e is not None else None, ename,
                           oldme.ename if oldme is not None else None, pname))

            value = unicode(pv.value, 'utf-8')
            if to_python:
                yield flavor.toPython(
                    value, eval_state.mainparser, propval=pv)
            else:
                for t in flavor.tokenize(
                    value, eval_state.mainparser, propval=pv):
                    yield t
            # If we haven't thrown an exception yet, sancheck some things
            # before cleaning up.  We don't do this in finally, because
            # we don't really want to fail asserts if something below
            # us already had an assertion failure.
            if added_current:
                assert eval_state.current_stack[-1] == orig_e, (
                    eval_state.current_stack, eval_state.current_stack[-1],
                    orig_e)
                assert eval_state.current_stack[:-1] == old_current, (eval_state.current_stack, old_current)
        finally:
            eval_state.me = oldme
            eval_state.poverrides = oldpover
            eval_state.eoverrides = oldeover
            eval_state.moverrides = oldmover
            eval_state.current_stack = old_current
            #print >>sys.stderr, "end of recurse", ename, pname

class nice_gen(object):
    def __init__(self, gen):
        self.__gen = gen
    def __iter__(self):
        return self
    def next(self):
        return self.__gen.next()
    def __add__(self, other):
        return chain(self, other)

class attrelm(object):
    def __init__(self, element, toPython, args=None, owner=None):
        assert hasattr(element, 'ename') or hasattr(element, 'element'), element
        self.element__ = element
        self.__toPython = toPython
        self.__args = args
        self.__owner = owner
    def unwrap__(self):
        if self.__args is None and self.__owner is None:
            return self.element__
        else:
            return Reference(self.element__, self.__args, self.__owner)
    def __getattr__(self, a):
        if a is None:
            raise KeyError()
        if not isinstance(a, unicode):
            a = unicode(a, 'utf-8')
        if self.__toPython:
            ret = to_python(self.element__, a)
            #print >>sys.stderr, "attrelm", self.element__, a, repr(ret), structure.get_flavor(a)
            # wrap elements we get back
            return baz_eval_wrap(ret, self.__toPython)
        else:
            return nice_gen(recurse(self.element__, a))
    def __str__(self):
        return '<@Element %s>' % self.element__.ename
    def __eq__(self, other):
        return isinstance(other,attrelm) and self.element__ == other.element__
    def __ne__(self, other):
        return not self.__eq__(other)
    def __hash__(self):
        return hash(self.element__)

class _elemdict(object):
    def __init__(self, toPython):
        self.__toPython = toPython
        self.__locals = {}
    def __contains__(self, key):
        if key in self.__locals or key == custom.PARENT_ELEMENT:
            return True
        if not isinstance(key, unicode):
            key = unicode(key,'utf-8')
        if key is None:
            return False
        if not key[0].isupper():
            eval_state.memoable_stack[-1].add(('eoverride', key))
            eval_state.memoable_stack[-1].add(('moverride', key))
            if key in eval_state.eoverrides or key in eval_state.moverrides:
                return True
            curr = get_current_for_prop(key)
            foo.append(curr)
            if curr is not None:
                return True
        if hasattr(macros, key):
            return True
        e = structure.get_element(key)
        if e is not None:
            return True
        else:
            if hasattr(eval_state, 'dependencies'):
                eval_state.dependencies.addExistsDep(key)
            return False
    def __getitem__(self,key):
        foo.append(key)
        if key in self.__locals:
            return self.__locals[key]
        if key == custom.PARENT_ELEMENT:
            eval_state.dependencies.addParentDep(eval_state.me)
            return attrelm(eval_state.me.get_parent(), self.__toPython)
        key = unicode(key, 'utf-8')
        if key not in self:
            foo.append(False)
            raise KeyError("No element '%s' exists!"%key)
        if not key[0].isupper() and key in eval_state.moverrides:
            return eval_state.moverrides[key]
        elif key[0] == '_' and key[1:] in eval_state.moverrides:
            return eval_state.moverrides[key[1:]]
        elif key not in eval_state.eoverrides and hasattr(macros, key):
            return tokens_text(getattr(macros, key)(None, None))
        else:
            if not key[0].isupper() and key not in eval_state.eoverrides:
                curr = get_current_for_prop(key)
                if curr is not None:
                    assert curr.has_propval(key)
                    return getattr(attrelm(curr, self.__toPython), key)
            return attrelm(get_element(key), self.__toPython)
    def __setitem__(self, key, value):
        self.__locals[key] = value
    def __delitem__(self, key):
        del self.__locals[key]
    def get(self,key,default=None):
        try:
            return self[key]
        except KeyError:
            return default
foo = []
# Macro-creating helpers
def environment(name, texname=None, tagname=None, texmode='environment',
                doc = ''):
    if texname is None:
        texname = name
    def env(arglist, content):
        args = parse_macro_args(arglist)
        subclasses = []
        for key,val in args.items():
            if key == 'subclass':
                subclasses.append(baz_eval(val))
            else:
                raise WikiException("Unknown arg '%s' in %s environment!"
                                % (key,name))
        if eval_state.format == FORMATS['html']:
            if tagname is not None:
                yield Literal('<%s>' % tagname)
                for t in content:
                    yield t
                yield Entity(ENV_BREAK, True)
                yield Literal('</%s>' % tagname)
            else:
                yield Literal('<div class="%s%s">' % (
                    name, ''.join(' '+c for c in subclasses)))
                for t in content:
                    yield t
                yield Entity(ENV_BREAK, True)
                yield Literal('</div>')
        elif eval_state.format == FORMATS['tex']:
            if texmode == 'environment':
                yield Literal('\\begin{%s}\n' % texname)
                for t in content:
                    yield t
                yield Entity(ENV_BREAK, True)
                yield Literal('\\end{%s}' % texname)
            else:
                yield Literal('\\%s{' % texname)
                for t in content:
                    yield t
                yield Entity(ENV_BREAK, True)
                yield Literal('}')
        else:
            for t in content:
                yield t
            yield Entity(ENV_BREAK, True)
    env.__doc__ = doc
    return env
def atom(default="", doc='', **kw):
    def ato(arglist, content=None):
        for k in kw:
            if eval_state.format == FORMATS[k]:
                yield Literal(kw[k])
                return
        else:
            yield Text(default)
    ato.__doc__ = doc
    return ato
def leafprop(pname):
    def lp(arglist):
        return recurse(custom.LEAF_ELEMENT, pname)
    return lp
def divided(name,divider,*inner):
    """A macro that turns something like <<card>>Foo<<flip/>>Bar<</card>>
    into
    <div class="card">
    <div class="front">Foo</div>
    <div class="back">Bar</div>
    </div>"""
    def div(arglist, content):
        parts = safesplit(content, (divider,))
        args = parse_macro_args(arglist)
        subclasses = []
        for key,val in args.items():
            if key == 'subclass':
                subclasses.append(baz_eval(val))
            else:
                raise WikiException("Unknown arg '%s' in %s divided!"
                                % (key,name))
        if len(parts) > len(inner):
            for e in _error("Too many %ss in a %s!"%(divider,name)):
                yield e
        else:
            if eval_state.format == FORMATS['html']:
                yield Literal('<div class="%s%s">' % (name,''.join(
                    ' '+c for c in subclasses)))
                for i in xrange(len(parts)):
                    yield Literal('<div class="%s">' % (inner[i]))
                    for t in parts[i]:
                        yield t
                    yield Literal('</div>')
                yield Literal('</div>')
            elif eval_state.format == FORMATS['tex']:
                yield Literal('\\begin{%s}\n' % (name))
                for i in xrange(len(parts)):
                    yield Literal('\\%s{' % (inner[i]))
                    for t in parts[i]:
                        yield t
                    yield Literal('}')
                yield Literal('\n\\end{%s}' % (name))
            else:
                for p in parts:
                    for t in p:
                        yield t
    return div

def inline_format(style, arg=None, doc=''):
    def fmt(arglist, content=[]):
        #if style == FOOTNOTE: assert False, eval_state.format.link_toks
        yield Start(style, arg)
        for t in content:
            yield t
        yield End(style, arg)
    fmt.__doc__ = doc
    fmt.hidden = 'latexalike'
    return fmt

def arg_index(num):
    def argi(arglist, content=None):
        """For references with arguments."""
        arg_name = u'arg%d' % num
        eval_state.memoable_stack[-1].add(('moverride', arg_name))
        if arg_name in eval_state.moverrides:
            for t in string_tokenize(eval_state.moverrides[arg_name].raw):
                yield t
        elif u'1' in eval_state.moverrides or u'arg1' in eval_state.moverrides:
            # We have some explicit num_ofs, just not this high
            pass
        else:
            #assert False, (num, eval_state.moverrides)
            raise WikiException("No arg %s: %s" % (num, eval_state.moverrides))
    return argi

def illegal(name,*container):
    def ill(arglist, content=None):
        return _error('"%s" may only be used in a %s!'
                      % (name, ' or '.join('"%s"' % c for c in container)))
    ill.hidden = 'illegal'
    return ill

# Helpers used by our macros
class noescape(object):
    """Wrap a format such that it's escape method is a nop."""
    def __init__(self,format):
        self.__format = format
    def escape(self,text):
        return text
    def __getattr__(self,attr):
        return getattr(self.__format,attr)

class maxwe(object):
    def __cmp__(self,other):
        return 1
MAXWE = maxwe()

class Macros(object):
    #@staticmethod
    #def nowiki(arglist, content):
    #    """Prevent wiki markup in the content from being evaluated.
    #
    #    Unlike ##~{{{}}}##, the font style is unchanged."""
    #    return parser.escape(content)
    @staticmethod
    def par(arglist):
        """Start a new paragraph."""
        yield Entity(ENV_BREAK)
    @staticmethod
    def block(arglist, content):
        """Treat the content as its own block, separate from surroundings."""
        for t in content:
            yield t

    @staticmethod
    def format(arglist, content):
        """Include the markup, unsanitized, in the given format, with
        macros evaluated.
    
        Use like {{{<<format "html">><b>Raw HTML</b><</format>>}}}."""
        args = parse_macro_args(arglist)
        fmt = baz_eval(args[1])
        if eval_state.format == FORMATS[fmt]:
            for t in content:
                if t.op == TEXT:
                    yield Literal(t.arg)
                else:
                    yield t

    @staticmethod
    def let(arglist, content):
        """Define local names for elements or other data.

        Use like {{{<<let current=ExamplePC foo="bar">>...<</let>>}}}."""
        args = parse_macro_args(arglist)
        oldmover = dict(eval_state.moverrides)
        oldeover = dict(eval_state.eoverrides)
        for k,v in args.items():
            # TODO: come up with better evaulation logic with the lazy
            #       evaluate magic/baz_eval
            v = baz_eval(v)
            if hasattr(v, 'ename'):
                eval_state.eoverrides[k] = v
            else:
                assert (isinstance(v, unicode)
                        or hasattr(v, '__call__')
                        or v is None), (v, eval_state.moverrides)
                eval_state.moverrides[k] = v
        for t in content:
            yield t
        eval_state.eoverrides = oldeover
        eval_state.moverrides = oldmover

    @staticmethod
    def plet(arglist, content):
        """Define local prop names.

        Use like {{{<<plet p="name">><<Bob.*p/>><</plet>>}}}."""
        args = parse_macro_args(arglist)
        oldpover = dict(eval_state.poverrides)
        pstubst = {}
        for k,v in args.items():
            # TODO: come up with better evaulation logic with the lazy
            #       evaluate magic/baz_eval
            v = baz_eval(v)
            psubst['*' + k] = v
        with poverride(psubst):
            for t in content:
                yield t
        
    @staticmethod
    def content(arglist):
        """When evaluating a reference, the content of the macro.

        For self-closing macros, content evaluates to None.  When not in
        reference context, raises an exception."""
        raise NoContentException()

    @staticmethod
    def _if(arglist, content):
        """Evaluate the content only if a condition is true.

        Can contain {{{<<elif another condition/>>}}} and/or {{{<<else/>>}}}.
        For example:
        {{{
        <<if leaf.gender == "male">>
          he
        <<elif leaf.gender == "female"/>>
          she
        <<else/>>
          they
        <</if>>
        }}}"""
        parttags = safesplit_tags(content, ('else', 'elif'))
        cond = u' '.join(arglist)
        while True:
            val = baz_eval(cond)
            #assert False, (arglist, val)
            if val:
                break
            else:
                parttags = parttags[1:]
            if len(parttags) <= 0:
                return
            name, arglist = parttags[0][1]
            if name == 'else':
                break
            else:
                cond = u' '.join(arglist)
        for t in parttags[0][0]:
            yield t
    # You don't need to put staticmethod() when you add things to this at runtime...
    _else = staticmethod(illegal('else','if'))
    _elif = staticmethod(illegal('elif','if'))

    @staticmethod
    def link(arglist, content=None, style=LINK):
        """Builds a link dynamically, evaluating and joining arguments.

        Use like {{{<<link "/print/" leaf.username ".pdf">>Packet<</link>>}}}
        or {{{<<link i.prefix ":" i.examplepage />>}}}."""
        #print >>sys.stderr, "LINK", repr(arglist), repr(content)
        args = parse_macro_args(arglist)
        dest = ''.join(unicode_eval(args[a]) for a in sorted(args))
        #print >>sys.stderr, "LINK dest=", dest
        if content is not None:
            yield Start(style, dest)
            for t in content:
                yield t
            yield End(style, dest)
        else:
            yield Entity(style, dest)

    @staticmethod
    def image(arglist, content=None):
        return Macros.link(arglist, content, style=IMAGE)

    @staticmethod
    def count(arglist, content=None):
        """Counts the number of elements that would be matched by foreach."""
        return Macros.foreach(arglist, content=content, count=True)

    #@staticmethod
    #def sum(arglist, content=None):
    #    total = 0
    #    for t in Macros.foreach(arglist, content=content):
    #        #assert t.op == TEXT, repr(t)
    #        if t.op == TEXT:
    #            try:
    #                total += int(t.arg)
    #            except ValueError:
    #                #assert False, repr(t.arg)
    #                pass
    #    return [Text(unicode(total))]

    @staticmethod
    def foreach(arglist, content=None, count=False):
        """Evaluate the content once for each member of some group.

        The basic form is {{{<<foreach group var ...>>...<</foreach>>}}}.
        "group" can either be an element, in which case the content is
        evaluated with var set to each of that element's descendents in
        turn, or a particular element's prop (generally of flavor references),
        in which case the content is evaluated with var set to each of the
        elements referenced in that prop value, in turn.

        Nested loops can be created in a single macro by adding
        {{{foreach group2 var2}}} after the first var name.
        You can also sum counts by doing something like:
        {{{<<foreach Object o count o.stuff i .../>>}}}.

        Parameters after the var name can be conditions filtering the
        specified group; these should generally be in parentheses.

        foreach also takes a large number of optional keyword parameters,
        which take a value after an =:

        orderBy: a prop the group members have they should be ordered by

        groupBy: the group members should be grouped by the specified
                 prop.  If the value contains a comma, the markup after
                 the comma is used as a group header.

        ifNone: markup that should be evaluated if the group is empty.

        asTree: put the evaluations of the content in a nested list
                structure reflecting the inheritance tree of the group.
                Doesn't support nested loops.

        inRowsOf: put the evaluations in a table with the specified number
                  of columns.  {{{<<break/>>}}} may be used to break an
                  evaluation into several rows.

        recursive: if the group was based on a references-flavor prop and
                   group members have that prop, evaluate the content
                   for the elements referenced in their values for the prop,
                   and so on.

        childrenOnly: use the specified element's children only, not all of
                      its descendents.
        """
        args = parse_macro_args(arglist)
        if 1 not in args or 2 not in args:
            raise WikiException("{{{foreach}}} takes at least two parameters!  See [[edit:Macros]]!")
        #print >>sys.stderr, "In foreach", args
        # We need to be able to iterate over this multiple times
        typ = args[1]
        del args[1]
        nam = args[2]
        del args[2]

        subtyps = []
        subnams = []
        next = 3
        while next in args and args[next] in ('foreach', 'count'):
            if args[next] == 'count':
                count = True
            del args[next]
            subtyps.append(args[next + 1])
            del args[next + 1]
            subnams.append(args[next + 2])
            del args[next + 2]
            next += 3

        if content is None:
            if not count:
                raise WikiException("{{{foreach}}} must have content!  See [[edit:Macros]]!")
            else:
                content = []
        content = list(content)

        restr = set()
        orderBy = []
        groupBy = None
        asTree = False
        recursive = False
        childrenOnly = False
        ifNone = None
        inRowsOf = None
        for key,arg in args.items():
            if not isinstance(key,int):
                if key.lower() == 'orderby':
                    orderBy +=arg.strip().split()
                elif key.lower() == 'groupby':
                    if ',' in arg:
                        var,groupMarkup = arg.split(',',1)
                    else:
                        var = arg
                        groupMarkup = ''
                    var = var.strip()
                    if ' ' in var:
                        var = var.split()
                        groupBy = var[0]
                        groupName = var[1]
                    else:
                        groupBy = groupName = var
                elif key.lower() == 'ifnone':
                    ifNone = baz_eval(arg)
                elif key.lower() == 'astree':
                    asTree = baz_eval(arg)
                elif key.lower() == 'recursive':
                    recursive = baz_eval(arg)
                elif key.lower() == 'childrenonly':
                    childrenOnly = baz_eval(arg)
                elif key.lower() == 'inrowsof':
                    inRowsOf = baz_eval(arg)
                else:
                    for e in _error("Unknown foreach parameter '%s'!" % key):
                        yield e
            else:
                restr.add(arg)

        # TODO(xavid): create separate "for each ancestor var" and
        #              "for var in list" syntaxes.
        list_or_elm = baz_eval(typ)
        #print >>sys.stderr, "l_o_e", typ, list_or_elm
        if not asTree:
            def get_list(list_or_elm):
                if hasattr(list_or_elm, 'ename'):
                    # Now do a traversal:
                    if childrenOnly:
                        eval_state.dependencies.addChildrenDep(list_or_elm)
                        return list_or_elm.get_children()
                    else:
                        return _list_descendants(list_or_elm)
                else:
                    assert isinstance(list_or_elm, list), repr(list_or_elm)
                    return list_or_elm
            def do_overrides(subst):
                for substnam in subst:
                    if (hasattr(subst[substnam], 'ename') or
                        hasattr(subst[substnam], 'element')):
                        eval_state.eoverrides[substnam] = subst[substnam]
                    else:
                        assert isinstance(subst[substnam], (unicode, types.FunctionType)), (subst, substnam)
                        eval_state.moverrides[substnam] = subst[substnam]
            with benchmarking('foreach get_list %s' % typ):
                lst = get_list(list_or_elm)
            nambits = nam.split(',')
            if len(nambits) == 1:
                for e in lst:
                    assert e is not None, (typ, lst, list_or_elm)
                subst_list = [{nam: e} for e in lst]
            else:
                for e in lst:
                    for f in e:
                        assert f is not None, lst
                if not all(len(nambits) == len(e) for e in lst):
                    raise WikiException("Length error unpacking for %s!" % nam)
                subst_list = [dict(zip(nambits, e)) for e in lst]
            
            old_eover = dict(eval_state.eoverrides)
            old_mover = dict(eval_state.moverrides)

            # Compute any nested loop levels
            for styp, snam in zip(subtyps, subnams):
                new_subst_list = []
                for subst in subst_list:
                    do_overrides(subst)
                    new_lst = get_list(baz_eval(styp))
                    for new_e in new_lst:
                        if new_e is None:
                            raise WikiException("Got None evaluating %s!"
                                                % (styp))
                        new_subst = dict(subst)
                        new_subst[snam] = new_e
                        new_subst_list.append(new_subst)
                subst_list = new_subst_list

            # Handle filters
            if len(restr) > 0:
                with benchmarking('foreach filtering %s' % restr):
                    filtlst = []
                    for subst in subst_list:
                        do_overrides(subst)
                        try:
                            if all(baz_eval(r) for r in restr):
                                filtlst.append(subst)
                        except NoContentException:
                            pass
                    subst_list = filtlst

            if count:
                yield Text(unicode(len(subst_list)))
            elif len(subst_list) == 0 and ifNone is not None:
                for t in tokenize(ifNone, makeRestricted):
                    yield t
            else:
                # Sort by the nam element always
                for f in reversed(orderBy):
                    subst_list.sort(key=lambda subst:
                                    to_python(subst[nambits[0]], f)
                                    if subst[nambits[0]].has_propval(f)
                                    else MAXWE)

                def stickon(subst, cont):
                    do_overrides(subst)
                    for t in cont:
                        yield t
                if groupBy:
                    groups = {}
                    for subst in subst_list:
                        e = subst[nam]
                        val = e[groupBy] if groupBy in e else MAXWE
                        if val in groups:
                            groups[val].append(subst)
                        else:
                            groups[val] = [subst]
                    for g in sorted(groups.keys()):
                        assert isinstance(g, unicode), repr(g)
                        eval_state.moverrides[groupName] = (g if g != MAXWE
                                                            else u"???")
                        if groupMarkup:
                            for t in tokenize(groupMarkup, makeRestricted):
                                yield t
                        for subst in groups[g]:
                            for t in stickon(subst, content):
                                yield t
                else:
                    if inRowsOf:
                        this_row = []
                        content_bits = safesplit(content, 'break')
                        content = content_bits[0]
                        content_bits = content_bits[1:]
                        def end_row():
                            yield End(TABLE_ROW)
                            for c in content_bits:
                                yield Start(TABLE_ROW)
                                for s in this_row:
                                    yield Start(TABLE_CELL)
                                    for t in stickon(s, c):
                                        yield t
                                    yield End(TABLE_CELL)
                                yield End(TABLE_ROW)
                            del this_row[:]
                    for subst in subst_list:
                        if inRowsOf:
                            if len(this_row) == 0:
                                yield Start(TABLE_ROW)
                            yield Start(TABLE_CELL)
                        #with benchmarking(u'foreach subst=%s' % subst):
                        for t in stickon(subst, content):
                            yield t
                        if inRowsOf:
                            yield End(TABLE_CELL)
                            this_row.append(subst)
                            if len(this_row) == inRowsOf:
                                for t in end_row():
                                    yield t
                    if inRowsOf and len(this_row) > 0:
                        for t in end_row():
                            yield t
            eval_state.eoverrides = old_eover
            eval_state.moverrides = old_mover            
        else: # asTree
            def dig(e,depth):
                assert e.ename is not None
                eval_state.dependencies.addChildrenDep(e)
                old_overrides = dict(eval_state.eoverrides)
                eval_state.eoverrides[nam] = e
                yield Start(UNORDERED_ITEM, depth)
                for t in content:
                    yield t
                yield End(UNORDERED_ITEM, depth)
                eval_state.eoverrides = old_overrides

                lst = e.getChildren()
                if count:
                    yield Text(unicode(len(lst)))
                elif len(lst) > 0:
                    for f in reversed(orderBy):
                        lst.sort(key=lambda e:e[f] if f in e else MAXWE)
                    for e in lst:
                        for y in dig(e,depth+1):
                            yield y
            assert hasattr(list_or_elm, 'ename')
            for t in dig(list_or_elm, 1):
                yield t
    _break = staticmethod(illegal('break', 'foreach', 'similar'))

    @staticmethod
    def contexteval(arglist=None):
        # TODO(xavid): actually explain this better.
        """Call as {{{<<contexteval/>>}}} in an element's substitution
        to allow that element to serve as a context for property/macro
        evaluation."""
        leaf = get_element(u'leaf')
        ccontent = get_moverride(u'content')
        
        # This is for members/contacts lists.
        is_refs = False
        #print >>sys.stderr, "contexteval", get_context()

        eover_dict = {}
        current = get_current_element()
        if hasattr(leaf, 'element'):
            if leaf.owner is not None:
                eover_dict[u'owner'] = leaf.owner
            leaf = leaf.element
        if (get_overridden_element(u'owner') is None and current != leaf
            and u'owner' not in eover_dict):
            eover_dict[u'owner'] = current

        with eoverride(eover_dict):
            if get_context('in_get_refs'):
                is_refs = True
                href = leaf.ename
                i = 1
                while get_moverride(u'arg%d' % i):
                    href += u'/' + tokens_text(
                        string_tokenize(get_moverride(u'arg%d' % i).raw))
                    i += 1
                yield Start(LINK, href)

            with context({'in_get_refs': False}):
                with push_current(leaf):
                    if ccontent is not None:
                        cnt = 0
                        for t in ccontent(None, None):
                            cnt += 1
                            yield t
                        if cnt == 0:
                            for t in recurse(leaf, u'name'): yield t
                    else:
                        for t in recurse(leaf, u'name'): yield t

            if is_refs:
                yield End(LINK, href)
            #assert False, eval_state.eoverrides

    @staticmethod
    def comment(arglist, content=None):
        """Suppress any content.

        Useful for leaving comments only relevant to people editing a prop's
        value."""
        return ()

    @staticmethod
    def withheaders(arglist, content): # More descriptive name
        """Renders the (up to) four arguments as headers around the content."""
        args = parse_macro_args(arglist)
        headfoot = ["","","",""]
        for key,val in args.items():
            if key in (1,2,3,4):
                headfoot[key-1] = baz_eval(val)
            else:
                raise WikiException("Unknown arg '%s' in card!"
                                    % (key))
        cont = list(content)
        if cont:
            if eval_state.format == FORMATS['html']:
                yield Literal('''<div class="page">
<div class="lheader">%s</div>
<div class="rheader">%s</div>
<div class="lfooter">%s</div>
<div class="rfooter">%s</div>
''' % tuple(headfoot))
                for t in cont:
                    yield t
                yield Literal('''
</div>
''')
            elif eval_state.format == FORMATS['tex']:
                yield Literal(r'''\cleardoublepage
\resetnumbering
\headers{%s}{%s}{%s}{%s}
''' % tuple(headfoot))
                for t in cont:
                    yield t
                yield Literal(r'''
\cleardoublepage
''')
    
    center = staticmethod(environment('center',tagname='center',
                                      doc="""Center the content."""))
    right = staticmethod(environment('right',
                                     texname='flushright',
                                     doc="""Right-justify the content."""))
    left = staticmethod(environment('left',
                                    texname='flushleft',
                                    doc="""Left-justify the content."""))
    dbox = staticmethod(environment(
        'dbox', texmode='command',
        doc="""Put a double box around the content"""))

    figure = staticmethod(environment('figure', doc="Make a floating figure."))

    clearpage = staticmethod(atom(html=u"<hr />",tex=r'\clearpage',
                                  doc="Start a new logical page."))
    cleardoublepage = staticmethod(atom(html=u"<hr />",tex=r'\cleardoublepage',
                                        doc="Start a new piece of paper."))

    smallskip = staticmethod(atom(html=u'<div class="gap"></div>',
                              tex=r'\par\smallskip ',
                              doc="Insert a small gap."))
    medskip = staticmethod(atom(html=u'<div class="gap"></div>',
                              tex=r'\par\medskip ',
                              doc="Insert a medium gap."))
    bigskip = staticmethod(atom(html=u'<div class="gap"></div>',
                              tex=r'\par\bigskip ',
                              doc="Insert a big gap."))

    tableofcontents = staticmethod(atom(html=u'',
                                        tex=u'\tableofcontents',
                                        doc="Insert a table of contents."))

    # LaTeX-alike macro punct
    ldots = staticmethod(atom(u'\u2026', doc="Ellipsis."))

    # LaTeX-alike non-punct special chars
    ae = staticmethod(atom(u'\u00E6', doc="ae"))
    AE = staticmethod(atom(u'\u00C6', doc="AE"))

    # LaTeX-alike macros
    textbf = staticmethod(inline_format(BOLD, doc="Bold."))
    textit = staticmethod(inline_format(ITALIC, doc="Italic."))
    emph = staticmethod(inline_format(ITALIC, doc="Italic."))
    texttt = staticmethod(inline_format(MONOSPACE, doc="Monospace."))
    uline = staticmethod(inline_format(UNDERLINE, doc="Underline."))
    sout = staticmethod(inline_format(STRIKE, doc="Strike."))
    footnote = staticmethod(inline_format(FOOTNOTE, doc="Footnote."))

    Tiny = staticmethod(inline_format(SIZE, -5, doc="Small text."))
    tiny = staticmethod(inline_format(SIZE, -4, doc="Small text."))
    SMALL = staticmethod(inline_format(SIZE, -3, doc="Small text."))
    Small = staticmethod(inline_format(SIZE, -2, doc="Small text."))
    small = staticmethod(inline_format(SIZE, -1, doc="Small text."))
    normalsize = staticmethod(inline_format(SIZE, 0, doc="Normal font size."))
    large = staticmethod(inline_format(SIZE, 1, doc="Large text."))
    Large = staticmethod(inline_format(SIZE, 2, doc="Rather large text."))
    LARGE = staticmethod(inline_format(SIZE, 3, doc="Very large text."))
    huge = staticmethod(inline_format(SIZE, 4, doc="Huge text."))
    Huge = staticmethod(inline_format(SIZE, 5, doc="Very huge text."))

    @staticmethod
    def noindent(arglist, content=None):
        """No indentation."""
        if content is None:
            yield Entity(NOINDENT)
        else:
            yield Start(NOINDENT)
            for t in content:
                yield t
            yield End(NOINDENT)
    
    @staticmethod
    def _eval(arglist):
        """Evaluate the given expression and output its value.

        E.g., {{{<<eval 3 + 5/>>}}} will output 8."""
        result = unicode_eval(u' '.join(arglist))
        assert isinstance(result, unicode), result
        yield Text(result)

    @staticmethod
    def raw(arglist):
        """Return a macro's value as a string, unevaluated.

        Useful for args of macro-like elements."""
        argstr = u' '.join(arglist).strip()
        eval_state.memoable_stack[-1].add(('moverride', argstr))
        if (argstr in eval_state.moverrides
            and hasattr(eval_state.moverrides[argstr], 'raw')):
            yield Text(eval_state.moverrides[argstr].raw)
        else:
            yield Text(unicode_eval(argstr))

    @staticmethod
    def cache(arglist, content=None, editable=False):
        """Evaluate the specified prop of an element cachably.

        This means the evaluation will ignore any context, but that it can
        take advantage of caching.  Limited context can be provided in
        the form of {{{let}}}-style keyword parameters."""
        args = parse_macro_args(arglist)
        if 1 not in args: # or 2 in args:
            raise WikiException(
                "##cache## takes only one non-keyword parameter!")

        ename, prop_name = args[1].split('.')
        element = get_element(ename)
        #print >>sys.stderr, "CACHING %s.%s" % (element.ename, prop_name)
        del args[1]
        let = {}
        for k,v in args.items():
            let[k] = baz_eval(v)

        if hasattr(element, 'element'):
            # flavors.Reference
            for i in xrange(len(element.args)):
                let[str(i+1)] = element.args[i]
            element = element.element
        
        import conversion
        flat = conversion.flatten_filters({'let': let})
        
        ce = conversion.get_from_cache(element.ename, prop_name,
                                       format=eval_state.extension,
                                       cache_tag=flat)
        if ce is None:
            pv = element.get_propval(prop_name)
            if pv is None:
                raise WikiException('##%s## has no property ##%s##!'
                                    % (ename, prop_name))
            result, deps, metadata = evaluate(pv,
                                              eval_state.extension,
                                              let=let,
                                              reentrant=True)
            if dependencies.DISCORDIA not in deps:
                conversion.cache_propval(element.ename, prop_name,
                                         eval_state.extension+flat,
                                         result, deps, metadata)
        else:
            result = ce['value']
            deps = ce['dependencies']
            metadata = ce['metadata']
            pv = None
        for d in deps:
            if d == dependencies.DISCORDIA:
                eval_state.dependencies.makeUncacheable()
                break
            elif d == dependencies.OMNISCIENCE:
                eval_state.dependencies.makeRestricted()
            else:
                assert len(d) == 3, d
                eval_state.dependencies.addRawDep(*d)
        for i in metadata['images']:
            eval_state.images.append(i)
        if editable:
            eval_state.dependencies.makeRestricted()            
            yield Literal('<div class="editable">')
        if isinstance(result, basestring):
            yield Literal(result)
        else:
            # This is a list of arbitrary objects from an ExtractingFormat
            for i in result:
                yield Literal(i)
        if editable:
            # This macro/dep should not be in bazbase
            from bazki.util import jsarg_escape
            if pv is None:
                pv = element.get_propval(prop_name)
            val = unicode(pv.value, 'utf-8')
            yield Literal(u'</div><a onclick="editize(\'%s.%s\', \'%s\', this); return false" class="editlink" id="editize-%s.%s" href="#">'
                          % (
                element.ename, prop_name,
                jsarg_escape(val),
                element.ename, prop_name))
            yield Text(u'edit')
            yield Literal('</a>')

    @staticmethod
    def editable(arglist):
        """Like {{{<<cache>>}}}, but adds an edit link."""
        editable = eval_state.format == FORMATS['html']
        for t in Macros.cache(arglist, editable=editable):
            yield t

    # TODO(xavid): combine with other foreach syntax
    @staticmethod
    def foreachpropin(arglist, content):
        """Evaluate the content once for each prop in the given element.

        Use like
        {{{<<foreachpropin Gizmo p>> * <<Gizmo.p/>><</foreachpropin>>}}}.

        Can be given an extra ##flavor=whatever## arg to limit it to
        props of a given flavor."""
        args = parse_macro_args(arglist)
        if 1 not in args or 2 not in args:
            raise WikiException(
                "##foreachpropin## takes two parameters!")
        ename = args[1]
        var = args[2]
        if 'flavor' in args:
            flavor = baz_eval(args['flavor'])
        else:
            flavor = None

        # We need to loop through content more than once
        content = list(content)

        element = get_element(ename)
        for pname in element.list_props():
            if pname != get_propname(u'this'):
                if flavor is not None:
                    if structure.get_prop(pname).flavor != flavor:
                        continue
                with poverride({'*' + var: pname}):
                    for t in content:
                        yield t

    @staticmethod
    def foreachimage(arglist, content):
        args = parse_macro_args(arglist)
        if 1 not in args or 2 not in args:
            raise WikiException("{{{foreachimage}}} takes at least two parameters!")

        old_images = eval_state.images
        eval_state.images = []

        propval = baz_eval(args[1])
        nam = args[2]
        content = list(content)

        latched_images = eval_state.images
        eval_state.images = old_images

        for i in latched_images:
            with moverride({nam: i}):
                for t in content:
                    yield t

    @staticmethod
    def sourceof(arglist):
        """Render the source of the given prop of an element.

        Use like {{{<<sourceof Gizmo.whatzit/>>}}}."""
        args = parse_macro_args(arglist)
        if 1 not in args or len(args) != 1:
            raise WikiException(
                "##sourceof## takes only one parameter!")

        ename, prop_name = args[1].split('.')
        element = get_element(ename)
        prop_name = get_propname(prop_name)

        if prop_name not in element:
            raise WikiException(
                "##%s## has no prop ##%s## to get the source of!"
                % (element.ename, prop_name))

        yield Start(CODEBLOCK)
        yield Text(unicode(element[prop_name].value, 'utf-8'))
        yield End(CODEBLOCK)

    @staticmethod
    def propname(arglist):
        """Render the propname of the given prop; useful in
        {{{<<foreachpropin>>}}}."""
        args = parse_macro_args(arglist)
        if 1 not in args or len(args) != 1:
            raise WikiException(
                "##propname## takes only one parameter!")

        prop_name = get_propname(args[1])

        yield Text(prop_name)

    @staticmethod
    def foreachmacro(arglist, content):
        """Evaluate content once for each built-in macro.

        Sets the argument to the name of the built-in macro.  Optionally,
        a second argument selects a class of macro hidden by default."""
        args = parse_macro_args(arglist)
        if 1 not in args or len(args) not in (1,2):
            raise WikiException(
                "##foreachmacro## takes one or two parameters!")
        
        var = args[1]
        expected_hidden = args[2] if 2 in args else None
        for m in dir(macros):
            if not m.startswith('_') and getattr(getattr(macros, m), 'hidden',
                                                 None) == expected_hidden:
                if m.endswith('_'):
                    m = m[:-1]
                with moverride({var: m}):
                    for t in content:
                        yield t

    @staticmethod
    def macrodoc(arglist):
        """Given a built-in macro, render its help text."""
        args = parse_macro_args(arglist)
        if 1 not in args or len(args) != 1:
            raise WikiException(
                "##macrodoc## takes only one parameter!")
        
        name = args[1]
        eval_state.memoable_stack[-1].add(('moverride', name))
        if name in eval_state.moverrides:
            name = eval_state.moverrides[name]

        if hasattr(macros, name+'_'):
            name = name+'_'
        elif not hasattr(macros, name):
            raise WikiException("No such macro ##%s##!" % name)

        doc = getattr(getattr(macros, name), '__doc__', None)
        if doc is not None:
            for t in tokenize(doc, makeRestricted):
                yield t

    @staticmethod
    def macro(arglist, content=None):
        """Evaluate the built-in macro named by the given expression."""
        argstr = u' '.join(arglist).strip()
        if ' ' in argstr:
            name, argstr = argstr.split(' ', 1)
        else:
            name = argstr
            argstr = ""
        eval_state.memoable_stack[-1].add(('moverride', name))
        if name in eval_state.moverrides:
            name = eval_state.moverrides[name]

        if content is not None:
            return getattr(macros, name)(argstr, content)
        else:
            return getattr(macros, name)(argstr)

    @staticmethod
    def default(arglist, content=""):
        """Special indicator macro.  Propvals that //start// with this macro
        will be considered 'abstract', and the interface will encourage
        users to override them in children.  Has no effect on evaluation."""
        for t in content:
            yield t

    @staticmethod
    def year(arglist):
        """Evauates to the current year."""
        from datetime import datetime
        yield Text(unicode(datetime.now().year))

    @staticmethod
    def label(arglist, content):
        if not get_context('in_get_refs'):
            lab = tokens_text(content)
            assert lab not in eval_state.label_map, (lab, eval_state.label_map)
            eval_state.label_map[lab] = eval_state.figure
        return ()

    @staticmethod
    def ref(arglist, content):
        lab = tokens_text(content)
        yield Entity(REF, lab)

    @staticmethod
    def subfigure(arglist, content):
        import traceback
        #traceback.print_stack()
        import conversion, translators
        ename, pname_with_filters = u' '.join(arglist).split('.', 1)
        filters = {}
        pname = conversion.extract_filters(pname_with_filters, filters)
        width, height = conversion.cache_and_get_dimensions(ename, pname,
                                                            filters)
        assert width is not None and height is not None, (width, height, ename,
                                                          pname, filters)
        # TODO(xavid): better understand whether and/or why we need this list()
        eval_state.subfigures.append((ename+'.'+pname_with_filters,
                                      width, height, list(content)))
        return ()

    @staticmethod
    def table(arglist, content):
        """Start a table, with various options.

        border: include a border, default True.

        align: cell alignment, default 'left'.

        figure: Treat this as the numbered figure, with each column as
                subfigures labeled in the second row.

        font: a macro name to wrap each cell with
        """
        # TODO(xavid): way to switch into x (equal width) mode (mode: 'equal')
        args = parse_macro_args(arglist)
        border = baz_eval(args['border']) if 'border' in args else True
        align = baz_eval(args['align']) if 'align' in args else 'left'
        figure = baz_eval(args['figure']) if 'figure' in args else None
        font = baz_eval(args['font']) if 'font' in args else None

        if figure is True:
            figure = eval_state.last_figure + 1
        if not get_context('in_get_refs'):
            eval_state.last_figure = figure
        eval_state.subfigures = []

        params = {}
        if not border:
            params['border'] = False
        if align != 'left':
            params['align'] = align

        # TODO(xavid): DTWT for nested tables
        row = 0
        subfigure_ord = ord('a')
        yield Start(TABLE, params)
        for t in content:
            if t.op == END and t.style == TABLE_CELL:
                if font:
                    yield End(MACRO, font)
                eval_state.figure = None
            yield t
            if t.op == START:
                if t.style == TABLE_ROW:
                    row += 1
                elif t.style == TABLE_CELL:
                    if font:
                        yield Start(MACRO, (font, None))
                    if row == 2 and figure is not None:
                        label = u'%s%s' % (figure, unichr(subfigure_ord))
                        yield Text(u'(%s) ' % (label,))
                        eval_state.figure = label
                        subfigure_ord += 1

        if len(eval_state.subfigures) > 0:
            imgs = []
            caps = []
            specs = []
            for img, width, height, caption in eval_state.subfigures:
                imgs.append(img)
                specs.append('w{%spt}' % (width * latex.PX_TO_PT + 5))
                caps.append(caption)
            yield Start(TABLE_ROW)
            for img in imgs:
                yield Start(TABLE_CELL)
                yield Entity(IMAGE, img)
                yield End(TABLE_CELL)
            yield End(TABLE_ROW)
            yield Start(TABLE_ROW)
            for cap in caps:
                yield Start(TABLE_CELL)
                if font:
                    yield Start(MACRO, (font, None))
                label = u'%s%s' % (figure, unichr(subfigure_ord))
                yield Text(u'(%s) ' % (label,))
                eval_state.figure = label
                subfigure_ord += 1
                for t in cap:
                    yield t
                if font:
                    yield End(MACRO, font)
                eval_state.figure = None
                yield End(TABLE_CELL)
            yield End(TABLE_ROW)
            params['specs'] = specs
        del eval_state.subfigures
        
        yield End(TABLE, params)

    @staticmethod
    def tablerow(arglist, content):
        yield Start(TABLE_ROW)
        for t in content:
            yield t
        yield End(TABLE_ROW)

    @staticmethod
    def tablecell(arglist, content):
        args = parse_macro_args(arglist)
        start_dict = {}
        if 'colspan' in args:
            start_dict['colspan'] = baz_eval(args['colspan'])
        yield Start(TABLE_CELL, start_dict)
        for t in content:
            yield t
        yield End(TABLE_CELL)

    _1 = staticmethod(arg_index(1))
    _2 = staticmethod(arg_index(2))
    _3 = staticmethod(arg_index(3))
    _4 = staticmethod(arg_index(4))
    _5 = staticmethod(arg_index(5))
    _6 = staticmethod(arg_index(6))
    _7 = staticmethod(arg_index(7))
    _8 = staticmethod(arg_index(8))
    _9 = staticmethod(arg_index(9))

def link_func(href, sty=LINK):
    implicit = False
    if (':' not in href and not href.startswith('/')) or href[0].isupper():
        if '/' not in href and ' ' not in href:
            # Normal context-based interpretation
            if (eval_state.topthis is not None
                and structure.get_prop(eval_state.topthis).visible):
                href = 'get:'+href
            else:
                href = 'edit:'+href
            implicit = True
        elif '/' in href and (' ' not in href
                              or href.find(' ') > href.find('/')):
            href = 'get:'+href
            implicit = True
    if ':' in href:
        pref,rest = href.split(':',1)
        rest = rest.strip()
        if pref in ('get','edit'):
            ext = None
            if '.' in rest and not rest.endswith('.'):
                ename,pname = rest.split('.',1)
                if '.' in pname:
                    pname, ext = pname.rsplit('.', 1)
                    if ext[0].isdigit() and ext.endswith('in'):
                        pname = pname + '.' + ext
                        ext = None
                    else:
                        ext = '.' + ext
            else:
                ename = rest
                pname = None
            args = None
            render_func = render_propval
            if '/' in ename:
                args = ename.split('/')
                ename = args.pop(0)
                mover = {}
                for i in xrange(len(args)):
                    mover[unicode(i+1)] = args[i]
                def render_with_content(element, prop_name):
                    return render_propval(element, prop_name,
                                          moverrides=mover)
                render_func = render_with_content
            if ename in eval_state.eoverrides:
                elm = eval_state.eoverrides[ename]
                if hasattr(elm, 'element'):
                    # Reference
                    # TODO(xavid): how should args/owner be handled here?
                    elm = elm.element
                ename = elm.ename
            else:
                elm = structure.get_element(ename)
            if sty == IMAGE:
                eval_state.images.append('%s.%s' % (ename, pname))
            with push_current(elm):
                if pref == 'get':
                    text, info = custom.product_link(
                        ename, pname, eval_state.dependencies,
                        render_func, extension=ext, args=args)
                elif pref == 'edit':
                    text, info = custom.edit_link(
                        ename, pname, eval_state.dependencies,
                        render_func, extension=ext)
            if ename not in eval_state.mentions:
                eval_state.mentions.append(ename)
            return text, info
        elif not rest.startswith('//'):
            # Check interwiki
            iw = structure.get_element(u'InterwikiConfig')
            if iw is None:
                eval_state.dependencies.addExistsDep(u'InterwikiConfig')
            else:
                eval_state.dependencies.addDep(iw.get_propval(u'prefix'))
                eval_state.dependencies.addDep(iw.get_propval(u'url'))
                if iw.has_propval(u'prefix') and iw.has_propval(u'url'):
                    for i in _list_descendants(iw):
                        eval_state.dependencies.addDep(i.get_propval(u'prefix'))
                        if unicode(i.get_prop(u'prefix'), 'utf-8') == pref:
                            eval_state.dependencies.addDep(i.get_propval(u'url'))
                            return (href, dict(url=unicode(i.get_prop(u'url'),
                                                           'utf-8')+rest,
                                               style='external'))
    # None matched
    if href.startswith('/'):
        abshref = custom.local_link(href)
        style = 'internal'
    else:
        abshref = href
        if '://' in href:
            style = 'external'
        else:
            style = 'internal'
    return href, dict(url=abshref, style=style)

def _error(message):
    if eval_state.catch_errors:
        yield Error('[error]')
    else:
        eval_state.dependencies.makeRestricted()
        eval_state.errors.append(message)
        yield Start(ERROR)
        for t in tokenize(message, makeRestricted):
            yield t
        yield End(ERROR)

MACRO_NAME_PAT = re.compile('^([^\s#]+)(?:\#\S+)?')

def macro_func(name, arglist=[], content=None):
    #print >>sys.stderr, "macro_func", name, arglist, content
    try:
        # Allow macros to have #foo suffix, to allow nesting
        m=MACRO_NAME_PAT.match(name)
        if m is None:
            raise WikiException("Weird macro name '%s'!" % name)
        name = m.group(1)
        origname = name
        arglist = arglist or []
        maced = False
        if '.' not in name and not name[0].isupper() and not arglist:
            eval_state.memoable_stack[-1].add(('moverride', name))
            overridable = True
        else:
            overridable = False
        if ('.' not in name
            and (name[0].isupper()
                 or (overridable and name not in eval_state.moverrides))
            and not hasattr(macros,name)
            and not hasattr(macros,'_'+name)):
            if get_overridden_element(name) is None:
                curr = get_current_for_prop(name)
                if curr is not None:
                    assert curr.has_propval(name)
                    eval_state.dependencies.addExistsDep(name)
                    elm,pnam = curr, name
                else:
                    elm = name
                    pnam = custom.SUBSTITUTION_PROP
            else:
                elm = name
                pnam = custom.SUBSTITUTION_PROP
            maced=True
        elif '.' in name:
            elm,pnam = name.strip().rsplit('.', 1)
            if '.' in elm:
                # This could be a reference, where foo.owner itself refers
                # to an element.
                elm = baz_eval(elm)
                if elm is None:
                    raise WikiException("%s evaluates to None!" % name)
        else:
            # Non-element-reference case
            if (not name[0].isupper() and overridable
                and name in eval_state.moverrides):
                ove = eval_state.moverrides[name]
                if hasattr(ove, '__call__'):
                    for t in ove(arglist, content):
                        yield t
                elif ove is None:
                    pass
                else:
                    yield Text(ove)
                return
            if hasattr(macros,'_'+name):
                name = '_'+name
            if hasattr(macros, name):
                if content is not None:
                    gen = getattr(macros,name)(arglist=arglist,content=content)
                else:
                    gen = getattr(macros,name)(arglist=arglist)
                if not hasattr(gen, '__iter__'):
                    for e in _error(
                        'Error: {{{<<%s %s>>}}} return non-iterator "%s"!'
                        % (name, u' '.join(arglist), repr(gen))):
                        yield e
                    return
                for t in gen:
                    yield t
            else:
                for e in _error('Error: no such macro %s!' % name):
                    yield e
            return # End non-element-reference case
        #print >>sys.stderr, "maced:", name

        # Element reference case
        if pnam != custom.ELEMENT_NAME_PROP:
            flav = structure.get_flavor(get_propname(pnam))
            assert flav is not None, (pnam, flav)
            if flav.is_ref:
                elm = to_python(elm, pnam)
                if elm is None:
                    raise WikiException(
                        "%s is a reference that evaluates to None!" % name)
                pnam = custom.SUBSTITUTION_PROP

        try:
            try:
                assert elm is not None, name
                r = recurse(elm, pnam, content=content,
                            arglist=arglist)
                for t in r:
                    yield t
            except ElementNotFound:
                # This hack is a DWIM for Gameki.
                if isinstance(elm, unicode) and len(elm) >= 2 and elm[0].islower() and elm[1].isupper():
                    r = recurse(elm[1:], pnam, content=content,
                                arglist=arglist)
                    for t in r:
                        yield t
                else:
                    raise
        except ElementNotFound:
            if maced:
                leaf = eval_state.eoverrides[custom.LEAF_ELEMENT]
                if leaf is not None:
                    eval_bit = '%s.%s' % (leaf.ename,
                                          eval_state.poverrides['this'])
                else:
                    eval_bit = 'wikitext'
                for e in _error(
                    "No element or macro ##%s## exists evaluating %s! (Tried %s.)"
                    % (origname,
                       eval_bit,
                       name)):
                    yield e
            else:
                assert False, (elm, eval_state.eoverrides)
                for e in _error("No element ##%s## exists!" % elm):
                    yield e
    except NoContentException:
        raise
    except WikiException,e:
        for e in _error(e.args[0]):
            yield e

# Set or add to this to change what macros are available
macros = Macros()

# These accent macrs can't be set directly
def accent(subst):
    def amac(arglist, content):
        s = tokens_text(content)
        import unicodedata
        # TODO(xavid): use XeLaTeX, which handles unicode better, and stop
        #              doing this.
        yield Text(unicodedata.normalize('NFC', s+subst))
    return amac
ACCENT_MACROS = {"'": u'\u0301',
                 '`': u'\u0300',
                 '"': u'\u0308',
                 '~': u'\u0303',
                 '=': u'\u0304',
                 }
for sym in ACCENT_MACROS:
    setattr(macros, sym, accent(ACCENT_MACROS[sym]))

# This is a bit of a hack; we could allow some state to get passed through
# AbstractMarkup.evaluate...
eval_state = threading.local()

def init_eval_state(element, propname, extension, flavor, let={},
                    catch_errors=False):
    eval_state.dependencies = dependencies.Dependencies()
    eval_state.topthis = propname
    if propname is not None:
        eval_state.dependencies.addDep(element.get_propval(propname))
    eval_state.me = element
    eval_state.eoverrides = {
        custom.CURRENT_ELEMENT: element,
        custom.LEAF_ELEMENT: element
        }
    if element is None:
        eval_state.current_stack = []
    else:
        eval_state.current_stack = [element]
    eval_state.moverrides = {u'args': u""}
    eval_state.poverrides = {u'this': propname}
    for k,v in let.items():
        if hasattr(v, 'ename'):
            eval_state.eoverrides[k] = v
        else:
            assert isinstance(v, unicode), let
            eval_state.moverrides[k] = v
    eval_state.images = []
    eval_state.mentions = []
    eval_state.errors = []
    eval_state.context = {}
    eval_state.memoable_stack = [set()]
    eval_state.last_figure = 0
    eval_state.label_map = {}
    
    eval_state.extension = extension
    if extension in FORMATS:
        fmt = FORMATS[extension]
        mainparser = Parser(fmt, macro_func, link_func)
        eval_state.format = fmt
        eval_state.mainparser = mainparser
    else:
        assert flavor.binary
        # binary flavors interpret a string as a parser, for great justice.
        eval_state.mainparser = extension
    eval_state.catch_errors = catch_errors

# See placeholder() in redbeans/formats.py.
PLACEHOLDER_PAT = re.compile('<<<([^>]+)>>>')

def unplaceholder(match):
    try:
        return eval_state.label_map[match.group(1)]
    except KeyError:
        #assert False, (match.group(), match.group(1))
        return "???"

def evaluate(thing, extension, toPython=False, let={}, element=None,
             flavor=None, reentrant=False, catchErrors=False):
    if hasattr(thing, 'element'):
        element = thing.element
        propval = thing
        propname = propval.propname
        assert element is not None
        assert propname is not None
        assert propval.value is not None, propval
        if flavor is None:
            flavor = structure.get_flavor(propval.propname)
        if flavor.binary:
            wikitext = propval.value
        else:
            wikitext = unicode(propval.value, 'utf-8')
        prop_owned = flavor.owned
    else:
        propval = None
        propname = None
        wikitext = thing
        prop_owned = False
        assert flavor is not None

    if not reentrant:
        assert not hasattr(eval_state, 'dependencies'), repr(eval_state.__dict__)
    else:
        old_eval_state = dict(eval_state.__dict__)

    init_eval_state(element, propname, extension, flavor, let,
                    catch_errors=catchErrors)
    if prop_owned:
        eval_state.eoverrides[u'owner'] = element
    try:
        desc = ("%s.%s" % (propval.element.ename, propval.propname)
                if propval is not None else 'wikitext')
        if not toPython:
            with benchmarking('evaluating '+desc):
                res = flavor.evaluate(wikitext, eval_state.mainparser,
                                      propval=propval)
            # It might not be a string for an ExtractingFormat.
            if isinstance(res, basestring):
                res = PLACEHOLDER_PAT.sub(unplaceholder, res)
        else:
            with benchmarking('python-converting '+desc):
                res = flavor.toPython(wikitext, eval_state.mainparser,
                                      propval=propval)

    except NoContentException,e:
        raise
        if toPython:
            raise
        makeRestricted()
        res = eval_state.mainparser.render([
            Text(u"Content unspecified, raw markup: "),
            Start(CODEBLOCK),
            Text(wikitext),
            End(CODEBLOCK)])
    except WikiException,e:
        if toPython:
            raise
        res = eval_state.mainparser.render(
            _error(u"Wiki Error: {{{%s}}}" % (e.args[0],)))
    finally:
        deps = eval_state.dependencies
        images = eval_state.images
        errors = eval_state.errors
        
        clear_eval_state()
        if reentrant:
            eval_state.__dict__.update(old_eval_state)

    # rendered result, dependencies, metadata
    metadata = {'images': images}
    if errors:
        metadata['errors'] = errors
    return res, deps, metadata

def clear_eval_state():
    eval_state.__dict__.clear()

def get_eval_state_delta(which, text_or_elm, propname=None):
    if propname is not None:
        assert not structure.get_flavor(propname).binary
        text = unicode(text_or_elm[propname].value, 'utf-8')
        eval_state.dependencies.addDep(text_or_elm[propname])
    else:
        assert isinstance(text_or_elm, unicode)
        text = text_or_elm
    old_mentions = eval_state.mentions
    old_errors = eval_state.errors
    setattr(eval_state, which, [])
    eval_state.mainparser.parse(text, format=Format())
    ret = getattr(eval_state, which)
    eval_state.mentions = old_mentions
    eval_state.errors = old_errors
    return ret

def get_format():
    return eval_state.format

def get_topthis():
    return eval_state.topthis

def addDeps(deps):
    eval_state.dependencies.update(deps)

def makeRestricted():
    eval_state.dependencies.makeRestricted()