from __future__ import with_statement from __future__ import absolute_import import sys from . import wiki, flavors, db, dependencies, structure, formats from .translators import Intermed, TRANSLATORS from .filters import FILTERS from .benchmark import benchmarking from .cache import (cached_formats, get_from_cache, cache_propval, invalidate_cache) TXT=u"txt" class ConversionFailedException(formats.FormatException): pass def flatten_filters(filters): # metadata is not a cachable filter, somehow assert 'metadata' not in filters mfilters = dict(filters) if 'let' in mfilters: assert 'elet' not in mfilters assert 'mlet' not in mfilters elet = {} mlet = {} for k,v in mfilters['let'].items(): if hasattr(v, 'ename'): elet[k] = v else: assert isinstance(v, unicode), repr(v) assert ';' not in v assert '|' not in v assert '^' not in v mlet[k] = v del mfilters['let'] if len(elet) > 0: mfilters['elet'] = ';'.join("%s=%s" % (k,v.ename) for k,v in elet.items()) if len(mlet) > 0: mfilters['mlet'] = ';'.join("%s=%s" % (k,v) for k,v in mlet.items()) assert all(isinstance(v, basestring) for v in mfilters.values()), mfilters if len(mfilters) > 0: return '^'+'^'.join('%s=%s'%(k, mfilters[k]) for k in sorted(mfilters)) else: return '' # These extensions require proper image metadata for resulting forms # to be produced properly, and thus can't be cached if they have images. # TODO(xavid): either cache image metadata or do something more clever. IMAGE_METADATA_FORMATS = ('tex', '.tex') # Formats where cached values should be treated as utf-8-encoded Unicode, # not raw bytes in a str. UNICODE_FORMATS = ('txt', 'html', 'tex') def apply_filters(propval, rendering, rendering_ext, deps, metadata, flist, cache_tag=None): if rendering_ext in UNICODE_FORMATS: assert isinstance(rendering, unicode), repr(rendering) else: assert isinstance(rendering, str), repr(rendering) if propval is not None: ename = propval.element.ename prop_name = propval.propname else: ename = prop_name = None im = Intermed(metadata) im.setData(rendering, rendering_ext) im.addDeps(deps) for f in flist: # Only hold on to data that's already in memory, # for efficiency. (The only case we care about, # html => .html, will be in memory.) # We're not cachable if we have a dependency on DISCORDIA # or we have filters we haven't dealt with or image metadata # we need. if (cache_tag is not None and not im.isPath() and not dependencies.DISCORDIA in im.getDeps() and len(im.metadata().get('filters', {})) == 0 and (im.getExtension() not in IMAGE_METADATA_FORMATS or len(im.metadata().get('images', [])) == 0)): oldext = im.getExtension() olddeps = set(im.getDeps()) olddata = im.asData() oldmetadata = dict(im.metadata()) else: olddata = None f(im) # Cache it if we just developed a dependency on DISCORDIA and # we had some prior cacheable data. if (olddata is not None and dependencies.DISCORDIA in im.getDeps()): cache_propval(ename, prop_name, oldext, olddata, olddeps, oldmetadata, cache_tag=cache_tag) filters = im.metadata().get('filters', {}) for filt in list(filters): FILTERS[im.getExtension()][filt](im,filters[filt]) # im now holds the return data and extension, let's see # if we can cache it value = im.asData() ext = im.getExtension() deps = im.getDeps() if propval is not None: if not structure.get_flavor(propval.propname).binary: if ext in UNICODE_FORMATS: assert isinstance(value, unicode), [repr(value), propval.propname] else: assert isinstance(value, str), [repr(value), propval.propname] # Do dependencies if cache_tag is not None: if (dependencies.DISCORDIA not in deps and (ext not in IMAGE_METADATA_FORMATS or len(im.metadata().get('images', [])) == 0)): cache_propval(ename, prop_name, ext, value, deps, im.metadata(), cache_tag=cache_tag) else: assert ext != (TXT,) return im def cached(ename, prop_name, format): entry = get_from_cache(ename, prop_name, format=format) if entry is None: return None else: return entry['value'] def convert_any(ename, prop_name, dests, filters={}, reentrant=False, method='convert', offset=0): #with benchmarking('%sing %s.%s as %s' # % (method, ename, prop_name, # dests), # offset=offset + 1): if True: explicit_metadata = {} if filters: filters = dict(filters) # metadata is defined to not affect anything cacheable, somehow. if 'metadata' in filters: explicit_metadata = filters['metadata'] del filters['metadata'] flat = flatten_filters(filters) else: flat = '' cached, deps_info = cached_formats(ename, prop_name, flat) for f in dests: if f in cached: ce = get_from_cache(ename, prop_name, f, flat, deps_info=deps_info) assert ce is not None, (ename, prop_name, f, flat, cached) assert 'metadata' in ce, ce im = Intermed(ce['metadata']) im.setData(ce['value'], f) im.cached = (ename, prop_name, f, flat, 1) return im benchmarking.info('%s not in cache data for %s/%s%s' % ( dests, ename, prop_name, flat)) propval = structure.get_propval(ename, prop_name) if propval is None: raise KeyError("%s has no propval %s!" % (ename, prop_name)) exts = structure.get_flavor(propval.propname).getExtensions(propval) new = dict((d,([],d)) for d in dests) tried = set() while len(new) > 0: tried.update(new.keys()) for n in new: if n in exts or n+flat in cached: flist,dest = new[n] let = filters.pop('let', {}) # TODO(xavid): This logic seems really iffy. if n+flat in cached: ce = get_from_cache(ename, prop_name, n, flat, deps_info=deps_info) assert ce is not None, (cached, n+flat) value = ce['value'] deps = ce['dependencies'] metadata = dict(ce['metadata']) cached = (ename, prop_name, n, flat, 2) else: value,deps,metadata = wiki.evaluate( propval, n, let=let, reentrant=reentrant) cached = False metadata.update(explicit_metadata) metadata['filters'] = filters metadata['let'] = let metadata['element'] = ename im = apply_filters(propval, value, n, deps, metadata, flist, flat) im.cached = cached return im newnew = {} for n in new: flist,dest = new[n] if n in TRANSLATORS: for t in TRANSLATORS[n]: if t not in tried: newnew[t] = ([TRANSLATORS[n][t]]+flist,dest) new = newnew raise ConversionFailedException( "Couldn't convert from %s (flavor %s) to any of %s!" % (', '.join(exts), structure.get_prop(propval.propname).flavor, ', '.join(dests))) def convert_markup(markup, dest, global_metadata={}, cacheable_as=None): """Convert a string of explicit markup to the given dest format.""" assert not dest.startswith('_'), dest if cacheable_as is not None: ce = get_from_cache(None, None, format=dest, cache_tag=cacheable_as) if ce is not None: if dest in UNICODE_FORMATS: return unicode(ce['value'],'utf-8') else: return ce['value'] new = {dest: []} tried = set() while len(new) > 0: tried.update(new.keys()) for n in new: if n in flavors.FORMATS: flist = new[n] rendered, deps, metadata = wiki.evaluate(markup, n, element=None, flavor=flavors.text) metadata.update(global_metadata) im = apply_filters(None, rendered, n, deps, metadata, flist, cacheable_as) assert im.getExtension() == dest, (im.getExtension(), dest) return im.asData() newnew = {} for n in new: flist = new[n] if n in TRANSLATORS: for t in TRANSLATORS[n]: if t not in tried: newnew[t] = [TRANSLATORS[n][t]]+flist new = newnew raise ConversionFailedException("Couldn't convert list to %s!" % dest) def render_propval(ename, prop_name, format=TXT, filters={}): return convert_any(ename, prop_name, [format], filters=filters).asData() def render(element, prop_name, format=TXT, filters={}): return render_propval(element.ename, prop_name, format, filters=filters) def render_raw(ename, prop_name): # No assertion for efficiency, but this documents that prop_name is assumed # to be of flavor raw, and thus not have external dependencies. return render_propval(ename, prop_name) def to_python(element, prop_name, format=TXT, flavor=None): if flavor is not None: flavor = flavors.FLAVORS[flavor] pv = element.get_propval(prop_name) if pv is None: return None val, deps, metadata = wiki.evaluate(pv, format, toPython=True, flavor=flavor) return val def cache_and_get_dimensions(ename, pname, filters, format=None): if format: measurables = (formats.get_measurable_format(format),) else: measurables = formats.MEASURABLE_FORMATS.keys() # We precache the image so we can get its size, # to enable LaTeX to scale it optimally and for # HTML browser size hinting. # We'll need it cached anyways. # If conversion doesn't autogen height and width # metadata (or if we're loading from cache and # we haven't started caching metadata yet), we # can determine it fresh. try: imgim = convert_any(ename, pname, measurables, filters, reentrant=True) except formats.FormatException: # Not actually a valid image; punt. #pass raise else: md = imgim.metadata() if 'height' in md and 'width' in md: assert md['width'] is not None and md['height'] is not None return md['width'], md['height'] #assert info['width'] == 100 or info['height'] == 100, (info, md) else: # TODO(xavid): once we start caching metadata, # these should be cached try: width, height = formats.get_dimensions( imgim) except formats.FormatException: # Not actually a valid image; punt. #pass raise else: assert width is not None and height is not None return width, height #assert info['width'] == 100 or info['height'] == 100, (info, imgim.cached) imgim.nix() return None, None def extract_filters(word, filters): bits = word.split('^') word = bits[0] for b in bits[1:]: if '=' in b: k,v = b.split('=',1) else: k = b[0] v = b[1:] filters[k] = v return word