import sys import codecs import yaml # Need to override internal method, so can't use C version. # (Could use CLoader, but mumble mumble exception reporting.) # TODO(xavid): Investiage whether the C versions would save a lot of time, # and if so figure out a way to use them. from yaml import Loader, Dumper from bazbase import translators from bazbase.flavors import str_to_bool class BazDumper(Dumper): def analyze_scalar(self, scalar): ret = Dumper.analyze_scalar(self, scalar) # TODO(xavid): Are there more cases I should suppress this? if scalar: ret.allow_block = True return ret class InvalidBazFile(Exception): pass INCLUDE = u'!include' class Include(object): def __init__(self, val, ext): self.val = val self.ext = ext def __eq__(self, other): return isinstance(other, Include) and other.val == self.val def expect(seq, evt): e = seq.next() if not isinstance(e, evt): raise InvalidBazFile("Expected %s, got %s"%(evt,e)) return e def default_include_name(ename, pname, ext, previous=None): assert ext.startswith('.'), ext if previous is not None and previous.endswith(ext): return previous return "%s.%s%s" % (ename, pname, ext) def style_for_value(value): if '\n' in value or len(value) > 60: return '|' else: return None def parse_h(curre, handler, seq): currp = None if hasattr(handler, 'get_new_properties'): get_extra = lambda: handler.get_new_properties(curre) else: get_extra = lambda: None seen = {curre: set()} handler.start_element(curre) yield expect(seq, yaml.StreamStartEvent) e = seq.next() if isinstance(e, yaml.StreamEndEvent): # Was empty file extra = get_extra() if extra: # If we're adding new props, we need to make this look like # a proper mapping yield yaml.DocumentStartEvent(explicit=False) seq = iter([yaml.MappingStartEvent(anchor=None, tag=None, implicit=True, flow_style=False), yaml.MappingEndEvent(), yaml.DocumentEndEvent()]) get_extra = lambda: extra else: yield e handler.end_element(curre) return elif isinstance(e, yaml.DocumentStartEvent): yield e else: raise InvalidBazFile("Got %s at start of document." % e) e = seq.next() if isinstance(e, yaml.ScalarEvent): if e.value == '': extra = get_extra() if extra: # If we're adding new props, we need to make this look like # a proper mapping yield yaml.MappingStartEvent(anchor=None, tag=None, implicit=True, flow_style=False) seq = iter([yaml.MappingEndEvent(), expect(seq, yaml.DocumentEndEvent)]) get_extra = lambda: extra else: yield e yield expect(seq, yaml.DocumentEndEvent) handler.end_element(curre) return else: raise InvalidBazFile("Non-empty scalar '%s' at start of mapping." % e.value) elif isinstance(e, yaml.MappingStartEvent): yield e else: raise InvalidBazFile("Got %s at start of mapping." % e) try: for e in seq: if isinstance(e, yaml.MappingEndEvent): # Write any remaining props extra = get_extra() if extra: for p in extra: assert isinstance(p, unicode), repr(p) yield yaml.ScalarEvent(anchor=None, tag=None, implicit=(True, True), style=None, value=p) if isinstance(extra[p], unicode): yield yaml.ScalarEvent( anchor=None, tag=None, implicit=(True, True), style=style_for_value(extra[p]), value=extra[p]) else: ext, val = extra[p] assert isinstance(val, str), repr(extra[p]) name = default_include_name(curre, p, ext) name = handler.set_include(name, val, new=True) assert '/' not in name, name assert name.endswith(ext), (name, ext) yield yaml.ScalarEvent(anchor=None, tag=INCLUDE, implicit=(False, False), value=name) yield e break elif not isinstance(e, yaml.ScalarEvent): if (isinstance(e, yaml.SequenceStartEvent) and currp is not None): raise InvalidBazFile("Unexpected sequence in value for " "%s.%s; values starting with a [ " "must be quoted." % (curre, currp)) elif (isinstance(e, yaml.MappingStartEvent) and currp is not None): raise InvalidBazFile("Unexpected mapping in value for " "%s.%s; values starting with a { " "must be quoted." % (curre, currp)) else: raise InvalidBazFile("Unexpected yaml event: %s"%e) if currp is not None: if e.tag == INCLUDE: if '/' in e.value: raise InvalidBazFile("Includes must not be in a different directory! (%s in %s.%s)" % (e.value, curre, currp)) if e.value.endswith('.yaml'): raise InvalidBazFile("Includes must not end in .yaml! (%s in %s.%s)" % (e.value, curre, currp)) val = handler.get_include(e.value) if val is None: raise InvalidBazFile("Couldn't read include '%s' for %s.%s!" % (e.value, curre, currp)) val = Include(val, '.' + e.value.rsplit('.', 1)[-1]) else: assert e.tag in (None, u'!'), e.tag val = e.value if currp in seen[curre]: raise InvalidBazFile("Prop '%s' set more than once in %s!" % (currp, curre)) seen[curre].add(currp) oldcurrp = currp currp = None ret = handler.prop(curre, oldcurrp, val) if ret is not None: assert isinstance(ret, (unicode, Include)), repr(ret) assert isinstance(val, (unicode, Include)), repr(val) if ret != val: if isinstance(ret, Include): oldname = e.value if e.tag == INCLUDE else None name = default_include_name( curre, oldcurrp, ret.ext, oldname) name = handler.set_include(name, ret.val, new=(name != oldname)) assert '/' not in name, name e.tag = INCLUDE e.implicit = (False, False) e.style = None e.value = name else: e.tag = None e.implicit = (True, True) e.style = style_for_value(ret) e.value = ret if e.style is None: e.style = '' yield currpe yield e elif curre is not None: assert e.tag in (None, u'!'), e.tag if curre is False: # Lines after the end of a .list raise InvalidBazFile("Lines after end of file!") currp = e.value currpe = e continue except yaml.scanner.ScannerError, e: if e.problem_mark: suffix = '\n' + str(e.problem_mark) else: suffix = '' if (e.problem and e.problem.endswith("but found '*'") and currp is not None): raise InvalidBazFile("Malformed alias in value for %s.%s; values starting with a * must be quoted.%s" % (curre, currp, suffix)) elif (e.problem and e.problem == "mapping keys are not allowed here" and currp is not None): raise InvalidBazFile("Unexpected mapping key in value for %s.%s; values starting with a ? must be quoted.%s" % (curre, currp, suffix)) elif (e.problem and e.problem == "mapping values are not allowed here"): raise InvalidBazFile("Unexpected mapping value in %s; values containing a : must be quoted.%s" % (curre, suffix)) else: raise # TODO(xavid): Loop here for list-style files. yield expect(seq, yaml.DocumentEndEvent) yield expect(seq, yaml.StreamEndEvent) # Check to make sure we ended in a good state if currp is not None: raise InvalidBazFile("%s.%s unclosed!"%(curre,currp)) # We're golden! handler.end_element(curre) if False: def p(s): print >>sys.stderr,repr(s) return s else: p = lambda s:s # handler's methods only have a return value if outf is specified. # Yes, I realize this is a wonky interface. def parse(path, handler, inf=None, outf=None): if inf is None: inf = codecs.open(path,"r","utf-8") assert path.endswith('.yaml') curre = path.rsplit('/',1)[-1].rsplit('.',1)[0] try: iterator = parse_h(curre, handler, yaml.parse(inf, Loader=Loader)) if outf: y = [p(s) for s in iterator] yaml.emit(y, outf, Dumper=BazDumper) else: for e in iterator: pass except yaml.YAMLError, exc: print >>sys.stderr, path if hasattr(exc, 'problem_mark'): print >>sys.stderr,exc.problem_mark,dir(exc.problem_mark) exc.problem_mark.name = path raise def interpret_as_prop(attr, value): assert attr in ('flavor', 'default', 'visible', 'comment') if attr == 'visible': return str_to_bool(value) elif attr == 'default': return value.encode('utf-8') else: return value