import sys import yaml # Need to override internal method, so can't use C version. # (Could use CLoader, but mumble mumble exception reporting.) # TODO(xavid): Investiage whether the C versions would save a lot of time, # and if so figure out a way to use them. from yaml import Loader, Dumper from bazbase import translators class BazDumper(Dumper): def analyze_scalar(self, scalar): ret = Dumper.analyze_scalar(self, scalar) # TODO(xavid): Are there more cases I should suppress this? if scalar: ret.allow_block = True return ret class InvalidBazFile(Exception): pass INCLUDE = u'!include' class Include(str): pass def expect(seq, evt): e = seq.next() if not isinstance(e, evt): raise InvalidBazFile("Expected %s, got %s"%(evt,e)) return e def default_include_name(ename, pname, value, previous=None): ext = translators.extension(value) if previous is not None and previous.endswith(ext): return previous return "%s.%s%s" % (ename, pname, ext) def style_for_value(value): if '\n' in value or len(value) > 60: return '|' else: return None def parse_h(curre, handler, seq): currp = None quenche = False if hasattr(handler, 'get_new_properties'): get_extra = lambda: handler.get_new_properties(curre) else: get_extra = lambda: None handler.start_element(curre) yield expect(seq, yaml.StreamStartEvent) e = seq.next() if isinstance(e, yaml.StreamEndEvent): # Was empty file extra = get_extra() if extra: # If we're adding new props, we need to make this look like # a proper mapping yield yaml.DocumentStartEvent(explicit=False) seq = iter([yaml.MappingStartEvent(anchor=None, tag=None, implicit=True, flow_style=False), yaml.MappingEndEvent(), yaml.DocumentEndEvent()]) get_extra = lambda: extra else: yield e handler.end_element(curre) return elif isinstance(e, yaml.DocumentStartEvent): yield e else: raise InvalidBazFile("Got %s at start of document." % e) e = seq.next() if isinstance(e, yaml.ScalarEvent): if e.value == '': extra = get_extra() if extra: # If we're adding new props, we need to make this look like # a proper mapping yield yaml.MappingStartEvent(anchor=None, tag=None, implicit=True, flow_style=False) seq = iter([yaml.MappingEndEvent(), expect(seq, yaml.DocumentEndEvent)]) get_extra = lambda: extra else: yield e yield expect(seq, yaml.DocumentEndEvent) handler.end_element(curre) return else: raise InvalidBazFile("Non-empty scalar '%s' at start of mapping." % e.value) elif isinstance(e, yaml.MappingStartEvent): yield e else: raise InvalidBazFile("Got %s at start of mapping." % e) try: for e in seq: if isinstance(e, yaml.MappingEndEvent): # Write any remaining props extra = get_extra() if extra: for p in extra: assert isinstance(p, unicode), repr(p) yield yaml.ScalarEvent(anchor=None, tag=None, implicit=(True, True), style=None, value=p) if isinstance(extra[p], unicode): yield yaml.ScalarEvent( anchor=None, tag=None, implicit=(True, True), style=style_for_value(extra[p]), value=extra[p]) else: assert isinstance(extra[p], str), repr(extra[p]) name = default_include_name(curre, p, extra[p]) name = handler.set_include(name, extra[p], new=True) assert '/' not in name, name yield yaml.ScalarEvent(anchor=None, tag=INCLUDE, implicit=(False, False), value=name) yield e break elif not isinstance(e, yaml.ScalarEvent): if (isinstance(e, yaml.SequenceStartEvent) and currp is not None): raise InvalidBazFile("Unexpected sequence in value for " "%s.%s; values starting with a [ " "must be quoted." % (curre, currp)) elif (isinstance(e, yaml.MappingStartEvent) and currp is not None): raise InvalidBazFile("Unexpected mapping in value for " "%s.%s; values starting with a { " "must be quoted." % (curre, currp)) else: raise InvalidBazFile("Unexpected yaml event: %s"%e) if currp is not None: if e.tag == INCLUDE: if '/' in e.value: raise InvalidBazFile("Includes must not be in a different directory! (%s in %s.%s)" % (e.value, curre, currp)) if e.value.endswith('.yaml'): raise InvalidBazFile("Includes must not end in .yaml! (%s in %s.%s)" % (e.value, curre, currp)) val = handler.get_include(e.value) if val is None: raise InvalidBazFile("Couldn't read include '%s' for %s.%s!" % (e.value, curre, currp)) else: assert e.tag in (None, u'!'), e.tag val = e.value oldcurrp = currp currp = None if quenche: continue val = val.encode('utf-8') ret = handler.prop(curre, oldcurrp, val) if ret is not None: if ret != val: if isinstance(ret, Include): oldname = e.value if e.tag == INCLUDE else None name = default_include_name( curre, oldcurrp, val, oldname) name = handler.set_include(name, ret, new=(name != oldname)) assert '/' not in name, name e.tag = INCLUDE e.implicit = (False, False) e.style = None e.value = name else: e.tag = None e.implicit = (True, True) e.style = style_for_value(ret) e.value = ret if e.style is None: e.style = '' yield currpe yield e elif curre is not None: assert e.tag in (None, u'!'), e.tag if curre is False: # Lines after the end of a .list raise InvalidBazFile("Lines after end of file!") currp = e.value currpe = e continue except yaml.scanner.ScannerError, e: if (e.problem and e.problem.endswith("but found '*'") and currp is not None): raise InvalidBazFile("Malformed alias in value for %s.%s; values starting with a * must be quoted." % (curre, currp)) elif (e.problem and e.problem == "mapping keys are not allowed here" and currp is not None): raise InvalidBazFile("Unexpected mapping key in value for %s.%s; values starting with a ? must be quoted." % (curre, currp)) else: raise # TODO(xavid): Loop here for list-style files. yield expect(seq, yaml.DocumentEndEvent) yield expect(seq, yaml.StreamEndEvent) # Check to make sure we ended in a good state if currp is not None: raise InvalidBazFile("%s.%s unclosed!"%(curre,currp)) # We're golden! handler.end_element(curre) if False: def p(s): print >>sys.stderr,repr(s) return s else: p = lambda s:s # handler's methods only have a return value if outf is specified. # Yes, I realize this is a wonky interface. def parse(path, handler, inf=None, outf=None): if inf is None: inf = codecs.open(path,"r","utf-8") assert path.endswith('.yaml') curre = path.rsplit('/',1)[-1].rsplit('.',1)[0] try: iterator = parse_h(curre, handler, yaml.parse(inf, Loader=Loader)) if outf: y = [p(s) for s in iterator] yaml.emit(y, outf, Dumper=BazDumper) else: for e in iterator: pass except yaml.YAMLError, exc: print >>sys.stderr, path if hasattr(exc, 'problem_mark'): print >>sys.stderr,exc.problem_mark,dir(exc.problem_mark) exc.problem_mark.name = path raise