from __future__ import with_statement import sys, os, pwd, tempfile, threading, shutil, warnings, random, string import pysvn from bazbase import config from bazbase.model import Element, Prop from bazbase.db import TransactionAborted from bazbase.flavors import FLAVORS from bazbase.benchmark import benchmarking from bazbase import custom as basecust from bazjunk.path import makedirs from .share import svndriven from . import custom, format state = threading.local() def checkouts_dir(): return "/tmp/%s.%s.bazki/checkouts" % (pwd.getpwuid(os.getuid())[0], basecust.APP_NAME) def locked_checkouts_dir(): return "/tmp/%s.%s.bazki/locked_checkouts" % (pwd.getpwuid(os.getuid())[0], basecust.APP_NAME) def lock_one_of(dirs): d = random.choice(dirs) nd = os.path.join(locked_checkouts_dir(), d) os.rename(os.path.join(checkouts_dir(), d), nd) return nd def update(client, tempd, rev): if client.info(tempd).revision.number != rev.number: client.update(tempd, revision=rev) def new_checkout_dir(): nd = os.path.join(locked_checkouts_dir(), ''.join(random.choice(string.lowercase) for x in xrange(32))) os.makedirs(nd) return nd def start_checkout(client, revision, repopath): try: nd = lock_one_of(os.listdir(checkouts_dir())) except (IndexError, OSError): nd = new_checkout_dir() thread = threading.Thread( target=lambda cl, tempd, rev: cl.checkout(repopath, tempd, revision=rev), args=(client, nd, revision)) else: thread = threading.Thread(target=update, args=(client, nd, revision)) if thread is not None: thread.start() return nd, thread def return_checkout(dir): makedirs(checkouts_dir()) os.rename(dir, os.path.join(checkouts_dir(), os.path.basename(dir))) def prefetch_checkout(): try: client = pysvn.Client() dirs = os.listdir(checkouts_dir()) if len(dirs) > 2: nd = lock_one_of(dirs) client.update(nd) else: nd = new_checkout_dir() client.checkout('file:///'+os.path.abspath(custom.REPOSITORY), nd) return_checkout(nd) except OSError: # If we lost a race, we don't care. pass def clear_checkouts(): if os.path.exists(checkouts_dir()): shutil.rmtree(checkouts_dir()) if os.path.exists(locked_checkouts_dir()): shutil.rmtree(locked_checkouts_dir()) def _mkdir_p(path): bits = path.split('/') pathpnt = '' for b in bits: pathpnt += b+'/' if not os.path.exists(pathpnt): state.client.mkdir(pathpnt,"") # helper func def _setaddfile(path,val): #print path dir,fil = path.rsplit('/',1) _mkdir_p(dir) with open(path,'w') as fil: fil.write(val+'\n') try: state.client.add(path) except pysvn.ClientError: pass class InvalidState(Exception): pass class LockFailed(Exception): pass def _elementpath(element,parent=None,symlink=False,orgmode=None): above = [] if orgmode is None: orgmode = element.orgmode while True: if len(above) == 0: if element.hasChildren(): above = [element.ename,element.ename+'.yaml'] else: above = [element.ename+'.yaml'] else: above = [element.ename] + above if parent is not None: element = parent parent = None else: element = element.parent if element is None: break elif element is False: return None orgmode = element.orgmode return os.path.join(*above) def notify(dct): if pysvn.wc_notify_state.conflicted in (dct['content_state'], dct['prop_state']): state.conflict = True def begin(): username = custom.get_username() if username: username = username.encode('utf-8') else: username = 'nobody' state.client = pysvn.Client() state.client.callback_notify = notify state.client.set_default_username(username) state.deltas = {} state.lock_state = False state.hostname = os.uname()[1] state.conflict = False repopath = 'file:///'+os.path.abspath(custom.REPOSITORY) revision = state.client.info2(repopath, recurse=False)[0][1]['rev'] state.tempd, state.checkout_thread = start_checkout(state.client, revision, repopath) return revision.number def setprop(element, pname, val): path = _elementpath(element) assert isinstance(val, str), repr(val) prop = Prop.get(pname) if (prop.default == val and element.parent is not None and pname in element.parent): state.deltas.setdefault(path, {}).setdefault(element.ename, {})[pname] = None else: if not FLAVORS[prop.flavor].binary: val = unicode(val, 'utf-8') state.deltas.setdefault(path, {}).setdefault(element.ename, {})[pname] = val def delete(element,pname): path = _elementpath(element) state.deltas.setdefault(path,{}).setdefault(element.ename,{})[pname]=None def esetattr(element,attr,val,bootstrapping=False): if attr == 'parent': assert element is not None assert isinstance(val,Element) or val is None,val.__class__ olddelts = {} if element.parent == val and not bootstrapping: return if element.parent is not False: oldpath = _elementpath(element) if (oldpath in state.deltas and element.ename in state.deltas[oldpath]): olddelts = state.deltas[oldpath][element.ename] newpath = _elementpath(element,parent=val) if newpath.endswith('.list') and (element.parent is False or not oldpath.endswith('.list')): if element.parent is not False: edelete(element) state.deltas.setdefault(newpath,{}).setdefault('parentage',{}).setdefault(val,{})[element.ename] = olddelts elif (element.parent is not False and oldpath.endswith('.list') and not newpath.endswith('.list')): do_something() elif (element.parent is not False and oldpath.endswith('.list') and newpath.endswith('.list')): do_something_else() else: # Just a simple listless move assert newpath not in state.deltas,(element,newpath,state.deltas[newpath]) if val is not None and not val.hasChildren() and val.orgmode != 'toplevel': # Transform from foo.yaml to foo/foo.yaml oldparpath = os.path.dirname(newpath)+'.yaml' newparpath = os.path.dirname(newpath)+'/'+val.ename +'.yaml' state.checkout_thread.join() state.client.mkdir(state.tempd+'/' +os.path.dirname(newpath),"") state.client.move(state.tempd+'/'+oldparpath, state.tempd+'/'+newparpath,force=True) assert newparpath not in state.deltas if oldparpath in state.deltas: state.deltas[newparpath] = state.deltas[oldparpath] del state.deltas[oldparpath] if element.parent is False or bootstrapping: state.checkout_thread.join() _setaddfile(state.tempd+'/'+newpath,'') elif oldpath != newpath: state.checkout_thread.join() if oldpath.endswith("%s/%s.yaml" % (element.ename, element.ename)): assert newpath.endswith("%s/%s.yaml" % (element.ename, element.ename)) oldpath = os.path.dirname(oldpath) newpath = os.path.dirname(newpath) if element.parent.countDescendants() == 1: # We're the last child oldparpath = os.path.join(os.path.dirname(path), element.parent.ename+'.yaml') newparpath = os.path.join(os.path.dirname( os.path.dirname(path)), element.parent.ename+'.yaml') state.client.move(oldparpath, newparpath, force=True) state.client.remove([os.path.dirname(oldparpath)]) state.client.move(state.tempd+'/'+oldpath, state.tempd+'/'+newpath,force=True) # Make a symlink for toplevel elements if element.orgmode == 'toplevel': state.checkout_thread.join() if element.parent != val: oldsymlink = _elementpath(element,symlink=True) state.client.remove(state.tempd+'/'+oldsymlink) newsymlink = _elementpath(element,parent=val,symlink=True) if os.path.exists(state.tempd+'/'+newsymlink): os.unlink(state.tempd+'/'+newsymlink) dontadd=True else: dontadd=False os.symlink('../'*newsymlink.count('/')+newpath, state.tempd+'/'+newsymlink) if not dontadd: state.client.add(state.tempd+'/'+newsymlink) elif attr == 'ename': if element.hasChildren(): fil1 = os.path.dirname(_elementpath(element)) fil2 = os.path.join(os.path.dirname(fil1), val) else: fil1 = _elementpath(element) fil2 = os.path.join(os.path.dirname(fil1), val+'.yaml') assert fil1 != fil2, (fil1, fil2) state.checkout_thread.join() state.client.move(state.tempd + '/' + fil1, state.tempd + '/' + fil2, force=True) elif attr == 'orgmode': # TODO(xavid): Handle orgmode! assert False, "orgmode" was = _elementpath(element) will = _elementpath(element,orgmode=val) if was != will: assert will not in state.deltas if was in state.deltas: state.deltas[will] = state.deltas[was] del state.deltas[was] state.checkout_thread.join() if element.orgmode == 'list': assert False elif val == 'list': assert False elif val == 'toplevel': assert element.orgmode == 'normal' if was.endswith('.def'): # Move to toplevel and make symlink state.client.move(state.tempd+'/'+os.path.dirname(was), state.tempd+'/'+os.path.dirname(will), force=True) for k in state.deltas.keys(): if k.startswith(os.path.dirname(was)+'/'): new = k[len(os.path.dirname(was)):] new = os.path.dirname(will)+new assert new not in state.deltas state.deltas[new] = state.deltas[k] del state.deltas[k] else: assert was.endswith('.elm') # Make dir, move, make symlink state.client.mkdir(state.tempd+'/'+os.path.dirname(will), "") state.client.move(state.tempd+'/'+was, state.tempd+'/'+will,force=True) # Symlink sympath = _elementpath(element,orgmode=val,symlink=True) os.symlink('../'*sympath.count('/')+os.path.dirname(will), state.tempd+'/'+sympath) state.client.add(state.tempd+'/'+sympath) elif val == 'normal': assert element.orgmode == 'toplevel' if will.endswith('.def'): # Move from toplevel and nix symlink state.client.move(state.tempd+'/'+os.path.dirname(was), state.tempd+'/'+os.path.dirname(will), force=True) for k in state.deltas.keys(): if k.startswith(os.path.dirname(was)+'/'): new = k[len(os.path.dirname(was)):] new = os.path.dirname(will)+new assert new not in state.deltas state.deltas[new] = state.deltas[k] del state.deltas[k] else: assert will.endswith('.elm') # Move, rmdir, nix symlink state.client.move(state.tempd+'/'+was, state.tempd+'/'+will,force=True) state.client.remove([state.tempd+'/'+os.path.dirname(was)]) # Nix symlink sympath = _elementpath(element,symlink=True) state.client.remove(state.tempd+'/'+sympath) else: raise InvalidState("Invalid orgmode '%s'!" % val) else: raise Exception('Unknown eattr %s!'%attr) def edelete(element): rpath = _elementpath(element) state.checkout_thread.join() path = os.path.join(state.tempd,rpath) if path.endswith('%s/%s.yaml' % (element.ename, element.ename)): raise InvalidState("Removing a node with children!") elif path.endswith('.list'): if os.path.basename(path) == element.ename+'.list': assert not element.hasChildren() state.client.remove([path],force=True) else: state.deltas.setdefault(path,{})[element.ename] = None if rpath in state.deltas and element.ename in state.deltas[rpath]: del state.deltas[rpath][element.ename] return # skip deltanukage else: state.client.remove([path], force=True) if element.parent.countDescendants() == 1: # We're the last child oldparpath = os.path.join(os.path.dirname(path), element.parent.ename+'.yaml') newparpath = os.path.join(os.path.dirname(os.path.dirname(path)), element.parent.ename+'.yaml') state.client.move(oldparpath, newparpath, force=True) state.client.remove([os.path.dirname(oldparpath)]) if rpath in state.deltas: del state.deltas[rpath] def psetattr(prop,attr,val): assert attr in ('flavor','default','comment','pname','visible') state.checkout_thread.join() if attr == 'pname': fil1 = 'props/%s.yaml' % (prop.name) fil2 = 'props/%s.yaml' % (val) assert fil2 not in state.deltas if os.path.exists(state.tempd+'/'+fil1): state.client.move(state.tempd+'/'+fil1, state.tempd+'/'+fil2,force=True) if fil1 in state.deltas: state.deltas[fil2] = state.deltas[fil1] del state.deltas[fil1] else: _setaddfile(os.path.join(state.tempd,path),'') else: path = 'props/%s.yaml' % (prop.name) if not isinstance(val, unicode): val = unicode(str(val), 'utf-8') state.deltas.setdefault(path,{}).setdefault(prop.name,{})[unicode( attr, 'utf-8')] = val if not os.path.exists(state.tempd+'/'+path): _setaddfile(os.path.join(state.tempd,path),'') def pdelete(prop): state.checkout_thread.join() fil = 'props/%s.yaml' % (prop.name) state.client.remove([state.tempd+'/'+fil], force=True) if fil in state.deltas: del state.deltas[fil] class parse_handler(object): def __init__(self,subs,path): self.subs = subs self.dir = state.tempd+'/'+os.path.dirname(path) self.noprop = False def prop(self, e, p, contents): assert not self.noprop if e in self.subs and p in self.subs[e]: ret = self.subs[e][p] del self.subs[e][p] return ret else: return contents def start_element(self,e,parent=None): if e in self.subs and self.subs[e] is None: return False else: return True def end_element(self, e): pass def get_new_properties(self,e): self.noprop = True # Things can be None if they're new and defaulting, say for a new # element. return dict((p, self.subs[e][p]) for p in self.subs[e] if self.subs[e][p] is not None) def get_new_children(self,e): if e in self.parentage: return self.parentage[e] else: return [] def get_include(self, path): path = os.path.join(self.dir, path) with open(path, 'rb') as fil: return fil.read() def set_include(self, path, val, new): path = os.path.join(self.dir, path) if val is not None: if new: cnt = 1 # Start numbering at 2 origpath = path while os.path.exists(path): cnt += 1 path = ('.%s.' % cnt).join(origpath.rsplit('.', 1)) _setaddfile(path, val) else: assert not new if os.path.exists(path): state.client.remove([path], force=True) return os.path.basename(path) def commit(): assert not svndriven() with benchmarking('waiting for checkout'): state.checkout_thread.join() try: # Do the runaround of state.deltas for path in state.deltas: inf = open(state.tempd+'/'+path) # We replace the old file with a new file with the same path, # for ease of processing os.unlink(state.tempd+'/'+path) outf = open(state.tempd+'/'+path,mode='w') elms = state.deltas[path] if path.endswith('.yaml'): assert len(elms) == 1 format.parse(state.tempd+'/'+path, parse_handler(state.deltas[path],path),inf,outf) inf.close() outf.close() # -xavid: too slow for a rare-case fix ##update here to make sure no transactions are in progress and to merge ##harmless changes #state.client.update(state.tempd) #if state.conflict: # raise TransactionAborted("Subversion conflict!") except: state.revert_thread = threading.Thread(target=do_revert, args=(state.client,state.tempd)) state.revert_thread.start() raise else: log_message = "Web Edit: " + custom.get_commit_message() # commit changes revision = state.client.checkin(state.tempd, log_message=log_message) return_checkout(state.tempd) if revision is None: return None else: return revision.number def do_revert(client,tempd): client.revert(tempd, recurse=True) # Remove unversioned files. for status in client.status(tempd, get_all=False): if not status.is_versioned: p = os.path.join(tempd, status.path) if os.path.isdir(p): shutil.rmtree(p) else: os.unlink(p) else: raise Exception("Reverted client has file in unknown status: %s" % status) return_checkout(tempd) def abort(): assert not svndriven() try: state.checkout_thread.join() except AttributeError: pass try: state.revert_thread = threading.Thread(target=do_revert, args=(state.client,state.tempd)) except AttributeError: pass else: state.revert_thread.start() def get_revision(): client = pysvn.Client() # TODO(xavid): unify with impl above repopath = 'file:///'+os.path.abspath(custom.REPOSITORY) revision = client.info2(repopath, recurse=False)[0][1]['rev'] return revision.number