From 81c14cf81970b92f37ef0b72848f3e31aceac692 Mon Sep 17 00:00:00 2001 From: "Eric S. Raymond" Date: Wed, 10 Nov 2010 06:40:25 -0500 Subject: [PATCH] This code parses an import stream from stdin and dos nothing. It's the deserialization code from reposurgeon with everything unnecessary stripped out. --- rcs-fast-import | 525 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 525 insertions(+) create mode 100755 rcs-fast-import diff --git a/rcs-fast-import b/rcs-fast-import new file mode 100755 index 0000000..98cc9dc --- /dev/null +++ b/rcs-fast-import @@ -0,0 +1,525 @@ +#!/usr/bin/env python +# +# rcs-fast-import - break a git import stream into corresponding RCS files. +# +# By ESR, November 2010. BSD terms apply. +# +# Requires Python 2.5 or newer. +# +import sys, os, cStringIO, subprocess +import time, calendar, re, signal, copy, shlex +import email.utils + +version="0.1" + +verbose = 0 +DEBUG_SHUFFLE = 2 # Debug directory handling +DEBUG_DELETE = 2 # Debug canonicalization after deletes +DEBUG_COMMANDS = 2 # Show commands as they are executed + +class Baton: + "Ship progress indications to stdout." + def __init__(self, prompt, endmsg='done', enable=False): + self.prompt = prompt + self.endmsg = endmsg + if enable: + self.stream = sys.stdout + else: + self.stream = None + def __enter__(self): + if self.stream: + self.stream.write(self.prompt + "...") + if os.isatty(self.stream.fileno()): + self.stream.write(" \010") + self.stream.flush() + self.count = 0 + self.time = time.time() + return self + def twirl(self, ch=None): + if self.stream is None: + return + if os.isatty(self.stream.fileno()): + if ch: + self.stream.write(ch) + else: + self.stream.write("-/|\\"[self.count % 4]) + self.stream.write("\010") + self.stream.flush() + self.count = self.count + 1 + return + def __exit__(self, type, value, traceback): + if type == KeyboardInterrupt: + self.endmsg = "interrupted" + if type == FatalException: + self.endmsg = "aborted by error" + if self.stream: + self.stream.write("...(%2.2f sec) %s.\n" \ + % (time.time() - self.time, self.endmsg)) + return False + +def nuke(directory, legend): + "Remove a (large) directory, with a progress indicator." + with Baton(legend, enable=verbose>=DEBUG_SHUFFLE) as baton: + for root, dirs, files in os.walk(directory, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + baton.twirl() + for name in dirs: + os.rmdir(os.path.join(root, name)) + baton.twirl() + try: + os.rmdir(directory) + except OSError: + pass + +class Date: + "A time/date in local time. Preserves TZ information bu doesn't use it." + def __init__(self, text): + "Recognize date formats that exporters or email programs might emit." + # First, look for git's preferred format. + text = text.strip() + if re.match(r"[0-9]+\s*[+-][0-9]+$", text): + (self.timestamp, self.timezone) = text.split() + self.timestamp = int(self.timestamp) + return + # If that didn't work, look for an RFC822 date, which git also + # accepts. Note, there could be edge cases that Python's parser + # handles but git doesn't. + try: + self.timestamp = int(time.mktime(email.utils.parsedate(text))) + self.timezone = text.split()[5] + return + except TypeError: + # time.mktime throws this when it gets None: + # TypeError: argument must be 9-item sequence, not None + pass + # Date format not recognized + raise FatalException("'%s' is not a valid timestamp" % text) + def rfc822(self): + "Format as an RFC822 timestamp." + return time.strftime("%a %d %b %Y %H:%M:%S", time.localtime(self.timestamp)) + " " + self.timezone + def delta(self, other): + return other.timestamp - self.timestamp + def __str__(self): + "Format as a git timestamp." + return str(self.timestamp) + " " + self.timezone + def __cmp__(self, other): + return cmp(self.timestamp, other.timestamp) + +class Attribution: + "Represents an attribution of a repo action to a person and time." + def __init__(self, person=None): + self.name = self.email = self.date = None + if person: + # First, validity-check the email address + (self.name, self.email) = email.utils.parseaddr(person) + if not self.name or not self.email: + FatalException("can't recognize address in attribution") + # Attribution format is actually stricter than RFC822; + # needs to have a following date in the right place. + person = person.replace(" <", "|").replace("> ", "|") + try: + self.date = Date(person.strip().split("|")[2]) + except (ValueError, IndexError): + raise FatalException("malformed attribution %s" % person) + def __eq__(self, other): + "Compare attributions after canonicalization." + return (self.name == other.name + and self.email == other.email + and self.date == other.date) + def __str__(self): + return self.name + " <" + self.email + "> " + str(self.date) + +class Blob: + "Represent a detached blob of data referenced by a mark." + def __init__(self, repo): + self.repo = repo + self.mark = None + self.path = None # First in-repo path associated with this blob + self.color = None + def blobfile(self): + return self.repo.subdir() + "/blob-" + `id(self)` + "-" + self.mark + def release(self): + os.remove(self.blobfile()) + self.path = None # Marked for deletion + def __str__(self): + with open(self.blobfile()) as dp: + content = dp.read() + return "blob\nmark %s\ndata %d\n%s\n" % (self.mark, len(content), content) + +class Tag: + "Represents an annotated tag." + def __init__(self, name, committish, tagger, content): + self.name = name + self.committish = committish + self.tagger = tagger + self.comment = content + self.color = None + def __str__(self): + st = "tag %s\nfrom %s\n" % (self.name, self.committish) + if self.tagger: + st += "tagger %s\n" % self.tagger + st += "data %d\n%s\n" % (len(self.comment), self.comment,) + return st + +class Reset: + "Represents a branch creation." + def __init__(self): + self.ref = None + self.committish = None + def __str__(self): + st = "reset %s\n" % self.ref + if self.committish: + st += "from %s\n\n" % self.committish + return st + +class FileOp: + "Represent a primitive operation on a file." + modify_re = re.compile(r"(M) ([0-9]+) (\S+) (.*)") + def __init__(self, opline, commit): + self.commit = commit # Only used for debugging. + if opline.startswith("M"): + m = FileOp.modify_re.match(opline) + if not m: + raise FatalException("bad format of M line: %s" % `opline`) + (self.op, self.mode, self.ref, self.path) = m.groups() + elif opline[0] == "D": + (self.op, self.path) = ("D", opline[2:].strip()) + elif opline[0] in ("R", "C"): + (self.op, self.source, self.target) = shlex.split(opline) + elif opline == "deleteall": + self.op = "deleteall" + else: + raise FatalException("unexpected fileop %s while parsing" % opline) + def paths(self): + "return the set of all paths touched by this file op." + if self.op in ("M", "D"): + return set([self.path]) + elif self.op in ("R", "C"): + return set([self.source, self.target]) + def relevant(self, other): + "Do two fileops touch the same file(s)?" + if self.op == "deleteall" or other.op == "deleteall": + return True + else: + return self.paths() & other.paths() + return st + +class Commit: + "Generic commit object." + def __init__(self, repo): + self.repo = repo + self.mark = None # Mark name of commit (may be None) + self.authors = [] # Authors of commit + self.committer = None # Person responsible for committing it. + self.comment = None # Commit comment + self.parent_marks = [] # List of parent nodes + self.branch = None # branch name + self.fileops = [] # blob and file operation list + self.properties = {} # commit properties (extension) + def children(self): + "Get a list of this commit's children." + return filter(lambda e:self.mark in e.parent_marks,self.repo.commits()) + def parents(self): + "Get a list of this commit's parents." + return filter(lambda e:e.mark in self.parent_marks,self.repo.commits()) + def cliques(self): + "Return a dictionary mapping filenames to associated M cliques." + cliques = {} + for (i, fileop) in enumerate(self.fileops): + if fileop.op == "M": + if fileop.path not in cliques: + cliques[fileop.path] = [] + cliques[fileop.path].append(i) + return cliques + +class Passthrough: + "Represents a passthrough line." + def __init__(self, line): + self.text = text + +class FatalException: + "Unrecoverable error." + def __init__(self, msg): + self.msg = msg + +class Repository: + "Generic repository object." + def __init__(self): + self.name = None + self.readtime = None + self.readsize = 0 + self.vcs = None + self.sourcedir = None + self.events = [] # A list of the events encountered, in order + self.nmarks = 0 + self.branches = set([]) + self.import_line = 0 + self.inline_count = 0 + self.preserve_set = set([]) + self.case_coverage = set([]) + self.basedir = os.getcwd() + def cleanup(self): + nuke(self.subdir(), "rcs-fast-import: cleaning up %s" % self.subdir()) + def subdir(self, name=None): + if name is None: + name = self.name + if not name: + return os.path.join(self.basedir, ".rs" + `os.getpid()`) + else: + return os.path.join(self.basedir, ".rs" + `os.getpid()`+ "-" + name) + def error(self, msg, atline=True): + if atline: + raise FatalException(msg + " at line " + `self.import_line`) + else: + raise FatalException(msg) + def warn(self, msg, atline=True): + if atline and self.import_line: + print "rcs-fast-import: " + msg + " at line " + `self.import_line` + else: + print "rcs-fast-import: " + msg + def find(self, mark): + "Find an object by mark" + for (i, e) in enumerate(self.events): + if hasattr(e, "mark") and mark == e.mark: + return i + return None + def fast_import(self, fp, progress=False): + "Initialize repo object from fast-import stream." + try: + try: + if verbose >= DEBUG_SHUFFLE: + self.warn("repository fast import creates " + self.subdir()) + os.mkdir(self.subdir()) + except OSError: + self.error("can't create operating directory", atline=False) + with Baton("rcs-fast-import: from %s" % os.path.relpath(fp.name), enable=progress) as baton: + self.import_line = 0 + linebuffers = [] + ncommits = 0 + def read_data(dp, line=None): + if not line: + line = readline() + if line.startswith("data <<"): + delim = line[7:] + while True: + dataline = fp.readline() + if dataline == delim: + break + elif not dataline: + raise FatalException("EOF while reading blob") + elif line.startswith("data"): + try: + count = int(line[5:]) + dp.write(fp.read(count)) + except ValueError: + self.error("bad count in data") + else: + self.error("malformed data header %s" % `line`) + line = readline() + if line != '\n': + pushback(line) # Data commands optionally end with LF + return dp + def readline(): + if linebuffers: + line = linebuffers.pop() + else: + self.import_line += 1 + line = fp.readline() + self.readsize += len(line) + return line + def pushback(line): + linebuffers.append(line) + while True: + line = readline() + if not line: + break + elif not line.strip(): + continue + elif line.startswith("blob"): + blob = Blob(self) + line = readline() + if line.startswith("mark"): + blob.mark = line[5:].strip() + read_data(open(blob.blobfile(), "w")).close() + self.nmarks += 1 + else: + self.error("missing mark after blob") + self.events.append(blob) + baton.twirl() + elif line.startswith("data"): + self.error("unexpected data object") + elif line.startswith("commit"): + commitbegin = self.import_line + commit = Commit(self) + commit.branch = line.split()[1] + self.branches.add(commit.branch) + ncommits += 1 + while True: + line = readline() + if not line: + break + elif line.startswith("mark"): + commit.mark = line[5:].strip() + self.nmarks += 1 + elif line.startswith("author"): + try: + commit.authors.append(Attribution(line[7:])) + except ValueError: + self.error("malformed author line") + elif line.startswith("committer"): + try: + commit.committer = Attribution(line[10:]) + except ValueError: + self.error("malformed committer line") + elif line.startswith("property"): + fields = line.split(" ") + if len(fields) < 3: + self.error("malformed property line") + elif len(fields) == 3: + self.properties[fields[1]] = True + else: + name = fields[1] + length = int(fields[2]) + value = " ".join(fields[3:]) + if len(value) < length: + value += fp.read(length-len(value)) + if fp.read(1) != '\n': + self.error("trailing junk on property value") + elif len(value) == length + 1: + value = value[:-1] # Trim '\n' + else: + self.error("garbage length field on property line") + commit.properties[name] = value + elif line.startswith("data"): + dp = read_data(cStringIO.StringIO(), line) + commit.comment = dp.getvalue() + dp.close() + elif line.startswith("from") or line.startswith("merge"): + commit.parent_marks.append(line.split()[1]) + # Handling of file ops begins. + elif line[0] in ("C", "D", "R"): + commit.fileops.append(FileOp(line, commit)) + elif line == "filedeleteall\n": + commit.fileops.append(FileOp("filedeleteall"), commit) + elif line[0] == "M": + fileop = FileOp(line, commit) + if commit.mark is None: + self.warn("unmarked commit") + commit.fileops.append(fileop) + if fileop.ref[0] == ':': + for obj in self.events: + if isinstance(obj, Blob) and obj.mark == fileop.ref: + obj.path = fileop.path + fileop.copyname = obj.blobfile() + break + else: + self.error("no blob matches commit reference to %s" % fileop.ref) + elif fileop.ref == 'inline': + fileop.copyname = "inline-" + `id(self)` + "-" + `self.inline_count` + read_data(open(os.path.join(self.subdir(), fileop.copyname), "w")).close() + self.inline_count += 1 + else: + self.error("unknown content type in filemodify") + # Handling of file ops ends. + elif line.isspace(): + # This handles slightly broken + # exporters like the bzr-fast-export + # one that may tack an extra LF onto + # the end of data objects. With it, + # we don't drop out of the + # commit-processing loop until we see + # a *nonblank* line that doesn't match + # a commit subpart. + continue + else: + # Dodgy bzr autodetection hook... + if not self.vcs: + if "branch-nick" in commit.properties: + for vcs in vcstypes: + if vcs.name == "bzr": + self.vcs = vcs + break + pushback(line) + break + if not (commit.mark and commit.committer): + self.import_line = commitbegin + self.error("missing required fields in commit") + self.events.append(commit) + baton.twirl() + elif line.startswith("reset"): + reset = Reset() + reset.ref = line[6:].strip() + line = readline() + if line.startswith("from"): + reset.committish = line[5:].strip() + else: + pushback(line) + self.events.append(reset) + baton.twirl() + elif line.startswith("tag"): + tagger = None + tagname = line[4:].strip() + line = readline() + if line.startswith("from"): + referent = line[5:].strip() + else: + self.error("missing from after tag") + line = readline() + if line.startswith("tagger"): + try: + tagger = Attribution(line[7:]) + except ValueError: + self.error("malformed tagger line") + else: + self.warn("missing tagger after from in tag") + pushback(line) + dp = read_data(cStringIO.StringIO()) + self.events.append(Tag(tagname, + referent, tagger, dp.getvalue())) + baton.twirl() + else: + # Simply pass through any line we don't understand. + self.events.append(line) + self.import_line = 0 + self.readtime = time.time() + except KeyboardInterrupt: + nuke(self.subdir(), "rcs-fast-import: import interrupted, removing %s" % self.subdir()) + raise KeyboardInterrupt + def commits(self): + "Return a list of the repository commit objects." + return filter(lambda e: isinstance(e, Commit), self.events) + # Container emulation methods + def __len__(self): + return len(self.events) + def __getitem__(self, i): + return self.events[i] + def __setitem__(self, i, v): + self.events[i] = v + +def complain(msg): + print >>sys.stderr, "rcs-fast-import:", msg + +def announce(msg): + print "rcs-fast-import:", msg + +def do_or_die(cmd, legend=""): + "Either execute a command or raise a fatal exception." + if legend: + legend = " " + legend + if verbose >= DEBUG_COMMANDS: + announce("executing '%s'%s" % (cmd, legend)) + try: + retcode = subprocess.call(cmd, shell=True) + if retcode < 0: + raise FatalException("child was terminated by signal %d." % -retcode) + elif retcode != 0: + raise FatalException("child returned %d." % retcode) + except OSError, e: + raise FatalException("execution of %s%s failed: %s" % (cmd, legend, e)) + +if __name__ == '__main__': + repo = Repository() + repo.fast_import(sys.stdin) + +# end -- 2.43.0