--- /dev/null
+#!/usr/bin/env python
+#
+# rcs-fast-import - break a git import stream into corresponding RCS files.
+#
+# By ESR, November 2010. BSD terms apply.
+#
+# Requires Python 2.5 or newer.
+#
+import sys, os, cStringIO, subprocess
+import time, calendar, re, signal, copy, shlex
+import email.utils
+
+version="0.1"
+
+verbose = 0
+DEBUG_SHUFFLE = 2 # Debug directory handling
+DEBUG_DELETE = 2 # Debug canonicalization after deletes
+DEBUG_COMMANDS = 2 # Show commands as they are executed
+
+class Baton:
+ "Ship progress indications to stdout."
+ def __init__(self, prompt, endmsg='done', enable=False):
+ self.prompt = prompt
+ self.endmsg = endmsg
+ if enable:
+ self.stream = sys.stdout
+ else:
+ self.stream = None
+ def __enter__(self):
+ if self.stream:
+ self.stream.write(self.prompt + "...")
+ if os.isatty(self.stream.fileno()):
+ self.stream.write(" \010")
+ self.stream.flush()
+ self.count = 0
+ self.time = time.time()
+ return self
+ def twirl(self, ch=None):
+ if self.stream is None:
+ return
+ if os.isatty(self.stream.fileno()):
+ if ch:
+ self.stream.write(ch)
+ else:
+ self.stream.write("-/|\\"[self.count % 4])
+ self.stream.write("\010")
+ self.stream.flush()
+ self.count = self.count + 1
+ return
+ def __exit__(self, type, value, traceback):
+ if type == KeyboardInterrupt:
+ self.endmsg = "interrupted"
+ if type == FatalException:
+ self.endmsg = "aborted by error"
+ if self.stream:
+ self.stream.write("...(%2.2f sec) %s.\n" \
+ % (time.time() - self.time, self.endmsg))
+ return False
+
+def nuke(directory, legend):
+ "Remove a (large) directory, with a progress indicator."
+ with Baton(legend, enable=verbose>=DEBUG_SHUFFLE) as baton:
+ for root, dirs, files in os.walk(directory, topdown=False):
+ for name in files:
+ os.remove(os.path.join(root, name))
+ baton.twirl()
+ for name in dirs:
+ os.rmdir(os.path.join(root, name))
+ baton.twirl()
+ try:
+ os.rmdir(directory)
+ except OSError:
+ pass
+
+class Date:
+ "A time/date in local time. Preserves TZ information bu doesn't use it."
+ def __init__(self, text):
+ "Recognize date formats that exporters or email programs might emit."
+ # First, look for git's preferred format.
+ text = text.strip()
+ if re.match(r"[0-9]+\s*[+-][0-9]+$", text):
+ (self.timestamp, self.timezone) = text.split()
+ self.timestamp = int(self.timestamp)
+ return
+ # If that didn't work, look for an RFC822 date, which git also
+ # accepts. Note, there could be edge cases that Python's parser
+ # handles but git doesn't.
+ try:
+ self.timestamp = int(time.mktime(email.utils.parsedate(text)))
+ self.timezone = text.split()[5]
+ return
+ except TypeError:
+ # time.mktime throws this when it gets None:
+ # TypeError: argument must be 9-item sequence, not None
+ pass
+ # Date format not recognized
+ raise FatalException("'%s' is not a valid timestamp" % text)
+ def rfc822(self):
+ "Format as an RFC822 timestamp."
+ return time.strftime("%a %d %b %Y %H:%M:%S", time.localtime(self.timestamp)) + " " + self.timezone
+ def delta(self, other):
+ return other.timestamp - self.timestamp
+ def __str__(self):
+ "Format as a git timestamp."
+ return str(self.timestamp) + " " + self.timezone
+ def __cmp__(self, other):
+ return cmp(self.timestamp, other.timestamp)
+
+class Attribution:
+ "Represents an attribution of a repo action to a person and time."
+ def __init__(self, person=None):
+ self.name = self.email = self.date = None
+ if person:
+ # First, validity-check the email address
+ (self.name, self.email) = email.utils.parseaddr(person)
+ if not self.name or not self.email:
+ FatalException("can't recognize address in attribution")
+ # Attribution format is actually stricter than RFC822;
+ # needs to have a following date in the right place.
+ person = person.replace(" <", "|").replace("> ", "|")
+ try:
+ self.date = Date(person.strip().split("|")[2])
+ except (ValueError, IndexError):
+ raise FatalException("malformed attribution %s" % person)
+ def __eq__(self, other):
+ "Compare attributions after canonicalization."
+ return (self.name == other.name
+ and self.email == other.email
+ and self.date == other.date)
+ def __str__(self):
+ return self.name + " <" + self.email + "> " + str(self.date)
+
+class Blob:
+ "Represent a detached blob of data referenced by a mark."
+ def __init__(self, repo):
+ self.repo = repo
+ self.mark = None
+ self.path = None # First in-repo path associated with this blob
+ self.color = None
+ def blobfile(self):
+ return self.repo.subdir() + "/blob-" + `id(self)` + "-" + self.mark
+ def release(self):
+ os.remove(self.blobfile())
+ self.path = None # Marked for deletion
+ def __str__(self):
+ with open(self.blobfile()) as dp:
+ content = dp.read()
+ return "blob\nmark %s\ndata %d\n%s\n" % (self.mark, len(content), content)
+
+class Tag:
+ "Represents an annotated tag."
+ def __init__(self, name, committish, tagger, content):
+ self.name = name
+ self.committish = committish
+ self.tagger = tagger
+ self.comment = content
+ self.color = None
+ def __str__(self):
+ st = "tag %s\nfrom %s\n" % (self.name, self.committish)
+ if self.tagger:
+ st += "tagger %s\n" % self.tagger
+ st += "data %d\n%s\n" % (len(self.comment), self.comment,)
+ return st
+
+class Reset:
+ "Represents a branch creation."
+ def __init__(self):
+ self.ref = None
+ self.committish = None
+ def __str__(self):
+ st = "reset %s\n" % self.ref
+ if self.committish:
+ st += "from %s\n\n" % self.committish
+ return st
+
+class FileOp:
+ "Represent a primitive operation on a file."
+ modify_re = re.compile(r"(M) ([0-9]+) (\S+) (.*)")
+ def __init__(self, opline, commit):
+ self.commit = commit # Only used for debugging.
+ if opline.startswith("M"):
+ m = FileOp.modify_re.match(opline)
+ if not m:
+ raise FatalException("bad format of M line: %s" % `opline`)
+ (self.op, self.mode, self.ref, self.path) = m.groups()
+ elif opline[0] == "D":
+ (self.op, self.path) = ("D", opline[2:].strip())
+ elif opline[0] in ("R", "C"):
+ (self.op, self.source, self.target) = shlex.split(opline)
+ elif opline == "deleteall":
+ self.op = "deleteall"
+ else:
+ raise FatalException("unexpected fileop %s while parsing" % opline)
+ def paths(self):
+ "return the set of all paths touched by this file op."
+ if self.op in ("M", "D"):
+ return set([self.path])
+ elif self.op in ("R", "C"):
+ return set([self.source, self.target])
+ def relevant(self, other):
+ "Do two fileops touch the same file(s)?"
+ if self.op == "deleteall" or other.op == "deleteall":
+ return True
+ else:
+ return self.paths() & other.paths()
+ return st
+
+class Commit:
+ "Generic commit object."
+ def __init__(self, repo):
+ self.repo = repo
+ self.mark = None # Mark name of commit (may be None)
+ self.authors = [] # Authors of commit
+ self.committer = None # Person responsible for committing it.
+ self.comment = None # Commit comment
+ self.parent_marks = [] # List of parent nodes
+ self.branch = None # branch name
+ self.fileops = [] # blob and file operation list
+ self.properties = {} # commit properties (extension)
+ def children(self):
+ "Get a list of this commit's children."
+ return filter(lambda e:self.mark in e.parent_marks,self.repo.commits())
+ def parents(self):
+ "Get a list of this commit's parents."
+ return filter(lambda e:e.mark in self.parent_marks,self.repo.commits())
+ def cliques(self):
+ "Return a dictionary mapping filenames to associated M cliques."
+ cliques = {}
+ for (i, fileop) in enumerate(self.fileops):
+ if fileop.op == "M":
+ if fileop.path not in cliques:
+ cliques[fileop.path] = []
+ cliques[fileop.path].append(i)
+ return cliques
+
+class Passthrough:
+ "Represents a passthrough line."
+ def __init__(self, line):
+ self.text = text
+
+class FatalException:
+ "Unrecoverable error."
+ def __init__(self, msg):
+ self.msg = msg
+
+class Repository:
+ "Generic repository object."
+ def __init__(self):
+ self.name = None
+ self.readtime = None
+ self.readsize = 0
+ self.vcs = None
+ self.sourcedir = None
+ self.events = [] # A list of the events encountered, in order
+ self.nmarks = 0
+ self.branches = set([])
+ self.import_line = 0
+ self.inline_count = 0
+ self.preserve_set = set([])
+ self.case_coverage = set([])
+ self.basedir = os.getcwd()
+ def cleanup(self):
+ nuke(self.subdir(), "rcs-fast-import: cleaning up %s" % self.subdir())
+ def subdir(self, name=None):
+ if name is None:
+ name = self.name
+ if not name:
+ return os.path.join(self.basedir, ".rs" + `os.getpid()`)
+ else:
+ return os.path.join(self.basedir, ".rs" + `os.getpid()`+ "-" + name)
+ def error(self, msg, atline=True):
+ if atline:
+ raise FatalException(msg + " at line " + `self.import_line`)
+ else:
+ raise FatalException(msg)
+ def warn(self, msg, atline=True):
+ if atline and self.import_line:
+ print "rcs-fast-import: " + msg + " at line " + `self.import_line`
+ else:
+ print "rcs-fast-import: " + msg
+ def find(self, mark):
+ "Find an object by mark"
+ for (i, e) in enumerate(self.events):
+ if hasattr(e, "mark") and mark == e.mark:
+ return i
+ return None
+ def fast_import(self, fp, progress=False):
+ "Initialize repo object from fast-import stream."
+ try:
+ try:
+ if verbose >= DEBUG_SHUFFLE:
+ self.warn("repository fast import creates " + self.subdir())
+ os.mkdir(self.subdir())
+ except OSError:
+ self.error("can't create operating directory", atline=False)
+ with Baton("rcs-fast-import: from %s" % os.path.relpath(fp.name), enable=progress) as baton:
+ self.import_line = 0
+ linebuffers = []
+ ncommits = 0
+ def read_data(dp, line=None):
+ if not line:
+ line = readline()
+ if line.startswith("data <<"):
+ delim = line[7:]
+ while True:
+ dataline = fp.readline()
+ if dataline == delim:
+ break
+ elif not dataline:
+ raise FatalException("EOF while reading blob")
+ elif line.startswith("data"):
+ try:
+ count = int(line[5:])
+ dp.write(fp.read(count))
+ except ValueError:
+ self.error("bad count in data")
+ else:
+ self.error("malformed data header %s" % `line`)
+ line = readline()
+ if line != '\n':
+ pushback(line) # Data commands optionally end with LF
+ return dp
+ def readline():
+ if linebuffers:
+ line = linebuffers.pop()
+ else:
+ self.import_line += 1
+ line = fp.readline()
+ self.readsize += len(line)
+ return line
+ def pushback(line):
+ linebuffers.append(line)
+ while True:
+ line = readline()
+ if not line:
+ break
+ elif not line.strip():
+ continue
+ elif line.startswith("blob"):
+ blob = Blob(self)
+ line = readline()
+ if line.startswith("mark"):
+ blob.mark = line[5:].strip()
+ read_data(open(blob.blobfile(), "w")).close()
+ self.nmarks += 1
+ else:
+ self.error("missing mark after blob")
+ self.events.append(blob)
+ baton.twirl()
+ elif line.startswith("data"):
+ self.error("unexpected data object")
+ elif line.startswith("commit"):
+ commitbegin = self.import_line
+ commit = Commit(self)
+ commit.branch = line.split()[1]
+ self.branches.add(commit.branch)
+ ncommits += 1
+ while True:
+ line = readline()
+ if not line:
+ break
+ elif line.startswith("mark"):
+ commit.mark = line[5:].strip()
+ self.nmarks += 1
+ elif line.startswith("author"):
+ try:
+ commit.authors.append(Attribution(line[7:]))
+ except ValueError:
+ self.error("malformed author line")
+ elif line.startswith("committer"):
+ try:
+ commit.committer = Attribution(line[10:])
+ except ValueError:
+ self.error("malformed committer line")
+ elif line.startswith("property"):
+ fields = line.split(" ")
+ if len(fields) < 3:
+ self.error("malformed property line")
+ elif len(fields) == 3:
+ self.properties[fields[1]] = True
+ else:
+ name = fields[1]
+ length = int(fields[2])
+ value = " ".join(fields[3:])
+ if len(value) < length:
+ value += fp.read(length-len(value))
+ if fp.read(1) != '\n':
+ self.error("trailing junk on property value")
+ elif len(value) == length + 1:
+ value = value[:-1] # Trim '\n'
+ else:
+ self.error("garbage length field on property line")
+ commit.properties[name] = value
+ elif line.startswith("data"):
+ dp = read_data(cStringIO.StringIO(), line)
+ commit.comment = dp.getvalue()
+ dp.close()
+ elif line.startswith("from") or line.startswith("merge"):
+ commit.parent_marks.append(line.split()[1])
+ # Handling of file ops begins.
+ elif line[0] in ("C", "D", "R"):
+ commit.fileops.append(FileOp(line, commit))
+ elif line == "filedeleteall\n":
+ commit.fileops.append(FileOp("filedeleteall"), commit)
+ elif line[0] == "M":
+ fileop = FileOp(line, commit)
+ if commit.mark is None:
+ self.warn("unmarked commit")
+ commit.fileops.append(fileop)
+ if fileop.ref[0] == ':':
+ for obj in self.events:
+ if isinstance(obj, Blob) and obj.mark == fileop.ref:
+ obj.path = fileop.path
+ fileop.copyname = obj.blobfile()
+ break
+ else:
+ self.error("no blob matches commit reference to %s" % fileop.ref)
+ elif fileop.ref == 'inline':
+ fileop.copyname = "inline-" + `id(self)` + "-" + `self.inline_count`
+ read_data(open(os.path.join(self.subdir(), fileop.copyname), "w")).close()
+ self.inline_count += 1
+ else:
+ self.error("unknown content type in filemodify")
+ # Handling of file ops ends.
+ elif line.isspace():
+ # This handles slightly broken
+ # exporters like the bzr-fast-export
+ # one that may tack an extra LF onto
+ # the end of data objects. With it,
+ # we don't drop out of the
+ # commit-processing loop until we see
+ # a *nonblank* line that doesn't match
+ # a commit subpart.
+ continue
+ else:
+ # Dodgy bzr autodetection hook...
+ if not self.vcs:
+ if "branch-nick" in commit.properties:
+ for vcs in vcstypes:
+ if vcs.name == "bzr":
+ self.vcs = vcs
+ break
+ pushback(line)
+ break
+ if not (commit.mark and commit.committer):
+ self.import_line = commitbegin
+ self.error("missing required fields in commit")
+ self.events.append(commit)
+ baton.twirl()
+ elif line.startswith("reset"):
+ reset = Reset()
+ reset.ref = line[6:].strip()
+ line = readline()
+ if line.startswith("from"):
+ reset.committish = line[5:].strip()
+ else:
+ pushback(line)
+ self.events.append(reset)
+ baton.twirl()
+ elif line.startswith("tag"):
+ tagger = None
+ tagname = line[4:].strip()
+ line = readline()
+ if line.startswith("from"):
+ referent = line[5:].strip()
+ else:
+ self.error("missing from after tag")
+ line = readline()
+ if line.startswith("tagger"):
+ try:
+ tagger = Attribution(line[7:])
+ except ValueError:
+ self.error("malformed tagger line")
+ else:
+ self.warn("missing tagger after from in tag")
+ pushback(line)
+ dp = read_data(cStringIO.StringIO())
+ self.events.append(Tag(tagname,
+ referent, tagger, dp.getvalue()))
+ baton.twirl()
+ else:
+ # Simply pass through any line we don't understand.
+ self.events.append(line)
+ self.import_line = 0
+ self.readtime = time.time()
+ except KeyboardInterrupt:
+ nuke(self.subdir(), "rcs-fast-import: import interrupted, removing %s" % self.subdir())
+ raise KeyboardInterrupt
+ def commits(self):
+ "Return a list of the repository commit objects."
+ return filter(lambda e: isinstance(e, Commit), self.events)
+ # Container emulation methods
+ def __len__(self):
+ return len(self.events)
+ def __getitem__(self, i):
+ return self.events[i]
+ def __setitem__(self, i, v):
+ self.events[i] = v
+
+def complain(msg):
+ print >>sys.stderr, "rcs-fast-import:", msg
+
+def announce(msg):
+ print "rcs-fast-import:", msg
+
+def do_or_die(cmd, legend=""):
+ "Either execute a command or raise a fatal exception."
+ if legend:
+ legend = " " + legend
+ if verbose >= DEBUG_COMMANDS:
+ announce("executing '%s'%s" % (cmd, legend))
+ try:
+ retcode = subprocess.call(cmd, shell=True)
+ if retcode < 0:
+ raise FatalException("child was terminated by signal %d." % -retcode)
+ elif retcode != 0:
+ raise FatalException("child returned %d." % retcode)
+ except OSError, e:
+ raise FatalException("execution of %s%s failed: %s" % (cmd, legend, e))
+
+if __name__ == '__main__':
+ repo = Repository()
+ repo.fast_import(sys.stdin)
+
+# end
projects / rcs-fast-import.git / commitdiff