#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # Usage: logimport [options] # Options: # --svn=PATH Use a non-default svn binary # --debug Enable debug-level logging to logimport.debug.log # --sqldebug Enable SQL-level logging to logimport.sql.log # # Converts the history of the repository at into a # single-tree directory index. import logging import subprocess import sys try: from lxml.etree import iterparse except ImportError: from xml.etree.ElementTree import iterparse from dirindex import Dirent, Index, Revision def parse(index, stream): kindmap = {"dir": Dirent.DIR, "file": Dirent.FILE} version = None revcount = 0 for event, logentry in iterparse(stream): if logentry.tag != "logentry": continue version = int(logentry.get("revision")) revcount += 1 if revcount == 1: logging.info("initial: r%d", version) else: logger = not revcount % 1000 and logging.info or logging.debug logger("%d: r%d", revcount, version) created = logentry.find("date") if created is not None: created = created.text else: created = "" author = logentry.find("author") if author is not None: author = author.text log = logentry.find("msg") if log is not None: log = log.text with Revision(index, version, created, author, log) as rev: actionmap = dict(A = (rev.add, True), R = (rev.replace, True), M = (rev.modify, False), D = (rev.delete, False)) for path in logentry.findall("paths/path"): abspath = path.text action = path.get("action") handler, newnode = actionmap[action] if not newnode: logging.debug(" %s %s", action, abspath) handler(abspath) continue kindstr = path.get("kind") kind = kindmap[kindstr] frompath = path.get("copyfrom-path") if frompath is not None: fromrev = int(path.get("copyfrom-rev")) logging.debug(" %s %-4s %s <- %s@%d", action, kindstr, abspath, frompath, fromrev) handler(abspath, kind, frompath, fromrev) else: logging.debug(" %s %-4s %s", action, kindstr, abspath) handler(abspath, kind) if version is not None: logging.info("final: r%d", version) def logimport(database, url, svn): try: index = Index(database) index.cursor.execute("PRAGMA journal_mode = MEMORY") index.cursor.execute("PRAGMA locking_mode = EXCLUSIVE") index.cursor.execute("PRAGMA synchronous = OFF") index.cursor.execute("PRAGMA cache_size = -100000") index.initialize() svnlog = subprocess.Popen( [svn, "log", "-v", "--xml", "-r1:HEAD", url], stdout = subprocess.PIPE) parse(index, svnlog.stdout) return svnlog.wait() except: logging.exception("logimport failed") try: svnlog.wait() except: pass return 2 def main(): import logging.config from optparse import OptionParser from dirindex import SQLobject parser = OptionParser("Usage: %prog [options] ") parser.add_option("--svn", action="store", default="svn", help="Use a non-default svn binary", metavar="PATH") parser.add_option("--debug", action="store_true", default=False, help="Enable debug-level logging to logimport.debug.log") parser.add_option("--sqldebug", action="store_true", default=False, help="Enable SQL-level logging to logimport.debug.log") opts, args = parser.parse_args() if len(args) != 2: parser.error("wrong number of arguments") database, url = args logconfig = { "version": 1, "formatters": { "console": {"format": "%(levelname)-7s %(message)s"}, "logfile": {"format": "%(asctime)s %(levelname)-7s %(message)s"}}, "handlers": { "console": { "class": "logging.StreamHandler", "level": logging.INFO, "stream": sys.stderr, "formatter": "console"}}, "root": { "level": logging.INFO, "handlers": ["console"]}} handlers = logconfig["root"]["handlers"] if opts.debug: logconfig["root"]["level"] = logging.DEBUG logconfig["handlers"]["debug"] = { "class": "logging.FileHandler", "level": logging.DEBUG, "mode": "w", "filename": "./logimport.debug.log", "formatter": "logfile"} handlers.append("debug") if opts.sqldebug: logconfig["root"]["level"] = SQLobject.LOGLEVEL logconfig["handlers"]["sqldebug"] = { "class": "logging.FileHandler", "level": SQLobject.LOGLEVEL, "mode": "w", "filename": "./logimport.sql.log", "formatter": "logfile"} handlers.append("sqldebug") logging.config.dictConfig(logconfig) sys.exit(logimport(database, url, opts.svn)) if __name__ == "__main__": main()