reposurgeon 3.37-1 / usr / bin / repotool

#!/bin/bash
#
# repotool - query and manipulate multiple repository types in a uniform way.
#
operation=$1
shift;

TMPDIR=${TMPDIR:=/tmp}

case $operation in
initialize)
    project=$1
    if [ -z "$1" ]
    then
	echo "repotool: initialize requires a project name."
	exit 1
    fi
    if [ -f Makefile ]
    then
	echo "repotool: a Makefile already exists here."
	exit 1
    fi
    if [ -f "${project}.opts" ]
    then
	echo "repotool: a project options file already exists here."
	exit 1
    fi
    if [ -f "${project}.lift" ]
    then
	echo "repotool: a project lift file already exists here."
	exit 1
    fi
    shift
    if [ -z "$1" ]
    then
	/bin/echo -n "repotool: what VCS do you want to convert from? "
	read source_vcs
    else
	source_vcs=$1
    fi
    case $source_vcs in
	cvs|svn|git|bzr|hg|darcs) ;;
	*) echo "repotool: unknown source VCS type ${source_vcs}"; exit 1 ;;
    esac
    shift
    if [ -z "$1" ]
    then
	/bin/echo -n "repotool: what VCS do you want to convert to? "
	read target_vcs
    else
	target_vcs=$1
    fi
    case $target_vcs in
	cvs|svn|git|bzr|hg|darcs) ;;
	*) echo "repotool: unknown target VCS type ${target_vcs}"; exit 1 ;;
    esac 
    echo "repotool: generating Makefile, some variables in it need to be set."
    cat >Makefile <<EOF
# Makefile for $project conversion using reposurgeon
#
# Steps to using this:
# 1. Make sure reposurgeon and repotool are on your \$PATH.
# 2. For svn, set REMOTE_URL to point at the remote repository
#    you want to convert.
# 3. For cvs, set CVS_HOST to the repo hostname and CVS_MODULE to the module,
#    then uncomment the line that builds REMOTE_URL 
#    Note: for CVS hosts other than Sourceforge or Savannah you will need to 
#    include the path to the CVS modules directory after the hostname.
# 4. Set any required read options, such as --user-ignores or --nobranch,
#    by setting READ_OPTIONS.
# 5. Run 'make stubmap' to create a stub author map.
# 6. (Optional) set REPOSURGEON to point at a faster cython build of the tool.
# 7. Run 'make' to build a converted repository.
#
# The reason both first- and second-stage stream files are generated is that,
# especially with Subversion, making the first-stage stream file is often
# painfully slow. By splitting the process, we lower the overhead of
# experiments with the lift script.
#
# For a production-quality conversion you will need to edit the map
# file and the lift script.  During the process you can set EXTRAS to
# name extra metadata such as a comments mailbox.
#
# Afterwards, you can use the headcompare and tagscompare productions
# to check your work.
#

EXTRAS = 
REMOTE_URL = svn://svn.debian.org/${project}
#REMOTE_URL = https://${project}.googlecode.com/svn/
CVS_HOST = ${project}.cvs.sourceforge.net
#CVS_HOST = cvs.savannah.gnu.org
CVS_MODULE = ${project}
#REMOTE_URL = cvs://\$(CVS_HOST)/${project}\\#\$(CVS_MODULE)
READ_OPTIONS =
VERBOSITY = "verbose 1"
REPOSURGEON = reposurgeon

# Configuration ends here

.PHONY: local-clobber remote-clobber gitk gc compare clean dist stubmap
# Tell make not to auto-remove tag directories, because it only tries rm 
# and hence fails
.PRECIOUS: ${project}-%-checkout ${project}-%-${target_vcs}

default: ${project}-${target_vcs}

# Build the converted repo from the second-stage fast-import stream
${project}-${target_vcs}: ${project}.fi
	rm -fr ${project}-${target_vcs}; \$(REPOSURGEON) "read <${project}.fi" "prefer ${target_vcs}" "rebuild ${project}-${target_vcs}"

# Build the second-stage fast-import stream from the first-stage stream dump
${project}.fi: ${project}.${source_vcs} ${project}.opts ${project}.lift ${project}.map \$(EXTRAS)
	\$(REPOSURGEON) \$(VERBOSITY) "script ${project}.opts" "read \$(READ_OPTIONS) <${project}.${source_vcs}" "authors read <${project}.map" "sourcetype ${source_vcs}" "prefer git" "script ${project}.lift" "legacy write >${project}.fo" "write >${project}.fi"

# Build the first-stage stream dump from the local mirror
${project}.${source_vcs}: ${project}-mirror
	(cd ${project}-mirror/ >/dev/null; repotool export) >${project}.${source_vcs}

# Build a local mirror of the remote repository
${project}-mirror:
	repotool mirror \$(REMOTE_URL) ${project}-mirror

# Make a local checkout of the source mirror for inspection
${project}-checkout: ${project}-mirror
	cd ${project}-mirror >/dev/null; repotool checkout ../${project}-checkout

# Make a local checkout of the source mirror for inspection at a specific tag
${project}-%-checkout: ${project}-mirror
	cd ${project}-mirror >/dev/null; repotool checkout ../${project}-\$*-checkout \$*

# Force rebuild of first-stage stream from the local mirror on the next make
local-clobber: clean
	rm -fr ${project}.fi ${project}-${target_vcs} *~ .rs* ${project}-conversion.tar.gz ${project}-*-${target_vcs}

# Force full rebuild from the remote repo on the next make.
remote-clobber: local-clobber
	rm -fr ${project}.${source_vcs} ${project}-mirror ${project}-checkout ${project}-*-checkout

# Get the (empty) state of the author mapping from the first-stage stream
stubmap: ${project}.${source_vcs}
	\$(REPOSURGEON) "read \$(READ_OPTIONS) <${project}.${source_vcs}" "authors write >${project}.map"

# Compare the histories of the unconverted and converted repositories at head
# and all tags.
EXCLUDE = -x CVS -x .${source_vcs} -x .${target_vcs}
EXCLUDE += -x .${source_vcs}ignore -x .${target_vcs}ignore
headcompare: ${project}-mirror ${project}-${target_vcs}
	repotool compare \$(EXCLUDE) ${project}-mirror ${project}-${target_vcs}
tagscompare: ${project}-mirror ${project}-${target_vcs}
	repotool compare-tags \$(EXCLUDE) ${project}-mirror ${project}-${target_vcs}
branchescompare: ${project}-mirror ${project}-${target_vcs}
	repotool compare-branches \$(EXCLUDE) ${project}-mirror ${project}-${target_vcs}
allcompare: ${project}-mirror ${project}-${target_vcs}
	repotool compare-all \$(EXCLUDE) ${project}-mirror ${project}-${target_vcs}

# General cleanup and utility
clean:
	rm -fr *~ .rs* ${project}-conversion.tar.gz *.${source_vcs} *.fi *.fo

# Bundle up the conversion metadata for shipping
SOURCES = Makefile ${project}.lift ${project}.map \$(EXTRAS)
${project}-conversion.tar.gz: \$(SOURCES)
	tar --dereference --transform 's:^:${project}-conversion/:' -czvf ${project}-conversion.tar.gz \$(SOURCES)

dist: ${project}-conversion.tar.gz
EOF
    if [ $target_vcs = git ]
    then
	cat >>Makefile <<EOF

#
# The following productions are git-specific
#

# Browse the generated git repository
gitk: ${project}-git
	cd ${project}-git; gitk --all

# Run a garbage-collect on the generated git repository.  Import doesn't.
# This repack call is the active part of gc --aggressive.  This call is
# tuned for very large repositories.
gc: ${project}-git
	cd ${project}-git; time git -c pack.threads=1 repack -AdF --window=1250 --depth=250
EOF
    fi
    echo "# Pre-read options for reposurgeon go here." >${project}.opts
    echo "# Lift commands for ${project}" >${project}.lift
    ;;
export)
    cvs=no	
    if [ -d CVSROOT ]
    then
    	    cvs=yes
    else
	    for file in *,v; do
		    if [ -f "$file" ]; then
			    cvs=yes
			    break
		    fi
	    done
    fi
    if [ "$cvs" = 'yes' ]
    then
       find -name \*,v | cvs-fast-export -q --reposurgeon
    elif [ -d locks ]
    then
	# Subversion dump rather than the git conversion because the
	# git conversion is lossy.
	svnadmin -q dump .
    elif [ -d .git ]
    then
	    git fast-export --all
    elif [ -d .bzr ]
    then
	    bzr fast-export --no-plain .
    elif [ -d .hg ]
    then
    	    # Use the extractor in reposurgeon itself.
	    reposurgeon "read ." "prefer git" "write -"
    elif [ -d _darcs ]
    then
	    darcs fastconvert export
    else
	echo "repotool: does not look like a repository directory of known type."
	exit 1
    fi
    ;;
mirror)
    operand=$1
    shift
    mirrordir=$1

    if case ${operand} in
	   svn://*|svn+ssh://*|https://*|http://*) true;;
	   file://*) [ -d "${operand#file://}"/locks ];;
	   *) false;;
       esac
    then
	if [ "$mirrordir" ]
	then
	   local=$mirrordir
	else
	    local=`basename $operand`-mirror
	fi
	svnadmin create $local
	cat >$local/hooks/pre-revprop-change <<EOF
#!/bin/sh
exit 0;
EOF
	chmod a+x $local/hooks/pre-revprop-change
	svnsync init file://${PWD}/$local $operand
	svnsync synchronize file://${PWD}/$local
    elif [ -d "$operand/locks" ]
    then
	svnsync synchronize file://${PWD}/$operand
    elif [ `expr "$operand" : "cvs://"` = 6 ]
    then
	if [ "$mirrordir" ]
	then
	   local=$mirrordir
	else
	    local=`echo basename $operand | sed -e /.*#/s///`-mirror
        fi
	mkdir $local
	cvssync -c -o $local "$operand"
	echo "$operand" >$local/.cvssync
    elif [ -f $operand/.cvssync ]
    then
	cvssync -c -o $operand `cat $operand/.cvssync`
    else
	echo "repotool: $operand does not look like a repository mirror."
	exit 1
    fi
    ;;
tags)
    if [ -d CVSROOT ]
    then
        # Will screw up if any tag is not common to all files
	module=`ls -1 | grep -v CVSROOT`
	cvs -Q -d:local:${PWD} rlog -h $module 2>&1 \
	| awk -F"[.:]" '/^\t/&&$(NF-1)!=0{print $1}' | awk '{print $1}' | sort -u
    elif [ -d locks ]
    then
	    svn ls "^/tags"
    elif [ -d .git ]
    then
	    git tag -l
    elif [ -d .bzr ]
    then
	    bzr tags
    elif [ -d .hg ]
    then
	    hg tags --quiet
    elif [ -d _darcs ]
    then
	    darcs show tags
    else
        echo "repotool: tags listing not supported for this repository type."
        exit 1
    fi
    ;;
branches)
    if [ -d CVSROOT ]
    then
        # Will screw up if any tag is not common to all files
	module=`ls -1 | grep -v CVSROOT`
	cvs -Q -d:local:${PWD} rlog -h $module 2>&1 \
	| awk -F"[.:]" '/^\t/&&$(NF-1)==0{print $1}' | awk '{print $1}' | sort -u
    elif [ -d locks ]
    then
	    svn ls "^/branches"
    elif [ -d .git ]
    then
	    git branch -l | cut -c 3- | grep -v '^master$'
    elif [ -d .bzr ]
    then
	    bzr branches | cut -c 3-
    elif [ -d .hg ]
    then
	    hg branches --template '{branch}\n' | grep -v '^default$'
    else
        echo "repotool: branch listing not supported for this repository type."
        exit 1
    fi
    ;;
checkout)
    outdir=$1
    if [ -z "$outdir" ]
    then
	echo "repotool: target directory is required for checkout."
	exit 1
    else
	outdirpath=`realpath $1`
    fi
    shift
    rev=$1
    if [ -d CVSROOT ]
    then
	module=`ls -1 | grep -v CVSROOT`
	if [ "$rev" ]
	then
	    rev="-r $rev"
	fi
	# By choosing -kb we get binary files right, but won't
	# suppress any expanded keywords that might be lurking
	# in masters.   
	cvs -Q -d:local:${PWD} co -P $rev -d $outdirpath -kb $module
    elif [ -d locks ]
    then
	if [ "$rev" ]
	then
	    rev="-d $rev"
	fi
	svn co -q $rev file://${PWD} $outdir
    elif [ -d .git ]
    then
	git clone --quiet --shared --no-checkout . $outdir
	if [ -z "$rev" ]
	then
	    rev="master"
	fi
	cd $outdir >/dev/null; git checkout --quiet $rev
    elif [ -d .bzr ]
    then
	echo "Not yet supported."
	exit 1
    elif [ -d .hg ]
    then
	if [ "$rev" ]
	then
	    rev="-u $rev"
	fi
	hg clone -q $rev . $outdir
    elif [ -d _darcs ]
    then
	echo "Not yet supported."
	exit 1
    else
        echo "repotool: tags listing not supported for this repository type."
        exit 1
    fi
    ;;
compare)
    exclude=
    while getopts qux: opt
    do
	case $opt in
	    x) exclude="$exclude --exclude=$OPTARG" ;;
	    u) diffopts=-u;;
	    q) diffopts=-q;;
	esac
    done
    shift $(($OPTIND - 1))
    repo1=`realpath $1`
    shift
    repo2=`realpath $1`
    shift
    rev=$1
    if [ -z "$repo1" -o -z "$repo2" ]
    then
	echo "repotool: two repositories are required for comparison."
	exit 1
    fi
    here=$PWD
    rm -fr ${TMPDIR}/source $TMPDIR/target
    cd $repo1; repotool checkout $TMPDIR/source $rev
    cd $repo2; repotool checkout $TMPDIR/target $rev
    cd $here
    diff $exclude --ignore-matching-lines=' @(#) ' --ignore-matching-lines='$Id.*$' --ignore-matching-lines='$Header.*$' --ignore-matching-lines='$Log.*$' $diffopts -r $TMPDIR/source $TMPDIR/target| sed \
			-e '/Only in .tmp./s//Only in /' \
			-e '/: /s::/:' \
			-e '/source./s//source: /' \
		        -e '/target./s//target: /'
    ;;
compare-tags)
    exclude=
    diffopts=-q
    while getopts qux: opt
    do
	case $opt in
	    x) exclude="$exclude -x $OPTARG" ;;
	    u) diffopts=-u;;
	    q) diffopts=-q;;
	esac
    done
    shift $(($OPTIND - 1))
    repo1=$1
    repopath1=`realpath $1`
    shift
    repo2=$1
    repopath2=`realpath $1`
    shift
    if [ -z "$repo1" -o -z "$repo2" ]
    then
	echo "repotool: two repositories are required for comparison."
	exit 1
    fi
    #echo "----------------------------------------------------------------"
    #echo "HEAD:"
    #repotool compare $exclude $repo1 $repo2
    here=$PWD
    cd $repopath1; repotool tags | sort >$TMPDIR/tags1
    cd $repopath2; repotool tags | sort >$TMPDIR/tags2
    echo "----------------------------------------------------------------"
    echo "Tags only in $repo1:"; comm -23 $TMPDIR/tags1 $TMPDIR/tags2
    echo "----------------------------------------------------------------"
    echo "Tags only in $repo2:"; comm -13 $TMPDIR/tags1 $TMPDIR/tags2
    cd $here
    for tag in `comm -12 $TMPDIR/tags1 $TMPDIR/tags2`
    do
	echo "----------------------------------------------------------------"
	echo "${tag}:"
	repotool compare $diffopts $exclude $repo1 $repo2 $tag
    done
    ;;
compare-branches)
    exclude=
    diffopts=-q
    while getopts qux: opt
    do
	case $opt in
	    x) exclude="$exclude -x $OPTARG" ;;
	    u) diffopts=-u;;
	    q) diffopts=-q;;
	esac
    done
    shift $(($OPTIND - 1))
    repo1=$1
    repopath1=`realpath $1`
    shift
    repo2=$1
    repopath2=`realpath $1`
    shift
    if [ -z "$repo1" -o -z "$repo2" ]
    then
	echo "repotool: two repositories are required for comparison."
	exit 1
    fi
    here=$PWD
    cd $repopath1; repotool branches | sort >$TMPDIR/branches1
    cd $repopath2; repotool branches | sort >$TMPDIR/branches2
    echo "----------------------------------------------------------------"
    echo "Branches only in $repo1:"; comm -23 $TMPDIR/branches1 $TMPDIR/branches2
    echo "----------------------------------------------------------------"
    echo "Branches only in $repo2:"; comm -13 $TMPDIR/branches1 $TMPDIR/branches2
    cd $here
    for branch in `comm -12 $TMPDIR/branches1 $TMPDIR/branches2`
    do
	echo "----------------------------------------------------------------"
	echo "${branch}:"
	repotool compare $diffopts $exclude $repo1 $repo2 $branch
    done
    ;;
compare-all)
    exclude=
    diffopts=-q
    while getopts qux: opt
    do
	case $opt in
	    x) exclude="$exclude -x $OPTARG" ;;
	    u) diffopts=-u;;
	    q) diffopts=-q;;
	esac
    done
    shift $(($OPTIND - 1))
    repo1=$1
    repopath1=`realpath $1`
    shift
    repo2=$1
    repopath2=`realpath $1`
    shift
    if [ -z "$repo1" -o -z "$repo2" ]
    then
	echo "repotool: two repositories are required for comparison."
	exit 1
    fi
    echo "----------------------------------------------------------------"
    echo "HEAD:"
    repotool compare $diffopts $exclude $repo1 $repo2
    here=$PWD
    # Does not distinguish branches from tags, as that is more useful for
    # testing converted CVS repositories.
    (cd $repopath1; repotool tags; repotool branches) | sort >$TMPDIR/tags1
    (cd $repopath2; repotool tags; repotool branches) | sort >$TMPDIR/tags2
    echo "----------------------------------------------------------------"
    echo "Tags/branches only in $repo1:"; comm -23 $TMPDIR/tags1 $TMPDIR/tags2
    echo "----------------------------------------------------------------"
    echo "Tags/branches only in $repo2:"; comm -13 $TMPDIR/tags1 $TMPDIR/tags2
    cd $here
    for tag in `comm -12 $TMPDIR/tags1 $TMPDIR/tags2`
    do
	echo "----------------------------------------------------------------"
	echo "${tag}:"
	repotool compare $diffopts $exclude $repo1 $repo2 $tag
    done
    ;;
*)
    echo "repotool: unknown action '$operation'."
esac

exit 0

# end