From 68b45e161630289b4f25341f5d895e0f557f8d80 Mon Sep 17 00:00:00 2001 From: Sitaram Chamarty Date: Fri, 12 Aug 2011 22:08:28 +0530 Subject: [PATCH] (new mirroring) bulk of the changes are here: - post-receive now just calls mirror-push - mirror-push is a medium complex shell script (all that backgrounding etc., can't be done so easily in God's first language!) - mirror-shell is now a perl program that does a few different things (receive mirror-pushes, command line re-sync, re-sync requests from a slave, etc) - auth-command changes to reject/redirect non-native pushes --- hooks/common/post-receive.mirrorpush | 28 +++-- src/gl-auth-command | 19 ++-- src/gl-mirror-push | 88 +++++++++++++++ src/gl-mirror-shell | 153 +++++++++++++++++++++++---- 4 files changed, 243 insertions(+), 45 deletions(-) create mode 100755 src/gl-mirror-push diff --git a/hooks/common/post-receive.mirrorpush b/hooks/common/post-receive.mirrorpush index 6be6d81..42ee5b4 100755 --- a/hooks/common/post-receive.mirrorpush +++ b/hooks/common/post-receive.mirrorpush @@ -8,19 +8,15 @@ # if you don't do this, git-shell sometimes dies of a signal 13 (SIGPIPE) [ -t 0 ] || cat >/dev/null -if [ -n "$GL_SLAVES" ] -then - for mirror in $GL_SLAVES - do - if git push --mirror $mirror:$GL_REPO.git - then - : - else - ssh $mirror mkdir -p $GL_REPO.git - ssh $mirror git init --bare $GL_REPO.git - ssh $mirror "cd $GL_REPO.git; git config receive.fsckObjects true" - git push --mirror $mirror:$GL_REPO.git || - echo "WARNING: mirror push to $mirror failed" - fi - done -fi >&2 +# even slaves have post-receive hooks, but due to the way the push happens, we +# don't have GL_REPO set. So we detect that generic situation and bail... +[ -n "$GL_BYPASS_UPDATE_HOOK" ] && exit 0 +# CAUTION: this means that a server-side push (bypassing gitolite) will not be +# mirrored automatically because (a) we don't know GL_REPO (we can deduce it +# but we won't!), and (b) we can't distinguish easily between that and this +# case (the slave receiving a mirror push case) + +[ -z "$GL_REPO" ] && { echo $0: GL_REPO not set -- this is BAD >&2; exit 1; } +[ -z "$GL_BINDIR" ] && { echo $0: GL_BINDIR not set -- this is BAD >&2; exit 1; } + +$GL_BINDIR/gl-mirror-push $GL_REPO diff --git a/src/gl-auth-command b/src/gl-auth-command index 60f0e40..c12d7f5 100755 --- a/src/gl-auth-command +++ b/src/gl-auth-command @@ -93,10 +93,6 @@ unless ($ENV{SSH_ORIGINAL_COMMAND}) { $ENV{SSH_ORIGINAL_COMMAND} = 'info'; } -# slave mode should not do much -die "server is in slave mode; you can only fetch\n" - if ($GL_SLAVE_MODE and $ENV{SSH_ORIGINAL_COMMAND} !~ /^(info|expand|get|git-upload-)/); - # admin defined commands; please see doc/admin-defined-commands.mkd if ($GL_ADC_PATH and -d $GL_ADC_PATH) { try_adc(); # if it succeeds, this also 'exec's out @@ -139,6 +135,18 @@ $ENV{GL_REPO}=$repo; # the real git commands (git-receive-pack, etc...) # ---------------------------------------------------------------------------- +# we know the user and repo; we just need to know what perm he's trying for +# (aa == attempted access; setting this makes some later logic simpler) +my $aa = ($verb =~ $R_COMMANDS ? 'R' : 'W'); + +# writes may get redirected under certain conditions +if ( $GL_HOSTNAME and $aa eq 'W' and mirror_mode($repo) =~ /^slave of (\S+)/ ) { + my $master = $1; + die "$ABRT $GL_HOSTNAME not the master, please push to $master\n" unless mirror_redirectOK($repo, $GL_HOSTNAME); + print STDERR "$GL_HOSTNAME ==== $user ($repo) ===> $master\n"; + exec("ssh", $master, "USER=$user", "SOC=$ENV{SSH_ORIGINAL_COMMAND}"); +} + # first level permissions check my ($perm, $creator, $wild); @@ -150,9 +158,6 @@ if ( $GL_ALL_READ_ALL and $verb =~ $R_COMMANDS and -d "$REPO_BASE/$repo.git") { # it was missing, and you have create perms, so create it new_wild_repo($repo, $user) if ($perm =~ /C/); -# we know the user and repo; we just need to know what perm he's trying for -# (aa == attempted access) -my $aa = ($verb =~ $R_COMMANDS ? 'R' : 'W'); die "$aa access for $repo DENIED to $user (Or there may be no repository at the given path. Did you spell it correctly?)\n" unless $perm =~ /$aa/; diff --git a/src/gl-mirror-push b/src/gl-mirror-push new file mode 100755 index 0000000..4dd98e7 --- /dev/null +++ b/src/gl-mirror-push @@ -0,0 +1,88 @@ +#!/bin/sh + +# arguments: reponame, list of slaves + +# optional flag after reponame: "-fg" to run in foreground. This is only +# going to be given by one specific invocation, and if given will only work +# for one slave. + +# if list of slaves not given, get it from '...slaves' config + +die() { echo gl-mirror-push${hn:+ on $hn}: "$@" >&2; exit 1; } +get_rc_val() { ${0%/*}/gl-query-rc $1; } + +# ---------- + +# is mirroring even enabled? +hn=`get_rc_val GL_HOSTNAME` +[ -z "$hn" ] && exit + +# we should not be invoked directly from the command line +[ -z "$GL_LOG" ] && die fatal: do not run $0 directly + +# ---------- + +# get repo name then check if it's a local or slave (ie we're not the master) +[ -z "$1" ] && die fatal: missing reponame argument +repo=$1; shift + +REPO_BASE=`get_rc_val REPO_BASE` +cd $REPO_BASE/$repo.git +gmm=`git config --get gitolite.mirror.master` + +# is it local? (remember, empty/undef ==> local +gmm=${gmm:-local} +[ "$gmm" = "local" ] && exit + +# is it a slave? +[ "$hn" = "$gmm" ] || die fatal: wrong master. Try $gmm... + +# ---------- + +# now see if we want to be foregrounded. Fg mode accepts only one slave +[ "$1" = "-fg" ] && { + [ -z "$2" ] && die fatal: missing slavename argument + [ -n "$3" ] && die fatal: too many slavenames + git push --mirror $2:$repo 2>&1 | sed -e "s/^/$hn:/" + exit +} + +# ---------- + +# normal (self-backgrounding) mode. Any number of slaves. If none are given, +# use the slave list from the repo config + +export slaves +if [ -n "$1" ] +then + slaves="$*" +else + slaves=`git config --get gitolite.mirror.slaves` +fi + +# ---------- + +# print out the job ID, then redirect all 3 FDs +export job_id=$$ # can change to something else if needed +echo "($job_id&) $hn ==== ($repo) ===>" $slaves >&2 +logfile=${GL_LOG/%.log/-mirror-pushes.log} +exec >>$logfile 2>&1 ' + + for s in $slaves + do + [ "$s" = "$hn" ] && continue # skip ourselves + git push --mirror $s:$repo + done 2>&1 | sed -e "s/^/ /" + echo `date +%T` '===>' $slaves + echo + ) 2>&1 | sed -e "s/^/$job_id:/" & # background the whole thing +) diff --git a/src/gl-mirror-shell b/src/gl-mirror-shell index e72f4ad..52741c1 100755 --- a/src/gl-mirror-shell +++ b/src/gl-mirror-shell @@ -1,30 +1,139 @@ -#!/bin/bash +#!/usr/bin/perl -export GL_BYPASS_UPDATE_HOOK -GL_BYPASS_UPDATE_HOOK=1 +# terminology: +# native repo: a repo for which we are the master; pushes happen here +# authkeys: shorthand for ~/.ssh/authorized_keys -get_rc_val() { - ${0%/*}/gl-query-rc $1 +# this is invoked in one of two ways: + +# (1) locally, from a shell script or command line + +# (2) from a remote server, via authkeys, with one argument (the name of the +# sending server), similar to what happens with normal users and the +# 'gl-auth-command' program. SSH_ORIGINAL_COMMAND will then contain the +# actual command that the remote sent. +# +# Currently, these commands are (a) 'info', (b) 'git-receive-pack' when a +# mirror push is *received* by a slave, (c) 'request-push' sent by a slave +# (possibly via an ADC) when the slave finds itself out of sync, (d) a +# redirected push, from a user pushing to a slave, which is represented not by +# a command per se but by starting with "USER=..." + +use strict; +use warnings; + +# ---------------------------------------------------------------------------- +# this section of code snarfed from gl-auth-command +# XXX add this program to 'that bindir thing' in doc/developer-notes.mkd +BEGIN { + $0 =~ m|^(/)?(.*)/| and $ENV{GL_BINDIR} = ($1 || "$ENV{PWD}/") . $2; } -REPO_BASE=$( get_rc_val REPO_BASE) -REPO_UMASK=$(get_rc_val REPO_UMASK) +use lib $ENV{GL_BINDIR}; -umask $REPO_UMASK +use gitolite_rc; +use gitolite_env; +use gitolite; -if echo $SSH_ORIGINAL_COMMAND | egrep git-upload\|git-receive >/dev/null -then +setup_environment(); +die "fatal: GL_HOSTNAME not set in rc; mirroring disabled\n" unless $GL_HOSTNAME; - # the (special) admin post-update hook needs these, so we cheat - export GL_RC - export GL_ADMINDIR - export GL_BINDIR - GL_RC=$( get_rc_val GL_RC) - GL_ADMINDIR=$(get_rc_val GL_ADMINDIR) - GL_BINDIR=$( get_rc_val GL_BINDIR) +my $soc = $ENV{SSH_ORIGINAL_COMMAND} || ''; - SSH_ORIGINAL_COMMAND=`echo $SSH_ORIGINAL_COMMAND | sed -e "s:':'$REPO_BASE/:"` - exec git shell -c "$SSH_ORIGINAL_COMMAND" -else - bash -c "cd $REPO_BASE; $SSH_ORIGINAL_COMMAND" -fi +# ---------------------------------------------------------------------------- + +# deal with local invocations first + +# on the "master", run from a shell, for one specific repo, with an optional +# list of slaves, like so: +# gl-mirror-shell request-push some-repo [optional-list-of-slaves] +if ( ($ARGV[0] || '') eq 'request-push' and not $soc) { + shift; + # rest of the arguments are fit to go directly to gl-mirror-push + # (reponame, optional list of slaves) + system("gl-mirror-push", @ARGV); + + exit; +} + +unless (@ARGV) { print STDERR "fatal: missing argument\n"; exit 1; } + +# ---------- + +# now the remote invocations; log it, then get the sender name +my $sender = shift; +$ENV{GL_USER} ||= "host:$sender"; +log_it(); + +# ---------- + +# our famous 'info' command +if ($soc eq 'info') { + print STDERR "Hello $sender, I am $GL_HOSTNAME\n"; + + exit; +} + +# ---------- + +# when running on the "slave", we have to "receive" the `git push --mirror` +# from a master. Check that the repo is indeed a slave and the sender is the +# correct master before allowing the push. + +if ($soc =~ /^git-receive-pack '(\S+)'$/) { + my $repo = $1; + my $mm = mirror_mode($repo); + + # reminder: we're not going through the slave-side gl-auth-command. This + # is a server-to-server transaction, with an authenticated sender. + # Authorisation consists of checking to make sure our config says this + # sender is indeed the master for this repo + die "$ABRT fatal: $GL_HOSTNAME <==//== $sender mirror-push rejected: $repo is $mm\n" unless $mm eq "slave of $sender"; + print STDERR "$GL_HOSTNAME <=== ($repo) ==== $sender\n"; + + $ENV{GL_BYPASS_UPDATE_HOOK} = 1; + # replace the repo path with the full path and hand off to git-shell + # m-TODO: the admin repo will need more stuff :) + $soc =~ s(')('$ENV{GL_REPO_BASE_ABS}/); + exec("git", "shell", "-c", $soc); +} + +# ---------- + +# a slave may have found itself out of sync (perhaps the network was down at +# the time of the last push to the master), and now wants to request a sync. +# This is similar to the "local invocation" described above, but we check the +# sender name against gitolite.mirror.slaves to prevent some random slave from +# asking for a repo it should not be having! + +if ($soc =~ /^request-push (\S+)$/) { + my $repo = $1; + die "$ABRT fatal: $GL_HOSTNAME ==//==> $sender refused: not in slave list\n" unless mirror_listslaves($repo) =~ /(^|\s)$sender(\s|$)/; + print STDERR "$GL_HOSTNAME ==== ($repo) ===> $sender\n"; + # just one sender, and we've checked that he is "on the list". Foreground... + system("$ENV{GL_BINDIR}/gl-mirror-push", $repo, "-fg", $sender); + + exit; +} + +# ---------- + +# experimental feature... + +# when running on the "master", receive a redirected push from a slave. This +# is disabled by default and needs to be explicitly enabled on both the master +# and the slave. SEE DOCUMENTATION FOR CAVEATS AND CAUTIONS. + +if ($soc =~ /^USER=(\S+) SOC=(git-receive-pack '(\S+)')$/) { + + my $user = $1; + $ENV{SSH_ORIGINAL_COMMAND} = $2; + my $repo = $3; + die "$ABRT fatal: $GL_HOSTNAME <==//== $sender redirected push rejected\n" unless mirror_redirectOK($repo, $sender); + print STDERR "$GL_HOSTNAME <=== $user ($repo) ==== $sender\n"; + + my $pgm = $0; + $pgm =~ s([^/]+$)(gl-auth-command); + + exec($pgm, $user); +}