commit 784bac5f84c7b4b02e11e74070d57c944d1dc291 Author: Linus Nordberg linus@nordberg.se Date: Sat Mar 15 16:42:50 2014 +0100
Add tools directory with scripts for nightly builds. --- tools/continuous-builds/README | 14 ++++++ tools/continuous-builds/build-tbb.sh | 48 +++++++++++++++++++++ tools/continuous-builds/checkout-tbb.sh | 8 ++++ tools/continuous-builds/park-nightly.sh | 42 ++++++++++++++++++ tools/continuous-builds/setup-gitian-build-env.sh | 3 ++ 5 files changed, 115 insertions(+)
diff --git a/tools/continuous-builds/README b/tools/continuous-builds/README new file mode 100644 index 0000000..c60f19f --- /dev/null +++ b/tools/continuous-builds/README @@ -0,0 +1,14 @@ +Nightly builds can be produced and uploaded to a publishing host by +running build-tbb.sh. The checkout-tbb.sh script can be used to check +out the latest master before starting a build. + +The publishing host should handle uploaded tar file using the +park-nightly.sh script through the command option in sshd's +AuthorizedKeysFile. + + command="[ $(hostname) = perdulce ] && ~/park-nightly.sh",no-pty ssh-rsa ... + +In order to not grow the disk usage too much on the publishing host, +the following script can be run by cron: +https://github.com/boklm/prune-old-builds. + diff --git a/tools/continuous-builds/build-tbb.sh b/tools/continuous-builds/build-tbb.sh new file mode 100755 index 0000000..c44e0fd --- /dev/null +++ b/tools/continuous-builds/build-tbb.sh @@ -0,0 +1,48 @@ +#! /bin/sh +# usage: +# build-tbb.sh [TARGET [PUBLISH-HOST [PUBLISH-KEY [BUILDDIR [DESTDIR [N]]]]]] +# +# Build TARGET in BUILDDIR, which will end up in DESTDIR +# Try doing it N times. +# Upload result to PUBLISH-HOST using SSH key PUBLISH-KEY. + +# TODO: +# - if there's no new commits, don't build but make sure there's a symlink on perdulce + +TARGET=$1; [ -z "$TARGET" ] && TARGET=nightly +PUBLISH_HOST=$2; [ -z "$PUBLISH_HOST" ] && PUBLISH_HOST=perdulce.torproject.org +PUBLISH_KEY=$3; [ -z "$PUBLISH_KEY" ] && PUBLISH_KEY=~/.ssh/perdulce-upload +BUILDDIR=$4; [ -z "$BUILDDIR" ] && BUILDDIR=~/usr/src/tor-browser-bundle/gitian +DESTDIR=$5; [ -z "$DESTDIR" ] && DESTDIR=$BUILDDIR/tbb-$TARGET +N=$6; [ -z "$N" ] && N=16 + +logfile=$(date -u +%s).log + +. ~/setup-gitian-build-env.sh +cd $BUILDDIR || exit 1 +status=init +n=0 +while [ $status != done ]; do + n=$(expr $n + 1) + printf "%s: Starting build number %d. target=$TARGET.\n" $0 $n | tee -a $logfile + date | tee -a $logfile + killall qemu-system-i386 qemu-system-x86_64 + make $TARGET > build-$(date -u +%s).log && status=done + printf "%s: Tried building $TARGET %d times. Status: %s.\n" $0 $n $status | tee -a $logfile + TARGET=build-nightly + [ $n -ge $N ] && break +done + +if [ $status = done ]; then + NEWDESTDIR=$DESTDIR-$(date +%F) + echo "$0: renaming $DESTDIR -> $NEWDESTDIR" | tee -a $logfile + mv $DESTDIR $NEWDESTDIR + cd $NEWDESTDIR || exit 3 + sha256sum *.tar.xz *.zip *.exe > sha256sums.txt + gpg -a --clearsign --local-user 0x984496E7 sha256sums.txt || exit 2 + cd .. + D=$(basename $NEWDESTDIR) + tar cf - $D/sha256sums* $D/*.tar.xz $D/*.zip $D/*.exe | ssh -i $PUBLISH_KEY $PUBLISH_HOST | tee -a $logfile +else + echo "$0: giving up after $n tries" | tee -a $logfile +fi diff --git a/tools/continuous-builds/checkout-tbb.sh b/tools/continuous-builds/checkout-tbb.sh new file mode 100755 index 0000000..f8a7929 --- /dev/null +++ b/tools/continuous-builds/checkout-tbb.sh @@ -0,0 +1,8 @@ +#! /bin/sh + +BUILDDIR=$1 +[ -z "$BUILDDIR" ] && BUILDDIR=~/usr/src/tor-browser-bundle/gitian + +cd $BUILDDIR || exit 1 +git checkout master +git pull diff --git a/tools/continuous-builds/park-nightly.sh b/tools/continuous-builds/park-nightly.sh new file mode 100755 index 0000000..1ac5b2c --- /dev/null +++ b/tools/continuous-builds/park-nightly.sh @@ -0,0 +1,42 @@ +#! /bin/sh +# usage: park-nightly.sh [DIR] + +V=1 #debug is enabled by default for now + +if [ "$1" = "-v" ]; then + V=1 + shift +fi + +DIR=$1 +[ -z "$DIR" ] && DIR=~/public_html/builds + +DSTDIR=tbb-nightly-$(date +%F) +[ -z "$V" ] || echo "Aiming to fill up $DSTDIR" + +do_check() { + [ -z "$1" ] || cd $1 || exit 5 + [ -z "$V" ] || echo "Verifying sha256sums.txt" + gpg -q --verify sha256sums.txt.asc > /dev/null || exit 3 + [ -z "$V" ] || echo "Checking sha256sums.txt" + sha256sum --strict --quiet -c sha256sums.txt || exit 4 +} + +if [ -d $DIR/$DSTDIR ] && [ -e $DIR/$DSTDIR/tbb-nightly.stamp ]; then + [ -z "$V" ] || echo "Files already here, just doing the checking" + do_check $DIR/$DSTDIR + exit +fi + +[ -d .staging ] || mkdir .staging +chmod 700 .staging; cd .staging +[ -z "$V" ] || echo "Saving files to disk" +TAROPT=x +[ -z "$V" ] || TAROPT=${TAROPT}v +tar $TAROPT -f - || exit 6 +touch $DSTDIR/tbb-nightly.stamp + +do_check $DSTDIR || exit 2 +[ -d $DIR/$DSTDIR ] && [ -e $DIR/$DSTDIR/tbb-nightly.stamp ] && rm -rf $DIR/$DSTDIR +cd ..; mv $DSTDIR $DIR/ || exit 1 +[ -z "$V" ] || echo "All good, all good" diff --git a/tools/continuous-builds/setup-gitian-build-env.sh b/tools/continuous-builds/setup-gitian-build-env.sh new file mode 100644 index 0000000..e9f435a --- /dev/null +++ b/tools/continuous-builds/setup-gitian-build-env.sh @@ -0,0 +1,3 @@ +# source me! +export LC_ALL=C +export NUM_PROCS=10