From 5aa20920226e95138cfaf77e12b3df08a9d1f97a Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Wed, 19 Aug 2015 13:21:34 +0000 Subject: [PATCH] git-tools: concurrency for pbuilder builds SA-768 --- id-deb-build/id-pbuild-root.sh | 87 ++++++++++++++++++++++++++++++++++++++++ id-deb-build/id-pbuild.sh | 50 +++++++++++++---------- id-deb-build/pbuilderrc | 4 +- 3 files changed, 117 insertions(+), 24 deletions(-) create mode 100755 id-deb-build/id-pbuild-root.sh diff --git a/id-deb-build/id-pbuild-root.sh b/id-deb-build/id-pbuild-root.sh new file mode 100755 index 0000000..e0d1e14 --- /dev/null +++ b/id-deb-build/id-pbuild-root.sh @@ -0,0 +1,87 @@ +#!/bin/sh +usage() +{ + cat <&2 +fi +if test -z "$DIST"; then + echo "id-pbuild-root.sh: missing --dist" + exit 1 +fi +if test -z "$ARCH"; then + echo "id-pbuild-root.sh: missing --arch" + exit 1 +fi +if test -z "$DSC"; then + echo "id-pbuild-root.sh: missing --dsc" + exit 1 +fi +if test ! -x /usr/bin/lxc-attach; then + echo "No lxc-attach. Probably not running on mochi!!" + exit 1 +fi +export DIST ARCH # export for pbuilder to see +echo "$DIST-$ARCH begin" +if $update; then + if ! lxc-attach -n pbuilder -- pbuilder --update --override-config >update-${DIST}-${ARCH}.log 2>&1; then + if ! lxc-attach -n pbuilder -- pbuilder --create >create-${DIST}-${ARCH}.log 2>&1; then + echo "$DIST-$ARCH failed. Inspect deb-src/create-${DIST}-${ARCH}.log" + exit 2 + fi + if ! lxc-attach -n pbuilder -- pbuilder --update --override-config >update-${DIST}-${ARCH}.log 2>&1; then + echo "$DIST-$ARCH failed. Inspect deb-src/update-${DIST}-${ARCH}.log" + exit 2 + fi + fi +fi +if ! lxc-attach -n pbuilder -- pbuilder --build $DSC >build-${DIST}-${ARCH}.log 2>&1; then + echo "$DIST-$ARCH failed. Inspect deb-src/build-${DIST}-${ARCH}.log" + exit 3 +fi +echo "$DIST-$ARCH done" +exit 0 + +# Local Variables: +# mode:shell-script +# sh-indentation: 4 +# sh-basic-offset: 4 +# End: + diff --git a/id-deb-build/id-pbuild.sh b/id-deb-build/id-pbuild.sh index 6676f99..2d588b1 100755 --- a/id-deb-build/id-pbuild.sh +++ b/id-deb-build/id-pbuild.sh @@ -7,15 +7,18 @@ Options: [--debian=distros] [--ubuntu=distros] [--centos=distros] + [--concurrency=level] + [--update=true|false] [--upload] EOF exit $1 } D0=`dirname $0` ID_DEB_BUILD=`cd $D0; pwd` - +update=true do_help=false upload=false +concurrency=8 ARCHS="i386 amd64" while test $# -gt 0; do case "$1" in @@ -32,9 +35,15 @@ while test $# -gt 0; do --centos=*) CENTOS_DIST_A=$optarg ;; + --concurrency=*) + concurrency=$optarg + ;; --upload) upload=true ;; + --update=*) + update=$optarg + ;; -*) do_help=yes ;; @@ -224,31 +233,28 @@ if $debian; then else USE_ARCHS=$ARCHS fi + rm -f ~/.pbuilderrc + i=0 + cat ${ID_DEB_BUILD}/pbuilderrc > ~/.pbuilderrc for dist in ${DEBIAN_DIST} ${UBUNTU_DIST}; do for arch in ${USE_ARCHS}; do - echo -n "dist=$dist arch=$arch" - rm -f ~/.pbuilderrc - echo "DIST=$dist" > ~/.pbuilderrc - echo "ARCH=$arch" >> ~/.pbuilderrc - cat ${ID_DEB_BUILD}/pbuilderrc >> ~/.pbuilderrc - echo -n " update" - if ! sudo lxc-attach -n pbuilder -- pbuilder --update --override-config >update-${dist}-${arch}.log 2>&1; then - echo -n " create" - if ! sudo lxc-attach -n pbuilder -- pbuilder --create >create-${dist}-${arch}.log 2>&1; then - echo " failed. Inspect deb-src/create-${dist}-${arch}.log" - exit 1 - fi - echo -n " update" - if ! sudo lxc-attach -n pbuilder -- pbuilder --update --override-config >update-${dist}-${arch}.log 2>&1; then - echo " failed. Inspect deb-src/update-${dist}-${arch}.log" + ${ID_DEB_BUILD}/id-pbuild-root.sh --update=$update --dsc=$DSC --arch=$arch --dist=$dist & + i=`expr $i + 1` + if test $i -ge $concurrency; then + i=0 + wait + if ! test -f ${dist}-${arch}/*.dsc; then + echo "FAILED: No .dsc file for ${dist}-${arch}" exit 1 - fi + fi fi - echo -n " build" - if sudo lxc-attach -n pbuilder -- pbuilder --build $DSC >build-${dist}-${arch}.log 2>&1; then - echo " done" - else - echo " failed. Inspect deb-src/build-${dist}-${arch}.log" + done + done + wait + for dist in ${DEBIAN_DIST} ${UBUNTU_DIST}; do + for arch in ${USE_ARCHS}; do + if ! test -f ${dist}-${arch}/*.dsc; then + echo "FAILED: No .dsc file for ${dist}-${arch}" exit 1 fi done diff --git a/id-deb-build/pbuilderrc b/id-deb-build/pbuilderrc index a3a6848..0772d4d 100644 --- a/id-deb-build/pbuilderrc +++ b/id-deb-build/pbuilderrc @@ -3,8 +3,8 @@ # https://wiki.ubuntu.com/PbuilderHowto #DIST=`dpkg-parsechangelog | awk '/^Distribution: / {print $2}'` -: ${DIST:=$(lsb_release --short --codename)} -: ${ARCH:=$(dpkg --print-architecture)} +# : ${DIST:=$(lsb_release --short --codename)} +# : ${ARCH:=$(dpkg --print-architecture)} NAME="$DIST-$ARCH" DISTRIBUTION="$DIST" # We avoid keyring in assignment below. bug 4122 -- 1.7.10.4