From: Adam Dickmeiss <adam@indexdata.dk>
Date: Wed, 19 Aug 2015 13:21:34 +0000 (+0000)
Subject: git-tools: concurrency for pbuilder builds SA-768
X-Git-Url: http://git.indexdata.com/?p=git-tools-moved-to-github.git;a=commitdiff_plain;h=5aa20920226e95138cfaf77e12b3df08a9d1f97a

git-tools: concurrency for pbuilder builds SA-768
---

diff --git a/id-deb-build/id-pbuild-root.sh b/id-deb-build/id-pbuild-root.sh
new file mode 100755
index 0000000..e0d1e14
--- /dev/null
+++ b/id-deb-build/id-pbuild-root.sh
@@ -0,0 +1,87 @@
+#!/bin/sh
+usage()
+{
+    cat <<EOF
+Usage: id-pbuild-root.sh [OPTIONS]
+Options:
+        [--arch=architecture]
+        [--dist=distribution]
+        [--dsc=dscfile]
+        [--update=true|false]
+EOF
+    exit $1
+}
+D0=`dirname $0`
+
+do_help=false
+update=true
+while test $# -gt 0; do
+    case "$1" in
+        -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+        *) optarg= ;;
+    esac
+    case $1 in
+        --dist=*)
+            DIST=$optarg
+            ;;
+        --arch=*)
+            ARCH=$optarg
+            ;;
+        --dsc=*)
+            DSC=$optarg
+            ;;
+        --update=*)
+            update=$optarg
+            ;;
+        -*)
+            do_help=yes
+            ;;
+    esac
+    shift
+done
+if test "$do_help" = "yes"; then
+    usage 1 1>&2
+fi
+if test -z "$DIST"; then
+    echo "id-pbuild-root.sh: missing --dist"
+    exit 1
+fi
+if test -z "$ARCH"; then
+    echo "id-pbuild-root.sh: missing --arch"
+    exit 1
+fi
+if test -z "$DSC"; then
+    echo "id-pbuild-root.sh: missing --dsc"
+    exit 1
+fi
+if test ! -x /usr/bin/lxc-attach; then
+    echo "No lxc-attach. Probably not running on mochi!!"
+    exit 1
+fi
+export DIST ARCH # export for pbuilder to see
+echo "$DIST-$ARCH begin"
+if $update; then
+    if ! lxc-attach -n pbuilder -- pbuilder --update --override-config >update-${DIST}-${ARCH}.log 2>&1; then
+        if ! lxc-attach -n pbuilder -- pbuilder --create >create-${DIST}-${ARCH}.log 2>&1; then
+            echo "$DIST-$ARCH failed. Inspect deb-src/create-${DIST}-${ARCH}.log"
+            exit 2
+        fi
+        if ! lxc-attach -n pbuilder -- pbuilder --update --override-config >update-${DIST}-${ARCH}.log 2>&1; then
+            echo "$DIST-$ARCH failed. Inspect deb-src/update-${DIST}-${ARCH}.log"
+           exit 2
+       fi
+   fi
+fi
+if ! lxc-attach -n pbuilder -- pbuilder --build $DSC >build-${DIST}-${ARCH}.log 2>&1; then
+    echo "$DIST-$ARCH failed. Inspect deb-src/build-${DIST}-${ARCH}.log"
+    exit 3
+fi
+echo "$DIST-$ARCH done"
+exit 0
+
+# Local Variables:
+# mode:shell-script
+# sh-indentation: 4
+# sh-basic-offset: 4
+# End:
+
diff --git a/id-deb-build/id-pbuild.sh b/id-deb-build/id-pbuild.sh
index 6676f99..2d588b1 100755
--- a/id-deb-build/id-pbuild.sh
+++ b/id-deb-build/id-pbuild.sh
@@ -7,15 +7,18 @@ Options:
         [--debian=distros]
         [--ubuntu=distros]
         [--centos=distros]
+        [--concurrency=level]
+        [--update=true|false]
         [--upload]
 EOF
     exit $1
 }
 D0=`dirname $0`
 ID_DEB_BUILD=`cd $D0; pwd`
-
+update=true
 do_help=false
 upload=false
+concurrency=8
 ARCHS="i386 amd64"
 while test $# -gt 0; do
     case "$1" in
@@ -32,9 +35,15 @@ while test $# -gt 0; do
         --centos=*)
             CENTOS_DIST_A=$optarg
             ;;
+        --concurrency=*)
+            concurrency=$optarg
+            ;;
         --upload)
             upload=true
             ;;
+        --update=*)
+            update=$optarg
+            ;;
         -*)
             do_help=yes
             ;;
@@ -224,31 +233,28 @@ if $debian; then
     else
 	USE_ARCHS=$ARCHS
     fi
+    rm -f ~/.pbuilderrc
+    i=0
+    cat ${ID_DEB_BUILD}/pbuilderrc > ~/.pbuilderrc
     for dist in ${DEBIAN_DIST} ${UBUNTU_DIST}; do
 	for arch in ${USE_ARCHS}; do
-	    echo -n "dist=$dist arch=$arch"
-	    rm -f ~/.pbuilderrc
-	    echo "DIST=$dist" > ~/.pbuilderrc
-	    echo "ARCH=$arch" >> ~/.pbuilderrc
-	    cat ${ID_DEB_BUILD}/pbuilderrc >> ~/.pbuilderrc
-	    echo -n " update"
-	    if ! sudo lxc-attach -n pbuilder -- pbuilder --update --override-config >update-${dist}-${arch}.log 2>&1; then
-		echo -n " create"
-		if ! sudo lxc-attach -n pbuilder -- pbuilder --create >create-${dist}-${arch}.log 2>&1; then
-		    echo " failed. Inspect deb-src/create-${dist}-${arch}.log"
-		    exit 1
-		fi
-		echo -n " update"
-		if ! sudo lxc-attach -n pbuilder -- pbuilder --update --override-config >update-${dist}-${arch}.log 2>&1; then
-		    echo " failed. Inspect deb-src/update-${dist}-${arch}.log"
+	    ${ID_DEB_BUILD}/id-pbuild-root.sh --update=$update --dsc=$DSC --arch=$arch --dist=$dist &
+	    i=`expr $i + 1`
+	    if test $i -ge $concurrency; then
+		i=0
+		wait
+	        if ! test -f ${dist}-${arch}/*.dsc; then
+		    echo "FAILED: No .dsc file for ${dist}-${arch}"
 		    exit 1
-		fi
+	        fi
 	    fi
-	    echo -n " build"
-	    if sudo lxc-attach -n pbuilder -- pbuilder --build $DSC >build-${dist}-${arch}.log 2>&1; then
-		echo " done"
-	    else
-		echo " failed. Inspect deb-src/build-${dist}-${arch}.log"
+	done
+    done
+    wait
+    for dist in ${DEBIAN_DIST} ${UBUNTU_DIST}; do
+	for arch in ${USE_ARCHS}; do
+	    if ! test -f ${dist}-${arch}/*.dsc; then
+		echo "FAILED: No .dsc file for ${dist}-${arch}"
 		exit 1
 	    fi
 	done
diff --git a/id-deb-build/pbuilderrc b/id-deb-build/pbuilderrc
index a3a6848..0772d4d 100644
--- a/id-deb-build/pbuilderrc
+++ b/id-deb-build/pbuilderrc
@@ -3,8 +3,8 @@
 # https://wiki.ubuntu.com/PbuilderHowto
 #DIST=`dpkg-parsechangelog | awk '/^Distribution: / {print $2}'`
 
-: ${DIST:=$(lsb_release --short --codename)}
-: ${ARCH:=$(dpkg --print-architecture)}
+# : ${DIST:=$(lsb_release --short --codename)}
+# : ${ARCH:=$(dpkg --print-architecture)}
 NAME="$DIST-$ARCH"
 DISTRIBUTION="$DIST"
 # We avoid keyring in assignment below. bug 4122