openstack-pkg-tools 54 / usr / bin / pkgos-setup-sbuild

#!/bin/sh

set -e
set -x

#################################
# AUTOMATICALLY DETECT ENV HERE #
#################################

if ! [ -r /etc/pkgos/pkgos.conf ] ; then
	echo "Could not read /etc/pkgos/pkgos.conf"
	exit 1
else
	. /etc/pkgos/pkgos.conf
fi

detect_env () {
	DEB_RELEASE=`lsb_release -a | grep Codename: | awk '{print $2}'`
	DEB_RELEASE_NUM=`lsb_release -a | grep Release: | awk '{print $2}'`
	DEB_DIST_TYPE=$(lsb_release -a | grep "Distributor ID:" | awk '{print $3}')
	APT="apt-get install -y"
	echo 'APT::Install-Recommends "0";' >/etc/apt/apt.conf.d/80norecommends
}

configure_hostname () {
	DEFROUTE_IF=`LC_ALL=C /sbin/route | grep default |awk -- '{ print $8 }' | cut -d" " -f1`
	if [ -n "${DEFROUTE_IF}" ] ; then
		DEFROUTE_IP=`LC_ALL=C ip addr show "${DEFROUTE_IF}" | grep inet | head -n 1 | awk '{print $2}' | cut -d/ -f1 | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'`
		if [ -n "${DEFROUTE_IP}" ] ; then
			echo "Detected IP: ${DEFROUTE_IP}"
			echo "127.0.0.1       localhost.localdomain localhost
${DEFROUTE_IP}	${HOST_FQDN} ${TARGET_OPENSTACK_REL}-${TARGET_DISTRO}

# The following lines are desirable for IPv6 capable hosts
::1     ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
" >/etc/hosts
		fi
	fi
	echo $HOST_FQDN >/etc/hostname
	hostname `cat /etc/hostname`
}

install_all_software () {
	if [ "${DEB_DIST_TYPE}" = "Debian" ] ; then
		ADDED_DEB_PKG="linux-image-amd64 jenkins-cli"
	fi
	${APT} ${ADDED_DEB_PKG} sbuild apache2 screen joe apache2 pure-ftpd ftp most \
		man-db git-buildpackage debhelper eatmydata build-essential python-setuptools \
		fakeroot python3-all python-all python3-setuptools pristine-tar dh-autoreconf ssl-cert \
		dh-python dh-systemd python-sphinx sudo debootstrap openstack-pkg-tools \
		lintian lsb-release postfix gem2deb
	if [ "${DEV_OR_JENKINS}" = "jenkins" ] ; then
		${APT} jenkins jenkins-job-builder
	fi
}

configure_apache () {
	a2enmod proxy
	a2enmod proxy_http
	a2enmod ssl
	a2enmod headers
	if [ "${DEB_RELEASE}" = "precise" ] ; then
		APACHE_SSL_VHOST_CONF=default-ssl
		FORWARD_TO_ADDR=ip6-localhost
		DEFAULT_SITE=default
		DEFAULT_SSL_SITE=default-ssl
	elif [ "${DEB_RELEASE}" = "wheezy" ] ; then
		APACHE_SSL_VHOST_CONF=default-ssl
		FORWARD_TO_ADDR=localhost
		DEFAULT_SITE=default
		DEFAULT_SSL_SITE=default-ssl
	else
		APACHE_SSL_VHOST_CONF=default-ssl.conf
		FORWARD_TO_ADDR=localhost
		DEFAULT_SITE=000-default.conf
		DEFAULT_SSL_SITE=default-ssl.conf
	fi
	a2ensite ${DEFAULT_SSL_SITE}
	APACHE_SSL_VHOST_CONF_FULL_PATH=/etc/apache2/sites-available/${APACHE_SSL_VHOST_CONF}
	echo "<IfModule mod_ssl.c>
<VirtualHost *:443>
	ServerAdmin webmaster@localhost
	ErrorLog \${APACHE_LOG_DIR}/error.log
	CustomLog \${APACHE_LOG_DIR}/access.log combined

	SSLEngine on
	SSLCertificateFile      /etc/ssl/certs/ssl-cert-snakeoil.pem
	SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key

	<FilesMatch \"\.(cgi|shtml|phtml|php)\$\">
                                SSLOptions +StdEnvVars
                </FilesMatch>
                <Directory /usr/lib/cgi-bin>
                                SSLOptions +StdEnvVars
	</Directory>

	# Jenkins proxy (and reverse)
	ProxyPass / http://${FORWARD_TO_ADDR}:8080/ nocanon
	ProxyPassReverse / http://${FORWARD_TO_ADDR}:8080/ 
	ProxyRequests Off
	AllowEncodedSlashes NoDecode
	Header edit Location ^http://${HOST_FQDN}/ https://${HOST_FQDN}/
	RequestHeader set X-Forwarded-Proto \"https\"
	RequestHeader set X-Forwarded-Port \"443\"
	ProxyPreserveHost on
	SetOutputFilter INFLATE;proxy-html;DEFLATE
	SetEnv proxy-nokeepalive 1

	ServerAdmin webmaster@localhost
</VirtualHost>
</IfModule>
" >${APACHE_SSL_VHOST_CONF_FULL_PATH}
	a2ensite ${APACHE_SSL_VHOST_CONF}
	invoke-rc.d apache2 restart
	if [ ! -e /var/www/html/debian ] && [ ! -h /var/www/html/debian ] ; then
		ln -s /home/ftp/debian /var/www/html/debian
	fi
}

configure_pure () {
	if getent passwd ftp >/dev/null ; then
		echo "FTP user already existing"
	else
		/usr/sbin/useradd -m ftp
	fi
	chown ${THE_DEV_USER}:${THE_DEV_USER} /home/ftp
	rm -f /etc/pure-ftpd/conf/NoAnonymous
	invoke-rc.d pure-ftpd stop
	sleep 2
	invoke-rc.d pure-ftpd start
}

install_jenkins_plugins () {
	jenkins-cli -s https://${HOST_FQDN}/ -noCertificateCheck install-plugin instant-messaging
	jenkins-cli -s https://${HOST_FQDN}/ -noCertificateCheck install-plugin ircbot
}

configure_jenkins_dotgitconfig () {
	echo "[user]
	email = zigo@debian.org
	name = Thomas Goirand  
[gitreview]
	username = thomas-goirand
[alias]
	wdiff = diff --color-words
	wshow = show --color-words
[color]
	ui = true" >/var/lib/jenkins/.gitconfig
}

configure_jenkins_sudoers () {
	echo "jenkins ALL = NOPASSWD: /usr/bin/sbuild-update -udcar ${TARGET_DISTRO}-amd64" >/etc/sudoers.d/jenkins
	chmod 440 /etc/sudoers.d/jenkins
}

# @param: $1 homedir of the GPG user
gen_the_dev_user_gpg_key () {
	GPG_USER_HOMEDIR=${1}
	mkdir -p ${GPG_USER_HOMEDIR}/.gnupg
	chmod 700 ${GPG_USER_HOMEDIR}/.gnupg
	chmod 700 ${GPG_USER_HOMEDIR}/.gnupg
	if ! [ -r ${GPG_USER_HOMEDIR}/.gnupg/gpg.conf ] ; then
		echo "keyserver hkp://pool.sks-keyservers.net
personal-digest-preferences SHA256
default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed
cert-digest-algo SHA256
" >${GPG_USER_HOMEDIR}/.gnupg/gpg.conf
	fi
	if ! [ -r ${GPG_USER_HOMEDIR}/.gnupg/pkgos-gen-key-batchfile ] ; then
		echo "
%echo Generating a basic OpenPGP key
Key-Type: RSA
Key-Length: 4096
Name-Real: Autogenerated key
Name-Email: ${THE_DEV_USER}@"`hostname --fqdn`"
Expire-Date: 0
" >${GPG_USER_HOMEDIR}/.gnupg/pkgos-gen-key-batchfile
	fi
	chown -R ${THE_DEV_USER}:${THE_DEV_USER} ${GPG_USER_HOMEDIR}/.gnupg
	su ${THE_DEV_USER} -c 'gpg --gen-key --batch '${GPG_USER_HOMEDIR}'/.gnupg/pkgos-gen-key-batchfile'
}

configure_jenkins_gpg_key () {
	# Generate a self-signed gpg key so that we can sign packages
	if ! [ -e /var/lib/jenkins/.gnupg/secring.gpg ] ; then
		gen_the_dev_user_gpg_key /var/lib/jenkins
	fi
}


check_user_gpg_key () {
	if ! [ -d /home/${THE_DEV_USER}/.gnupg ] ; then
		echo "There's no /home/${THE_DEV_USER}/.gnupg folder,"
		echo "do you want this script to generate one? If you"
		echo "don't, the script will exit."
		echo -n "Generate a GPG key (Y/n)?"
		read GEN_KEY
		if [ -z "${GEN_KEY}" ] || [ "${GEN_KEY}" = "y" ] || [ "${GEN_KEY}" = "Y" ] ; then
			echo "Genrating a gpg key for you..."
			gen_the_dev_user_gpg_key /home/${THE_DEV_USER}
		else
			echo "No gpg key: exiting..."
			exit 1
		fi
	fi
	GPG_KEY_ID=$(su ${THE_DEV_USER} -c "gpg --list-keys ${THE_DEV_USER} | grep ^pub | awk '{print \$2}' | cut -d/ -f2 | head -n 1")
	if [ -z "${GPG_KEY_ID}" ] ; then
		echo "Cloud not find key ID, but there's a /home/${THE_DEV_USER}/.gnupg"
		echo -n "Generate a GPG key (Y/n)?"
		read GEN_KEY
		if [ -z "${GEN_KEY}" ] || [ "${GEN_KEY}" = "y" ] || [ "${GEN_KEY}" = "Y" ] ; then
			echo "Genrating a gpg key for you..."
			gen_the_dev_user_gpg_key /home/${THE_DEV_USER}
		else
			echo "No gpg key: exiting..."
			exit 1
		fi
	fi
	GPG_KEY_ID=$(su ${THE_DEV_USER} -c "gpg --list-keys ${THE_DEV_USER} | grep ^pub | awk '{print \$2}' | cut -d/ -f2 | head -n 1")
	echo "===> Key ID: ${GPG_KEY_ID}"
}

build_ostack_archive_keyring_package () {
	# Export the jenkins GPG key as pubkey.gpg in debian/dists/pubkey.gpg
	mkdir -p ${REPO_ROOT}/debian/dists
	chown -R ${THE_DEV_USER}:${THE_DEV_USER} ${REPO_ROOT}/debian/dists
	chown ${THE_DEV_USER}:${THE_DEV_USER} ${REPO_ROOT}/debian
	su ${THE_DEV_USER} -c "gpg --export -a ${THE_DEV_USER}" >${REPO_ROOT}/debian/dists/pubkey.gpg
	# Create a Debian package out of it, called openstack-debian-archive-keyring
	# and put it in the newly created Debian repository.
	# Yes, a working, Debian-policy-compliant package is really only a few lines of shell... :)
	TMPDIR=`mktemp -d`
	MYCWD=`pwd`
	cd ${TMPDIR}
	chown ${THE_DEV_USER}:${THE_DEV_USER} .
	VER=0.1
	NAME=${TARGET_OPENSTACK_REL}-${TARGET_DISTRO}-archive-keyring
	rm -rf ${NAME}-${VER}
	mkdir -p ${NAME}-${VER}/debian/source
	cd ${NAME}-${VER}
	export DEBFULLNAME="Debian OpenStack Jenkins"
	export DEBEMAIL="${THE_DEV_USER}@${HOST_FQDN}"
	dch --create --package ${NAME} -D unstable --noquery --newversion 0.1 -m "Automatic archive package build."
	sed -i 's/MAINTAINER <EMAIL>/Debian OpenStack <'${THE_DEV_USER}'@'${HOST_FQDN}'>/' debian/changelog
	echo "3.0 (native)" >debian/source/format
	echo 9 >debian/compat
	echo "#!/usr/bin/make -f
%:
	dh \$@
" >debian/rules
	echo "Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: ${NAME}
Source: See the openstack-pkg-tools package

Files: *
Copyright: (c) 2015, Thomas Goirand <zigo@debian.org>
License: Apache-2
 Licensed under the Apache License, Version 2.0 (the \"License\");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 .
    http://www.apache.org/licenses/LICENSE-2.0
 .
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an \"AS IS\" BASIS,  
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 .
 On Debian-based systems the full text of the Apache version 2.0 license
 can be found in /usr/share/common-licenses/Apache-2.0.
" >debian/copyright
	echo "Source: ${NAME}
Section: net
Priority: extra
Maintainer: PKG OpenStack <openstack-devel@lists.alioth.debian.org>
Uploaders: Thomas Goirand <zigo@debian.org>
Build-Depends: debhelper (>= 9)
Standards-Version: 3.9.6

Package: ${NAME}
Architecture: all
Depends: \${misc:Depends}
Description: OpenStack ${TARGET_OPENSTACK_REL} ${TARGET_DISTRO} archive keyring
 OpenStack ${TARGET_OPENSTACK_REL} ${TARGET_DISTRO} archive keyring
">debian/control
	cp ${REPO_ROOT}/debian/dists/pubkey.gpg .
	echo "pubkey.gpg /usr/share/${NAME}" >debian/${NAME}.install
	echo "#!/bin/sh
set -e

if [ \"\$1\" = \"configure\" ] || [ \"\$1\" = \"reconfigure\" ] ; then
	apt-key add /usr/share/${NAME}/pubkey.gpg
fi

#DEBHELPER#

exit 0
" >debian/${NAME}.postinst
	chown -R ${THE_DEV_USER}:${THE_DEV_USER} .
	su ${THE_DEV_USER} -c dpkg-buildpackage || true
	FIRST_LETTER=`echo ${TARGET_OPENSTACK_REL} | awk '{print substr($0,0,2)}'`
	cd ..
	MYDEST=${REPO_ROOT}/debian/pool/${REPO_DEST}/main/${FIRST_LETTER}/${NAME}
	MYDEST2=${REPO_ROOT}/debian/pool/${REPO_NOCHANGE_BACKPORT_DEST}/main/${FIRST_LETTER}/${NAME}
	for i in ${MYDEST} ${MYDEST2} ; do
		su ${THE_DEV_USER} -c "mkdir -p ${i}"
		cp *.changes *.tar.xz *.deb *.dsc ${i}
		chown -R ${THE_DEV_USER}:${THE_DEV_USER} ${i}
	done
	cd ${MYCWD}
	rm -r ${TMPDIR}
}

configure_sbuildrc () {
	GPG_KEY_ID=$(su ${THE_DEV_USER} -c "gpg --list-keys ${THE_DEV_USER} | grep ^pub | awk '{print \$2}' | cut -d/ -f2")
	DOT_SBUILDRC_PATH=${THE_USER_HOMEDIR}/.sbuildrc
	if ! [ -r "${DOT_SBUILDRC_PATH}" ] ; then
		echo "# don't remove this, Perl needs it:

\$build_arch_all = 1;
\$build_source = 1;
\$distribution = '"${TARGET_DISTRO}"';
\$run_lintian = 0;

# Don't sign packages:
#\$pgp_options = '-us -uc';

# FIX THIS !!!
\$key_id = '${GPG_KEY_ID}';

1;
" >${DOT_SBUILDRC_PATH}
	fi
}

configure_sbuild () {
	USE_OVERLAY=yes
	# Setup the gpg key for sbuild
	mkdir -p /root/.gnupg
	chmod 600 /root/.gnupg
	gpg --list-keys
	sbuild-update --keygen

	# Add jenkins as a sbuild user
	sbuild-adduser ${THE_DEV_USER}

	if [ "${USE_OVERLAY}" = "yes" ] ; then
		# Create the actual schroot env
		if ! [ -e /var/lib/sbuild/${TARGET_DISTRO} ] ; then
			# Using AUFS or overlay
			sbuild-createchroot ${TARGET_DISTRO} /var/lib/sbuild/${TARGET_DISTRO} ${CLOSEST_DEBIAN_MIRROR}
		fi
	else
		# Create the actual schroot env
		if ! [ -e /var/lib/sbuild/${TARGET_DISTRO}-amd64.tar.gz ] ; then
			# Using tarballs
			sbuild-createchroot --make-sbuild-tarball=/var/lib/sbuild/${TARGET_DISTRO}-amd64.tar.gz ${TARGET_DISTRO} `mktemp -d` ${CLOSEST_DEBIAN_MIRROR}
		fi
	fi

	# Make sure git-buildpackage is using sbuild
	sed -i 's/^[ #\t]*builder[ #\t]*=.*/builder = sbuild -v --no-apt-update/' /etc/git-buildpackage/gbp.conf
	sed -i 's/^[ #\t]*cleaner[ #\t]*=.*/cleaner = \/bin\/true/' /etc/git-buildpackage/gbp.conf

	# Make sure that /dev/shm is mounted in the chroot, otherwise anything which
	# uses /dev/shm (like python-taskflow, etc.) will fail to build
	sed -i 's|#/dev/shm|/dev/shm|' /etc/schroot/default/fstab

	# Install the juno-jessie-archive-keyring package in the repository
	FIRST_LETTER=`echo ${TARGET_OPENSTACK_REL} | awk '{print substr($0,0,2)}'`
	NAME=${TARGET_OPENSTACK_REL}-${TARGET_DISTRO}-archive-keyring
	VERS=0.1
	MYPKG_FILE_NAME=${NAME}_0.1_all.deb
	LOCATION=debian/pool/${TARGET_DISTRO}-${TARGET_OPENSTACK_REL}-backports/main/${FIRST_LETTER}/${NAME}/${MYPKG_FILE_NAME}
	if [ "${USE_OVERLAY}" = "yes" ] ; then
		cp /home/ftp/${LOCATION} /var/lib/sbuild/${TARGET_DISTRO}
		chroot /var/lib/sbuild/${TARGET_DISTRO} dpkg -i ${MYPKG_FILE_NAME}
		rm /var/lib/sbuild/${TARGET_DISTRO}/${MYPKG_FILE_NAME}
	else
		schroot -c source:${TARGET_DISTRO}-amd64-sbuild -u root -- apt-get install -y wget
		schroot -c source:${TARGET_DISTRO}-amd64-sbuild -u root -- wget http://localhost/${LOCATION}
		schroot -c source:${TARGET_DISTRO}-amd64-sbuild -u root -- dpkg -i ${MYPKG_FILE_NAME}
	fi

	# Since we have already a first package (the archive-keyring one),
	# let's scan the tree in /home/ftp/debian/pool, and build a valid Debian repo

	MYCWD=`pwd`
	cd ${THE_USER_HOMEDIR}
	su ${THE_DEV_USER} -c pkgos-scan-repo
	su ${THE_DEV_USER} -c pkgos-scan-repo ${REPO_NOCHANGE_BACKPORT_DEST}
	cd ${MYCWD}

	# Add a hook to have both our repositories used inside the sbuild chroot
	echo "#!/bin/sh

set -e

. \"\$SETUP_DATA_DIR/common-data\"
. \"\$SETUP_DATA_DIR/common-functions\"
. \"\$SETUP_DATA_DIR/common-config\"   

if [ \$STAGE = \"setup-start\" ] || [ \$STAGE = \"setup-recover\" ]; then
	echo \"deb http://localhost/debian ${TARGET_DISTRO}-${TARGET_OPENSTACK_REL}-backports main\" >\${CHROOT_PATH}/etc/apt/sources.list.d/openstack.list
	echo \"deb http://localhost/debian ${REPO_NOCHANGE_BACKPORT_DEST} main\" >\${CHROOT_PATH}/etc/apt/sources.list.d/openstack-backports.list
fi" >>/etc/schroot/setup.d/80sources
	chmod +x /etc/schroot/setup.d/80sources
}

configure_jenkins_job_builder () {
	GENERATED_PASSWORD=$(dd if=/dev/random bs=64 count=1 2>|/dev/null | md5sum | awk '{print $1}')
	echo "[job_builder]
ignore_cache=True
keep_descriptions=False
include_path=/usr/local/bin
recursive=False
allow_duplicates=False

[jenkins]
user=zigo
password=${JENKINS_JOBS_BUILDER_PASS}
url=http://localhost:8080/
" >/etc/jenkins_jobs/jenkins_jobs.ini
	echo "" >job.yaml
	for i in `cat /etc/pkgos/build-list` ; do
		echo "- job:
    name: $i
    builders:
        - shell: 'pkgos-bop-jenkins "${i}"'
    auth-token: g5rjtpms5emw
    logrotate:
        numToKeep: 4
    publishers:
        - ircbot:
            strategy: all
            notify-start: true
            message-type: summary
    wrappers:
        - ansicolor:
            colormap: xterm
" >>job.yaml
	done
	jenkins-jobs update job.yaml
}

restart_jenkins () {
	service jenkins restart
}

##############################
# ACTUAL START OF THE SCRIPT #
##############################

usage () {
	echo "Wrong usage: $0 [dev|jenkins] <username>"
	echo "the username 2nd param is only if you use the dev mode"
	exit 1
}

if [ "${1}" = "dev" ] ; then
	echo "Setting-up a developer machine."
	DEV_OR_JENKINS=dev
	shift
	if [ -z "${1}" ] ; then
		usage
	fi
	THE_DEV_USER=${1}
	shift
	if [ -n "${1}" ] ; then
		usage
	fi
	THE_USER_HOMEDIR=/home/${THE_DEV_USER}
elif [ "${1}" = "jenkins" ] ; then
	echo "Setting-up a Jenkins build server."
	DEV_OR_JENKINS=jenkins
	THE_DEV_USER=jenkins
	shift
	if [ -n "${1}" ] ; then
		usage
	fi
	THE_USER_HOMEDIR=/var/lib/jenkins
else
	usage
fi

detect_env
[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_hostname
install_all_software
configure_apache
configure_pure
# This often fails because the list of plugins isn't fetched by Jenkins
#install_jenkins_plugins
[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_jenkins_dotgitconfig
[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_jenkins_sudoers
[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_jenkins_gpg_key
[ "${DEV_OR_JENKINS}" = "dev" ] && check_user_gpg_key
configure_sbuildrc
build_ostack_archive_keyring_package
configure_sbuild
[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_jenkins_job_builder
# This is needed so that jenkins can login into sbuild
[ "${DEV_OR_JENKINS}" = "jenkins" ] && restart_jenkins