#!/bin/bash # Update GSA Ubuntu Repositories or create a local repository # # Author: Leonardo Tonetto (tonetto@linux.vnet.ibm.com) # Revisor: Arif Ali (aali@ocf.co.uk) # # # Getting Started: # - Clone the xcat-core git repository under a directory named "xcat-core/src" # - make sure reprepro is installed on the build machine # - Run this script from the local git repository you just created. # ./build-ubunturepo -c BUILDALL=1 # Usage: attr=value attr=value ... ./build-ubunturepo { -c | -d } # PROMOTE=1 - if the attribute "PROMOTE" is specified, means an official dot release. This does not # actually build xcat, just uploads the most recent snap build to http://xcat.org/files/xcat/ . # If not specified, a snap build is assumed, which uploads to https://xcat.org/files/xcat/ # PREGA=1 - use this option with PROMOTE=1 on a branch that already has a released dot release, but this # build is a GA candidate build, not to be released yet. This will result in the tarball # being uploaded to http://xcat.org/files/xcat/repos/apt # (but the tarball file name will be like a released tarball, not a snap build). # When you are ready to release this build, use PROMOTE=1 without PREGA # BUILDALL=1 - build all rpms, whether they changed or not. Should be used for snap builds that are in # prep for a release. # GPGSIGN=0 - Do not sign the repo in the end of the build. The repo will be signed by default # # LOG= - provide an LOG file option to redirect some output into log file # # DEST= - provide a directory to contains the build result # # For the dependency packages 1. All the xcat dependency deb packages should be uploaded to # "pokgsa/projects/x/xcat/build/ubuntu/xcat-dep/debs/" on GSA # 2. run ./build-ubunturepo -d # # 3. the built xcat-dep deb packages tarball can be found in "../../xcat-dep" # related to the path of this script ############################ printusage() { printf "Usage: %s {-c | -d} \n" $(basename $0) >&2 echo " -c : Build the xcat-core packages and create the repo" echo " -d : Create the xcat-dep repo." } # For the purpose of getting the distribution name if [[ ! -f /etc/lsb-release ]]; then echo "ERROR: Could not find /etc/lsb-release, is this script executed on a Ubuntu machine?" exit 1 fi . /etc/lsb-release # Check the necessary packages before starting the build declare -a packages=( "reprepro" "devscripts" "debhelper" "libsoap-lite-perl" "libdbi-perl" "quilt" "git") for package in ${packages[@]}; do RC=`dpkg -l | grep $package >> /dev/null 2>&1; echo $?` if [[ ${RC} != 0 ]]; then echo "ERROR: Could not find $package, install using 'apt-get install $package' to continue" exit 1 fi done # Process cmd line variable assignments, assigning each attr=val pair to a variable of same name for i in $*; do echo $i | grep '=' -q if [ $? != 0 ];then continue fi # upper case the variable name varstring=`echo "$i"|cut -d '=' -f 1|tr '[a-z]' '[A-Z]'`=`echo "$i"|cut -d '=' -f 2` export $varstring done # Supported distributions dists="saucy trusty utopic xenial bionic" c_flag= # xcat-core (trunk-delvel) path d_flag= # xcat-dep (trunk) path r_flag= #genesis base rpm package path while getopts 'cdr:' OPTION do case $OPTION in c) c_flag=1 ;; d) d_flag=1 ;; r) r_flag=1 genesis_rpm_path="$OPTARG" ;; ?) printusage exit 2 ;; esac done shift $(($OPTIND - 1)) if [ -z "$c_flag" -a -z "$d_flag" ];then printusage exit 2 fi if [ "$c_flag" -a "$d_flag" ];then printusage exit 2 fi if [ -z "$BUILDALL" ]; then BUILDALL=1 fi # Find where this script is located to set some build variables old_pwd=`pwd` cd `dirname $0` curdir=`pwd` # Use flock to only one person build at the same time # Get a lock, so can not do 2 builds at once exec 8>/var/lock/xcatbld.lock if ! flock -n 8; then echo "ERROR: Can't get lock /var/lock/xcatbld.lock. Someone else must be doing a build right now. Exiting...." exit 1 fi # for the git case, query the current branch and set REL (changing master to devel if necessary) function setbranch { # Get the current branch name branch=`git rev-parse --abbrev-ref HEAD` if [ "$branch" = "master" ]; then REL="devel" elif [ "$branch" = "HEAD" ]; then # Special handling when in a 'detached HEAD' state branch=`git describe --abbrev=0 HEAD` [[ -n "$branch" ]] && REL=`echo $branch|cut -d. -f 1,2` else REL=$branch fi } export HOME=/root WGET_CMD="wget" if [ ! -z ${LOG} ]; then WGET_CMD="wget -o ${LOG}" fi if [ "$GPGSIGN" = "0" ];then echo "GPGSIGN=$GPGSIGN specified, skip gnupg key downloading" else #sync the gpg key to the build machine local gsa_url=http://pokgsa.ibm.com/projects/x/xcat/build/linux mkdir -p $HOME/.gnupg for key_name in pubring.gpg secring.gpg trustdb.gpg; do if [ ! -f $HOME/.gnupg/$key_name ] || [ `wc -c $HOME/.gnupg/$key_name|cut -f 1 -d' '` == 0 ]; then rm -f $HOME/.gnupg/$key_name ${WGET_CMD} -P $HOME/.gnupg $gsa_url/keys/$key_name chmod 600 $HOME/.gnupg/$key_name fi done fi REL=xcat-core if [ "$c_flag" ] then setbranch package_dir_name=debs$REL #define the dep source code path, core build target path and dep build target path if [ -z "$DEST" ]; then local_core_repo_path="$curdir/../../xcat-core" PKGDIR="../../$package_dir_name" else local_core_repo_path="$DEST/$package_dir_name/xcat-core" PKGDIR="$DEST/$package_dir_name/$package_dir_name" fi if [ ! -d "$PKGDIR" ];then mkdir -p "$PKGDIR" fi echo "#############################################################" echo "Building xcat-core on branch ($REL) to $local_core_repo_path" echo "#############################################################" if [ "$PROMOTE" != 1 ]; then code_change=0 update_log='' if [ -z "$GITUP" ];then update_log=../coregitup echo "git pull > $update_log" git pull > $update_log else update_log=$GITUP fi if ! grep -q 'Already up-to-date' $update_log; then code_change=1 fi ver=`cat Version` short_ver=`cat Version|cut -d. -f 1,2` short_short_ver=`cat Version|cut -d. -f 1` build_time=`date` build_machine=`hostname` commit_id_long=`git rev-parse HEAD` commit_id="${commit_id_long:0:7}" if [ $code_change == 0 -a "$UP" != 1 -a "$BUILDALL" != 1 ]; then echo "Nothing new detected. Exiting...." exit 0 fi echo "###############################" echo "# Building xcat-core packages #" echo "###############################" #the package type: local | snap | alpha #the build introduce string build_string="Snap_Build" xcat_release="snap$(date '+%Y%m%d%H%M')" pkg_version="${ver}-${xcat_release}" packages="xCAT-client xCAT-genesis-scripts perl-xCAT xCAT-server xCAT xCATsn xCAT-test xCAT-buildkit xCAT-vlan xCAT-confluent xCAT-probe" target_archs=(amd64 ppc64el) for file in $packages do file_low=`echo $file | tr '[A-Z]' '[a-z]'` if [ "$file" = "xCAT" -o "$file" = "xCAT-genesis-scripts" -o "$file" = "xCATsn" ]; then target_archs="amd64 ppc64el" else target_archs="all" fi for target_arch in $target_archs do if grep -q $file $update_log || [ "$BUILDALL" == 1 -o "$file" = "perl-xCAT" ]; then rm -f $PKGDIR/${file_low}_*.$target_arch.deb cd $file CURDIR=$(pwd) dch -v $pkg_version -b -c debian/changelog $build_string if [ "$target_arch" = "all" ]; then #xcat probe use some functions shipped by xCAT, for below reasons we need to copy files to xCAT-probe directory #1 make xcat probe code to be self-contained #2 don't maintain two files for each script #3 symbolic link can't work during package if [ $file_low = "xcat-probe" ]; then mkdir -p ${CURDIR}/lib/perl/xCAT/ cp -f ${CURDIR}/../perl-xCAT/xCAT/NetworkUtils.pm ${CURDIR}/lib/perl/xCAT/ cp -f ${CURDIR}/../perl-xCAT/xCAT/GlobalDef.pm ${CURDIR}/lib/perl/xCAT/ cp -f ${CURDIR}/../perl-xCAT/xCAT/ServiceNodeUtils.pm ${CURDIR}/lib/perl/xCAT/ fi CURDIR=$(pwd) cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 # Magic string used here sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control dpkg-buildpackage -rfakeroot -uc -us mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control else if [ "$file" = "xCAT-genesis-scripts" ]; then echo "Rename control file to build pkg: mv ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control" cp ${CURDIR}/debian/control-${target_arch} ${CURDIR}/debian/control elif [ "$file" = "xCAT" ]; then # shipping bmcsetup and getipmi scripts as part of postscripts files=("bmcsetup" "getipmi") for f in "${files[@]}"; do cp ${CURDIR}/../xCAT-genesis-scripts/usr/bin/$f ${CURDIR}/postscripts/$f sed -i "s/xcat.genesis.$f/$f/g" ${CURDIR}/postscripts/$f done fi CURDIR=$(pwd) cp ${CURDIR}/debian/control ${CURDIR}/debian/control.save.998 # Magic string used here sed -i -e "s#>= 2.13-snap000000000000#= ${pkg_version}#g" ${CURDIR}/debian/control dpkg-buildpackage -rfakeroot -uc -us -a$target_arch mv ${CURDIR}/debian/control.save.998 ${CURDIR}/debian/control if [ "$file" = "xCAT-genesis-scripts" ]; then echo "Move control file back: mv ${CURDIR}/debian/control ${CURDIR}/debian/control-${target_arch}" rm ${CURDIR}/debian/control elif [ "$file" = "xCAT" ]; then files=("bmcsetup" "getipmi") for f in "${files[@]}"; do rm -f ${CURDIR}/postscripts/$f done fi fi rc=$? if [ $rc -gt 0 ]; then echo "Error: $file build package failed exit code $rc" exit $rc fi cd - find $file -maxdepth 3 -type d -name "${file_low}*" | grep debian | xargs rm -rf find $file -maxdepth 3 -type f -name "files" | grep debian | xargs rm -rf mv ${file_low}* $PKGDIR/ fi done done find $PKGDIR/* ! -name *.deb | xargs rm -f fi if [ "$PROMOTE" = 1 ]; then upload_dir="xcat-core" tar_name="xcat-core-$ver.tar.bz2" else upload_dir="core-snap" tar_name="core-debs-snap.tar.bz2" fi echo "#################################" echo "# Creating xcat-core repository #" echo "#################################" #clean the repo directory if [ -e $local_core_repo_path ]; then rm -rf $local_core_repo_path fi mkdir -p $local_core_repo_path cd $local_core_repo_path mkdir conf for dist in $dists; do # for all releases moving forward, support amd64 and ppc64el tmp_out_arch="amd64 ppc64el" if [ "$dist" = "saucy" ]; then # for older releases of Ubuntu that does not support ppc64el tmp_out_arch="amd64" fi cat << __EOF__ >> conf/distributions Origin: xCAT internal repository Label: xcat-core bazaar repository Codename: $dist Architectures: $tmp_out_arch Components: main Description: Repository automatically genereted conf __EOF__ if [ "$GPGSIGN" = "0" ];then #echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" echo "" >> conf/distributions else echo "SignWith: 5619700D" >> conf/distributions echo "" >> conf/distributions fi done cat << __EOF__ > conf/options verbose ask-passphrase basedir . __EOF__ #import the deb packages into the repo amd_files=`ls ../$package_dir_name/*.deb | grep -v "ppc64el"` all_files=`ls ../$package_dir_name/*.deb` for dist in $dists; do deb_files=$all_files if [ "$dist" = "saucy" ]; then # for older releases of Ubuntu that does not support ppc64el deb_files=$amd_files fi for file in $deb_files; do reprepro -b ./ includedeb $dist $file; done done #create the mklocalrepo script cat << '__EOF__' > mklocalrepo.sh . /etc/lsb-release cd `dirname $0` host_arch=`uname -m` if [ "$host_arch" != "ppc64le" ];then host_arch="amd64" else host_arch="ppc64el" fi echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-core.list __EOF__ chmod 775 mklocalrepo.sh # # Add a buildinfo file into the tar.bz2 file to track information about the build # BUILDINFO=$local_core_repo_path/buildinfo echo "VERSION=$ver" > $BUILDINFO echo "RELEASE=$xcat_release" >> $BUILDINFO echo "BUILD_TIME=$build_time" >> $BUILDINFO echo "BUILD_MACHINE=$build_machine" >> $BUILDINFO echo "COMMIT_ID=$commit_id" >> $BUILDINFO echo "COMMIT_ID_LONG=$commit_id_long" >> $BUILDINFO #create the xcat-core.list file cd ../ if ! grep xcat /etc/group ; then groupadd xcat fi chgrp -R root xcat-core chmod -R g+w xcat-core #build the tar ball echo "Creating `pwd`/$tar_name ..." tar -hjcf $tar_name xcat-core chgrp root $tar_name chmod g+w $tar_name if [ -n "$DEST" ]; then ln -sf $(basename `pwd`)/$tar_name ../$tar_name if [ $? != 0 ]; then echo "ERROR: Failed to make symbol link $DEST/$tar_name" fi fi if [ ! -e core-snap ]; then ln -s xcat-core core-snap fi cd $old_pwd exit 0 fi if [ "$d_flag" ] then echo "################################" echo "# Creating xcat-dep repository #" echo "################################" #the path of ubuntu xcat-dep deb packages on GSA GSA="/gsa/pokgsa/projects/x/xcat/build/ubuntu/xcat-dep" if [ ! -d $GSA ]; then echo "build-ubunturepo: It appears that you do not have gsa installed to access the xcat-dep pkgs." exit 1; fi #define the dep source code path, core build target path and dep build target path if [ -z "$DEST" ]; then local_dep_repo_path="$curdir/../../xcat-dep/xcat-dep" else local_dep_repo_path="$DEST/xcat-dep/xcat-dep" fi # Sync from the GSA master copy of the dep rpms echo "Creating directory $local_dep_repo_path" mkdir -p $local_dep_repo_path/ echo "Syncing RPMs from $GSA/ to $local_dep_repo_path/../ ..." rsync -ilrtpu --delete $GSA/ $local_dep_repo_path/../ if [ $? -ne 0 ]; then echo "Error from rsync, cannot continue!" exit 1 fi #clean all old files if [ -e $local_dep_repo_path ];then rm -rf $local_dep_repo_path fi mkdir -p $local_dep_repo_path cd $local_dep_repo_path mkdir conf #create the conf/distributions file for dist in $dists; do tmp_out_arch="amd64 ppc64el" if [ "$dist" = "saucy" ]; then # for older releases of Ubuntu that does not support ppc64el tmp_out_arch="amd64" fi cat << __EOF__ >> conf/distributions Origin: xCAT internal repository Label: xcat-dep bazaar repository Codename: $dist Architectures: $tmp_out_arch Components: main Description: Repository automatically genereted conf __EOF__ if [ "$GPGSIGN" = "0" ];then echo "GPGSIGN=$GPGSIGN specified, the repo will not be signed" echo "" >> conf/distributions else echo "SignWith: 5619700D" >> conf/distributions echo "" >> conf/distributions fi done cat << __EOF__ > conf/options verbose ask-passphrase basedir . __EOF__ #import the deb packages into the repo amd_files=`ls ../debs/*.deb | grep -v "ppc64el"` all_files=`ls ../debs/*.deb` for dist in $dists; do deb_files=$all_files if [ "$dist" = "saucy" ]; then # for older releases of Ubuntu that does not support ppc64el deb_files=$amd_files fi for file in $deb_files; do reprepro -b ./ includedeb $dist $file; done done cat << '__EOF__' > mklocalrepo.sh . /etc/lsb-release cd `dirname $0` host_arch=`uname -m` if [ "$host_arch" != "ppc64le" ];then host_arch="amd64" else host_arch="ppc64el" fi echo deb [arch=$host_arch] file://"`pwd`" $DISTRIB_CODENAME main > /etc/apt/sources.list.d/xcat-dep.list __EOF__ chmod 775 mklocalrepo.sh cd .. if ! grep xcat /etc/group ; then groupadd xcat fi chgrp -R root xcat-dep chmod -R g+w xcat-dep #create the tar ball dep_tar_name=xcat-dep-ubuntu-`date +%Y%m%d%H%M`.tar.bz2 tar -hjcf $dep_tar_name xcat-dep chgrp root $dep_tar_name chmod g+w $dep_tar_name USER="xcat" SERVER="xcat.org" FILES_PATH="files" FRS="/var/www/${SERVER}/${FILES_PATH}" APT_DIR="${FRS}/xcat" APT_REPO_DIR="${APT_DIR}/repos/apt/devel" # Decide whether to upload the xcat-dep package or NOT (default is to NOT upload xcat-dep if [ "$UP" != "1" ]; then echo "Upload not specified, Done! (rerun with UP=1, to upload)" cd $old_pwd exit 0 fi #upload the dep packages i=0 echo "Uploading debs from xcat-dep to ${APT_REPO_DIR}/xcat-dep/ ..." while [ $((i+=1)) -le 5 ] && ! rsync -urLv --delete xcat-dep $USER@${SERVER}:${APT_REPO_DIR}/ do : ; done #upload the tarball i=0 echo "Uploading $dep_tar_name to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." while [ $((i+=1)) -le 5 ] && ! rsync -v --force $dep_tar_name $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ do : ; done #upload the README file cd debs i=0 echo "Uploading README to ${APT_DIR}/xcat-dep/2.x_Ubuntu/ ..." while [ $((i+=1)) -le 5 ] && ! rsync -v --force README $USER@${SERVER}:${APT_DIR}/xcat-dep/2.x_Ubuntu/ do : ; done fi cd $old_pwd exit 0