From 64e8468718e2272deeddf12d9192f30772d37b15 Mon Sep 17 00:00:00 2001 From: PktSurf Date: Thu, 11 May 2023 22:56:36 +0530 Subject: [PATCH] Merged autobuild into bldpkg, made changes related to it in bldpkg.conf --- autobuild | 134 -------- bldpkg | 930 +++++++++++++++++++++++++++++----------------------- bldpkg.conf | 6 +- 3 files changed, 527 insertions(+), 543 deletions(-) delete mode 100755 autobuild diff --git a/autobuild b/autobuild deleted file mode 100755 index a2b6c25..0000000 --- a/autobuild +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# -# Part of the SMLinux distribution -# http://git.pktsurf.in/smlinux -# -# autobuild version 0.100 -# Bash script to automatically build section-specific packages for SMLinux -# -# Copyright (c) 2023 PktSurf -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -############ - -sections=(base xorg gtk extra net xfce) - -buildcleanup() { - [[ -f $autobuildtemp ]] && rm -f $autobuildtemp - exit -} - -trap "buildcleanup" INT -trap "buildcleanup" EXIT - -err() { - printf "**ERROR**\n$@\n" - exit 1 -} - -warn() { - printf "[WARN]\n$@\n" -} - -info() { - printf "[INFO] $@\n" -} - -parenttmp="/tmp" -autobuild=1 -arch="$HOSTTYPE" -colours=0 -autobuildtemp="$(mktemp $parenttmp/SMBUILD.XXXXXX)" - -export parenttmp autobuild arch colours autobuildtemp - -for section in "${sections[@]}" ; do - - pkgdest="$parenttmp/sml/packages/$arch/$section" - logdir="$parenttmp/sml/sml-buildlogs/$arch/$section" - export section pkgdest - mkdir -p $pkgdest $logdir - - if [[ ! -d $section ]] ; then - err "Section directory '$section' not found!" - fi - - cd $section - - # This check compares a list of source directories with the list of the - # packages in the build list and warns of any missing package names - # in either of the two. - - dirtempfile=$(mktemp $parenttmp/DIRECTORYNAMES."$section".XXXXXX) - dirfiletemppath="$dirtempfile" - dirlist=$(find . -type d -maxdepth 1 -mindepth 1 | sed 's@./@@' | sort > $dirfiletemppath) - - packtempfile=$(mktemp $parenttmp/BUILDFILENAMES."$section".XXXXXX) - packfiletemppath="$packtempfile" - sort .buildlist.$section > $packfiletemppath - - directorycount="$( wc -l < $dirtempfile )" - buildlistcount="$( wc -l < $packtempfile )" - - # Get number of total packages - totalpkgnumber="$(wc -l < .buildlist.$section)" - export totalpkgnumber - - if diff -u "$dirfiletemppath" "$packfiletemppath" > /dev/null 2>&1 ; then - diffstatus="0" - else - diffstatus="1" - fi - - if [ "$diffstatus" != "0" ]; then - warn "In section '"$section"', the number of packages in the hidden file '.buildlist."$section"' is different to the number of package directories. Some packages may not have been added to this file/section directory. They are listed below:" - - diff -u "$dirfiletemppath" "$packfiletemppath" || true - diff -u "$packfiletemppath" "$dirfiletemppath" || true - - warn "Building anyways :-) " - sleep 2 - fi - rm -f $packfiletemppath $dirfiletemppath - - if [[ ! -f .buildlist.$section ]] ; then - err "Section build list file '.buildlist.$section' not found!" - fi - - while IFS="" read -r package || [[ -n $package ]] ; do - currentpkgnumber="$(grep -Ewn "^$package" .buildlist.$section | cut -d: -f 1)" - export currentpkgnumber - ( - cd $package - source $package.SMBuild - # Check if package installer exists or is installed on the system - # If the package installer does not exist in the set pkg destination, build it - if [ ! -f $pkgdest/$app-$version-*-$build.* ] ; then - bldpkg 2>&1 | tee -a $logdir/$app-$version-$build.log.txt - # If the package is not installed in the system, install it - if [ ! -f /var/log/packages/$app-$version-*-$build ] ; then - upgradepkg --install-new $pkgdest/$app-$version-*-$build.* - else - info "'$app-$version' already installed, skipping it" - fi - elif [ ! -f /var/log/packages/$app-$version-*-$build ] ; then - upgradepkg --install-new $pkgdest/$app-$version-*-$build.* - else - info "Already built '$app-$version', skipping it" - fi - ) - done < ".buildlist.$section" - - cd .. - -done diff --git a/bldpkg b/bldpkg index e059461..b4c01d2 100755 --- a/bldpkg +++ b/bldpkg @@ -3,7 +3,7 @@ # Part of the SMLinux distribution # http://git.pktsurf.in/smlinux # -# /bin/bldpkg version 0.101 +# /bin/bldpkg version 0.102 # Bash script to build SMLinux-specific packages # # Copyright (c) 2022-2023 PktSurf @@ -73,6 +73,9 @@ Building package 'alsa-lib' version '1.x'... Usage: + -a Switch mode to autobuild. Default is to build individual packages. + Requires sections variable in bldpkg.conf to be defined + -d Produce a package with debug symbols preserved, i.e., don't strip resulting ELF objects. Uses -g3 by default @@ -163,13 +166,30 @@ validatebldfile() { err "Invalid URL in the 'download' variable in the build file." fi fi + + # Check integrity of files defined in sha512sums variable which is expected + # in nearly every single package build file + if [[ -z $sha512sums ]] ; then + err "Please run 'bldpkg -g' to add sha512sums into '$buildfile'!" + else + eval sums=\"\$sha512sums\" + info "Verifying SHA512 checksums against source files..." + IFS=$'\n' + for src in $sums; do + if ! echo $src | sha512sum -c ; then + err "Checksums failed to match!" + fi + done + unset IFS + fi } # Function to error out the build in absence of a build file nopackagebuildfileerror() { err "No package build file to source from! Was expecting '$buildfile' to be present inside this directory '$PWD'. -Try -f if your build file has a different name (Not recommended)" +Try -f if your build file has a different name (Not recommended) +If you want to autobuild packages, use -a" } validatemakeflags() { @@ -179,7 +199,7 @@ validatemakeflags() { err "Invalid MAKEFLAGS '$makeflags0'!" fi } - + # Function for applying patches to the build in a more efficient manner applypatch() { # Take patch file name as the first argument @@ -220,8 +240,9 @@ compileonlyfor() { local archargument="$1" if [[ $archname != $archargument ]]; then - info "'$app' not supported on '$archname' and hence not not being built. Exiting." - exit 0 + info "'$app' not supported on your current CPU '$archname'. Build aborted." + buildskipped=1 + return 0 fi } @@ -293,7 +314,6 @@ mkandenterbuilddir() { # Now create all essential build-related directories mkdir -p "$tmp" "$pkg/install" "$pkgdocs" "$pkgdest" - info "Leaving source directory $srcdir" info "Entering build directory $tmp" cd "$tmp" } @@ -327,8 +347,15 @@ runtime() { fi } +getbuildfileanddir() { + srcdir="$PWD" + buildfile="${srcdir##*/}.SMBuild" +} + # Generate and insert sha512sums into the build file genchecksum() { + getbuildfileanddir + # File types whose checksums will go into the new build file info "Discarding any old sha512sums from '$buildfile'" sed -E -i \ @@ -344,10 +371,10 @@ genchecksum() { # For loop that searches for files matching the above extension and prints them to the bottom of the build file for file in "${files[@]}" ; do - if [[ -f $file ]] ; then - info "Adding $file" - sha512sum "$file" >> "$buildfile" - fi + if [[ -f $file ]] ; then + info "Adding $file" + sha512sum "$file" >> "$buildfile" + fi done printf '"' >> "$buildfile" @@ -458,7 +485,7 @@ promptuser() { fi if [[ $pkgstatus = 0 ]] ; then - exit 0 + return 0 fi } @@ -475,11 +502,7 @@ interruptoutput() { # Terminate sccache sccacheprocess stop - # If installprompt and extractprompt are set and the prompt is invoked after a successful build, hitting - # ctrl-C will only set the above sm variable repeatedly and won't return user to the shell because - # of the interrupt (SIGINT) trap set way below. Putting exit 0 is a decent way to get out of that prompt - - exit 0 + return 0 } # Function to perform post-compile tasks. @@ -541,11 +564,11 @@ EOF for manpagedir in $(find . -type d -name "man*") ; do ( cd $manpagedir for eachpage in $( find . -type l -maxdepth 1) ; do - ln -s $( readlink $eachpage ).gz $eachpage.gz - rm $eachpage + ln -s $( readlink $eachpage ).gz $eachpage.gz + rm $eachpage done - gzip -9 ./*.? >/dev/null 2>&1 || true - ) + gzip -9 ./*.? >/dev/null 2>&1 || true + ) done ) fi @@ -622,6 +645,7 @@ EOF --xform 'sx^\./\(.\)x\1x' \ --show-stored-names | "$compressor" "$compressoropts" > "$newpkglocation" pkgstatus=$? + #export pkgstatus echo info "SMLinux package '$app-$version-$arch-$build.$pkgext' successfully generated in $pkgdest." @@ -631,13 +655,6 @@ EOF info "Leaving staging directory $pkg" - # cd back to $srcdir when preservepackagedir is set to 0 to prevent this error: - # shell-init: error retrieving current directory: getcwd: cannot access parent directories: - # No such file or directory - info "Re-entering source directory $srcdir" - cd "$srcdir" - - # With SECONDS reset, the shell will add in a fresh value, which we can now use to ascertain # the packaging time, by again passing that value as an argument to the runtime function if [[ $showsummary = 1 ]] || [[ $htmloutput = 1 ]] ; then @@ -692,7 +709,7 @@ findelffiles() { buildfilecleanup() { # Discard all temporary files - rm "$parenttmp/BUILDING" + rm -f "$parenttmp/BUILDING" } prepbuildoutput() { @@ -823,12 +840,7 @@ prepbuildoutput() { ttime="$totaltime ( $compiletimeb Compile ) + ( $packagetimeb Packaging )" fi - # Output the section name if autobuildtemp is set. This means we are running an autobuild. - if [[ -n $autobuildtemp ]]; then - bsection="$section" - else - bsection="None" - fi + [[ -z $section ]] && section="None" # Output the build summary to the user on every build cat << EOF @@ -836,7 +848,7 @@ $colourc ----------------------------------------------------------------------- $colourc BUILD SUMMARY FOR PACKAGE $colourv'$app'$colourc VERSION $colourv'$version'$colourc TAG $colourv'$build' $colourc ------------------------------------------------------------------------------- $colourd $colourc Build Status: $colourd $bldstatus -$colourc Build Section: $colourd $bsection +$colourc Build Section: $colourd $section $colourc Total Time: $colourd $ttime $colourc Started: $colourd $commencedate $colourc Stopped: $colourd $finishdate @@ -863,6 +875,435 @@ EOF promptuser } +stage0prep() { + # Check if $parenttmp is set and is a directory + if [[ -z $parenttmp ]] ; then + err "parenttmp variable not set in bldpkg.conf." + elif [[ ! -d $parenttmp ]] ; then + err "parenttmp variable set to '$parenttmp' in bldpkg.conf is not a directory." + fi + + # Attempt to write to the $parenttmp directory. This directory is used for everything related to the + # build process outside the source directory $srcdir + if ! touch "$parenttmp/.parenttmpwritetest" ; then + err "Parent temp directory '$parenttmp' is not writable!" + else + # Discard the test file + rm "$parenttmp/.parenttmpwritetest" + fi + + printf "[INFO] Validating compressor '$compressor'..." + mkdir -p "$parenttmp/.tar.testdir" + touch "$parenttmp/.tar.testdir/.tar.testfile" + + if ! tar -cf - -C "$parenttmp/.tar.testdir" . | "$compressor" "$compressoropts" > "$parenttmp/.tar.testdir.$pkgext" ; then + err "Failed to validate '$compressor' options! Please check your bldpkg.conf" + fi + + printf " done\n" + rm -r "$parenttmp/.tar.testdir"* + + # Validate the TMPFS directory if usetmpfs is set to 1 and tmpfsdir variable is set. If it fails, + # declare a variable for the build summary. + if [[ $tmpfs = 1 ]] && [[ -n $tmpfsdir ]]; then + if [[ ! -d $tmpfsdir ]] || ! touch "$tmpfsdir/.tmpfswritetest" \ + || [[ $(findmnt -no TARGET $tmpfsdir) != $tmpfsdir ]] \ + || [[ $(findmnt -no FSTYPE $tmpfsdir) != tmpfs ]]; then + tmpfscheckfailed=1 + fi + + # Discard the file used to test the tmp directory + rm "$tmpfsdir/.tmpfswritetest" + + # Check the tmpfsdir for stale directories from previous pkg builds. If found, issue a warning. + if [[ $(find "$tmpfsdir" -type d -maxdepth 1 -name "*.src" -o -name "package-*" | wc -l) -gt 0 ]]; then + if [[ ! -d $tmpfsdir/package-$app ]] || [[ ! -d $tmpfsdir/$app.src ]] ; then + warn "TMPFS directory '$tmpfsdir' has stale directories from previous builds!" + fi + fi + fi + + # Validate system swap if swapcheck is defined and set to 1 + if [[ $swapcheck = 1 ]]; then + if inarray "${app}" "${packagesrequiringswap[@]}" ; then + + # Here we determine available system swap size needed to compile exceptional packages that pull + # in a lot of RAM. Those packages are listed under the packagesrequiringswap array in bldpkg.conf. + # Check whether swap is available on the system and if it is, determine its size. If its size + # is >= swapsize, we are all good. If it's less than swapsize, we exit with a status 1. + + swapcheck=$(grep "SwapFree" /proc/meminfo | awk '{print $2}') + [[ $swapcheck -lt $swapsize ]] && err "Insufficient swap size to build packages, advise increasing it." + fi + fi + + # Set the build and package staging directories. This is where package files that get "installed" go into, + # for example 'make install DESTDIR=$pkg' or 'DESTDIR="$pkg" ninja install'. + + # If tmpfs is set to 1 and tmpfscheckfailed variable is unset, determine + # if the $app is in the exception list and whether to build inside or outside the TMPFS directory. + if [[ $tmpfs = 1 ]] && [[ -z $tmpfscheckfailed ]] ; then + + # If $app is in the TMPFS exception list inside bldpkg.conf, compile it *OUTSIDE* the TMPFS + # directory, i.e the non-TMPFS directory, else compile it *INSIDE* the TMPFS directory. This if/else + # is solely for deciding whether $app is in the exception list or not. + if inarray "${app}" "${tmpfsexceptionlist[@]}" ; then + + info "'$app' is in tmpfs exception list. Falling back to non-tmpfs directory" + + # We DO NOT compile inside tmpfsdir + tmpfsenabledforthispackage=0 + + # In the absence of tmpfs, we use the normal directory + tmp="$nontmpfsdir/$app.src" + pkg="$nontmpfsdir/package-$app" + else + # We compile inside tmpfsdir. Set the tmpfsenabledforthispackage variable here to inform build + # summary function at the bottom + tmpfsenabledforthispackage=1 + + # Disable ccache + ccache=0 + + # Override preservebuilddir and preservepackagedir to remove both build and package staging directories + preservebuilddir=0 + preservepackagedir=0 + + # Get the directory from the tmpfsdir variable for extracting the source and set it as our build + # and staging directory + tmp="$tmpfsdir/$app.src" + pkg="$tmpfsdir/package-$app" + + fi + else + # If tmpfs is disabled, we compile in the non-TMPFS directory + tmp="$nontmpfsdir/$app.src" + pkg="$nontmpfsdir/package-$app" + fi + + if inarray "${pkg}" "${protecteddirectories[@]}" ; then + echo "############ ATTENTION ############" + err "'pkg' VARIABLE IS SET TO '$pkg' WHICH IS A PROTECTED DIRECTORY! EXITING!" + fi + + # Also check tmp directory in advance + if inarray "${tmp}" "${protecteddirectories[@]}" ; then + echo "############ ATTENTION ############" + err "'tmp' VARIABLE IS SET TO '$tmp' WHICH IS A PROTECTED DIRECTORY! EXITING!" + fi + + # Run a for loop to find compile and build-related files + for requiredfile in "${rqfiles[@]}"; do + if [[ ! -x $(type -p "$requiredfile") ]] ; then + err "Could not find required program '$requiredfile'!" + fi + done +} + +stage1prep() { + # Apply CPU-specific compiler variables defined inside bldpkg.conf + # noarch is set inside initfs, pkgtools, GTK themes and some other stuff. + + # Only set $arch if it's not set in the build file. We take the help of $HOSTTYPE variable set by the bash shell. + [[ -z $arch ]] && arch="$HOSTTYPE" + + for compilervariable in CC CXX CFLAGS ; do + [[ -z $compilervariable ]] && err "$compilervariable not set in bldpkg.conf!" + done + + # Clean out CFLAGS if $arch is set to noarch + if [[ $arch = noarch ]]; then + CFLAGS="" + elif [[ -n $debug ]]; then + CFLAGS="$gccdebug $CFLAGS" + fi + + CXXFLAGS="$CFLAGS" + export hostdist builddist CC CXX CFLAGS CXXFLAGS MAKEFLAGS + + # Validate available compilers. Quotes are important while passing C/CXXFLAGS + compilertestfile "$CC" "$CFLAGS" cc-test.c cc-test + compilertestfile "$CXX" "$CXXFLAGS" cxx-test.cpp cxx-test + + # Validate everything related to ccache if ccache is set + if [[ $ccache = 1 ]]; then + [[ ! -x $ccachebinpath ]] && err "Ccache binary does not exist/is not an executable!" + + checkcompilersymlink $ccachesymdirpath $ccachebinpath + + # Prepare $PATH and export it only if ccache is unset in the pkg build file + PATH="$ccachesymdirpath:$PATH" + export PATH + fi + + # Validate everything related to sccache if ccache is set + if [[ $sccache = 1 ]]; then + [[ ! -x $sccachebinpath ]] && err "$sccachebinpath was either not found or is not an executable!" + + # We expect the rustc wrapper to be defined in $HOME/.cargo/config.toml + if ! grep -q 'rustc-wrapper\ \= "\/bin\/sccache"' $HOME/.cargo/config.toml ; then + err 'sccache is not defined as a rustc wrapper! +Please add the following to your '"$HOME/"'.cargo/config.toml: +[build] +rustc-wrapper = "/bin/sccache"' + fi + + # Useful for rust-specific builds. + RUSTC_WRAPPER="$sccachebinpath" + export RUSTC_WRAPPER + + sccacheprocess start + fi + + # Validate everything related to distcc if distcc is set + if [[ $distcc = 1 ]] ; then + # Check if distcc exists and is an executable + [[ ! -x $distccbinpath ]] && err "Distcc binary was either not found or is not an executable" + + checkcompilersymlink $distccsymdirpath $distccbinpath + + # Prepare $PATH and export DISTCC_HOSTS and DISTCC_IO_TIMEOUT only if distcc is unset in the pkg build file + # netcat hosts inside $DISTCC_HOSTS by checking for an open port + info "Validating distcc hosts..." + # Check if we have nc + if [[ ! -x /bin/nc ]] ; then + warn "nc does not exist! Ignoring this..." + else + # Remove the common options along with the slash and the numbers after it + hosts=$(echo "$DISTCC_HOSTS" | sed -e 's@/[a-z0-9,]*@@g' -e 's@--randomize@@' -e 's@localhost@@') + for host in ${hosts[@]} ; do + # We only run distccd on TCP port 3632 + if ! /bin/nc -z -w 1 "$host" 3632 > /dev/null 2>&1 ; then + warn "Distcc host '$host' is OFFLINE! Rewriting DISTCC_HOSTS" + DISTCC_HOSTS=$(echo "$DISTCC_HOSTS" | sed "s@$host/[a-z0-9,]*@@") + fi + done + fi + + PATH="$distccsymdirpath:$PATH" + export DISTCC_HOSTS DISTCC_IO_TIMEOUT PATH + fi + + # If verbosebuild is set in bldpkg.conf or if verbosity is set to 1 from getopts, set V and + # VERBOSE environment variables for make and cmake build systems to pick up. + if [[ $verbosebuild = 1 ]] || [[ $getoptsverbosebuild = 1 ]] ; then + V=1 + VERBOSE=1 + else + V=0 + VERBOSE=0 + fi + export V VERBOSE + + # If $noautoconfsite is unset in an individual package build file, export CONFIG_SITE variable into the build + # environment for a package's configure script to pickup. Most autoconf-compatible configure scripts will + # automatically pick up this variable from the environment and speed up the initial configure process. + if [[ -z $noautoconfsite ]] ; then + if [[ -n $configsite ]] && [[ -e $configsite ]]; then + CONFIG_SITE="$configsite" + export CONFIG_SITE + fi + fi + + # Do a preliminary package dependency check if $checkdependencies is set to 1 in bldpkg.conf + if [[ $checkdependencies = 1 ]] ; then + info "Parsing $app 's dependency list..." + for packagedep in $requires; do + depcount=$(find /share/doc -type f -name "$packagedep.SMBuild" | wc -l) + # If count is 1, we are ok + if [[ $depcount = 1 ]] ; then + info "Found dependency '$packagedep'" + # If count is 0, we exit, because we are in trouble + elif [[ $depcount = 0 ]] ; then + err "Did not find dependency '$packagedep'" + # If count is greater than or equal to 2, we are in slightly less trouble + elif [[ $depcount -ge 2 ]] ; then + warn "Found multiple versions of '$packagedep' !" + sleep 0.5 + fi + done + fi +} + +autobuild() { + trap "interruptoutput" INT + + # We don't need colours and we set arch on purpose so that the packages land in the right directory + colours=0 + arch="$HOSTTYPE" + + # For loop to recurse through sections defined inside bldpkg.conf + for section in "${sections[@]}" ; do + + pkgdest="$parenttmp/sml/packages/$arch/$section" + logdir="$parenttmp/sml/sml-buildlogs/$arch/$section" + mkdir -p $logdir + + # If the defined section does not exist, abort + if [[ ! -d $section ]] ; then + err "Section directory '$section' not found!" + fi + + cd $section + + # This check compares a list of source directories with the list of the + # packages in the build list and warns of any missing package names + # in either of the two. + + dirtempfile=$(mktemp $parenttmp/DIRECTORYNAMES."$section".XXXXXX) + dirfiletemppath="$dirtempfile" + dirlist=$(find . -type d -maxdepth 1 -mindepth 1 | sed 's@./@@' | sort > $dirfiletemppath) + + packtempfile=$(mktemp $parenttmp/BUILDFILENAMES."$section".XXXXXX) + packfiletemppath="$packtempfile" + sort .buildlist.$section > $packfiletemppath + + directorycount="$( wc -l < $dirtempfile )" + buildlistcount="$( wc -l < $packtempfile )" + + # Get number of total packages + totalpkgnumber="$(wc -l < .buildlist.$section)" + + if diff -u "$dirfiletemppath" "$packfiletemppath" > /dev/null 2>&1 ; then + diffstatus="0" + else + diffstatus="1" + fi + + if [ "$diffstatus" != "0" ]; then + warn "In section '"$section"', the number of packages in the hidden file '.buildlist."$section"' is different to the number of package directories. Some packages may not have been added to this file/section directory. They are listed below:" + + diff -u "$dirfiletemppath" "$packfiletemppath" || true + diff -u "$packfiletemppath" "$dirfiletemppath" || true + + warn "Building anyways :-) " + sleep 2 + fi + rm -f $packfiletemppath $dirfiletemppath + + if [[ ! -f .buildlist.$section ]] ; then + err "Section build list file '.buildlist.$section' not found!" + fi + + while IFS="" read -r package || [[ -n $package ]] ; do + unset app build version buildskipped + currentpkgnumber="$(grep -Ewn "^$package" .buildlist.$section | cut -d: -f 1)" + + # We need this variable to switch back to the section directory inside the loop + currentpwd="$PWD" + + cd $package + + # Get the name of the buildfile and validate it + getbuildfileanddir + validatebldfile "$buildfile" + + # Check if package installer exists or is installed on the system + # If the package installer does not exist in the set pkg destination, build it + if [ ! -f $pkgdest/$app-$version-*-$build.* ] ; then + # Store the source directory path the build was initiated from + + # Presume that the build file name will match the name of the parent directory + # unless otherwise overridden using -f + stage0prep + touch "$parenttmp/BUILDING" + prepbuilddir + stage1prep + if [[ -z $buildskipped ]] ; then + trap "prepbuildoutput" EXIT + build + fi + + # If the package is not installed in the system, install it + if [ ! -f /var/log/packages/$app-$version-*-$build ] ; then + upgradepkg --install-new $pkgdest/$app-$version-*-$build.* + else + info "'$app-$version' already installed, skipping it" + fi + elif [ ! -f /var/log/packages/$app-$version-*-$build ] ; then + upgradepkg --install-new $pkgdest/$app-$version-*-$build.* + else + info "Already built '$app-$version', skipping it" + fi + + cd $currentpwd + + done < ".buildlist.$section" + + cd .. + + done +} + +manualbuild() { + trap "prepbuildoutput" EXIT + trap "interruptoutput" INT + + # Display the package and its version we are building or resuming + if [[ -z $resumepkgbuild ]] ; then + info "Building package '$app' version '$version' ..." + else + info "Resuming build of package '$app' version '$version' ..." + fi + + stage0prep + + # If htmloutput is set to 1, echo $app, $version and $build as file names inside the parent build directory. + # This will output into an HTML file so that the basic status of the build process (whether started, stopped, + # interrupted or failed) can be viewed in the web browser. + if [[ $htmloutput = 1 ]] ; then + + if [[ -n $autobuild ]] ; then + cat << EOF >> "$parenttmp/BUILDMONITOR" +$commencedate | Building package $currentpkgnumber / $totalpkgnumber: $app $version +EOF + else + cat << EOF >> "$parenttmp/BUILDMONITOR" +$commencedate | Building package $app $version +EOF + fi + + touch "$parenttmp/BUILDING" + fi + + # If $tmp exists, get the path to the $.app-$version.extraction.complete file + [[ -d $tmp ]] && buildresumepath="$(find $tmp -type f -name .$app-$version.extraction.complete)" + + # If the above file exists, $resumepkgbuild and $autobuild are unset + # and $autoresumepkgbuild is set to 1 in bldpkg.conf, prompt the user + if [[ -f $buildresumepath ]] && [[ -z $resumepkgbuild ]] && [[ $autoresumepkgbuild = 1 ]] && [[ -z $autobuild ]]; then + read -r -p "[NOTIFY] Would you like to resume building? " yn + case "$yn" in + N|n|No|no) info "Nope? Alright." ; + unset resumepkgbuild ;; + *) info "Wise choice :-) " + resumepkgbuild=1 ;; + esac + fi + + # If $resumepkgbuild variable is not set in getopts, only then execute prepbuilddir variable + [[ -z $resumepkgbuild ]] && prepbuilddir + + stage1prep + + # If $resumepkgbuild is set either in getopts or from the user prompt above, execute mkandenterbuilddir + # function to enter the build directory. This is being done because mkandenterbuilddir is part of prepbuilddir + # function in the build file and ignoring prepbuilddir will not cause mkandenterbuilddir to be invoked + # separately unless the build system is told to. + if [[ -n $resumepkgbuild ]] ; then + mkandenterbuilddir + + # fixbuilddirpermissions places a file ".$app-$version.extraction.complete". Only resume the build if that file exists + if [[ ! -f $buildresumepath ]] ; then + err "Can't resume build of '"$app"'! Are you certain the source was extracted completely?" + else + cd ${buildresumepath%/*} + fi + fi + + build +} + # Source the main configuration file holding all necessary values if [[ -f $HOME/.bldpkg.conf ]] ; then source "$HOME/.bldpkg.conf" @@ -877,16 +1318,10 @@ if [[ $ccache = 1 ]] && [[ $sccache = 1 ]] ; then err "Please only enable either ccache or sccache in bldpkg.conf at the same time, not both." fi -# Store the source directory path the build was initiated from -srcdir="$PWD" - -# Presume that the build file name will match the name of the parent directory -# unless otherwise overridden using -f -buildfile="${srcdir##*/}.SMBuild" - # While loop for providing handy arguments to users. Some will override bldpkg.conf. -while getopts ':def:ghj:o:rstvx' option; do +while getopts ':adef:ghj:o:rstvx' option; do case "$option" in + a) autobuildmode=1 ;; d) debug=1 ;; # Produce a debug build with -g3 e) extractprompt=0; autoextract=1 ;; # Automatically extract the final pkg installer inside user's PWD @@ -908,12 +1343,13 @@ done # For every argument passed, the number is incremented by 1. In our case, if OPTIND equals 1, no argument # was passed. We therefore expect a package build file to be present which will then be validated and sourced. if [[ $OPTIND = 1 ]] ; then + getbuildfileanddir if [[ ! -f $buildfile ]] ; then nopackagebuildfileerror elif ! validatebldfile "$buildfile" ; then err "'$buildfile' build file validation failed!" else - source "$buildfile" + manualbuild fi # If OPTIND is greater than 1, perform various checks and actions @@ -931,378 +1367,58 @@ elif [[ $OPTIND -gt 1 ]] ; then # Or fetch the number from nproc MAKEFLAGS="-j$(nproc --all)" fi - - # If $origbuildfile is set and is a file, check if $setbuildfile and $origbuildfile are the same - if [[ -n $origbuildfile ]] ; then - if [[ ! -f $origbuildfile ]] ; then - err "Original build file '$origbuildfile' does not exist!" - elif [[ -n $setbuildfile ]] && [[ -f $setbuildfile ]] ; then - if [[ $origbuildfile = $setbuildfile ]] ; then - err "Original build file and alternate build file are the same!" + if [[ $autobuildmode = 1 ]] ; then + autobuild + + elif [[ -z $autobuildmode ]] ; then + # Get the name of the default source file + getbuildfileanddir + + # If $origbuildfile is set and is a file, check if $setbuildfile and $origbuildfile are the same + if [[ -n $origbuildfile ]] ; then + if [[ ! -f $origbuildfile ]] ; then + err "Original build file '$origbuildfile' does not exist!" + + elif [[ -n $setbuildfile ]] && [[ -f $setbuildfile ]] ; then + if [[ $origbuildfile = $setbuildfile ]] ; then + err "Original build file and alternate build file are the same!" + fi + + elif ! validatebldfile "$origbuildfile" ; then + err "Alternative build file validation failed!" + fi + fi + + # If $setbuildfile is set and is a file, set buildfile to its value, source it and initialise the build + if [[ -n $setbuildfile ]] && [[ -f $setbuildfile ]] ; then + buildfile="$setbuildfile" + if ! validatebldfile "$buildfile" ; then + err "'$buildfile' validation failed!" + else + manualbuild fi - elif ! validatebldfile "$origbuildfile" ; then - err "Alternative build file validation failed!" - fi - fi - - # If $setbuildfile is set and is a file, set buildfile to its value, source it and initialise the build - if [[ -n $setbuildfile ]] && [[ -f $setbuildfile ]] ; then - buildfile="$setbuildfile" - if ! validatebldfile "$buildfile" ; then - err "'$buildfile' validation failed!" - else - source "$buildfile" - fi - - # If $setbuildfile is set but a file passed as an argument is not found, give an error - elif [[ -n $setbuildfile ]] && [[ ! -f $setbuildfile ]] ; then - err "Build file '$setbuildfile' not found!" + # If $setbuildfile is set but a file passed as an argument is not found, give an error + elif [[ -n $setbuildfile ]] && [[ ! -f $setbuildfile ]] ; then + err "Build file '$setbuildfile' not found!" # If the above two conditions don't meet, get the presumed $buildfile value as a file and source it - elif [[ -f $buildfile ]] ; then - if ! validatebldfile "$buildfile" ; then - err "'$buildfile' validation failed!" - else - source "$buildfile" - fi + elif [[ -f $buildfile ]] ; then + if ! validatebldfile "$buildfile" ; then + err "'$buildfile' validation failed!" + else + manualbuild + fi # If even that file is not found, throw an error and exit - else - nopackagebuildfileerror + else + nopackagebuildfileerror + fi fi fi -# Unset OPTIND and OPTARG to avoid problems if getopts is used again in the future +# Unset OPTIND and OPTARG unset OPTIND OPTARG -# Display the package and its version we are building or resuming -if [[ -z $resumepkgbuild ]] ; then - info "Building package '$app' version '$version' ..." -else - info "Resuming build of package '$app' version '$version' ..." -fi - -# Check integrity of files defined in sha512sums variable which is expected -# in nearly every single package build file -if [[ -z $sha512sums ]] ; then - err "Please run 'bldpkg -g' to add sha512sums into '$buildfile'!" -else - eval sums=\"\$sha512sums\" - info "Verifying SHA512 checksums against source files..." - IFS=$'\n' - for src in $sums; do - if ! echo $src | sha512sum -c ; then - err "Checksums failed to match!" - fi - done - unset IFS -fi - -# Check if $parenttmp is set and is a directory -if [[ -z $parenttmp ]] ; then - err "parenttmp variable not set in bldpkg.conf." -elif [[ ! -d $parenttmp ]] ; then - err "parenttmp variable set to '$parenttmp' in bldpkg.conf is not a directory." -fi - -# Attempt to write to the $parenttmp directory. This directory is used for everything related to the -# build process outside the source directory $srcdir -if ! touch "$parenttmp/.parenttmpwritetest" ; then - err "Parent temp directory '$parenttmp' is not writable!" -else - # Discard the test file - rm "$parenttmp/.parenttmpwritetest" -fi - -# If htmloutput is set to 1, echo $app, $version and $build as file names inside the parent build directory. -# This will output into an HTML file so that the basic status of the build process (whether started, stopped, -# interrupted or failed) can be viewed in the web browser. -if [[ $htmloutput = 1 ]] ; then - - if [[ -n $autobuild ]] ; then - cat << EOF >> "$parenttmp/BUILDMONITOR" -$commencedate | Building package $currentpkgnumber / $totalpkgnumber: $app $version -EOF - else - cat << EOF >> "$parenttmp/BUILDMONITOR" -$commencedate | Building package $app $version -EOF - fi - - touch "$parenttmp/BUILDING" -fi - -printf "[INFO] Validating compressor '$compressor'..." -mkdir -p "$parenttmp/.tar.testdir" -touch "$parenttmp/.tar.testdir/.tar.testfile" - -if ! tar -cf - -C "$parenttmp/.tar.testdir" . | "$compressor" "$compressoropts" > "$parenttmp/.tar.testdir.$pkgext" ; then - err "Failed to validate '$compressor' options! Please check your bldpkg.conf" -fi - -printf " done\n" -rm -r "$parenttmp/.tar.testdir"* - -# Validate the TMPFS directory if usetmpfs is set to 1 and tmpfsdir variable is set. If it fails, -# declare a variable for the build summary. -if [[ $tmpfs = 1 ]] && [[ -n $tmpfsdir ]]; then - if [[ ! -d $tmpfsdir ]] || ! touch "$tmpfsdir/.tmpfswritetest" \ - || [[ $(findmnt -no TARGET $tmpfsdir) != $tmpfsdir ]] \ - || [[ $(findmnt -no FSTYPE $tmpfsdir) != tmpfs ]]; then - tmpfscheckfailed=1 - fi - - # Discard the file used to test the tmp directory - rm "$tmpfsdir/.tmpfswritetest" - - # Check the tmpfsdir for stale directories from previous pkg builds. If found, issue a warning. - if [[ $(find "$tmpfsdir" -type d -maxdepth 1 -name "*.src" -o -name "package-*" | wc -l) -gt 0 ]]; then - if [[ ! -d $tmpfsdir/package-$app ]] || [[ ! -d $tmpfsdir/$app.src ]] ; then - warn "TMPFS directory '$tmpfsdir' has stale directories from previous builds!" - fi - fi -fi - -# Validate system swap if swapcheck is defined and set to 1 -if [[ $swapcheck = 1 ]]; then - if inarray "${app}" "${packagesrequiringswap[@]}" ; then - - # Here we determine available system swap size needed to compile exceptional packages that pull - # in a lot of RAM. Those packages are listed under the packagesrequiringswap array in bldpkg.conf. - # Check whether swap is available on the system and if it is, determine its size. If its size - # is >= swapsize, we are all good. If it's less than swapsize, we exit with a status 1. - - swapcheck=$(grep "SwapFree" /proc/meminfo | awk '{print $2}') - [[ $swapcheck -lt $swapsize ]] && err "Insufficient swap size to build packages, advise increasing it." - fi -fi - -# Set the build and package staging directories. This is where package files that get "installed" go into, -# for example 'make install DESTDIR=$pkg' or 'DESTDIR="$pkg" ninja install'. - -# If tmpfs is set to 1 and tmpfscheckfailed variable is unset, determine -# if the $app is in the exception list and whether to build inside or outside the TMPFS directory. -if [[ $tmpfs = 1 ]] && [[ -z $tmpfscheckfailed ]] ; then - - # If $app is in the TMPFS exception list inside bldpkg.conf, compile it *OUTSIDE* the TMPFS - # directory, i.e the non-TMPFS directory, else compile it *INSIDE* the TMPFS directory. This if/else - # is solely for deciding whether $app is in the exception list or not. - if inarray "${app}" "${tmpfsexceptionlist[@]}" ; then - - info "'$app' is in tmpfs exception list. Falling back to non-tmpfs directory" - - # We DO NOT compile inside tmpfsdir - tmpfsenabledforthispackage=0 - - # In the absence of tmpfs, we use the normal directory - tmp="$nontmpfsdir/$app.src" - pkg="$nontmpfsdir/package-$app" - else - # We compile inside tmpfsdir. Set the tmpfsenabledforthispackage variable here to inform build - # summary function at the bottom - tmpfsenabledforthispackage=1 - - # Disable ccache - ccache=0 - - # Override preservebuilddir and preservepackagedir to remove both build and package staging directories - preservebuilddir=0 - preservepackagedir=0 - - # Get the directory from the tmpfsdir variable for extracting the source and set it as our build - # and staging directory - tmp="$tmpfsdir/$app.src" - pkg="$tmpfsdir/package-$app" - - fi -else - # If tmpfs is disabled, we compile in the non-TMPFS directory - tmp="$nontmpfsdir/$app.src" - pkg="$nontmpfsdir/package-$app" -fi - -if inarray "${pkg}" "${protecteddirectories[@]}" ; then - echo "############ ATTENTION ############" - err "'pkg' VARIABLE IS SET TO '$pkg' WHICH IS A PROTECTED DIRECTORY! EXITING!" -fi - -# Also check tmp directory in advance -if inarray "${tmp}" "${protecteddirectories[@]}" ; then - echo "############ ATTENTION ############" - err "'tmp' VARIABLE IS SET TO '$tmp' WHICH IS A PROTECTED DIRECTORY! EXITING!" -fi - -# https://unix.stackexchange.com/questions/462392/bash-the-function-is-executed-twice -# https://stackoverflow.com/questions/9256644/identifying-received-signal-name-in-bash/9256709 -# We use two traps to identify the signals, EXIT and INT. -trap "prepbuildoutput" EXIT -trap "interruptoutput" INT - -# If $tmp exists, get the path to the $.app-$version.extraction.complete file -[[ -d $tmp ]] && buildresumepath="$(find $tmp -type f -name .$app-$version.extraction.complete)" - -# If the above file exists, $resumepkgbuild and $autobuild are unset -# and $autoresumepkgbuild is set to 1 in bldpkg.conf, prompt the user -if [[ -f $buildresumepath ]] && [[ -z $resumepkgbuild ]] && [[ $autoresumepkgbuild = 1 ]] && [[ -z $autobuild ]]; then - read -r -p "[NOTIFY] Would you like to resume building? " yn - case "$yn" in - N|n|No|no) info "Nope? Alright." ; - unset resumepkgbuild ;; - *) info "Wise choice :-) " - resumepkgbuild=1 ;; - esac -fi - -# If $resumepkgbuild variable is not set in getopts, only then execute prepbuilddir variable -[[ -z $resumepkgbuild ]] && prepbuilddir - -# If $resumepkgbuild is set either in getopts or from the user prompt above, execute mkandenterbuilddir -# function to enter the build directory. This is being done because mkandenterbuilddir is part of prepbuilddir -# function in the build file and ignoring prepbuilddir will not cause mkandenterbuilddir to be invoked -# separately unless the build system is told to. -if [[ -n $resumepkgbuild ]] ; then - mkandenterbuilddir - - # fixbuilddirpermissions places a file ".$app-$version.extraction.complete". Only resume the build if that file exists - if [[ ! -f $buildresumepath ]] ; then - err "Can't resume build of '"$app"'! Are you certain the source was extracted completely?" - else - cd ${buildresumepath%/*} - fi -fi - -# Run a for loop to find compile and build-related files -for requiredfile in "${rqfiles[@]}"; do - if [[ ! -x $(type -p "$requiredfile") ]] ; then - err "Could not find required program '$requiredfile'!" - fi -done - -# Apply CPU-specific compiler variables defined inside bldpkg.conf -# noarch is set inside initfs, pkgtools, GTK themes and some other stuff. - -# Only set $arch if it's not set in the build file. We take the help of $HOSTTYPE variable set by the bash shell. -[[ -z $arch ]] && arch="$HOSTTYPE" - -for compilervariable in CC CXX CFLAGS ; do - [[ -z $compilervariable ]] && err "$compilervariable not set in bldpkg.conf!" -done - -# Clean out CFLAGS if $arch is set to noarch -if [[ $arch = noarch ]]; then - CFLAGS="" -elif [[ -n $debug ]]; then - CFLAGS="$gccdebug $CFLAGS" -fi - -CXXFLAGS="$CFLAGS" -export hostdist builddist CC CXX CFLAGS CXXFLAGS MAKEFLAGS - -# Validate available compilers. Quotes are important while passing C/CXXFLAGS -compilertestfile "$CC" "$CFLAGS" cc-test.c cc-test -compilertestfile "$CXX" "$CXXFLAGS" cxx-test.cpp cxx-test - -# Validate everything related to ccache if ccache is set -if [[ $ccache = 1 ]]; then - [[ ! -x $ccachebinpath ]] && err "Ccache binary does not exist/is not an executable!" - - checkcompilersymlink $ccachesymdirpath $ccachebinpath - - # Prepare $PATH and export it only if ccache is unset in the pkg build file - PATH="$ccachesymdirpath:$PATH" - export PATH -fi - -# Validate everything related to sccache if ccache is set -if [[ $sccache = 1 ]]; then - [[ ! -x $sccachebinpath ]] && err "$sccachebinpath was either not found or is not an executable!" - - # We expect the rustc wrapper to be defined in $HOME/.cargo/config.toml - if ! grep -q 'rustc-wrapper\ \= "\/bin\/sccache"' $HOME/.cargo/config.toml ; then - err 'sccache is not defined as a rustc wrapper! -Please add the following to your '"$HOME/"'.cargo/config.toml: -[build] -rustc-wrapper = "/bin/sccache"' - fi - - # Useful for rust-specific builds. - RUSTC_WRAPPER="$sccachebinpath" - export RUSTC_WRAPPER - - sccacheprocess start -fi - -# Validate everything related to distcc if distcc is set -if [[ $distcc = 1 ]] ; then - # Check if distcc exists and is an executable - [[ ! -x $distccbinpath ]] && err "Distcc binary was either not found or is not an executable" - - checkcompilersymlink $distccsymdirpath $distccbinpath - - # Prepare $PATH and export DISTCC_HOSTS and DISTCC_IO_TIMEOUT only if distcc is unset in the pkg build file - # netcat hosts inside $DISTCC_HOSTS by checking for an open port - info "Validating distcc hosts..." - # Check if we have nc - if [[ ! -x /bin/nc ]] ; then - warn "nc does not exist! Ignoring this..." - else - # Remove the common options along with the slash and the numbers after it - hosts=$(echo "$DISTCC_HOSTS" | sed -e 's@/[a-z0-9,]*@@g' -e 's@--randomize@@' -e 's@localhost@@') - for host in ${hosts[@]} ; do - # We only run distccd on TCP port 3632 - if ! /bin/nc -z -w 1 "$host" 3632 > /dev/null 2>&1 ; then - warn "Distcc host '$host' is OFFLINE! Rewriting DISTCC_HOSTS" - DISTCC_HOSTS=$(echo "$DISTCC_HOSTS" | sed "s@$host/[a-z0-9,]*@@") - fi - done - fi - - PATH="$distccsymdirpath:$PATH" - export DISTCC_HOSTS DISTCC_IO_TIMEOUT PATH -fi - -# If verbosebuild is set in bldpkg.conf or if verbosity is set to 1 from getopts, set V and -# VERBOSE environment variables for make and cmake build systems to pick up. -if [[ $verbosebuild = 1 ]] || [[ $getoptsverbosebuild = 1 ]] ; then - V=1 - VERBOSE=1 -else - V=0 - VERBOSE=0 -fi -export V VERBOSE - -# If $noautoconfsite is unset in an individual package build file, export CONFIG_SITE variable into the build -# environment for a package's configure script to pickup. Most autoconf-compatible configure scripts will -# automatically pick up this variable from the environment and speed up the initial configure process. -if [[ -z $noautoconfsite ]] ; then - if [[ -n $configsite ]] && [[ -e $configsite ]]; then - CONFIG_SITE="$configsite" - export CONFIG_SITE - fi -fi - -# Do a preliminary package dependency check if $checkdependencies is set to 1 in bldpkg.conf -if [[ $checkdependencies = 1 ]] ; then - info "Parsing $app 's dependency list..." - for packagedep in $requires; do - depcount=$(find /share/doc -type f -name "$packagedep.SMBuild" | wc -l) - # If count is 1, we are ok - if [[ $depcount = 1 ]] ; then - info "Found dependency '$packagedep'" - # If count is 0, we exit, because we are in trouble - elif [[ $depcount = 0 ]] ; then - err "Did not find dependency '$packagedep'" - # If count is greater than or equal to 2, we are in slightly less trouble - elif [[ $depcount -ge 2 ]] ; then - warn "Found multiple versions of '$packagedep' !" - sleep 0.5 - fi - done -fi - -build - -# End script +# End of script diff --git a/bldpkg.conf b/bldpkg.conf index b3fe16f..40b658a 100644 --- a/bldpkg.conf +++ b/bldpkg.conf @@ -7,6 +7,8 @@ # Please refer to sample.SMBuild.explained in the buildsamples directory # for complete explanation. +sections=(base xorg gtk extra net xfce) + # Set the package extension. Makepkg will use this to figure out the # compression utility. Pick one out of tbz/tgz/txz/tlz. pkgext="tlz" @@ -109,7 +111,7 @@ nontmpfsdir="$parenttmp/sml" # openssh-8.1-x86_64-1sml.tlz should be placed in. Again, should be outside # tmpfs dir. Also declared and defined in autobuild section file, so values # may differ. This is for manual builds. -pkgdest=${pkgdest:-$parenttmp/sml/packages/} +pkgdest="$parenttmp/sml/packages/" # Whether you want to preserve the build directory which contains the extracted source, # for example /tmp/sml/openssh.src/openssh-8.1. usetmpfs variable, if set to 1, will @@ -135,7 +137,7 @@ showsummary=1 # Define whether to check for swap. 1 to enable, 0 to disable. swapcheck=0 -# If above variable is set to 1, define minimum swap size required for huge +# If above variable is set to 1, define swap size required for huge # compiles in KiB swapsize=2048000