pktsurf/smlinux
1
1
Fork 0
Code Issues 1 Pull requests Wiki Activity

Merged autobuild into bldpkg, made changes related to it in bldpkg.conf

Browse source
This commit is contained in:
PktSurf 2023-05-11 22:56:36 +05:30
parent 258fdc4b56
commit 64e8468718
3 changed files with 527 additions and 543 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

134
autobuild
View file

@ -1,134 +0,0 @@
#!/bin/bash
#
# Part of the SMLinux distribution
# http://git.pktsurf.in/smlinux
#
# autobuild version 0.100
# Bash script to automatically build section-specific packages for SMLinux
#
# Copyright (c) 2023 PktSurf <smlinux@pktsurf.in>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
############
sections=(base xorg gtk extra net xfce)
buildcleanup() {
[[ -f $autobuildtemp ]] && rm -f $autobuildtemp
exit
}
trap "buildcleanup" INT
trap "buildcleanup" EXIT
err() {
printf "**ERROR**\n$@\n"
exit 1
}
warn() {
printf "[WARN]\n$@\n"
}
info() {
printf "[INFO] $@\n"
}
parenttmp="/tmp"
autobuild=1
arch="$HOSTTYPE"
colours=0
autobuildtemp="$(mktemp $parenttmp/SMBUILD.XXXXXX)"
export parenttmp autobuild arch colours autobuildtemp
for section in "${sections[@]}" ; do
pkgdest="$parenttmp/sml/packages/$arch/$section"
logdir="$parenttmp/sml/sml-buildlogs/$arch/$section"
export section pkgdest
mkdir -p $pkgdest $logdir
if [[ ! -d $section ]] ; then
err "Section directory '$section' not found!"
fi
cd $section
# This check compares a list of source directories with the list of the
# packages in the build list and warns of any missing package names
# in either of the two.
dirtempfile=$(mktemp $parenttmp/DIRECTORYNAMES."$section".XXXXXX)
dirfiletemppath="$dirtempfile"
dirlist=$(find . -type d -maxdepth 1 -mindepth 1 | sed 's@./@@' | sort > $dirfiletemppath)
packtempfile=$(mktemp $parenttmp/BUILDFILENAMES."$section".XXXXXX)
packfiletemppath="$packtempfile"
sort .buildlist.$section > $packfiletemppath
directorycount="$( wc -l < $dirtempfile )"
buildlistcount="$( wc -l < $packtempfile )"
# Get number of total packages
totalpkgnumber="$(wc -l < .buildlist.$section)"
export totalpkgnumber
if diff -u "$dirfiletemppath" "$packfiletemppath" > /dev/null 2>&1 ; then
diffstatus="0"
else
diffstatus="1"
fi
if [ "$diffstatus" != "0" ]; then
warn "In section '"$section"', the number of packages in the hidden file '.buildlist."$section"' is different to the number of package directories. Some packages may not have been added to this file/section directory. They are listed below:"
diff -u "$dirfiletemppath" "$packfiletemppath" || true
diff -u "$packfiletemppath" "$dirfiletemppath" || true
warn "Building anyways :-) "
sleep 2
fi
rm -f $packfiletemppath $dirfiletemppath
if [[ ! -f .buildlist.$section ]] ; then
err "Section build list file '.buildlist.$section' not found!"
fi
while IFS="" read -r package || [[ -n $package ]] ; do
currentpkgnumber="$(grep -Ewn "^$package" .buildlist.$section | cut -d: -f 1)"
export currentpkgnumber
(
cd $package
source $package.SMBuild
# Check if package installer exists or is installed on the system
# If the package installer does not exist in the set pkg destination, build it
if [ ! -f $pkgdest/$app-$version-*-$build.* ] ; then
bldpkg 2>&1 | tee -a $logdir/$app-$version-$build.log.txt
# If the package is not installed in the system, install it
if [ ! -f /var/log/packages/$app-$version-*-$build ] ; then
upgradepkg --install-new $pkgdest/$app-$version-*-$build.*
else
info "'$app-$version' already installed, skipping it"
fi
elif [ ! -f /var/log/packages/$app-$version-*-$build ] ; then
upgradepkg --install-new $pkgdest/$app-$version-*-$build.*
else
info "Already built '$app-$version', skipping it"
fi
)
done < ".buildlist.$section"
cd ..
done

706
bldpkg
View file

@ -3,7 +3,7 @@
# Part of the SMLinux distribution # Part of the SMLinux distribution
# http://git.pktsurf.in/smlinux # http://git.pktsurf.in/smlinux
# #
# /bin/bldpkg version 0.101 # /bin/bldpkg version 0.102
# Bash script to build SMLinux-specific packages # Bash script to build SMLinux-specific packages
# #
# Copyright (c) 2022-2023 PktSurf <smlinux@pktsurf.in> # Copyright (c) 2022-2023 PktSurf <smlinux@pktsurf.in>
@ -73,6 +73,9 @@ Building package 'alsa-lib' version '1.x'...
Usage: Usage:
-a Switch mode to autobuild. Default is to build individual packages.
Requires sections variable in bldpkg.conf to be defined
-d Produce a package with debug symbols preserved, i.e., don't strip -d Produce a package with debug symbols preserved, i.e., don't strip
resulting ELF objects. Uses -g3 by default resulting ELF objects. Uses -g3 by default
@ -163,13 +166,30 @@ validatebldfile() {
err "Invalid URL in the 'download' variable in the build file." err "Invalid URL in the 'download' variable in the build file."
fi fi
fi fi
# Check integrity of files defined in sha512sums variable which is expected
# in nearly every single package build file
if [[ -z $sha512sums ]] ; then
err "Please run 'bldpkg -g' to add sha512sums into '$buildfile'!"
else
eval sums=\"\$sha512sums\"
info "Verifying SHA512 checksums against source files..."
IFS=$'\n'
for src in $sums; do
if ! echo $src | sha512sum -c ; then
err "Checksums failed to match!"
fi
done
unset IFS
fi
} }
# Function to error out the build in absence of a build file # Function to error out the build in absence of a build file
nopackagebuildfileerror() { nopackagebuildfileerror() {
err "No package build file to source from! err "No package build file to source from!
Was expecting '$buildfile' to be present inside this directory '$PWD'. Was expecting '$buildfile' to be present inside this directory '$PWD'.
Try -f <build_file> if your build file has a different name (Not recommended)" Try -f <build_file> if your build file has a different name (Not recommended)
If you want to autobuild packages, use -a"
} }
validatemakeflags() { validatemakeflags() {
@ -220,8 +240,9 @@ compileonlyfor() {
local archargument="$1" local archargument="$1"
if [[ $archname != $archargument ]]; then if [[ $archname != $archargument ]]; then
info "'$app' not supported on '$archname' and hence not not being built. Exiting." info "'$app' not supported on your current CPU '$archname'. Build aborted."
exit 0 buildskipped=1
return 0
fi fi
} }
@ -293,7 +314,6 @@ mkandenterbuilddir() {
# Now create all essential build-related directories # Now create all essential build-related directories
mkdir -p "$tmp" "$pkg/install" "$pkgdocs" "$pkgdest" mkdir -p "$tmp" "$pkg/install" "$pkgdocs" "$pkgdest"
info "Leaving source directory $srcdir"
info "Entering build directory $tmp" info "Entering build directory $tmp"
cd "$tmp" cd "$tmp"
} }
@ -327,8 +347,15 @@ runtime() {
fi fi
} }
getbuildfileanddir() {
srcdir="$PWD"
buildfile="${srcdir##*/}.SMBuild"
}
# Generate and insert sha512sums into the build file # Generate and insert sha512sums into the build file
genchecksum() { genchecksum() {
getbuildfileanddir
# File types whose checksums will go into the new build file # File types whose checksums will go into the new build file
info "Discarding any old sha512sums from '$buildfile'" info "Discarding any old sha512sums from '$buildfile'"
sed -E -i \ sed -E -i \
@ -458,7 +485,7 @@ promptuser() {
fi fi
if [[ $pkgstatus = 0 ]] ; then if [[ $pkgstatus = 0 ]] ; then
exit 0 return 0
fi fi
} }
@ -475,11 +502,7 @@ interruptoutput() {
# Terminate sccache # Terminate sccache
sccacheprocess stop sccacheprocess stop
# If installprompt and extractprompt are set and the prompt is invoked after a successful build, hitting return 0
# ctrl-C will only set the above sm variable repeatedly and won't return user to the shell because
# of the interrupt (SIGINT) trap set way below. Putting exit 0 is a decent way to get out of that prompt
exit 0
} }
# Function to perform post-compile tasks. # Function to perform post-compile tasks.
@ -622,6 +645,7 @@ EOF
--xform 'sx^\./\(.\)x\1x' \ --xform 'sx^\./\(.\)x\1x' \
--show-stored-names | "$compressor" "$compressoropts" > "$newpkglocation" --show-stored-names | "$compressor" "$compressoropts" > "$newpkglocation"
pkgstatus=$? pkgstatus=$?
#export pkgstatus
echo echo
info "SMLinux package '$app-$version-$arch-$build.$pkgext' successfully generated in $pkgdest." info "SMLinux package '$app-$version-$arch-$build.$pkgext' successfully generated in $pkgdest."
@ -631,13 +655,6 @@ EOF
info "Leaving staging directory $pkg" info "Leaving staging directory $pkg"
# cd back to $srcdir when preservepackagedir is set to 0 to prevent this error:
# shell-init: error retrieving current directory: getcwd: cannot access parent directories:
# No such file or directory
info "Re-entering source directory $srcdir"
cd "$srcdir"
# With SECONDS reset, the shell will add in a fresh value, which we can now use to ascertain # With SECONDS reset, the shell will add in a fresh value, which we can now use to ascertain
# the packaging time, by again passing that value as an argument to the runtime function # the packaging time, by again passing that value as an argument to the runtime function
if [[ $showsummary = 1 ]] || [[ $htmloutput = 1 ]] ; then if [[ $showsummary = 1 ]] || [[ $htmloutput = 1 ]] ; then
@ -692,7 +709,7 @@ findelffiles() {
buildfilecleanup() { buildfilecleanup() {
# Discard all temporary files # Discard all temporary files
rm "$parenttmp/BUILDING" rm -f "$parenttmp/BUILDING"
} }
prepbuildoutput() { prepbuildoutput() {
@ -823,12 +840,7 @@ prepbuildoutput() {
ttime="$totaltime ( $compiletimeb Compile ) + ( $packagetimeb Packaging )" ttime="$totaltime ( $compiletimeb Compile ) + ( $packagetimeb Packaging )"
fi fi
# Output the section name if autobuildtemp is set. This means we are running an autobuild. [[ -z $section ]] && section="None"
if [[ -n $autobuildtemp ]]; then
bsection="$section"
else
bsection="None"
fi
# Output the build summary to the user on every build # Output the build summary to the user on every build
cat << EOF cat << EOF
@ -836,7 +848,7 @@ $colourc -----------------------------------------------------------------------
$colourc BUILD SUMMARY FOR PACKAGE $colourv'$app'$colourc VERSION $colourv'$version'$colourc TAG $colourv'$build' $colourc BUILD SUMMARY FOR PACKAGE $colourv'$app'$colourc VERSION $colourv'$version'$colourc TAG $colourv'$build'
$colourc ------------------------------------------------------------------------------- $colourd $colourc ------------------------------------------------------------------------------- $colourd
$colourc Build Status: $colourd $bldstatus $colourc Build Status: $colourd $bldstatus
$colourc Build Section: $colourd $bsection $colourc Build Section: $colourd $section
$colourc Total Time: $colourd $ttime $colourc Total Time: $colourd $ttime
$colourc Started: $colourd $commencedate $colourc Started: $colourd $commencedate
$colourc Stopped: $colourd $finishdate $colourc Stopped: $colourd $finishdate
@ -863,191 +875,37 @@ EOF
promptuser promptuser
} }
# Source the main configuration file holding all necessary values stage0prep() {
if [[ -f $HOME/.bldpkg.conf ]] ; then # Check if $parenttmp is set and is a directory
source "$HOME/.bldpkg.conf" if [[ -z $parenttmp ]] ; then
elif [[ -f /etc/bldpkg.conf ]] ; then
source /etc/bldpkg.conf
else
err "Bldpkg configuration file '/etc/bldpkg.conf' not found!"
fi
# Error out if ccache and sccache are set in the /etc/bldpkg.conf
if [[ $ccache = 1 ]] && [[ $sccache = 1 ]] ; then
err "Please only enable either ccache or sccache in bldpkg.conf at the same time, not both."
fi
# Store the source directory path the build was initiated from
srcdir="$PWD"
# Presume that the build file name will match the name of the parent directory
# unless otherwise overridden using -f <buildfile>
buildfile="${srcdir##*/}.SMBuild"
# While loop for providing handy arguments to users. Some will override bldpkg.conf.
while getopts ':def:ghj:o:rstvx' option; do
case "$option" in
d) debug=1 ;; # Produce a debug build with -g3
e) extractprompt=0;
autoextract=1 ;; # Automatically extract the final pkg installer inside user's PWD
f) setbuildfile="$OPTARG" ;;
g) genchecksum ;;
h) help ;;
j) custommakeflags="$OPTARG" ;;
o) origbuildfile="$OPTARG" ;;
r) resumepkgbuild=1 ;;
s) showsummary=1 ;; # Show build summary at the end of the build irrespective of the build status
t) tmpfs=1 ;;
v) getoptsverbosebuild=1 ;;
x) set -xv ;; # Invoke bash's -x option for command tracing
*) help ;;
esac
done
# The getopts builtin sets the OPTIND environment variable whose value is set to 1 if no argument is given.
# For every argument passed, the number is incremented by 1. In our case, if OPTIND equals 1, no argument
# was passed. We therefore expect a package build file to be present which will then be validated and sourced.
if [[ $OPTIND = 1 ]] ; then
if [[ ! -f $buildfile ]] ; then
nopackagebuildfileerror
elif ! validatebldfile "$buildfile" ; then
err "'$buildfile' build file validation failed!"
else
source "$buildfile"
fi
# If OPTIND is greater than 1, perform various checks and actions
elif [[ $OPTIND -gt 1 ]] ; then
# If MAKEFLAGS and custommakeflags is set, let custommakeflags take precedence
if [[ -n $custommakeflags ]] ; then
validatemakeflags $custommakeflags
MAKEFLAGS="-j$custommakeflags"
elif [[ -n $MAKEFLAGS ]] ; then
validatemakeflags $MAKEFLAGS
elif [[ -z $MAKEFLAGS ]] ; then
# Or fetch the number from nproc
MAKEFLAGS="-j$(nproc --all)"
fi
# If $origbuildfile is set and is a file, check if $setbuildfile and $origbuildfile are the same
if [[ -n $origbuildfile ]] ; then
if [[ ! -f $origbuildfile ]] ; then
err "Original build file '$origbuildfile' does not exist!"
elif [[ -n $setbuildfile ]] && [[ -f $setbuildfile ]] ; then
if [[ $origbuildfile = $setbuildfile ]] ; then
err "Original build file and alternate build file are the same!"
fi
elif ! validatebldfile "$origbuildfile" ; then
err "Alternative build file validation failed!"
fi
fi
# If $setbuildfile is set and is a file, set buildfile to its value, source it and initialise the build
if [[ -n $setbuildfile ]] && [[ -f $setbuildfile ]] ; then
buildfile="$setbuildfile"
if ! validatebldfile "$buildfile" ; then
err "'$buildfile' validation failed!"
else
source "$buildfile"
fi
# If $setbuildfile is set but a file passed as an argument is not found, give an error
elif [[ -n $setbuildfile ]] && [[ ! -f $setbuildfile ]] ; then
err "Build file '$setbuildfile' not found!"
# If the above two conditions don't meet, get the presumed $buildfile value as a file and source it
elif [[ -f $buildfile ]] ; then
if ! validatebldfile "$buildfile" ; then
err "'$buildfile' validation failed!"
else
source "$buildfile"
fi
# If even that file is not found, throw an error and exit
else
nopackagebuildfileerror
fi
fi
# Unset OPTIND and OPTARG to avoid problems if getopts is used again in the future
unset OPTIND OPTARG
# Display the package and its version we are building or resuming
if [[ -z $resumepkgbuild ]] ; then
info "Building package '$app' version '$version' ..."
else
info "Resuming build of package '$app' version '$version' ..."
fi
# Check integrity of files defined in sha512sums variable which is expected
# in nearly every single package build file
if [[ -z $sha512sums ]] ; then
err "Please run 'bldpkg -g' to add sha512sums into '$buildfile'!"
else
eval sums=\"\$sha512sums\"
info "Verifying SHA512 checksums against source files..."
IFS=$'\n'
for src in $sums; do
if ! echo $src | sha512sum -c ; then
err "Checksums failed to match!"
fi
done
unset IFS
fi
# Check if $parenttmp is set and is a directory
if [[ -z $parenttmp ]] ; then
err "parenttmp variable not set in bldpkg.conf." err "parenttmp variable not set in bldpkg.conf."
elif [[ ! -d $parenttmp ]] ; then elif [[ ! -d $parenttmp ]] ; then
err "parenttmp variable set to '$parenttmp' in bldpkg.conf is not a directory." err "parenttmp variable set to '$parenttmp' in bldpkg.conf is not a directory."
fi fi
# Attempt to write to the $parenttmp directory. This directory is used for everything related to the # Attempt to write to the $parenttmp directory. This directory is used for everything related to the
# build process outside the source directory $srcdir # build process outside the source directory $srcdir
if ! touch "$parenttmp/.parenttmpwritetest" ; then if ! touch "$parenttmp/.parenttmpwritetest" ; then
err "Parent temp directory '$parenttmp' is not writable!" err "Parent temp directory '$parenttmp' is not writable!"
else else
# Discard the test file # Discard the test file
rm "$parenttmp/.parenttmpwritetest" rm "$parenttmp/.parenttmpwritetest"
fi
# If htmloutput is set to 1, echo $app, $version and $build as file names inside the parent build directory.
# This will output into an HTML file so that the basic status of the build process (whether started, stopped,
# interrupted or failed) can be viewed in the web browser.
if [[ $htmloutput = 1 ]] ; then
if [[ -n $autobuild ]] ; then
cat << EOF >> "$parenttmp/BUILDMONITOR"
<b>$commencedate | Building package $currentpkgnumber / $totalpkgnumber: <i><a href="/smlinux/pkgresults?pkg=$app&amp;smver=1.0&amp;arch=all&amp;resultnum=25">$app $version</a></i></b>
EOF
else
cat << EOF >> "$parenttmp/BUILDMONITOR"
<b>$commencedate | Building package <i><a href="/smlinux/pkgresults?pkg=$app&amp;smver=1.0&amp;arch=all&amp;resultnum=25">$app $version</a></i></b>
EOF
fi fi
touch "$parenttmp/BUILDING" printf "[INFO] Validating compressor '$compressor'..."
fi mkdir -p "$parenttmp/.tar.testdir"
touch "$parenttmp/.tar.testdir/.tar.testfile"
printf "[INFO] Validating compressor '$compressor'..." if ! tar -cf - -C "$parenttmp/.tar.testdir" . | "$compressor" "$compressoropts" > "$parenttmp/.tar.testdir.$pkgext" ; then
mkdir -p "$parenttmp/.tar.testdir"
touch "$parenttmp/.tar.testdir/.tar.testfile"
if ! tar -cf - -C "$parenttmp/.tar.testdir" . | "$compressor" "$compressoropts" > "$parenttmp/.tar.testdir.$pkgext" ; then
err "Failed to validate '$compressor' options! Please check your bldpkg.conf" err "Failed to validate '$compressor' options! Please check your bldpkg.conf"
fi fi
printf " done\n" printf " done\n"
rm -r "$parenttmp/.tar.testdir"* rm -r "$parenttmp/.tar.testdir"*
# Validate the TMPFS directory if usetmpfs is set to 1 and tmpfsdir variable is set. If it fails, # Validate the TMPFS directory if usetmpfs is set to 1 and tmpfsdir variable is set. If it fails,
# declare a variable for the build summary. # declare a variable for the build summary.
if [[ $tmpfs = 1 ]] && [[ -n $tmpfsdir ]]; then if [[ $tmpfs = 1 ]] && [[ -n $tmpfsdir ]]; then
if [[ ! -d $tmpfsdir ]] || ! touch "$tmpfsdir/.tmpfswritetest" \ if [[ ! -d $tmpfsdir ]] || ! touch "$tmpfsdir/.tmpfswritetest" \
|| [[ $(findmnt -no TARGET $tmpfsdir) != $tmpfsdir ]] \ || [[ $(findmnt -no TARGET $tmpfsdir) != $tmpfsdir ]] \
|| [[ $(findmnt -no FSTYPE $tmpfsdir) != tmpfs ]]; then || [[ $(findmnt -no FSTYPE $tmpfsdir) != tmpfs ]]; then
@ -1063,10 +921,10 @@ if [[ $tmpfs = 1 ]] && [[ -n $tmpfsdir ]]; then
warn "TMPFS directory '$tmpfsdir' has stale directories from previous builds!" warn "TMPFS directory '$tmpfsdir' has stale directories from previous builds!"
fi fi
fi fi
fi fi
# Validate system swap if swapcheck is defined and set to 1 # Validate system swap if swapcheck is defined and set to 1
if [[ $swapcheck = 1 ]]; then if [[ $swapcheck = 1 ]]; then
if inarray "${app}" "${packagesrequiringswap[@]}" ; then if inarray "${app}" "${packagesrequiringswap[@]}" ; then
# Here we determine available system swap size needed to compile exceptional packages that pull # Here we determine available system swap size needed to compile exceptional packages that pull
@ -1077,14 +935,14 @@ if [[ $swapcheck = 1 ]]; then
swapcheck=$(grep "SwapFree" /proc/meminfo | awk '{print $2}') swapcheck=$(grep "SwapFree" /proc/meminfo | awk '{print $2}')
[[ $swapcheck -lt $swapsize ]] && err "Insufficient swap size to build packages, advise increasing it." [[ $swapcheck -lt $swapsize ]] && err "Insufficient swap size to build packages, advise increasing it."
fi fi
fi fi
# Set the build and package staging directories. This is where package files that get "installed" go into, # Set the build and package staging directories. This is where package files that get "installed" go into,
# for example 'make install DESTDIR=$pkg' or 'DESTDIR="$pkg" ninja install'. # for example 'make install DESTDIR=$pkg' or 'DESTDIR="$pkg" ninja install'.
# If tmpfs is set to 1 and tmpfscheckfailed variable is unset, determine # If tmpfs is set to 1 and tmpfscheckfailed variable is unset, determine
# if the $app is in the exception list and whether to build inside or outside the TMPFS directory. # if the $app is in the exception list and whether to build inside or outside the TMPFS directory.
if [[ $tmpfs = 1 ]] && [[ -z $tmpfscheckfailed ]] ; then if [[ $tmpfs = 1 ]] && [[ -z $tmpfscheckfailed ]] ; then
# If $app is in the TMPFS exception list inside bldpkg.conf, compile it *OUTSIDE* the TMPFS # If $app is in the TMPFS exception list inside bldpkg.conf, compile it *OUTSIDE* the TMPFS
# directory, i.e the non-TMPFS directory, else compile it *INSIDE* the TMPFS directory. This if/else # directory, i.e the non-TMPFS directory, else compile it *INSIDE* the TMPFS directory. This if/else
@ -1117,95 +975,58 @@ if [[ $tmpfs = 1 ]] && [[ -z $tmpfscheckfailed ]] ; then
pkg="$tmpfsdir/package-$app" pkg="$tmpfsdir/package-$app"
fi fi
else else
# If tmpfs is disabled, we compile in the non-TMPFS directory # If tmpfs is disabled, we compile in the non-TMPFS directory
tmp="$nontmpfsdir/$app.src" tmp="$nontmpfsdir/$app.src"
pkg="$nontmpfsdir/package-$app" pkg="$nontmpfsdir/package-$app"
fi fi
if inarray "${pkg}" "${protecteddirectories[@]}" ; then if inarray "${pkg}" "${protecteddirectories[@]}" ; then
echo "############ ATTENTION ############" echo "############ ATTENTION ############"
err "'pkg' VARIABLE IS SET TO '$pkg' WHICH IS A PROTECTED DIRECTORY! EXITING!" err "'pkg' VARIABLE IS SET TO '$pkg' WHICH IS A PROTECTED DIRECTORY! EXITING!"
fi fi
# Also check tmp directory in advance # Also check tmp directory in advance
if inarray "${tmp}" "${protecteddirectories[@]}" ; then if inarray "${tmp}" "${protecteddirectories[@]}" ; then
echo "############ ATTENTION ############" echo "############ ATTENTION ############"
err "'tmp' VARIABLE IS SET TO '$tmp' WHICH IS A PROTECTED DIRECTORY! EXITING!" err "'tmp' VARIABLE IS SET TO '$tmp' WHICH IS A PROTECTED DIRECTORY! EXITING!"
fi
# https://unix.stackexchange.com/questions/462392/bash-the-function-is-executed-twice
# https://stackoverflow.com/questions/9256644/identifying-received-signal-name-in-bash/9256709
# We use two traps to identify the signals, EXIT and INT.
trap "prepbuildoutput" EXIT
trap "interruptoutput" INT
# If $tmp exists, get the path to the $.app-$version.extraction.complete file
[[ -d $tmp ]] && buildresumepath="$(find $tmp -type f -name .$app-$version.extraction.complete)"
# If the above file exists, $resumepkgbuild and $autobuild are unset
# and $autoresumepkgbuild is set to 1 in bldpkg.conf, prompt the user
if [[ -f $buildresumepath ]] && [[ -z $resumepkgbuild ]] && [[ $autoresumepkgbuild = 1 ]] && [[ -z $autobuild ]]; then
read -r -p "[NOTIFY] Would you like to resume building? " yn
case "$yn" in
N|n|No|no) info "Nope? Alright." ;
unset resumepkgbuild ;;
*) info "Wise choice :-) "
resumepkgbuild=1 ;;
esac
fi
# If $resumepkgbuild variable is not set in getopts, only then execute prepbuilddir variable
[[ -z $resumepkgbuild ]] && prepbuilddir
# If $resumepkgbuild is set either in getopts or from the user prompt above, execute mkandenterbuilddir
# function to enter the build directory. This is being done because mkandenterbuilddir is part of prepbuilddir
# function in the build file and ignoring prepbuilddir will not cause mkandenterbuilddir to be invoked
# separately unless the build system is told to.
if [[ -n $resumepkgbuild ]] ; then
mkandenterbuilddir
# fixbuilddirpermissions places a file ".$app-$version.extraction.complete". Only resume the build if that file exists
if [[ ! -f $buildresumepath ]] ; then
err "Can't resume build of '"$app"'! Are you certain the source was extracted completely?"
else
cd ${buildresumepath%/*}
fi fi
fi
# Run a for loop to find compile and build-related files # Run a for loop to find compile and build-related files
for requiredfile in "${rqfiles[@]}"; do for requiredfile in "${rqfiles[@]}"; do
if [[ ! -x $(type -p "$requiredfile") ]] ; then if [[ ! -x $(type -p "$requiredfile") ]] ; then
err "Could not find required program '$requiredfile'!" err "Could not find required program '$requiredfile'!"
fi fi
done done
}
# Apply CPU-specific compiler variables defined inside bldpkg.conf stage1prep() {
# noarch is set inside initfs, pkgtools, GTK themes and some other stuff. # Apply CPU-specific compiler variables defined inside bldpkg.conf
# noarch is set inside initfs, pkgtools, GTK themes and some other stuff.
# Only set $arch if it's not set in the build file. We take the help of $HOSTTYPE variable set by the bash shell. # Only set $arch if it's not set in the build file. We take the help of $HOSTTYPE variable set by the bash shell.
[[ -z $arch ]] && arch="$HOSTTYPE" [[ -z $arch ]] && arch="$HOSTTYPE"
for compilervariable in CC CXX CFLAGS ; do for compilervariable in CC CXX CFLAGS ; do
[[ -z $compilervariable ]] && err "$compilervariable not set in bldpkg.conf!" [[ -z $compilervariable ]] && err "$compilervariable not set in bldpkg.conf!"
done done
# Clean out CFLAGS if $arch is set to noarch # Clean out CFLAGS if $arch is set to noarch
if [[ $arch = noarch ]]; then if [[ $arch = noarch ]]; then
CFLAGS="" CFLAGS=""
elif [[ -n $debug ]]; then elif [[ -n $debug ]]; then
CFLAGS="$gccdebug $CFLAGS" CFLAGS="$gccdebug $CFLAGS"
fi fi
CXXFLAGS="$CFLAGS" CXXFLAGS="$CFLAGS"
export hostdist builddist CC CXX CFLAGS CXXFLAGS MAKEFLAGS export hostdist builddist CC CXX CFLAGS CXXFLAGS MAKEFLAGS
# Validate available compilers. Quotes are important while passing C/CXXFLAGS # Validate available compilers. Quotes are important while passing C/CXXFLAGS
compilertestfile "$CC" "$CFLAGS" cc-test.c cc-test compilertestfile "$CC" "$CFLAGS" cc-test.c cc-test
compilertestfile "$CXX" "$CXXFLAGS" cxx-test.cpp cxx-test compilertestfile "$CXX" "$CXXFLAGS" cxx-test.cpp cxx-test
# Validate everything related to ccache if ccache is set # Validate everything related to ccache if ccache is set
if [[ $ccache = 1 ]]; then if [[ $ccache = 1 ]]; then
[[ ! -x $ccachebinpath ]] && err "Ccache binary does not exist/is not an executable!" [[ ! -x $ccachebinpath ]] && err "Ccache binary does not exist/is not an executable!"
checkcompilersymlink $ccachesymdirpath $ccachebinpath checkcompilersymlink $ccachesymdirpath $ccachebinpath
@ -1213,10 +1034,10 @@ if [[ $ccache = 1 ]]; then
# Prepare $PATH and export it only if ccache is unset in the pkg build file # Prepare $PATH and export it only if ccache is unset in the pkg build file
PATH="$ccachesymdirpath:$PATH" PATH="$ccachesymdirpath:$PATH"
export PATH export PATH
fi fi
# Validate everything related to sccache if ccache is set # Validate everything related to sccache if ccache is set
if [[ $sccache = 1 ]]; then if [[ $sccache = 1 ]]; then
[[ ! -x $sccachebinpath ]] && err "$sccachebinpath was either not found or is not an executable!" [[ ! -x $sccachebinpath ]] && err "$sccachebinpath was either not found or is not an executable!"
# We expect the rustc wrapper to be defined in $HOME/.cargo/config.toml # We expect the rustc wrapper to be defined in $HOME/.cargo/config.toml
@ -1232,10 +1053,10 @@ rustc-wrapper = "/bin/sccache"'
export RUSTC_WRAPPER export RUSTC_WRAPPER
sccacheprocess start sccacheprocess start
fi fi
# Validate everything related to distcc if distcc is set # Validate everything related to distcc if distcc is set
if [[ $distcc = 1 ]] ; then if [[ $distcc = 1 ]] ; then
# Check if distcc exists and is an executable # Check if distcc exists and is an executable
[[ ! -x $distccbinpath ]] && err "Distcc binary was either not found or is not an executable" [[ ! -x $distccbinpath ]] && err "Distcc binary was either not found or is not an executable"
@ -1261,31 +1082,31 @@ if [[ $distcc = 1 ]] ; then
PATH="$distccsymdirpath:$PATH" PATH="$distccsymdirpath:$PATH"
export DISTCC_HOSTS DISTCC_IO_TIMEOUT PATH export DISTCC_HOSTS DISTCC_IO_TIMEOUT PATH
fi fi
# If verbosebuild is set in bldpkg.conf or if verbosity is set to 1 from getopts, set V and # If verbosebuild is set in bldpkg.conf or if verbosity is set to 1 from getopts, set V and
# VERBOSE environment variables for make and cmake build systems to pick up. # VERBOSE environment variables for make and cmake build systems to pick up.
if [[ $verbosebuild = 1 ]] || [[ $getoptsverbosebuild = 1 ]] ; then if [[ $verbosebuild = 1 ]] || [[ $getoptsverbosebuild = 1 ]] ; then
V=1 V=1
VERBOSE=1 VERBOSE=1
else else
V=0 V=0
VERBOSE=0 VERBOSE=0
fi fi
export V VERBOSE export V VERBOSE
# If $noautoconfsite is unset in an individual package build file, export CONFIG_SITE variable into the build # If $noautoconfsite is unset in an individual package build file, export CONFIG_SITE variable into the build
# environment for a package's configure script to pickup. Most autoconf-compatible configure scripts will # environment for a package's configure script to pickup. Most autoconf-compatible configure scripts will
# automatically pick up this variable from the environment and speed up the initial configure process. # automatically pick up this variable from the environment and speed up the initial configure process.
if [[ -z $noautoconfsite ]] ; then if [[ -z $noautoconfsite ]] ; then
if [[ -n $configsite ]] && [[ -e $configsite ]]; then if [[ -n $configsite ]] && [[ -e $configsite ]]; then
CONFIG_SITE="$configsite" CONFIG_SITE="$configsite"
export CONFIG_SITE export CONFIG_SITE
fi fi
fi fi
# Do a preliminary package dependency check if $checkdependencies is set to 1 in bldpkg.conf # Do a preliminary package dependency check if $checkdependencies is set to 1 in bldpkg.conf
if [[ $checkdependencies = 1 ]] ; then if [[ $checkdependencies = 1 ]] ; then
info "Parsing $app 's dependency list..." info "Parsing $app 's dependency list..."
for packagedep in $requires; do for packagedep in $requires; do
depcount=$(find /share/doc -type f -name "$packagedep.SMBuild" | wc -l) depcount=$(find /share/doc -type f -name "$packagedep.SMBuild" | wc -l)
@ -1301,8 +1122,303 @@ if [[ $checkdependencies = 1 ]] ; then
sleep 0.5 sleep 0.5
fi fi
done done
fi
}
autobuild() {
trap "interruptoutput" INT
# We don't need colours and we set arch on purpose so that the packages land in the right directory
colours=0
arch="$HOSTTYPE"
# For loop to recurse through sections defined inside bldpkg.conf
for section in "${sections[@]}" ; do
pkgdest="$parenttmp/sml/packages/$arch/$section"
logdir="$parenttmp/sml/sml-buildlogs/$arch/$section"
mkdir -p $logdir
# If the defined section does not exist, abort
if [[ ! -d $section ]] ; then
err "Section directory '$section' not found!"
fi
cd $section
# This check compares a list of source directories with the list of the
# packages in the build list and warns of any missing package names
# in either of the two.
dirtempfile=$(mktemp $parenttmp/DIRECTORYNAMES."$section".XXXXXX)
dirfiletemppath="$dirtempfile"
dirlist=$(find . -type d -maxdepth 1 -mindepth 1 | sed 's@./@@' | sort > $dirfiletemppath)
packtempfile=$(mktemp $parenttmp/BUILDFILENAMES."$section".XXXXXX)
packfiletemppath="$packtempfile"
sort .buildlist.$section > $packfiletemppath
directorycount="$( wc -l < $dirtempfile )"
buildlistcount="$( wc -l < $packtempfile )"
# Get number of total packages
totalpkgnumber="$(wc -l < .buildlist.$section)"
if diff -u "$dirfiletemppath" "$packfiletemppath" > /dev/null 2>&1 ; then
diffstatus="0"
else
diffstatus="1"
fi
if [ "$diffstatus" != "0" ]; then
warn "In section '"$section"', the number of packages in the hidden file '.buildlist."$section"' is different to the number of package directories. Some packages may not have been added to this file/section directory. They are listed below:"
diff -u "$dirfiletemppath" "$packfiletemppath" || true
diff -u "$packfiletemppath" "$dirfiletemppath" || true
warn "Building anyways :-) "
sleep 2
fi
rm -f $packfiletemppath $dirfiletemppath
if [[ ! -f .buildlist.$section ]] ; then
err "Section build list file '.buildlist.$section' not found!"
fi
while IFS="" read -r package || [[ -n $package ]] ; do
unset app build version buildskipped
currentpkgnumber="$(grep -Ewn "^$package" .buildlist.$section | cut -d: -f 1)"
# We need this variable to switch back to the section directory inside the loop
currentpwd="$PWD"
cd $package
# Get the name of the buildfile and validate it
getbuildfileanddir
validatebldfile "$buildfile"
# Check if package installer exists or is installed on the system
# If the package installer does not exist in the set pkg destination, build it
if [ ! -f $pkgdest/$app-$version-*-$build.* ] ; then
# Store the source directory path the build was initiated from
# Presume that the build file name will match the name of the parent directory
# unless otherwise overridden using -f <buildfile>
stage0prep
touch "$parenttmp/BUILDING"
prepbuilddir
stage1prep
if [[ -z $buildskipped ]] ; then
trap "prepbuildoutput" EXIT
build
fi
# If the package is not installed in the system, install it
if [ ! -f /var/log/packages/$app-$version-*-$build ] ; then
upgradepkg --install-new $pkgdest/$app-$version-*-$build.*
else
info "'$app-$version' already installed, skipping it"
fi
elif [ ! -f /var/log/packages/$app-$version-*-$build ] ; then
upgradepkg --install-new $pkgdest/$app-$version-*-$build.*
else
info "Already built '$app-$version', skipping it"
fi
cd $currentpwd
done < ".buildlist.$section"
cd ..
done
}
manualbuild() {
trap "prepbuildoutput" EXIT
trap "interruptoutput" INT
# Display the package and its version we are building or resuming
if [[ -z $resumepkgbuild ]] ; then
info "Building package '$app' version '$version' ..."
else
info "Resuming build of package '$app' version '$version' ..."
fi
stage0prep
# If htmloutput is set to 1, echo $app, $version and $build as file names inside the parent build directory.
# This will output into an HTML file so that the basic status of the build process (whether started, stopped,
# interrupted or failed) can be viewed in the web browser.
if [[ $htmloutput = 1 ]] ; then
if [[ -n $autobuild ]] ; then
cat << EOF >> "$parenttmp/BUILDMONITOR"
<b>$commencedate | Building package $currentpkgnumber / $totalpkgnumber: <i><a href="/smlinux/pkgresults?pkg=$app&amp;smver=1.0&amp;arch=all&amp;resultnum=25">$app $version</a></i></b>
EOF
else
cat << EOF >> "$parenttmp/BUILDMONITOR"
<b>$commencedate | Building package <i><a href="/smlinux/pkgresults?pkg=$app&amp;smver=1.0&amp;arch=all&amp;resultnum=25">$app $version</a></i></b>
EOF
fi
touch "$parenttmp/BUILDING"
fi
# If $tmp exists, get the path to the $.app-$version.extraction.complete file
[[ -d $tmp ]] && buildresumepath="$(find $tmp -type f -name .$app-$version.extraction.complete)"
# If the above file exists, $resumepkgbuild and $autobuild are unset
# and $autoresumepkgbuild is set to 1 in bldpkg.conf, prompt the user
if [[ -f $buildresumepath ]] && [[ -z $resumepkgbuild ]] && [[ $autoresumepkgbuild = 1 ]] && [[ -z $autobuild ]]; then
read -r -p "[NOTIFY] Would you like to resume building? " yn
case "$yn" in
N|n|No|no) info "Nope? Alright." ;
unset resumepkgbuild ;;
*) info "Wise choice :-) "
resumepkgbuild=1 ;;
esac
fi
# If $resumepkgbuild variable is not set in getopts, only then execute prepbuilddir variable
[[ -z $resumepkgbuild ]] && prepbuilddir
stage1prep
# If $resumepkgbuild is set either in getopts or from the user prompt above, execute mkandenterbuilddir
# function to enter the build directory. This is being done because mkandenterbuilddir is part of prepbuilddir
# function in the build file and ignoring prepbuilddir will not cause mkandenterbuilddir to be invoked
# separately unless the build system is told to.
if [[ -n $resumepkgbuild ]] ; then
mkandenterbuilddir
# fixbuilddirpermissions places a file ".$app-$version.extraction.complete". Only resume the build if that file exists
if [[ ! -f $buildresumepath ]] ; then
err "Can't resume build of '"$app"'! Are you certain the source was extracted completely?"
else
cd ${buildresumepath%/*}
fi
fi
build
}
# Source the main configuration file holding all necessary values
if [[ -f $HOME/.bldpkg.conf ]] ; then
source "$HOME/.bldpkg.conf"
elif [[ -f /etc/bldpkg.conf ]] ; then
source /etc/bldpkg.conf
else
err "Bldpkg configuration file '/etc/bldpkg.conf' not found!"
fi fi
build # Error out if ccache and sccache are set in the /etc/bldpkg.conf
if [[ $ccache = 1 ]] && [[ $sccache = 1 ]] ; then
err "Please only enable either ccache or sccache in bldpkg.conf at the same time, not both."
fi
# End script # While loop for providing handy arguments to users. Some will override bldpkg.conf.
while getopts ':adef:ghj:o:rstvx' option; do
case "$option" in
a) autobuildmode=1 ;;
d) debug=1 ;; # Produce a debug build with -g3
e) extractprompt=0;
autoextract=1 ;; # Automatically extract the final pkg installer inside user's PWD
f) setbuildfile="$OPTARG" ;;
g) genchecksum ;;
h) help ;;
j) custommakeflags="$OPTARG" ;;
o) origbuildfile="$OPTARG" ;;
r) resumepkgbuild=1 ;;
s) showsummary=1 ;; # Show build summary at the end of the build irrespective of the build status
t) tmpfs=1 ;;
v) getoptsverbosebuild=1 ;;
x) set -xv ;; # Invoke bash's -x option for command tracing
*) help ;;
esac
done
# The getopts builtin sets the OPTIND environment variable whose value is set to 1 if no argument is given.
# For every argument passed, the number is incremented by 1. In our case, if OPTIND equals 1, no argument
# was passed. We therefore expect a package build file to be present which will then be validated and sourced.
if [[ $OPTIND = 1 ]] ; then
getbuildfileanddir
if [[ ! -f $buildfile ]] ; then
nopackagebuildfileerror
elif ! validatebldfile "$buildfile" ; then
err "'$buildfile' build file validation failed!"
else
manualbuild
fi
# If OPTIND is greater than 1, perform various checks and actions
elif [[ $OPTIND -gt 1 ]] ; then
# If MAKEFLAGS and custommakeflags is set, let custommakeflags take precedence
if [[ -n $custommakeflags ]] ; then
validatemakeflags $custommakeflags
MAKEFLAGS="-j$custommakeflags"
elif [[ -n $MAKEFLAGS ]] ; then
validatemakeflags $MAKEFLAGS
elif [[ -z $MAKEFLAGS ]] ; then
# Or fetch the number from nproc
MAKEFLAGS="-j$(nproc --all)"
fi
if [[ $autobuildmode = 1 ]] ; then
autobuild
elif [[ -z $autobuildmode ]] ; then
# Get the name of the default source file
getbuildfileanddir
# If $origbuildfile is set and is a file, check if $setbuildfile and $origbuildfile are the same
if [[ -n $origbuildfile ]] ; then
if [[ ! -f $origbuildfile ]] ; then
err "Original build file '$origbuildfile' does not exist!"
elif [[ -n $setbuildfile ]] && [[ -f $setbuildfile ]] ; then
if [[ $origbuildfile = $setbuildfile ]] ; then
err "Original build file and alternate build file are the same!"
fi
elif ! validatebldfile "$origbuildfile" ; then
err "Alternative build file validation failed!"
fi
fi
# If $setbuildfile is set and is a file, set buildfile to its value, source it and initialise the build
if [[ -n $setbuildfile ]] && [[ -f $setbuildfile ]] ; then
buildfile="$setbuildfile"
if ! validatebldfile "$buildfile" ; then
err "'$buildfile' validation failed!"
else
manualbuild
fi
# If $setbuildfile is set but a file passed as an argument is not found, give an error
elif [[ -n $setbuildfile ]] && [[ ! -f $setbuildfile ]] ; then
err "Build file '$setbuildfile' not found!"
# If the above two conditions don't meet, get the presumed $buildfile value as a file and source it
elif [[ -f $buildfile ]] ; then
if ! validatebldfile "$buildfile" ; then
err "'$buildfile' validation failed!"
else
manualbuild
fi
# If even that file is not found, throw an error and exit
else
nopackagebuildfileerror
fi
fi
fi
# Unset OPTIND and OPTARG
unset OPTIND OPTARG
# End of script

6
bldpkg.conf
View file

@ -7,6 +7,8 @@
# Please refer to sample.SMBuild.explained in the buildsamples directory # Please refer to sample.SMBuild.explained in the buildsamples directory
# for complete explanation. # for complete explanation.
sections=(base xorg gtk extra net xfce)
# Set the package extension. Makepkg will use this to figure out the # Set the package extension. Makepkg will use this to figure out the
# compression utility. Pick one out of tbz/tgz/txz/tlz. # compression utility. Pick one out of tbz/tgz/txz/tlz.
pkgext="tlz" pkgext="tlz"
@ -109,7 +111,7 @@ nontmpfsdir="$parenttmp/sml"
# openssh-8.1-x86_64-1sml.tlz should be placed in. Again, should be outside # openssh-8.1-x86_64-1sml.tlz should be placed in. Again, should be outside
# tmpfs dir. Also declared and defined in autobuild section file, so values # tmpfs dir. Also declared and defined in autobuild section file, so values
# may differ. This is for manual builds. # may differ. This is for manual builds.
pkgdest=${pkgdest:-$parenttmp/sml/packages/} pkgdest="$parenttmp/sml/packages/"
# Whether you want to preserve the build directory which contains the extracted source, # Whether you want to preserve the build directory which contains the extracted source,
# for example /tmp/sml/openssh.src/openssh-8.1. usetmpfs variable, if set to 1, will # for example /tmp/sml/openssh.src/openssh-8.1. usetmpfs variable, if set to 1, will
@ -135,7 +137,7 @@ showsummary=1
# Define whether to check for swap. 1 to enable, 0 to disable. # Define whether to check for swap. 1 to enable, 0 to disable.
swapcheck=0 swapcheck=0
# If above variable is set to 1, define minimum swap size required for huge # If above variable is set to 1, define swap size required for huge
# compiles in KiB # compiles in KiB
swapsize=2048000 swapsize=2048000