From 80f14475022d14440b7cbc9611c94925605864f3 Mon Sep 17 00:00:00 2001 From: Victor Hu Date: Tue, 17 Apr 2018 11:04:36 -0400 Subject: [PATCH 1/2] Commit sample postinstall scripts for the cuda9 kernel parameters for diskless images. --- postscripts/README.md | 10 + postscripts/postinstall/README.md | 11 + postscripts/postinstall/compute.postinstall | 37 ++ postscripts/postinstall/functions | 361 ++++++++++++++++++ .../postinstall/postinstall.d/README.md | 5 + .../postinstall.d/install-cuda9-simple | 132 +++++++ 6 files changed, 556 insertions(+) create mode 100644 postscripts/README.md create mode 100644 postscripts/postinstall/README.md create mode 100755 postscripts/postinstall/compute.postinstall create mode 100755 postscripts/postinstall/functions create mode 100644 postscripts/postinstall/postinstall.d/README.md create mode 100755 postscripts/postinstall/postinstall.d/install-cuda9-simple diff --git a/postscripts/README.md b/postscripts/README.md new file mode 100644 index 0000000..3a0290c --- /dev/null +++ b/postscripts/README.md @@ -0,0 +1,10 @@ +# xCAT Postscripts + +To use: + +1. Clone this repo down to `/install/` on your management node + + ``` + git clone git@github.com:xcat2/xcat-extensions.git + ``` + diff --git a/postscripts/postinstall/README.md b/postscripts/postinstall/README.md new file mode 100644 index 0000000..27b169c --- /dev/null +++ b/postscripts/postinstall/README.md @@ -0,0 +1,11 @@ +# postinstall scripts + +The `compute.postinstall` script will look into the `postinstall.d` directory and execute any postscripts that have 755 permission. + +To add this to your `osimage` definition: + +``` +# Assumes that this repo has been cloned to `/install` on your management node +chdef -t osimage -p postinstall=/install/xcat-extensions/postscripts/postinstall/compute.postinstall +``` + diff --git a/postscripts/postinstall/compute.postinstall b/postscripts/postinstall/compute.postinstall new file mode 100755 index 0000000..dd97d6f --- /dev/null +++ b/postscripts/postinstall/compute.postinstall @@ -0,0 +1,37 @@ +#!/bin/sh +#-- this script is run after all packages from $profile.pkglist are installed +#-- +#-- it gets these arguments: +#-- +#-- $1 = install root (chroot directory for profile) +#-- $2 = OS version +#-- $3 = architecture +#-- $4 = profile name +#-- $5 = work dir (where genimage is located) +#-- +#-- + +#set -e + +thisdir=`dirname $0` +thisscript=`basename $0` + +source $thisdir/functions $* + +unset ARCH +unset XCAT_IN_CHROOT; + + +return_value=0 + +# RB: Custom post install gorp. +# return the last error found, or, zero if all cleared. +for s in $(find $thisdir/postinstall.d -type f -executable -maxdepth 1 | sort); do + echo "`date` Running postscript: $s" + rc=0 + $s $installroot $osver $arch $profile $workdir || rc=$?; + echo "postscript $s return with $rc"; + [[ $rc != 0 ]] && return_value=$rc; +done +exit $return_value; + diff --git a/postscripts/postinstall/functions b/postscripts/postinstall/functions new file mode 100755 index 0000000..16931ff --- /dev/null +++ b/postscripts/postinstall/functions @@ -0,0 +1,361 @@ +# +# common bash functions for postscripts... +# + +# global variables for the rest of these functions... +installroot=$1 +osver=$2 +arch=$3 +profile=$4 +workdir=$5 + + +chrooted=${XCAT_IN_CHROOT:-no}; +# chrooted takes over for XCAT_IN_CHROOT from this point on. +unset ARCH +unset XCAT_IN_CHROOT; + + +function set_master { + if [ ! -z $installroot ]; then + # make sure we have the master... + # where do we put this thingee... probably in the + # netboot postscript + if [ -z "$MASTER" ]; then + rc=0; + m=`lsdef -t site clustersite -i master -c` || rc=1; + if [ $rc != 0 ]; then + echo "lsdef failed to find master"; + exit 1; + fi + echo $m; + MASTER=${m#*=} + echo MASTER=$MASTER + fi + fi + + if [ -z "$MASTER" ]; then + echo "$MASTER not defined" # stop right here if this is not defined... + exit 1; + fi; + export MASTER + +} + + +# exit trap to undo any mounts done earlier +function mnt_exit_trap { + local SOPTS=$-; set +x; + set +e + [ -z "$installroot" ] && return + # pull the mounts out submounts to the install directory from the /proc/mounts file. + while read -r ignore1 dir ignore2 + do + [ "${dir:0:${#installroot}}" = "${installroot}" ] && umount -rl "$dir" + done /dev/null; + echo $pkgname; +} + + + +function ispkg_installed() { + rpmfile=$1; + echo $rpmfile; + if [[ -z $rpmfile ]]; then + echo "ispkg_installed parameter missing"; exit 1; + fi + { + rpm_info=`rpm -qip $rpmfile`; + pkgname=`echo $rpm_info | perl -nle '/Name : (\S+)/ && print "$1"'` + rpmversion=`echo $rpm_info | perl -nle '/Version : (\S+)/ && print "$1"'` + rpmrelease=`echo $rpm_info | perl -nle '/Release : (\S+)/ && print "$1"'` + } 2>/dev/null + + if [[ -z $pkgname ]]; then + echo "$rpmfile Package Name Not found"; exit 1; + fi + rc=0; + { + ins_info=`rpm -qi $pkgname 2>/dev/null` || rc=$?; + insversion=`echo $ins_info | perl -nle '/Version : (\S+)/ && print "$1"'` + insrelease=`echo $ins_info | perl -nle '/Release : (\S+)/ && print "$1"'` + #installok=`echo $ins_info | perl -nle '/Status: install ok/ && print "1"'`; + } 2>/dev/null + + + if [[ "$insversion" = "$rpmversion" ]] && [[ "$insrelease" = "$rpmrelease" ]]; then + echo "$rpmfile already installed"; + rc=0; + else + echo "$rpmfile not installed"; + rc=1; + fi + return $rc; +} + +# parse the manifest and return +# list of file names to get that we don't currently have... +# param $1 -- manifest file name. +# returns global wget_list -- list of packages to get +# returns global pack_list -- complete list of packages.. + +function parse_manifest { + pack_list=() + wget_list=(); + local mf=$1; + + while read line; do + mfstmd5=$(echo $line | cut -f1 -d' ') + package=$(echo $line | cut -f2 -d' ') + echo mfstmd5=$mfstmd5, package=$package + pack_list+=($package) + pkgmd5=none + if [ -e $package ]; then + pkgmd5=$(md5sum $package | cut -f1 -d' '); + fi + if [ "$pkgmd5" != "$mfstmd5" ]; then + wget_list+=($package); + fi + done < $mf + +} + +function retrieve_packages { + local func="${FUNCNAME[0]}()" + + if [ -z $1 ]; then + logError "$func Missing src_dir" + return 1 + fi + if [ -z $2 ]; then + logError "$func Missing dest_dir" + return 1 + fi + local src_dir=$1 + local dest_dir=$2 + + local src_dir="http://$MASTER/$src_dir" + mkdir -p $dest_dir + pushd $dest_dir + local mf=manifest.txt + if [ -e $mf ]; then + rm -f $mf + fi + logInfo "$func Getting $src_dir/$mf" + wget --quiet $src_dir/$mf + + parse_manifest $mf + ##what do we need to get.... + local p + for p in ${wget_list[@]} + do + if [ -e $p ] + then # remove the file prior to retrieving it, + rm -f $p # wget will not replace the file, it will add a second + fi # one with a new name. + logInfo "$func Getting $src_dir/$p" + wget --quiet $src_dir/$p + #ls -ald $PWD/$p + done +} + + +array_contains () { + local SOPTS=$-; set +x; + local array="$1[@]" + local seeking=$2 + local in=1 + for element in "${!array}"; do + if [[ $element == $seeking ]]; then + in=0 + break + fi + done + set -$SOPTS + return $in +} + +# read in a script list and return the values in the arrays indicated. +# will scan the directory, fill out the list and +# remove the blacklisted items... +# +function get_local_script_list { + postdir=$1; + if [[ ${#XCAT_PS_LOCAL[@]} == 0 ]]; then + for s in $(find $postdir -maxdepth 1 -type f -executable | sort); do + XCAT_PS_LOCAL+=(`basename $s`) + done + fi + nl=(); + for s in ${XCAT_PS_LOCAL[@]}; do + if ! array_contains XCAT_PS_LOCAL_BL $s; then + nl+=($s); + else + echo "$(basename ${BASH_SOURCE[1]}) blacklisted $s" + fi + done + XCAT_PS_LOCAL=(${nl[@]}) +} +function get_common_script_list { + postdir=$1; + #white list and black lists + if [[ ${#XCAT_PS_COMMON[@]} == 0 ]]; then + for s in $(find $postdir -maxdepth 1 -type f -executable | sort); do + XCAT_PS_COMMON+=(`basename $s`) + done + fi + nl=(); + for s in ${XCAT_PS_COMMON[@]}; do + if ! array_contains XCAT_PS_COMMON_BL $s; then + nl+=($s); + else + echo "$(basename ${BASH_SOURCE[1]}) blacklisted $s" + fi + done + XCAT_PS_COMMON=(${nl[@]}) +} + +function run_postscripts { + local postscripts="$1[@]" + local postdir=$2; + rvalue=0; + for n in "${!postscripts}"; do + s=$postdir/$n + echo "`date` Running postscript: $s" + rc=0 + $s $installroot $osver $arch $profile $workdir || rc=$?; + echo "postscript $s return with $rc"; + [[ $rc != 0 ]] && rvalue=$rc; + done + return $rvalue; + +} + + + diff --git a/postscripts/postinstall/postinstall.d/README.md b/postscripts/postinstall/postinstall.d/README.md new file mode 100644 index 0000000..2bc4103 --- /dev/null +++ b/postscripts/postinstall/postinstall.d/README.md @@ -0,0 +1,5 @@ +## Scripts + +* install-cuda9-simple + + This script works only for diskless images. Helps configure NVIDIA kerenl options that need to be active in the diskless image on boot. diff --git a/postscripts/postinstall/postinstall.d/install-cuda9-simple b/postscripts/postinstall/postinstall.d/install-cuda9-simple new file mode 100755 index 0000000..733239f --- /dev/null +++ b/postscripts/postinstall/postinstall.d/install-cuda9-simple @@ -0,0 +1,132 @@ +#!/bin/bash +# +#-- this script is run after all packages from $profile.pkglist are installed +#-- +#-- it gets these arguments: +#-- +#-- $1 = install root (chroot directory for profile) +#-- $2 = OS version +#-- $3 = architecture +#-- $4 = profile name +#-- $5 = work dir (where genimage is located) +#-- + +set -e + +thisdir=`dirname $0` +thisscript=`basename $0` +source $thisdir/../functions $* + +mount_dev +mount_proc + +run_chroot_if_required $thisdir/$thisscript + +if [ $chrooted == "no" ]; then + echo "Error: This must be run in chroot environment for diskless images" + exit 0 +fi + +# +# Setting up the CUDA paths +# +if [ ! -e /etc/profile.d/cuda.sh ]; then + cat >/etc/profile.d/cuda.sh <<"EOF" +CUDABIN=/usr/local/cuda/bin +CUDALIB=/usr/local/cuda/lib64 + +if ! echo $LD_LIBRARY_PATH | /bin/grep -q $CUDALIB +then + LD_LIBRARY_PATH=$LD_LIBRARY_PATH${LD_LIBRARY_PATH:+:}$CUDALIB +fi + +if ! echo $PATH | /bin/grep -q $CUDABIN +then + PATH=$PATH${PATH:+:}$CUDABIN +fi + +export LD_LIBRARY_PATH PATH +EOF +fi + +# uncomment this for debugging +# set -x; export PS4='+($(basename ${BASH_SOURCE}):${LINENO}):' + +# udev patch according to: +# https://github.ibm.com/DCS-research/WSC-coral/issues/224 +if [[ $(yum list cuda | awk '/cuda.ppc64le/ {print $2}') =~ ^9\.2\. ]]; then + if [ -e /lib/udev/rules.d/40-redhat.rules ]; then + sed -i /SUBSYSTEM==\"memory\"/d /lib/udev/rules.d/40-redhat.rules + fi +else + # hard coded the above for CUDA 9.2, print warning message incase this needs to be updated + logInfo "WARNING memory was NOT removed from /lib/udev/rules.d/40-redhat.rules, is this expected?" +fi + +# +# use dracut module-setup to enable the options in nvidia.conf file +# +nvidia_conf=/etc/modprobe.d/nvidia.conf +msetup=/usr/lib/dracut/modules.d/95nvidia/module-setup.sh +mkdir -p $(dirname $msetup); +cat >$msetup <<"EOF" +#!/bin/bash +# module setup file for dracut +# nvidia patch described in: +# https://github.ibm.com/DCS-research/WSC-coral/issues/324 + +check() { + + return 0 +} + +depends() { + return 0; +} + +installkernel() { + return 0; +} + +install() { + [ -d $initdir/etc/modprobe.d/ ] || mkdir $initdir/etc/modprobe.d + cat >$initdir/etc/modprobe.d/nvidia.conf <<"MPEOF" +#created by dracut module-setup.sh +options nvidia NVreg_EnableStreamMemOPs=1 NVreg_RegistryDwords="PeerMappingOverride=1" +MPEOF +} + +EOF + +chmod 755 $msetup +logInfo "Adding nvidia to dracutmodules in /etc/dracut.conf.d/nvidia_patch.conf..." +cat >/etc/dracut.conf.d/nvidia_patch.conf <$perst_svc < Date: Tue, 17 Apr 2018 11:14:42 -0400 Subject: [PATCH 2/2] Fix some wording in the top level Readme file --- postscripts/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/postscripts/README.md b/postscripts/README.md index 3a0290c..1e400db 100644 --- a/postscripts/README.md +++ b/postscripts/README.md @@ -1,10 +1,10 @@ -# xCAT Postscripts +# xCAT Postscripts Samples -To use: - -1. Clone this repo down to `/install/` on your management node +1. To use, clone this repo down to `/install/` on your management node ``` + cd /install + # may want to clone this read only git clone git@github.com:xcat2/xcat-extensions.git ```