-
Notifications
You must be signed in to change notification settings - Fork 0
/
sync-slim.sh
129 lines (119 loc) · 5.29 KB
/
sync-slim.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#!/bin/sh -x
datestamp=`date +%Y%m%d%H%M`
singimg="/data/yangw/images/centos6-cvmfs.atlas.cern.ch.mini.img"
latestsingimg="/data/yangw/images/centos6-cvmfs.atlas.cern.ch.mini.img"
cmt="x86_64-slc6-gcc49-opt"
script=$(readlink -f $0)
scriptdir=$(dirname $script)
exec > /tmp/sync-single-singularity.log 2>&1
date
cd /data/yangw/uncvmfs/root
if [ ! -f $latestsingimg ]; then
echo ""
echo "---------- Making a new singularity image"
echo ""
#cd /data/yangw/images
#cp /cvmfs/atlas.cern.ch/repo/images/singularity/x86_64-centos6.img $singimg
#singularity expand -s 460800 $latestsingimg
singularity create -s 102400 $latestsingimg
# add the following empty directories to the image
#mkdir -p workdir/usatlas
#cd workdir
#find . -type f -exec rm {} \;
#tar cf - * | singularity import $latestsingimg
#cd /data/yangw/uncvmfs/root
tar --no-acls --no-xattrs --exclude=cvmfs/atlas.cern.ch/repo -cvf - * | singularity import $latestsingimg
else
echo ""
echo "---------- Updatng singularity image"
echo ""
fi
echo ">>> rsync ATLASLocalRootBase"
# rsync-exclude.rules.txt needs to be copied everytime because each singularity rsync run maybe
# have a big time gap and this file may be deleted during the gap.
cp $scriptdir/rsync-exclude.rules.txt /tmp/rsync-exclude.rules.txt
# we can not exclude java and x86_64-slc6-gcc62-opt from ATLASLocalRootBase, or voms-proxy-info will
# not work (and rucio may not work either)
singularity exec -w -B /data/yangw/uncvmfs/root/cvmfs:/mnt $latestsingimg \
rsync -aO --no-o --no-g --delete -H --no-A --no-X -v \
--filter='+s **/x86_64-slc6-gcc62-opt' \
--filter='+s **/java' \
--filter='+s **/*.java' \
--filter='+s **/*.jar' \
--filter='. /tmp/rsync-exclude.rules.txt' \
--filter='-s /atlas.cern.ch/repo/ATLASLocalRootBase/logDir' \
--filter='-s /atlas.cern.ch/repo/ATLASLocalRootBase/x86_64-MacOS' \
--filter='+s /atlas.cern.ch/repo/ATLASLocalRootBase' \
--filter='P /atlas.cern.ch/repo/*' \
--filter='-s /atlas.cern.ch/repo/*' \
/mnt/ /cvmfs
echo ">>> rsync other things, excluding ATLASLocalRootBase"
cp $scriptdir/rsync-exclude.rules.txt /tmp/rsync-exclude.rules.txt
# sw/software and database/DBReleases will be rsync-ed later
singularity exec -w -B /data/yangw/uncvmfs/root/cvmfs:/mnt $latestsingimg \
rsync -aO --no-o --no-g --delete -H --no-A --no-X -v \
--filter='. /tmp/rsync-exclude.rules.txt' \
--filter='P /atlas.cern.ch/repo/ATLASLocalRootBase' \
--filter='-s /atlas.cern.ch/repo/sw/BOINC' \
--filter='-s /atlas.cern.ch/repo/sw/Generators' \
--filter='-s /atlas.cern.ch/repo/sw/arc' \
--filter='P /atlas.cern.ch/repo/sw/database/DBRelease' \
--filter='-s /atlas.cern.ch/repo/sw/database/*' \
--filter='-s /atlas.cern.ch/repo/sw/muon' \
--filter='-s /atlas.cern.ch/repo/sw/pacman*' \
--filter='P /atlas.cern.ch/repo/sw/software/21.0' \
--filter='-s /atlas.cern.ch/repo/sw/software/*' \
--filter='-s /atlas.cern.ch/repo/sw/tdaq' \
--filter='-s /atlas.cern.ch/repo/sw/tzero' \
--filter='+s /atlas.cern.ch/repo/sw' \
--filter='+s /atlas.cern.ch/repo/conditions' \
--filter='-s /atlas.cern.ch/repo/*' \
/mnt/ /cvmfs
echo ">>> rsync sw/software/xxx"
cp $scriptdir/rsync-exclude.rules.txt /tmp/rsync-exclude.rules.txt
singularity exec -w -B /data/yangw/uncvmfs/root/cvmfs:/mnt $latestsingimg \
rsync -aO --no-o --no-g --delete -H --no-A --no-X -v \
--filter='. /tmp/rsync-exclude.rules.txt' \
--filter='-s **/lcg/releases/R' \
--filter='-s **/21.0.[0-1]' \
--filter='-s **/21.0.[0-1].*' \
--filter='-s **/21.0.1[0-4]*' \
--filter='-s **/21.0.1[6-9]*' \
--filter='-s **/21.0.[2-9]*' \
--filter='+s /21.0' \
--filter='-s /*' \
/mnt/atlas.cern.ch/repo/sw/software/ /cvmfs/atlas.cern.ch/repo/sw/software
echo ">>> rsync sw/database/DBRelease"
cp $scriptdir/rsync-exclude.rules.txt /tmp/rsync-exclude.rules.txt
singularity exec -w -B /data/yangw/uncvmfs/root/cvmfs:/mnt $latestsingimg \
rsync -aO --no-o --no-g --delete -H --no-A --no-X -v -L \
--filter='. /tmp/rsync-exclude.rules.txt' \
--filter='+s /DBRelease/current' \
--filter='-s /DBRelease/*' \
--filter='+s /DBRelease' \
--filter='-s /*' \
/mnt/atlas.cern.ch/repo/sw/database/ /cvmfs/atlas.cern.ch/repo/sw/database
echo ">>> Update current DBRelease link"
singularity exec -w -B /data/yangw/uncvmfs/root/cvmfs:/mnt $latestsingimg sh -x <<EOF
#!/bin/sh
cd /mnt/atlas.cern.ch/repo/sw/database/DBRelease
tgtdbrel=\$(readlink current)
cd /cvmfs/atlas.cern.ch/repo/sw/database/DBRelease
rm -rf \$tgtdbrel
ln -s current \$tgtdbrel
EOF
echo ">>> shrine the image size"
tmpimg=$latestsingimg.TMP
dd if=$latestsingimg bs=1M count=1 | dd ibs=31 skip=1 of=$tmpimg
dd if=$latestsingimg bs=1M skip=1 of=$tmpimg oflag=append conv=notrunc
sync
resize2fs -f -M $tmpimg
dd if=/dev/zero bs=1M mount=100 of=$tmpimg oflag=append conv=notrunc
/sbin/e2fsck -fy $tmpimg
/sbin/resize2fs -f $tmpimg
/sbin/tune2fs -m 0 $tmpimg
rm $latestsingimg.$cmt
dd if=$latestsingimg ibs=31 count=1 > $latestsingimg.$cmt
dd if=$tmpimg bs=1M of=$latestsingimg.$cmt oflag=append conv=notrunc
rm $tmpimg
date