forked from tdozat/Parser-v3
-
Notifications
You must be signed in to change notification settings - Fork 0
/
slurm_train_dqm.sh
79 lines (70 loc) · 2.74 KB
/
slurm_train_dqm.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
#!/bin/bash
#SBATCH --partition=jag-urgent --qos=normal
#SBATCH --nodes=1
#SBATCH --job-name=tagparse
#SBATCH --ntasks-per-node=1
#SBATCH --mem=10240
#SBATCH --gres=gpu:1
#SBATCH --output=/sailhome/pengqi/logs/slurm-%j.out
#SBATCH [email protected]
#SBATCH --mail-type=FAIL
echo "SLURM_JOBID="$SLURM_JOBID
echo "SLURM_JOB_NODELIST"=$SLURM_JOB_NODELIST
echo "SLURM_NNODES"=$SLURM_NNODES
echo "SLURMTMPDIR="$SLURMTMPDIR
echo "working directory = "$SLURM_SUBMIT_DIR
LANGUAGE=$1
TREEBANK=$2
LC=$3
TB=$4
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/cuda/bin:$PATH
export CUDA_HOME=/usr/local/cuda
export PATH=/u/scr/pengqi/anaconda3_slurm/bin:$PATH
export LD_LIBRARY_PATH=/u/scr/pengqi/anaconda3_slurm/lib:$LD_LIBRARY_PATH
cd /u/scr/pengqi/Parser-v3
STACK=stack_dqm
mkdir -p $STACK
echo $LANGUAGE $TREEBANK $LC $TB
if [ ! -e $STACK/$LANGUAGE-$TREEBANK ]
then
touch $STACK/$LANGUAGE-$TREEBANK
if [ "$LC" == "zh" ]
then
pretrained_file=data/embeddings/"$LANGUAGE"T/"$LC".vectors.xz
elif [ "$LC" == "no" ]
then
if [ "$TB" == "bokmaal" ]
then
pretrained_file=data/embeddings/"$LANGUAGE"-Bokmaal/"$LC"_bokmaal.vectors.xz
else
pretrained_file=data/embeddings/"$LANGUAGE"-Nynorsk/"$LC"_nynorsk.vectors.xz
fi
else
pretrained_file=data/embeddings/$LANGUAGE/"$LC".vectors.xz
fi
basename=CoNLL18/UD_$LANGUAGE-$TREEBANK
basename1=CoNLL18_DQM/UD_$LANGUAGE-$TREEBANK
train_conllus=data/$basename/"$LC"_$TB-ud-train.conllu
dev_conllus=data/$basename/"$LC"_$TB-ud-dev.conllu
if [ ! -e $dev_conllus ]
then
dev_conllus=$train_conllus
fi
python main.py --save_dir saves/$basename1/Tagger train TaggerNetwork --DEFAULT train_conllus=$train_conllus dev_conllus=$dev_conllus --FormPretrainedVocab pretrained_file=$pretrained_file --config_file config/DQM17.cfg --force --noscreen
python main.py --save_dir saves/$basename1/Tagger run $train_conllus $dev_conllus
tagged_train_conllus=saves/$basename1/Tagger/parsed/$train_conllus
tagged_dev_conllus=saves/$basename1/Tagger/parsed/$dev_conllus
python scripts/reinsert_compounds.py $train_conllus $tagged_train_conllus
if [[ "$train_conllus" != "$dev_conllus" ]]; then
python scripts/reinsert_compounds.py $dev_conllus $tagged_dev_conllus
fi
python main.py --save_dir saves/$basename1/Parser train ParserNetwork --DEFAULT train_conllus=$tagged_train_conllus dev_conllus=$tagged_dev_conllus --FormPretrainedVocab pretrained_file=$pretrained_file --config_file config/DQM17.cfg --force --noscreen
python main.py --save_dir saves/$basename1/Parser run $tagged_train_conllu $tagged_dev_conllus
if [ $? -eq 0 ]
then
echo "Success" > $STACK/$LANGUAGE-$TREEBANK
else
echo "Failure" > $STACK/$LANGUAGE-$TREEBANK
fi
fi