-
Notifications
You must be signed in to change notification settings - Fork 6
/
.gitlab-ci.yml
121 lines (117 loc) · 10.8 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
stages:
- build_qa
- deploy_qa
- build_live
- deploy_live
build_qa:
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
services:
- docker:dind
tags:
- meedan
- meedan-labs
stage: build_qa
variables:
AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
script:
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $ECR_API_BASE_URL
- docker build --cache-from "$ECR_API_BASE_URL/qa/alegre/api:latest" -f production/Dockerfile -t "$ECR_API_BASE_URL/qa/alegre/api:$CI_COMMIT_SHA" -t "$ECR_API_BASE_URL/qa/alegre/api:latest" .
- docker push "$ECR_API_BASE_URL/qa/alegre/api:$CI_COMMIT_SHA"
- docker push "$ECR_API_BASE_URL/qa/alegre/api:latest"
only:
- develop
deploy_qa:
image: python:3-alpine
stage: deploy_qa
tags:
- meedan
- meedan-labs
variables:
AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
script:
- apk add --no-cache curl python3 py3-pip git
- pip install botocore==1.31.58
- pip install boto3==1.28.58
- pip install ecs-deploy==1.14.0
- pip install awscli==1.29.59
- aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /qa/alegre/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/qa/alegre/##' > env.qa.names
- for NAME in `cat env.qa.names`; do echo -n "-s qa-alegre-migration $NAME /qa/alegre/$NAME " >> qa-alegre-migration.env.args; done
- ecs update qa-alegre-migration --image qa-alegre-migration $QA_ECR_API_BASE_URL:$CI_COMMIT_SHA --exclusive-env -e qa-alegre-migration APP alegre -e qa-alegre-migration DEPLOY_ENV qa -e qa-alegre-migration AWS_REGION $AWS_DEFAULT_REGION -e qa-alegre-migration ALEGRE_PORT 8000 --exclusive-secrets `cat qa-alegre-migration.env.args`
- taskArn=$(aws ecs run-task --cluster ecs-qa --task-definition qa-alegre-migration --query 'tasks[].taskArn' --output text)
- echo "Migration task started - $taskArn"
- aws ecs wait tasks-stopped --cluster ecs-qa --tasks $taskArn
- echo "Migration task finished."
- for NAME in `cat env.qa.names`; do echo -n "-s qa-alegre-c $NAME /qa/alegre/$NAME " >> qa-alegre-c.env.args; done
- ecs deploy ecs-qa qa-alegre --diff --image qa-alegre-c $QA_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e qa-alegre-c APP alegre -e qa-alegre-c PERSISTENT_DISK_PATH /mnt/models/video -e qa-alegre-c DEPLOY_ENV qa -e qa-alegre-c ALEGRE_PORT 8000 --exclusive-secrets `cat qa-alegre-c.env.args`
- for NAME in `cat env.qa.names`; do echo -n "-s qa-alegre-indiansbert $NAME /qa/alegre/$NAME " >> qa-alegre-indiansbert.env.args; done
- ecs deploy ecs-qa qa-alegre-indiansbert --diff --image qa-alegre-indiansbert $QA_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e qa-alegre-indiansbert MODEL_NAME indiansbert -e qa-alegre-indiansbert SENTENCE_TRANSFORMERS_HOME /mnt/models/indiansbert-cache -e qa-alegre-indiansbert APP alegre -e qa-alegre-indiansbert DEPLOY_ENV qa -e qa-alegre-indiansbert ALEGRE_PORT 8000 --exclusive-secrets `cat qa-alegre-indiansbert.env.args`
- for NAME in `cat env.qa.names`; do echo -n "-s qa-alegre-meantokens $NAME /qa/alegre/$NAME " >> qa-alegre-meantokens.env.args; done
- ecs deploy ecs-qa qa-alegre-meantokens --diff --image qa-alegre-meantokens $QA_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e qa-alegre-meantokens MODEL_NAME meantokens -e qa-alegre-meantokens SENTENCE_TRANSFORMERS_HOME /mnt/models/meantokens-cache -e qa-alegre-meantokens APP alegre -e qa-alegre-meantokens DEPLOY_ENV qa -e qa-alegre-meantokens ALEGRE_PORT 8000 --exclusive-secrets `cat qa-alegre-meantokens.env.args`
- for NAME in `cat env.qa.names`; do echo -n "-s qa-alegre-fptg $NAME /qa/alegre/$NAME " >> qa-alegre-fptg.env.args; done
- ecs deploy ecs-qa qa-alegre-fptg --diff --image qa-alegre-fptg $QA_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e qa-alegre-fptg MODEL_NAME mdebertav3filipino -e qa-alegre-fptg SENTENCE_TRANSFORMERS_HOME /mnt/models/fptg-cache -e qa-alegre-fptg APP alegre -e qa-alegre-fptg DEPLOY_ENV qa -e qa-alegre-fptg ALEGRE_PORT 8000 --exclusive-secrets `cat qa-alegre-fptg.env.args`
- for NAME in `cat env.qa.names`; do echo -n "-s qa-alegre-multilingual $NAME /qa/alegre/$NAME " >> qa-alegre-multilingual.env.args; done
- ecs deploy ecs-qa qa-alegre-multilingual --diff --image qa-alegre-multilingual $QA_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e qa-alegre-multilingual MODEL_NAME paraphrasemultilingualmpnetbasev2 -e qa-alegre-multilingual SENTENCE_TRANSFORMERS_HOME /mnt/models/multilingual-cache -e qa-alegre-multilingual APP alegre -e qa-alegre-multilingual DEPLOY_ENV qa -e qa-alegre-multilingual ALEGRE_PORT 8000 --exclusive-secrets `cat qa-alegre-multilingual.env.args`
- for NAME in `cat env.qa.names`; do echo -n "-s qa-alegre-worker-c $NAME /qa/alegre/$NAME " >> qa-alegre-worker.env.args; done
- ecs deploy ecs-qa qa-alegre-worker --diff --image qa-alegre-worker-c $QA_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e qa-alegre-worker-c APP alegre -e qa-alegre-worker-c DEPLOY_ENV qa -e qa-alegre-worker-c ALEGRE_PORT 8000 --exclusive-secrets `cat qa-alegre-worker.env.args`
- echo "new Image was deployed $QA_ECR_API_BASE_URL:$CI_COMMIT_SHA"
only:
- develop
build_live:
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
services:
- docker:dind
tags:
- meedan
- meedan-labs
stage: build_live
variables:
AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
script:
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $ECR_API_BASE_URL
- docker build -f production/Dockerfile -t "$ECR_API_BASE_URL/live/alegre/api:$CI_COMMIT_SHA" .
- docker push "$ECR_API_BASE_URL/live/alegre/api:$CI_COMMIT_SHA"
only:
- master
deploy_live:
image: python:3-alpine
stage: deploy_live
when: manual
tags:
- meedan
- meedan-labs
variables:
AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
script:
- apk add --no-cache curl jq python3 py3-pip git
- pip install botocore==1.31.58
- pip install boto3==1.28.58
- pip install ecs-deploy==1.14.0
- pip install awscli==1.29.59
- aws ssm get-parameters-by-path --region $AWS_DEFAULT_REGION --path /live/alegre/ --recursive --with-decryption --output text --query "Parameters[].[Name]" | sed -E 's#/live/alegre/##' > env.live.names
- for NAME in `cat env.live.names`; do echo -n "-s live-alegre-migration $NAME /live/alegre/$NAME " >> live-alegre-migration.env.args; done
- ecs update live-alegre-migration --image live-alegre-migration $LIVE_ECR_API_BASE_URL:$CI_COMMIT_SHA --exclusive-env -e live-alegre-migration APP alegre -e live-alegre-migration DEPLOY_ENV live -e live-alegre-migration AWS_REGION $AWS_DEFAULT_REGION -e live-alegre-migration ALEGRE_PORT 8000 --exclusive-secrets `cat live-alegre-migration.env.args`
- taskArn=$(aws ecs run-task --cluster ecs-live --task-definition live-alegre-migration --query 'tasks[].taskArn' --output text)
- echo "Migration task started - $taskArn"
- aws ecs wait tasks-stopped --cluster ecs-live --tasks $taskArn
- echo "Migration task finished."
- for NAME in `cat env.live.names`; do echo -n "-s live-alegre-c $NAME /live/alegre/$NAME " >> live-alegre-c.env.args; done
- ecs deploy ecs-live live-alegre --image live-alegre-c $LIVE_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e live-alegre-c APP alegre -e live-alegre-c PERSISTENT_DISK_PATH /mnt/models/video -e live-alegre-c DEPLOY_ENV live -e live-alegre-c ALEGRE_PORT 8000 --exclusive-secrets `cat live-alegre-c.env.args`
- for NAME in `cat env.live.names`; do echo -n "-s live-alegre-indiansbert $NAME /live/alegre/$NAME " >> live-alegre-indiansbert.env.args; done
- ecs deploy ecs-live live-alegre-indiansbert --diff --image live-alegre-indiansbert $LIVE_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e live-alegre-indiansbert MODEL_NAME indiansbert -e live-alegre-indiansbert SENTENCE_TRANSFORMERS_HOME /mnt/models/indiansbert-cache -e live-alegre-indiansbert APP alegre -e live-alegre-indiansbert DEPLOY_ENV live -e live-alegre-indiansbert ALEGRE_PORT 8000 --exclusive-secrets `cat live-alegre-indiansbert.env.args`
- for NAME in `cat env.live.names`; do echo -n "-s live-alegre-meantokens $NAME /live/alegre/$NAME " >> live-alegre-meantokens.env.args; done
- ecs deploy ecs-live live-alegre-meantokens --diff --image live-alegre-meantokens $LIVE_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e live-alegre-meantokens MODEL_NAME meantokens -e live-alegre-meantokens SENTENCE_TRANSFORMERS_HOME /mnt/models/meantokens-cache -e live-alegre-meantokens APP alegre -e live-alegre-meantokens DEPLOY_ENV live -e live-alegre-meantokens ALEGRE_PORT 8000 --exclusive-secrets `cat live-alegre-meantokens.env.args`
- for NAME in `cat env.live.names`; do echo -n "-s live-alegre-fptg $NAME /live/alegre/$NAME " >> live-alegre-fptg.env.args; done
- ecs deploy ecs-live live-alegre-fptg --diff --image live-alegre-fptg $LIVE_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e live-alegre-fptg MODEL_NAME mdebertav3filipino -e live-alegre-fptg SENTENCE_TRANSFORMERS_HOME /mnt/models/fptg-cache -e live-alegre-fptg APP alegre -e live-alegre-fptg DEPLOY_ENV live -e live-alegre-fptg ALEGRE_PORT 8000 --exclusive-secrets `cat live-alegre-fptg.env.args`
- for NAME in `cat env.live.names`; do echo -n "-s live-alegre-multilingual $NAME /live/alegre/$NAME " >> live-alegre-multilingual.env.args; done
- ecs deploy ecs-live live-alegre-multilingual --diff --image live-alegre-multilingual $LIVE_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e live-alegre-multilingual MODEL_NAME paraphrasemultilingualmpnetbasev2 -e live-alegre-multilingual SENTENCE_TRANSFORMERS_HOME /mnt/models/multilingual-cache -e live-alegre-multilingual APP alegre -e live-alegre-multilingual DEPLOY_ENV live -e live-alegre-multilingual ALEGRE_PORT 8000 --exclusive-secrets `cat live-alegre-multilingual.env.args`
- for NAME in `cat env.live.names`; do echo -n "-s live-alegre-worker-c $NAME /live/alegre/$NAME " >> live-alegre-worker.env.args; done
- ecs deploy ecs-live live-alegre-worker --diff --image live-alegre-worker-c $LIVE_ECR_API_BASE_URL:$CI_COMMIT_SHA --timeout 1200 --exclusive-env -e live-alegre-worker-c APP alegre -e live-alegre-worker-c DEPLOY_ENV live -e live-alegre-worker-c ALEGRE_PORT 8000 --exclusive-secrets `cat live-alegre-worker.env.args`
- echo "new Image was deployed $LIVE_ECR_API_BASE_URL:$CI_COMMIT_SHA"
only:
- master