Merge pull request #36 from Sunbird-inQuiry/release-5.7.0

Merge Release 5.7.0 into Release-6.0.0
Sunbird-inQuiry · Aug 9, 2023 · 4e05cb5 · 4e05cb5
2 parents b26d45e + 2962a9d
commit 4e05cb5
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 7 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -21,7 +21,7 @@ jobs:
       - run:
           name: Execute coverage report
           command: |
-            mvn clean scoverage:report
+            mvn clean scoverage:report -DCLOUD_STORE_GROUP_ID=$CLOUD_STORE_GROUP_ID -DCLOUD_STORE_ARTIFACT_ID=$CLOUD_STORE_ARTIFACT_ID -DCLOUD_STORE_VERSION=$CLOUD_STORE_VERSION
       - run:
           name: Save test results
           command: |

diff --git a/jobs-core/pom.xml b/jobs-core/pom.xml
@@ -79,9 +79,9 @@
             <version>0.1.1</version>
         </dependency>
         <dependency>
-            <groupId>org.sunbird</groupId>
-            <artifactId>cloud-store-sdk_2.12</artifactId>
-            <version>1.4.3</version>
+            <groupId>${CLOUD_STORE_GROUP_ID}</groupId>
+            <artifactId>${CLOUD_STORE_ARTIFACT_ID}</artifactId>
+            <version>${CLOUD_STORE_VERSION}</version>
             <exclusions>
                 <exclusion>
                     <artifactId>log4j</artifactId>

diff --git a/jobs-distribution/Dockerfile b/jobs-distribution/Dockerfile
@@ -7,6 +7,7 @@ RUN apt-get install -y imagemagick
 COPY target/jobs-distribution-1.0.tar.gz /tmp
 USER flink
 RUN tar -xvf /tmp/jobs-distribution-1.0.tar.gz -C $FLINK_HOME/lib/
+RUN cp $FLINK_HOME/opt/flink-s3-fs-presto-1.13.5.jar $FLINK_HOME/lib/flink-aaa-s3-fs-presto-1.13.5.jar
 USER root
 RUN rm -f /tmp/jobs-distribution-1.0.tar.gz
 USER flink
diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
@@ -109,8 +109,16 @@ spec:
         workingDir: /opt/flink
         command: ["/opt/flink/bin/standalone-job.sh"]
         args: ["start-foreground",
-               "--job-classname={{ .Values.job_classname }}", 
+               "--job-classname={{ .Values.job_classname }}",
+               {{- if eq .Values.csp "oci" }}
+               "-Dpresto.s3.access-key={{ .Values.s3_access_key}}",
+               "-Dpresto.s3.secret-key={{ .Values.s3_secret_key }}",
+               "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}",
+               "-Dpresto.s3.region={{ .Values.s3_region }}",
+               "-Dpresto.s3.path-style-access={{ .Values.s3_path_style_access }}",
+               {{- else }}
                "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}",
+               {{- end}}
                "-Dweb.submit.enable=false",
                "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
                "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}",
@@ -183,7 +191,15 @@ spec:
         workingDir: {{ .Values.taskmanager.flink_work_dir }}
         command: ["/opt/flink/bin/taskmanager.sh"]
         args: ["start-foreground",
+          {{- if eq .Values.csp "oci"}}
+          "-Dpresto.s3.access.key={{ .Values.s3_access_key}}",
+          "-Dpresto.s3.secret.key={{ .Values.s3_secret_key }}",
+          "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}",
+          "-Dpresto.s3.endpoint={{ .Values.s3_region }}",
+          "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}",
+          {{- else}}
           "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}",
+          {{- end}}
           "-Dweb.submit.enable=false",
           "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
           "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager",

diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2
@@ -5,6 +5,16 @@ repository: {{flink_repository|default('data-pipeline')}}
 image_tag: {{ image_tag }}
 azure_account: {{ azure_account }}
 azure_secret: {{ azure_secret }}
+csp: {{cloud_service_provider}}
+s3_access_key: {{ cloud_public_storage_accountname }}
+s3_secret_key: {{cloud_public_storage_secret}}
+{% if cloud_service_provider == "oci" %}
+s3_endpoint: {{oci_flink_s3_storage_endpoint}}
+s3_region: {{s3_region}}
+s3_path_style_access: true
+{% else %}
+s3_endpoint: {{cloud_public_storage_endpoint}}
+{% endif %}
 serviceMonitor:
   enabled: {{ service_monitor_enabled | lower}}
 
@@ -74,6 +84,18 @@ base_config: |
     job {
       env = "{{ env_name }}"
       enable.distributed.checkpointing = true
+      {% if cloud_service_provider == "oci" %}
+            statebackend {
+              s3 {
+                storage {
+                  endpoint = "{{ oci_flink_s3_storage_endpoint }}"
+                  container = "{{ flink_container_name }}"
+                  checkpointing.dir = "checkpoint"
+                }
+              }
+              base.url = "s3://"${job.statebackend.s3.storage.container}"/"${job.statebackend.s3.storage.checkpointing.dir}
+            }
+      {% else %}
       statebackend {
         blob {
           storage {
@@ -84,6 +106,7 @@ base_config: |
         }
         base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir}
       }
+    {% endif %}
     }
     task {
       parallelism = 1

diff --git a/kubernetes/pipelines/build/Jenkinsfile b/kubernetes/pipelines/build/Jenkinsfile
@@ -21,13 +21,15 @@ node('build-slave') {
                    commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
                    build_tag = sh(script: "echo " + params.inquiry_release_tag.split('/')[-1] + "_" + commit_hash + "_" + env.BUILD_NUMBER, returnStdout: true).trim()
                    echo "build_tag: " + build_tag
+                   cloud_store_group_id = params.CLOUD_STORE_GROUP_ID
+                   cloud_store_artifact_id = params.CLOUD_STORE_ARTIFACT_ID
+                   cloud_store_version = params.CLOUD_STORE_VERSION
 
                    stage('Build') {
                        env.NODE_ENV = "build"
                        print "Environment will be : ${env.NODE_ENV}"
                        sh '/opt/apache-maven-3.6.3/bin/mvn3.6 -v'
-                       sh '/opt/apache-maven-3.6.3/bin/mvn3.6 clean install -DskipTests'
-
+                       sh '/opt/apache-maven-3.6.3/bin/mvn3.6 clean install -DskipTests -DCLOUD_STORE_GROUP_ID=' + cloud_store_group_id + ' -DCLOUD_STORE_ARTIFACT_ID=' + cloud_store_artifact_id + ' -DCLOUD_STORE_VERSION=' + cloud_store_version
                    }
 
                    stage('Package') {