Merge pull request #83 from Vivek-M-08/cbrelease-4.8.12

added the configs for survey status spark job

ansible/roles/data-products-deploy/defaults/main.yml

@@ -103,6 +103,8 @@ ml_report_path: "standalone-reports/ml-report"
 report_solution_ids: ""
 report_grace_period: "2"
 report_base_url_for_evidences: "www.https://igotkarmayogi.gov.in/"
+report_survey_question_report_columns_config: "{"reportColumns":{"createdBy":"UUID","organisation_name":"Organisation Name","organisation_id":"Organisation Id","surveyName":"Survey Name","surveyId":"Survey Id","surveySubmissionId":"Survey Submission Id","questionExternalId":"Question External Id","questionName":"Question","questionResponseLabel":"Answer","evidences":"Evidences","remarks":"Remarks"},"userProfileColumns":{"maskedPhone":"Masked Phone","firstName":"First Name","profileDetails-academics-nameOfQualification":"Academics Qualification","profileDetails-academics-type":"Academics Type","last_login":"Last Login"},"sortingColumns":"UUID,First Name,Masked Phone,Organisation Name,Organisation Id,Survey Name,Survey Id,Survey Submission Id,Question External Id,Question,Answer,Evidences,Remarks,Academics Qualification,Academics Type,Last Login"}"
+report_survey_status_report_columns_config: "{"reportColumns":{"createdBy":"UUID","organisation_name":"Organisation Name","survey_name":"Survey Name","surveyId":"Survey Id","survey_submission_id":"Survey Submission Id"},"userProfileColumns":{"firstName":"First Name","lastName":"Last Name","identifier":"Identifier","last_login":"Last Login"},"sortingColumns":"UUID,First Name,Last Name,Identifier,Last Login,Organisation Name,Survey Name,Survey Id,Survey Submission Id, Status of Submission, Submission Date"}"
 
 comms_console_prarambh_email_suffix: "[email protected]"
 comms_console_num_days_to_consider: "15"
@@ -265,6 +267,10 @@ run_survey_question_report_job:
   survey-question-report:
     hour: 11
     minute: 00
+run_survey_status_report_job:
+  survey-status-report:
+    hour: 11
+    minute: 30
 run_monitor_job:
   monitor-job-summ:
     hour: 03

ansible/roles/data-products-deploy/tasks/main.yml

@@ -297,6 +297,13 @@
     - spark1-jobs
     - default-jobs
 
+- name: Create daily cron jobs for survey-status-report
+  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh survey-status-report"
+  with_dict: "{{ run_survey_status_report_job }}"
+  tags:
+    - spark1-jobs
+    - default-jobs
+
 - name: Create monthly cron jobs for learner-leaderboard
   cron: name="{{env}}-{{ item.key }}" day={{ item.value.day }} minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh learner-leaderboard"
   with_dict: "{{ run_learner_leaderboard_job }}"

ansible/roles/data-products-deploy/templates/model-config.j2

@@ -146,8 +146,11 @@ config() {
         echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.kcm.KCMJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","appPostgresHost":"{{ app_postgres_host }}","appPostgresSchema":"sunbird","appPostgresUsername":"sunbird","appPostgresCredential":"sunbird","postgresCompetencyTable":"data_node","postgresCompetencyHierarchyTable":"node_mapping","dwPostgresHost":"{{ dw_postgres_host }}","dwPostgresSchema":"warehouse","dwPostgresUsername":"postgres","dwPostgresCredential":"{{ dw_postgres_credential }}","dwKcmDictionaryTable":"kcm_dictionary","dwKcmContentTable":"kcm_content_mapping","key":"{{ dp_storage_key_config }}","secret":"{{ dp_storage_secret_config }}","store":"{{ report_storage_type }}","container":"{{ s3_storage_container }}","cutoffTime":"60.0","kcmReportPath":"{{ kcm_report_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","org":"{{ dashboards_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"Data Warehouse Sync","deviceMapping":false}'
         ;;
         "survey-question-report")
-         echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.survey.question.QuestionReportJob","modelParams":{"debug":"false","validation":"false","solutionIDs":"'$reportSolutionIDs'","gracePeriod":"{{ report_grace_period }}","baseUrlForEvidences":"{{ report_base_url_for_evidences }}","mlReportPath":"{{ ml_report_path }}", "redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","sparkMongoConnectionHost":"192.168.3.178","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserFeedTable":"{{ dashboards_cassandra_user_feed_table }}","mongoDatabase":"{{ dashboards_mongo_survey_observation_db }}","mongoDBCollection":"{{ dashboards_mongo_survey_collection }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}"}}},"output":[],"parallelization":48,"appName":"Survey Question Report","deviceMapping":false}'
-         ;;
+        echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.survey.question.QuestionReportJob","modelParams":{"debug":"false","validation":"false","solutionIDs":"'$reportSolutionIDs'","gracePeriod":"{{ report_grace_period }}","baseUrlForEvidences":"{{ report_base_url_for_evidences }}","mlReportPath":"{{ ml_report_path }}","surveyQuestionReportColumnsConfig":"{{ report_survey_question_report_columns_config }}","surveyStatusReportColumnsConfig":"{{ report_survey_status_report_columns_config }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","sparkMongoConnectionHost":"192.168.3.178","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserFeedTable":"{{ dashboards_cassandra_user_feed_table }}","mongoDatabase":"{{ dashboards_mongo_survey_observation_db }}","mongoDBCollection":"{{ dashboards_mongo_survey_collection }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}"}}},"output":[],"parallelization":48,"appName":"Survey Question Report","deviceMapping":false}'
+        ;;
+        "survey-status-report")
+        echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.report.survey.status.StatusReportJob","modelParams":{"debug":"false","validation":"false","solutionIDs":"'$reportSolutionIDs'","gracePeriod":"{{ report_grace_period }}","baseUrlForEvidences":"{{ report_base_url_for_evidences }}","mlReportPath":"{{ ml_report_path }}","surveyQuestionReportColumnsConfig":"{{ report_survey_question_report_columns_config }}","surveyStatusReportColumnsConfig":"{{ report_survey_status_report_columns_config }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","sparkMongoConnectionHost":"192.168.3.178","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserFeedTable":"{{ dashboards_cassandra_user_feed_table }}","mongoDatabase":"{{ dashboards_mongo_survey_observation_db }}","mongoDBCollection":"{{ dashboards_mongo_survey_collection }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}"}}},"output":[],"parallelization":48,"appName":"Survey Status Report","deviceMapping":false}'
+        ;;
         "in-app-review-feed")
         echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.dashboard.survey.inappreview.InAppReviewJob","modelParams":{"debug":"false","validation":"false","reportSyncEnable":"{{ operational_report_sync_enable }}","redisHost":"{{ dashboards_redis_host }}","redisPort":"{{ dashboards_redis_port }}","redisDB":"{{ dashboards_redis_db }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","sparkDruidRouterHost":"{{ druid_router_host }}","sparkElasticsearchConnectionHost":"{{ single_node_es_host }}","fracBackendHost":"{{ dashboards_frac_backend_host }}","cassandraUserKeyspace":"{{ user_table_keyspace }}","cassandraCourseKeyspace":"{{ course_keyspace }}","cassandraHierarchyStoreKeyspace":"{{ hierarchy_store_keyspace }}","cassandraUserFeedKeyspace":"{{ dashboards_cassandra_user_feed_keyspace }}","cassandraUserTable":"{{ dashboards_cassandra_user_table }}","cassandraUserRolesTable":"{{ dashboards_cassandra_user_roles_table }}","cassandraOrgTable":"{{ dashboards_cassandra_org_table }}","cassandraUserEnrolmentsTable":"{{ dashboards_cassandra_user_enrolments_table }}","cassandraContentHierarchyTable":"{{ dashboards_cassandra_content_hierarchy_table }}","cassandraRatingSummaryTable":"{{ dashboards_cassandra_rating_summary_table }}","cassandraRatingsTable":"{{ dashboards_cassandra_ratings_table }}","cassandraUserAssessmentTable":"{{ dashboards_cassandra_user_assessment_table }}","cassandraOrgHierarchyTable":"{{ dashboards_cassandra_org_hierarchy_table }}","cassandraCourseBatchTable":"{{ dashboards_cassandra_course_batch_table }}","cassandraLearnerStatsTable":"{{ dashboards_cassandra_learner_stats_table }}","cassandraUserFeedTable":"{{ dashboards_cassandra_user_feed_table }}","mongoDatabase":"{{ dashboards_mongo_discussions_db }}","mongoDBCollection":"{{ dashboards_mongo_discussions_db_collection }}","platformRatingSurveyId":"{{ dashboards_platform_rating_survey_id }}","key":"{{ dp_storage_key_config }}","secret":"{{ dp_storage_secret_config }}","store":"{{ report_storage_type }}","container":"{{ s3_storage_container }}","mdoIDs":"'$reportMDOIDs'","userReportPath":"{{ user_report_path }}","userEnrolmentReportPath":"{{ user_enrolment_report_path }}","courseReportPath":"{{ course_report_path }}","cbaReportPath":"{{ cba_report_path }}","standaloneAssessmentReportPath":"{{ standalone_assessment_report_path }}","taggedUsersPath":"{{ tagged_users_path }}","sideOutput":{"brokerList":"'$brokerList'","compression":"{{ dashboards_broker_compression }}","topics":{"roleUserCount":"{{ dashboards_role_count_topic }}","orgRoleUserCount":"{{ dashboards_org_role_count_topic }}","allCourses":"{{ dashboards_courses_topic }}","userCourseProgramProgress":"{{ dashboards_user_course_program_progress_topic }}","fracCompetency":"{{ dashboards_frac_competency_topic }}","courseCompetency":"{{ dashboards_course_competency_topic }}","expectedCompetency":"{{ dashboards_expected_competency_topic }}","declaredCompetency":"{{ dashboards_declared_competency_topic }}","competencyGap":"{{ dashboards_competency_gap_topic }}","userOrg":"{{ dashboards_user_org_topic }}","userAssessment":"{{ dashboards_user_assessment_topic }}","assessment":"{{ dashboards_assessment_topic }}"}}},"output":[],"parallelization":16,"appName":"In App Review Trigger Job","deviceMapping":false}'
         ;;