Skip to content

Commit

Permalink
Merge pull request #34 from krgauraw/release-6.1.0
Browse files Browse the repository at this point in the history 
Issue #IQ-545 fix: updated code for hierarchy migration
  • Loading branch information
@sajeshkayyath
sajeshkayyath authored Aug 9, 2023
2 parents 84156eb + 03a27e1 commit 3416594
Show file tree
Hide file tree
Showing 9 changed files with 150 additions and 32 deletions.
3 changes: 3 additions & 0 deletions ...ator/src/main/scala/org/sunbird/job/quml/migrator/exceptions/QumlMigrationException.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package org.sunbird.job.quml.migrator.exceptions

class QumlMigrationException(message: String) extends java.lang.Exception(message)
2 changes: 1 addition & 1 deletion .../src/main/scala/org/sunbird/job/quml/migrator/function/QuestionSetMigrationFunction.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class QuestionSetMigrationFunction(config: QumlMigratorConfig, httpUtil: HttpUti
val objData = getObject(data.identifier, readerConfig)(neo4JUtil, cassandraUtil, config)
val messages: List[String] = validateQuestionSet(data.identifier, objData)(neo4JUtil)
if (messages.isEmpty) {
val migratedObj: ObjectData = migrateQuestionSet(objData)(definition).getOrElse(objData)
val migratedObj: ObjectData = migrateQuestionSet(objData)(definition, neo4JUtil).getOrElse(objData)
val status = migratedObj.metadata.getOrElse("status", "").asInstanceOf[String]
val qumlVersion: Double = migratedObj.metadata.getOrElse("qumlVersion", 1.0).asInstanceOf[Double]
val migrationVersion: Double = migratedObj.metadata.getOrElse("migrationVersion", 0.0).asInstanceOf[Double]
Expand Down
18 changes: 12 additions & 6 deletions quml-migrator/src/main/scala/org/sunbird/job/quml/migrator/helpers/QuestionMigrator.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.sunbird.job.domain.`object`.ObjectDefinition
import org.sunbird.job.quml.migrator.domain.{ExtDataConfig, ObjectData, ObjectExtData}
import org.sunbird.job.quml.migrator.exceptions.QumlMigrationException
import org.sunbird.job.quml.migrator.task.QumlMigratorConfig
import org.sunbird.job.util._

Expand Down Expand Up @@ -89,15 +90,18 @@ trait QuestionMigrator extends MigrationObjectReader with MigrationObjectUpdater
val migratedExtData = migrateExtData(data.identifier, extMeta)
migratedExtData.remove("primaryCategory")
val migrGrpahData: util.Map[String, AnyRef] = migrateGrpahData(data.identifier, jMap)
if(migrGrpahData.containsKey("bloomsLevel")) migrGrpahData.remove("bloomsLevel")
migrGrpahData.put("bloomsLevel", null)
migrGrpahData.put("version", null)
val updatedMeta: Map[String, AnyRef] = migrGrpahData.asScala.toMap ++ Map[String, AnyRef]("qumlVersion" -> 1.1.asInstanceOf[AnyRef], "schemaVersion" -> "1.1", "migrationVersion" -> 3.0.asInstanceOf[AnyRef])
logger.info("QuestionMigrator ::: migrateQuestion ::: migrated metadata :::: "+migrGrpahData)
logger.info("QuestionMigrator ::: migrateQuestion ::: migrated ext data :::: "+migratedExtData)
logger.info("QuestionMigrator ::: migrateQuestion ::: Completed Data Transformation For : " + data.identifier)
Some(new ObjectData(data.identifier, updatedMeta, Some(migratedExtData.asScala.toMap), data.hierarchy))
} catch {
case e: Exception => {
case e: java.lang.Exception => {
logger.info("QuestionMigrator ::: migrateQuestion ::: Failed Data Transformation For : " + data.identifier)
logger.info("QuestionMigrator ::: migrateQuestion ::: exception message :: "+ e.getMessage)
logger.info("QuestionMigrator ::: migrateQuestion ::: exception message :: "+ e.getLocalizedMessage)
e.printStackTrace()
val updatedMeta: Map[String, AnyRef] = data.metadata ++ Map[String, AnyRef]("migrationVersion" -> 2.1.asInstanceOf[AnyRef], "migrationError"->e.getMessage)
Some(new ObjectData(data.identifier, updatedMeta, data.extData, data.hierarchy))
Expand All @@ -114,8 +118,9 @@ trait QuestionMigrator extends MigrationObjectReader with MigrationObjectUpdater
} else data
} catch {
case e: Exception => {
logger.info(s"QuestionMigrator ::: migrateGrpahData ::: Error occurred while converting graph data for ${identifier} | Error: " +e.getMessage)
e.printStackTrace()
throw new Exception(s"Error Occurred While Converting Graph Data To Quml 1.1 Format for ${identifier}")
throw new QumlMigrationException(s"Error Occurred While Converting Graph Data To Quml 1.1 Format for ${identifier} | Error: "+e.getMessage)
}
}
}
Expand All @@ -134,14 +139,15 @@ trait QuestionMigrator extends MigrationObjectReader with MigrationObjectUpdater
data
} else data
} catch {
case e: Exception => {
case e: java.lang.Exception => {
e.printStackTrace()
throw new Exception(s"Error Occurred While Converting External Data To Quml 1.1 Format for ${identifier}")
logger.info(s"QuestionMigrator ::: migrateExtData ::: Error occurred while converting external data for ${identifier} | Error: " + e.getMessage )
throw new QumlMigrationException(s"Error Occurred While Converting External Data To Quml 1.1 Format for ${identifier} | Error: "+e.getMessage)
}
}
}

override def migrateQuestionSet(data: ObjectData)(implicit definition: ObjectDefinition): Option[ObjectData] = None
override def migrateQuestionSet(data: ObjectData)(implicit definition: ObjectDefinition, neo4JUtil: Neo4JUtil): Option[ObjectData] = None

def getFormatedData(data: String, dType: String): AnyRef = {
logger.info("getFormatedData ::: data ::: " + data)
Expand Down
54 changes: 38 additions & 16 deletions quml-migrator/src/main/scala/org/sunbird/job/quml/migrator/helpers/QuestionSetMigrator.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ import com.datastax.driver.core.Row
import com.datastax.driver.core.querybuilder.{Clause, Insert, QueryBuilder}
import org.apache.commons.lang3.StringUtils
import org.slf4j.LoggerFactory
import org.sunbird.job.domain.`object`.{ObjectDefinition}
import org.sunbird.job.domain.`object`.ObjectDefinition
import org.sunbird.job.quml.migrator.domain.{ExtDataConfig, ObjectData, ObjectExtData}
import org.sunbird.job.quml.migrator.exceptions.QumlMigrationException
import org.sunbird.job.quml.migrator.task.QumlMigratorConfig
import org.sunbird.job.util.{CassandraUtil, Neo4JUtil, ScalaJsonUtil}

Expand All @@ -18,6 +19,8 @@ trait QuestionSetMigrator extends MigrationObjectReader with MigrationObjectUpda

private[this] val logger = LoggerFactory.getLogger(classOf[QuestionSetMigrator])

val propsToRemove = List("outcomeDeclaration", "bloomsLevel", "maxScore","version")

def validateQuestionSet(identifier: String, obj: ObjectData)(implicit neo4JUtil: Neo4JUtil): List[String] = {
val messages = ListBuffer[String]()
if (obj.hierarchy.getOrElse(Map()).isEmpty) messages += s"""There is no hierarchy available for : $identifier"""
Expand Down Expand Up @@ -83,7 +86,7 @@ trait QuestionSetMigrator extends MigrationObjectReader with MigrationObjectUpda
result
}

override def migrateQuestionSet(data: ObjectData)(implicit definition: ObjectDefinition): Option[ObjectData] = {
override def migrateQuestionSet(data: ObjectData)(implicit definition: ObjectDefinition, neo4JUtil: Neo4JUtil): Option[ObjectData] = {
logger.info("QuestionSetMigrator ::: migrateQuestionSet ::: Stating Data Transformation For : " + data.identifier)
try {
val metaMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef]()
Expand All @@ -98,19 +101,20 @@ trait QuestionSetMigrator extends MigrationObjectReader with MigrationObjectUpda
val migrGrpahData: util.Map[String, AnyRef] = migrateGrpahData(data.identifier, metaMap)
val migrExtData: util.Map[String, AnyRef] = migrateExtData(data.identifier, extMeta)
val outcomeDeclaration: util.Map[String, AnyRef] = migrGrpahData.getOrDefault("outcomeDeclaration", Map[String, AnyRef]()).asInstanceOf[util.Map[String, AnyRef]]
migrGrpahData.remove("outcomeDeclaration")
propsToRemove.foreach(prop => migrGrpahData.put(prop, null))
migrExtData.put("outcomeDeclaration", outcomeDeclaration)
val migrHierarchy: util.Map[String, AnyRef] = migrateHierarchy(data.identifier, hierarchyData)

logger.info("migrateQuestionSet :: migrated graph data ::: " + migrGrpahData)
logger.info("migrateQuestionSet :: migrated ext data ::: " + migrExtData)
logger.info("migrateQuestionSet :: migrated hierarchy ::: " + migrHierarchy)
val updatedMeta: Map[String, AnyRef] = migrGrpahData.asScala.toMap ++ Map[String, AnyRef]("qumlVersion" -> 1.1.asInstanceOf[AnyRef], "schemaVersion" -> "1.1", "migrationVersion" -> 3.0.asInstanceOf[AnyRef])
logger.info("QuestionSetMigrator ::: migrateQuestionSet ::: Completed Data Transformation For : " + data.identifier)
Some(new ObjectData(data.identifier, updatedMeta, Some(migrExtData.asScala.toMap), Some(migrHierarchy.asScala.toMap)))
} catch {
case e: Exception => {
case e: java.lang.Exception => {
logger.info("QuestionSetMigrator ::: migrateQuestionSet ::: Failed Data Transformation For : " + data.identifier)
logger.info("QuestionSetMigrator ::: migrateQuestionSet ::: exception message :: "+ e.getMessage)
logger.info("QuestionSetMigrator ::: migrateQuestionSet ::: exception message :: "+ e.getLocalizedMessage)
val updatedMeta: Map[String, AnyRef] = data.metadata ++ Map[String, AnyRef]("migrationVersion" -> 2.1.asInstanceOf[AnyRef], "migrationError"->e.getMessage)
Some(new ObjectData(data.identifier, updatedMeta, data.extData, data.hierarchy))
}
Expand All @@ -127,9 +131,10 @@ trait QuestionSetMigrator extends MigrationObjectReader with MigrationObjectUpda
data
} else data
} catch {
case e: Exception => {
case e: java.lang.Exception => {
e.printStackTrace()
throw new Exception(s"Error Occurred While Converting Graph Data To Quml 1.1 Format for ${identifier}")
logger.info(s"QuestionSetMigrator :: migrateGrpahData ::: Error Occurred While Graph Data Transformation For ${identifier} | Error: "+ e.getMessage)
throw new QumlMigrationException(s"Error Occurred While Converting Graph Data To Quml 1.1 Format for ${identifier} | Error: "+e.getMessage)
}
}
}
Expand All @@ -141,14 +146,15 @@ trait QuestionSetMigrator extends MigrationObjectReader with MigrationObjectUpda
data
} else data
} catch {
case e: Exception => {
case e: java.lang.Exception => {
e.printStackTrace()
throw new Exception(s"Error Occurred While Converting External Data To Quml 1.1 Format for ${identifier}")
logger.info(s"QuestionSetMigrator :: migrateExtData ::: Error Occurred While External Data Transformation For ${identifier} | Error: "+ e.getMessage)
throw new QumlMigrationException(s"Error Occurred While Converting External Data To Quml 1.1 Format for ${identifier} | Error : "+e.getMessage)
}
}
}

def migrateHierarchy(identifier: String, data: util.Map[String, AnyRef]): util.Map[String, AnyRef] = {
def migrateHierarchy(identifier: String, data: util.Map[String, AnyRef])(implicit neo4JUtil: Neo4JUtil): util.Map[String, AnyRef] = {
try {
if (!data.isEmpty) {
logger.info(s"QuestionSetMigrator ::: migrateHierarchy ::: Hierarchy migration stated for ${identifier}")
Expand All @@ -168,24 +174,40 @@ trait QuestionSetMigrator extends MigrationObjectReader with MigrationObjectUpda
data
} else data
} catch {
case e: Exception => {
case e: java.lang.Exception => {
logger.info(s"QuestionSetMigrator :: migrateHierarchy ::: Error Occurred While Hierarchy Data Transformation For ${identifier} | Error: "+ e.getMessage)
e.printStackTrace()
throw new Exception(s"Error Occurred While Converting Hierarchy Data To Quml 1.1 Format for ${identifier}")
throw new QumlMigrationException(s"Error Occurred While Converting Hierarchy Data To Quml 1.1 Format for ${identifier} | Error: "+e.getMessage)
}
}
}

def migrateChildren(children: util.List[util.Map[String, AnyRef]]): Unit = {
def migrateChildren(children: util.List[util.Map[String, AnyRef]])(implicit neo4JUtil: Neo4JUtil): Unit = {
if (!children.isEmpty) {
children.foreach(ch => {
if (ch.containsKey("version")) ch.remove("version")
processBloomsLevel(ch)
processBooleanProps(ch)
if (StringUtils.equalsIgnoreCase("application/vnd.sunbird.questionset", ch.getOrDefault("mimeType", "").asInstanceOf[String])) {
if (ch.containsKey("version")) ch.remove("version")
processBloomsLevel(ch)
processBooleanProps(ch)
processTimeLimits(ch)
processInstructions(ch)
ch.put("schemaVersion", "1.1")
ch.put("qumlVersion", 1.1.asInstanceOf[AnyRef])
ch.put("migrationVersion", 3.0.asInstanceOf[AnyRef])
val nestedChildren = ch.getOrDefault("children", new util.ArrayList[java.util.Map[String, AnyRef]]).asInstanceOf[util.List[java.util.Map[String, AnyRef]]]
migrateChildren(nestedChildren)
} else {
val childrenId = ch.get("identifier").asInstanceOf[String].replace(".img","")
val chData = getMetadata(childrenId)(neo4JUtil)
val schemaVersion = chData.getOrElse("schemaVersion", "1.0").asInstanceOf[String]
val migrVer:Double = chData.getOrElse("migrationVersion", 1.0.asInstanceOf[AnyRef]).asInstanceOf[Double]
if(StringUtils.equalsIgnoreCase("1.1", schemaVersion) && List(3.0,3.1).contains(migrVer)) {
val chStr = ScalaJsonUtil.serialize(chData)
val chMap: util.Map[String, AnyRef] = mapper.readValue(chStr, classOf[util.Map[String, AnyRef]])
ch.putAll(chMap)
ch.remove("bloomsLevel")
ch.remove("version")
} else throw new QumlMigrationException(s"Please migrate children having identifier ${childrenId}")
}
})
}
Expand Down
4 changes: 2 additions & 2 deletions quml-migrator/src/main/scala/org/sunbird/job/quml/migrator/helpers/QumlMigrator.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import org.apache.commons.lang.StringUtils
import org.slf4j.LoggerFactory
import org.sunbird.job.domain.`object`.ObjectDefinition
import org.sunbird.job.quml.migrator.domain.ObjectData
import org.sunbird.job.util.JSONUtil
import org.sunbird.job.util.{JSONUtil, Neo4JUtil}

import java.util
import java.util.UUID
Expand All @@ -20,7 +20,7 @@ trait QumlMigrator {

def migrateQuestion(data: ObjectData)(implicit definition: ObjectDefinition): Option[ObjectData]

def migrateQuestionSet(data: ObjectData)(implicit definition: ObjectDefinition): Option[ObjectData]
def migrateQuestionSet(data: ObjectData)(implicit definition: ObjectDefinition, neo4JUtil: Neo4JUtil): Option[ObjectData]

def processMaxScore(data: util.Map[String, AnyRef]): Unit = {
if (data.containsKey("maxScore")) {
Expand Down
3 changes: 2 additions & 1 deletion quml-migrator/src/test/resources/test.cql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,5 @@ textAsBlob('1'),

INSERT INTO local_hierarchy_store.questionset_hierarchy(identifier, hierarchy) VALUES ('do_123', '{ "name": "QS1", "identifier": "do_123", "children": [{"identifier": "do_124", "objectType": "QuestionSet", "visibility": "Parent"}]}');
INSERT INTO local_hierarchy_store.questionset_hierarchy(identifier, hierarchy) VALUES ('do_321', '{ "name": "QS1", "identifier": "do_123", "children": [{"identifier": "do_124", "objectType": "QuestionSet", "visibility": "Parent"}]}');
INSERT INTO local_hierarchy_store.questionset_hierarchy(identifier, hierarchy) VALUES ('do_123.img', '{ "name": "QS1", "identifier": "do_123", "children": [{"identifier": "do_124", "objectType": "QuestionSet", "visibility": "Parent"}, {"identifier": "do_125", "objectType": "QuestionSet", "visibility": "Parent"}]}');
INSERT INTO local_hierarchy_store.questionset_hierarchy(identifier, hierarchy) VALUES ('do_123.img', '{ "name": "QS1", "identifier": "do_123", "children": [{"identifier": "do_124", "objectType": "QuestionSet", "visibility": "Parent"}, {"identifier": "do_125", "objectType": "QuestionSet", "visibility": "Parent"}]}');
INSERT INTO local_hierarchy_store.questionset_hierarchy(identifier, hierarchy) VALUES ('do_21385612453673369613688', '{"identifier":"do_21385612453673369613688","children":[{"parent":"do_21385612453673369613688","instructions":{"default":"sample instructions"},"code":"section-1","allowSkip":"Yes","containsUserData":"No","description":"Section-1","language":["English"],"mimeType":"application/vnd.sunbird.questionset","showHints":"No","createdOn":"2023-08-07T15:17:48.218+0000","objectType":"QuestionSet","scoreCutoffType":"AssessmentLevel","primaryCategory":"Practice Question Set","children":[{"parent":"do_21385612466764185613693","code":"q1","description":"Q1","language":["English"],"mimeType":"application/vnd.sunbird.question","createdOn":"2023-08-07T15:17:48.208+0000","objectType":"Question","primaryCategory":"Multiple Choice Question","contentDisposition":"inline","lastUpdatedOn":"2023-08-07T15:17:48.208+0000","contentEncoding":"gzip","showSolutions":"No","allowAnonymousAccess":"Yes","identifier":"do_21385612466755993613689","lastStatusChangedOn":"2023-08-07T15:17:48.208+0000","visibility":"Parent","showTimer":"No","index":1,"qType":"MCQ","maxScore":1,"languageCode":["en"],"bloomsLevel":"create","version":1,"versionKey":"1691421468218","showFeedback":"No","license":"CC BY 4.0","interactionTypes":["choice"],"depth":2,"compatibilityLevel":4,"name":"Q1","status":"Draft"},{"parent":"do_21385612466764185613693","code":"q2","description":"Q2","language":["English"],"mimeType":"application/vnd.sunbird.question","createdOn":"2023-08-07T15:17:48.217+0000","objectType":"Question","primaryCategory":"Subjective Question","contentDisposition":"inline","lastUpdatedOn":"2023-08-07T15:17:48.217+0000","contentEncoding":"gzip","showSolutions":"No","allowAnonymousAccess":"Yes","identifier":"do_21385612466763366413691","lastStatusChangedOn":"2023-08-07T15:17:48.217+0000","visibility":"Parent","showTimer":"Yes","index":2,"qType":"SA","maxScore":1,"languageCode":["en"],"version":1,"versionKey":"1691421468219","showFeedback":"No","license":"CC BY 4.0","depth":2,"compatibilityLevel":4,"name":"Q2","status":"Draft"}],"contentDisposition":"inline","lastUpdatedOn":"2023-08-07T15:17:48.218+0000","contentEncoding":"gzip","generateDIALCodes":"No","showSolutions":"No","trackable":{"enabled":"No","autoBatch":"No"},"allowAnonymousAccess":"Yes","identifier":"do_21385612466764185613693","lastStatusChangedOn":"2023-08-07T15:17:48.218+0000","requiresSubmit":"No","visibility":"Parent","showTimer":"Yes","index":1,"setType":"materialised","languageCode":["en"],"version":1,"versionKey":"1691421468218","showFeedback":"No","license":"CC BY 4.0","depth":1,"compatibilityLevel":5,"name":"Section-1","navigationMode":"non-linear","allowBranching":"No","shuffle":true,"status":"Draft"}]}');
9 changes: 4 additions & 5 deletions .../migrator/task/QuestionMigratorSpec.scala → ...grator/helpers/QuestionMigratorSpec.scala
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.sunbird.job.quml.migrator.task
package org.sunbird.job.quml.migrator.helpers

import akka.dispatch.ExecutionContexts
import com.typesafe.config.{Config, ConfigFactory}
Expand All @@ -10,7 +10,7 @@ import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
import org.scalatestplus.mockito.MockitoSugar
import org.sunbird.job.domain.`object`.{DefinitionCache, ObjectDefinition}
import org.sunbird.job.quml.migrator.domain.{ExtDataConfig, ObjectData, ObjectExtData}
import org.sunbird.job.quml.migrator.helpers.QuestionMigrator
import org.sunbird.job.quml.migrator.task.QumlMigratorConfig
import org.sunbird.job.util.{CassandraUtil, HttpUtil, Neo4JUtil}

import java.util
Expand Down Expand Up @@ -76,11 +76,10 @@ class QuestionMigratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers
val result: ObjectData = new TestQuestionMigrator().migrateQuestion(data).getOrElse(new ObjectData("do_234", Map()))
result.identifier should be("do_123")
result.metadata.contains("complexityLevel") should be(true)
result.metadata.contains("bloomsLevel") should be(false)
result.metadata.contains("bloomsLevel") should be(true)
result.metadata.get("bloomsLevel").get should be(null)
}



}

class TestQuestionMigrator extends QuestionMigrator {}
Loading

0 comments on commit 3416594

Please sign in to comment.