Dependency fixes

assessment-api/assessment-actors/src/test/resources/application.conf

@@ -287,6 +287,7 @@ cassandra.lpa.connection="127.0.0.1:9042,127.0.0.2:9042,127.0.0.3:9042"
 redis.host=localhost
 redis.port=6379
 redis.maxConnections=128
+redis.enable = false
 
 #Condition to enable publish locally
 content.publish_task.enabled=true

assessment-api/assessment-controllers/pom.xml

@@ -19,7 +19,7 @@
         <scala.maj.version>2.13</scala.maj.version>
         <scala.version>2.13.12</scala.version>
         <play2.version>3.0.5</play2.version>
-        <fasterxml.jackson.version>2.15.3</fasterxml.jackson.version>
+        <fasterxml.jackson.version>2.18.6</fasterxml.jackson.version>
     </properties>
 
     <dependencies>

assessment-api/assessment-service/conf/application.conf

@@ -362,9 +362,14 @@ cassandra.lp.connection="127.0.0.1:9042"
 content.keyspace = "content_store"
 
 # Redis Configuration
-redis.host="localhost"
-redis.port=6379
-redis.maxConnections=128
+# Disabled by default.
+redis.host = "localhost"
+redis.port = 6379
+redis.maxConnections = 128
+redis.enable = false
+
+# QuestionSet hierarchy caching (requires redis.enable=true)
+questionset.cache.enable = false
 
 # Graph Configuration
 graph.dir=/data/testingGraphDB

content-api/content-actors/src/test/resources/application.conf

@@ -287,6 +287,7 @@ cassandra.lpa.connection="127.0.0.1:9042,127.0.0.2:9042,127.0.0.3:9042"
 redis.host=localhost
 redis.port=6379
 redis.maxConnections=128
+redis.enable = false
 
 #Condition to enable publish locally
 content.publish_task.enabled=true

content-api/content-controllers/pom.xml

@@ -19,7 +19,7 @@
         <scala.maj.version>2.13</scala.maj.version>
         <scala.version>2.13.12</scala.version>
         <play2.version>3.0.5</play2.version>
-        <fasterxml.jackson.version>2.15.3</fasterxml.jackson.version>
+        <fasterxml.jackson.version>2.18.6</fasterxml.jackson.version>
     </properties>
 
     <dependencies>

content-api/content-service/conf/application.conf

@@ -393,9 +393,11 @@ cassandra.lp.connection="127.0.0.1:9042"
 cassandra.lpa.connection="127.0.0.1:9042"
 
 # Redis Configuration
-redis.host=localhost
-redis.port=6379
-redis.maxConnections=128
+# Disabled by default.
+redis.host = "localhost"
+redis.port = 6379
+redis.maxConnections = 128
+redis.enable = false
 
 #Condition to enable publish locally
 content.publish_task.enabled=true
@@ -575,13 +577,13 @@ cassandra.lp.consistency.level=QUORUM
 content.nested.fields="badgeAssertions,targets,badgeAssociations"
 
 content.cache.ttl=86400
-content.cache.enable=true
-collection.cache.enable=true
+content.cache.enable = false
+collection.cache.enable = false
 content.discard.status=["Draft","FlagDraft"]
 
 framework.categories_cached=["subject", "medium", "gradeLevel", "board"]
 framework.cache.ttl=86400
-framework.cache.read=true
+framework.cache.read = false
 
 
 # Max size(width/height) of thumbnail in pixels

content-api/hierarchy-manager/src/main/scala/org/sunbird/managers/HierarchyManager.scala

@@ -205,7 +205,8 @@ object HierarchyManager {
 
     @throws[Exception]
     def getPublishedHierarchy(request: Request)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = {
-        val redisHierarchy = RedisCache.get(hierarchyPrefix + request.get("rootId"))
+        val collectionCacheEnabled = Platform.getBoolean("collection.cache.enable", false)
+        val redisHierarchy = if (collectionCacheEnabled) RedisCache.get(hierarchyPrefix + request.get("rootId")) else ""
         val hierarchyFuture = if (StringUtils.isNotEmpty(redisHierarchy)) {
             Future(Map("content" -> JsonUtils.deserialize(redisHierarchy, classOf[java.util.Map[String, AnyRef]])).asJava)
         } else getCassandraHierarchy(request)

knowlg-service/conf/application.conf

@@ -467,9 +467,11 @@ cassandra.lp.connection="127.0.0.1:9042"
 cassandra.lpa.connection="127.0.0.1:9042"
 
 # Redis Configuration
-redis.host=localhost
-redis.port=6379
-redis.maxConnections=128
+# Disabled by default.
+redis.host = "localhost"
+redis.port = 6379
+redis.maxConnections = 128
+redis.enable = false
 
 #Condition to enable publish locally
 content.publish_task.enabled=true
@@ -653,13 +655,13 @@ cassandra.lp.consistency.level=QUORUM
 content.nested.fields="badgeAssertions,targets,badgeAssociations"
 
 content.cache.ttl=86400
-content.cache.enable=true
-collection.cache.enable=true
+content.cache.enable = false
+collection.cache.enable = false
 content.discard.status=["Draft","FlagDraft"]
 
 framework.categories_cached=["subject", "medium", "gradeLevel", "board"]
 framework.cache.ttl=86400
-framework.cache.read=true
+framework.cache.read = false
 
 
 # Max size(width/height) of thumbnail in pixels

knowlg-service/pom.xml

@@ -54,7 +54,7 @@
         <play2.plugin.version>1.0.0-rc5</play2.plugin.version>
         <sbt-compiler.plugin.version>1.0.0</sbt-compiler.plugin.version>
         <netty.version>4.1.129.Final</netty.version>
-        <fasterxml.jackson.version>2.15.3</fasterxml.jackson.version>
+        <fasterxml.jackson.version>2.18.6</fasterxml.jackson.version>
     </properties>
 
     <dependencyManagement>

ontology-engine/graph-core_2.13/pom.xml

@@ -63,7 +63,7 @@
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>
-            <version>1.33</version>
+            <version>2.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>

ontology-engine/graph-dac-api/src/main/java/org/sunbird/graph/service/operation/NodeAsyncOperations.java

@@ -83,6 +83,8 @@ public static Future<Node> addNode(String graphId, Node node) {
 
                 // Serialize complex objects (Maps, Lists, Arrays) to JSON strings —
                 Map<String, Object> metadata = setPrimitiveData(node.getMetadata());
+                // Sync serialized values back so the returned node's metadata reflects what was written to DB
+                node.getMetadata().putAll(metadata);
 
                 // Create vertex using Native API
                 JanusGraphVertex vertex = tx.addVertex(node.getObjectType());
@@ -208,6 +210,8 @@ public static Future<Node> upsertNode(String graphId, Node node, Request request
                 TelemetryManager.info("NodeAsyncOperations: Upserting Node with Status: "
                         + node.getMetadata().get("status") + " | ID: " + identifier);
                 Map<String, Object> metadata = setPrimitiveData(node.getMetadata());
+                // Sync serialized values back so the returned node's metadata reflects what was written to DB
+                node.getMetadata().putAll(metadata);
 
                 // Determine which properties to write.
                 Set<String> updatedFields = node.getUpdatedFields();

ontology-engine/graph-engine_2.13/src/main/scala/org/sunbird/graph/health/HealthCheckManager.scala

@@ -60,10 +60,12 @@ object HealthCheckManager extends CassandraConnector with RedisConnector {
     }
 
     private def checkRedisHealth(): Map[String, Any] = {
+        if (!isEnabled)
+            return Map("name" -> redisLabel, "healthy" -> true, "msg" -> "Redis not configured — skipped")
         try {
             val jedis = getConnection
-            jedis.close()
-            generateCheck(true, redisLabel)
+            if (null != jedis) { jedis.close(); generateCheck(true, redisLabel) }
+            else generateCheck(false, redisLabel)
         } catch {
             case e: Exception => generateCheck(false, redisLabel)
         }

ontology-engine/graph-engine_2.13/src/main/scala/org/sunbird/graph/schema/validator/FrameworkValidator.scala

@@ -32,46 +32,71 @@ trait FrameworkValidator extends IDefinition {
       val targetFwTerms = orgAndTargetTouple._2
       val masterCategories = orgAndTargetTouple._3
 
-      validateAndSetMultiFrameworks(node, orgFwTerms, targetFwTerms, masterCategories).map(_ => {
+      validateAndSetMultiFrameworks(node, orgFwTerms, targetFwTerms, masterCategories).flatMap(_ => {
         val framework: String = node.getMetadata.getOrDefault("framework", "").asInstanceOf[String]
         if (null != fwCategories && fwCategories.nonEmpty && framework.nonEmpty) {
-          //prepare data for validation
           val fwMetadata: Map[String, AnyRef] = node.getMetadata.asScala.filter(entry => fwCategories.contains(entry._1)).toMap
-          //validate data from cache
           if (fwMetadata.nonEmpty) {
-            val errors: util.List[String] = new util.ArrayList[String]
-            for (cat: String <- fwMetadata.keys) {
-              val value: AnyRef = fwMetadata.get(cat).get
-              //TODO: Replace Cache Call With FrameworkCache Implementation
+            Future.sequence(fwMetadata.keys.map { cat =>
+              val value: AnyRef = fwMetadata(cat)
               val cacheKey = "cat_" + framework + cat
-              val list: List[String] = RedisCache.getList(cacheKey)
-              val result: Boolean = value match {
-                case value: String => list.contains(value)
-                case value: util.List[String] => list.asJava.containsAll(value)
-                case value: Array[String] => value.forall(term => list.contains(term))
-                case _ => throw new ClientException("CLIENT_ERROR", "Validation Errors.", util.Arrays.asList("Please provide correct value for [" + cat + "]"))
-              }
-
-              if (!result) {
-                if (list.isEmpty) {
-                  errors.add(cat + " range data is empty from the given framework.")
-                } else {
-                  errors.add("Metadata " + cat + " should belong from:" + list.asJava)
+              val redisEnabled = Platform.getBoolean("redis.enable", false)
+              val cachedList: List[String] = if (redisEnabled) RedisCache.getList(cacheKey) else List()
+              val termsFuture: Future[List[String]] =
+                if (cachedList.nonEmpty) Future.successful(cachedList)
+                else getCategoryTermsFromDB(graphId, framework, cat).map { terms =>
+                  if (terms.nonEmpty && redisEnabled)
+                    RedisCache.saveList(cacheKey, terms)
+                  terms
+                }
+              termsFuture.map { list =>
+                val result: Boolean = value match {
+                  case v: String            => list.contains(v)
+                  case v: util.List[String] => list.asJava.containsAll(v)
+                  case v: Array[String]     => v.forall(list.contains)
+                  case _ => throw new ClientException("CLIENT_ERROR", "Validation Errors.",
+                      util.Arrays.asList("Please provide correct value for [" + cat + "]"))
                 }
+                if (!result) {
+                  if (list.isEmpty) cat + " range data is empty from the given framework."
+                  else "Metadata " + cat + " should belong from:" + list.asJava
+                } else ""
               }
+            }.toList).flatMap { errorList =>
+              val errors = errorList.filter(_.nonEmpty)
+              if (errors.nonEmpty)
+                throw new ClientException("CLIENT_ERROR", "Validation Errors.", errors.asJava)
+              super.validate(node, operation, setDefaultValue)
             }
-            if (!errors.isEmpty)
-              throw new ClientException("CLIENT_ERROR", "Validation Errors.", errors)
-          }
-        }
-        super.validate(node, operation)
-      }).flatten
+          } else super.validate(node, operation, setDefaultValue)
+        } else super.validate(node, operation, setDefaultValue)
+      })
     }).flatten
   }
 
+  private def getCategoryTermsFromDB(
+      graphId: String, framework: String, category: String
+  )(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[List[String]] = {
+    val mc = MetadataCriterion.create(new util.ArrayList[Filter]() {{
+      add(new Filter(SystemProperties.IL_FUNC_OBJECT_TYPE.name(), SearchConditions.OP_EQUAL, "Term"))
+      add(new Filter(SystemProperties.IL_SYS_NODE_TYPE.name(), SearchConditions.OP_EQUAL, "DATA_NODE"))
+      add(new Filter("category", SearchConditions.OP_EQUAL, category))
+      add(new Filter("status", SearchConditions.OP_NOT_EQUAL, "Retired"))
+    }})
+    val criteria = new SearchCriteria {{ addMetadata(mc); setCountQuery(false) }}
+    oec.graphService.getNodeByUniqueIds(graphId, criteria).map { nodes =>
+      nodes.asScala.filter { n =>
+        val id = n.getIdentifier
+        id != null && id.toLowerCase.startsWith(framework.toLowerCase + "_")
+      }.flatMap { n =>
+        Option(n.getMetadata.get("name")).map(_.asInstanceOf[String])
+      }.toList
+    }
+  }
+
   private def validateAndSetMultiFrameworks(node: Node, orgFwTerms: List[String], targetFwTerms: List[String], masterCategories: List[Map[String, AnyRef]])(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Map[String, AnyRef]] = {
     getValidatedTerms(node, orgFwTerms).map(orgTermMap => {
-     val jsonPropsType = schemaValidator.getAllPropsType.asScala 
+     val jsonPropsType = schemaValidator.getAllPropsType.asScala
      masterCategories.map(masterCategory => {
       val orgIdFieldName = masterCategory.getOrElse("orgIdFieldName", "").asInstanceOf[String]
       val code = masterCategory.getOrElse("code", "").asInstanceOf[String]
@@ -183,7 +208,8 @@ trait FrameworkValidator extends IDefinition {
           setCountQuery(false)
         }
       }
-      oec.graphService.getNodeByUniqueIds(node.getGraphId, searchCriteria).map(nodeList => {
+      val graphId = if (StringUtils.isNotBlank(node.getGraphId)) node.getGraphId else "domain"
+      oec.graphService.getNodeByUniqueIds(graphId, searchCriteria).map(nodeList => {
         if (CollectionUtils.isEmpty(nodeList))
           throw new ResourceNotFoundException("ERR_VALIDATING_CONTENT_FRAMEWORK", s"Nodes not found for Id's $ids ")
         val termMap = nodeList.asScala.map(node => node.getIdentifier -> node.getMetadata.getOrDefault("name", "")).toMap

ontology-engine/graph-engine_2.13/src/main/scala/org/sunbird/graph/schema/validator/PropAsEdgeValidator.scala

@@ -2,45 +2,73 @@ package org.sunbird.graph.schema.validator
 
 import org.apache.commons.collections4.CollectionUtils
 import org.sunbird.cache.impl.RedisCache
+import org.sunbird.common.Platform
 import org.sunbird.common.exception.ClientException
 import org.sunbird.graph.OntologyEngineContext
-import org.sunbird.graph.dac.model.Node
+import org.sunbird.graph.dac.model.{Filter, MetadataCriterion, Node, SearchConditions, SearchCriteria}
+import org.sunbird.graph.common.enums.SystemProperties
 import org.sunbird.graph.schema.IDefinition
 
 import scala.collection.convert.ImplicitConversions._
 import scala.concurrent.{ExecutionContext, Future}
+import scala.jdk.CollectionConverters._
 
 trait PropAsEdgeValidator extends IDefinition {
 
     val edgePropsKey = "edge.properties"
     val prefix = "edge_"
 
     @throws[Exception]
-    abstract override def validate(node: Node, operation: String, setDefaultValue: Boolean)(implicit ec: ExecutionContext, oec:OntologyEngineContext): Future[Node] = {
+    abstract override def validate(node: Node, operation: String, setDefaultValue: Boolean)(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Node] = {
         if (schemaValidator.getConfig.hasPath(edgePropsKey)) {
             val keys = CollectionUtils.intersection(node.getMetadata.keySet(), schemaValidator.getConfig.getObject(edgePropsKey).keySet())
             if (!keys.isEmpty) {
-                keys.toArray().toStream.map(key => {
-                    val cacheKey = prefix + schemaValidator.getConfig.getString(edgePropsKey + "." + key).toLowerCase
-                    val list = RedisCache.getList(cacheKey)
-                    if (CollectionUtils.isNotEmpty(list)) {
+                Future.sequence(keys.toArray().toList.map { key =>
+                    val objectType = schemaValidator.getConfig.getString(edgePropsKey + "." + key.toString)
+                    val cacheKey = prefix + objectType.toLowerCase
+                    val cachedList: List[String] = if (Platform.getBoolean("redis.enable", false))
+                        RedisCache.getList(cacheKey) else List()
+                    val resolvedGraphId = if (org.apache.commons.lang3.StringUtils.isNotBlank(node.getGraphId)) node.getGraphId else "domain"
+                    val listFuture: Future[List[String]] =
+                        if (cachedList.nonEmpty) Future.successful(cachedList)
+                        else getEdgeListFromDB(resolvedGraphId, objectType).map { list =>
+                            if (list.nonEmpty && Platform.getBoolean("redis.enable", false))
+                                RedisCache.saveList(cacheKey, list)
+                            list
+                        }
+                    listFuture.map { list =>
+                        if (list.isEmpty)
+                            throw new ClientException("ERR_EMPTY_EDGE_PROPERTY_LIST", "The list to validate input is empty.")
                         val value = node.getMetadata.get(key)
                         if (value.isInstanceOf[String]) {
-                            if(!list.contains(value.asInstanceOf[String]))
+                            if (!list.contains(value.asInstanceOf[String]))
                                 throw new ClientException("ERR_INVALID_EDGE_PROPERTY", key + " value should be one of " + list)
                         } else if (value.isInstanceOf[java.util.List[AnyRef]]) {
-                            val filteredSize = value.asInstanceOf[java.util.List[AnyRef]].toList.filter(e => list.contains(e)).size
-                            if(filteredSize != value.asInstanceOf[java.util.List[AnyRef]].size)
+                            val filteredSize = value.asInstanceOf[java.util.List[AnyRef]].toList.count(e => list.contains(e))
+                            if (filteredSize != value.asInstanceOf[java.util.List[AnyRef]].size)
                                 throw new ClientException("ERR_INVALID_EDGE_PROPERTY", key + " value should be any of " + list)
                         } else {
                             throw new ClientException("ERR_INVALID_EDGE_PROPERTY", key + " given datatype is invalid.")
                         }
-                    } else {
-                        throw new ClientException("ERR_EMPTY_EDGE_PROPERTY_LIST", "The list to validate input is empty.")
                     }
-                })
-            }
+                }).flatMap(_ => super.validate(node, operation, setDefaultValue))
+            } else super.validate(node, operation, setDefaultValue)
+        } else super.validate(node, operation, setDefaultValue)
+    }
+
+    private def getEdgeListFromDB(
+        graphId: String, objectType: String
+    )(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[List[String]] = {
+        val mc = MetadataCriterion.create(new java.util.ArrayList[Filter]() {{
+            add(new Filter(SystemProperties.IL_FUNC_OBJECT_TYPE.name(), SearchConditions.OP_EQUAL, objectType))
+            add(new Filter(SystemProperties.IL_SYS_NODE_TYPE.name(), SearchConditions.OP_EQUAL, "DATA_NODE"))
+            add(new Filter("status", SearchConditions.OP_EQUAL, "Live"))
+        }})
+        val criteria = new SearchCriteria {{ addMetadata(mc); setCountQuery(false) }}
+        oec.graphService.getNodeByUniqueIds(graphId, criteria).map { nodes =>
+            nodes.asScala.flatMap { n =>
+                Option(n.getMetadata.get("name")).map(_.asInstanceOf[String])
+            }.toList
         }
-        super.validate(node, operation)
     }
 }