From ecb75ff096c1fb9ca33fa5e5b6479a9fc59539f3 Mon Sep 17 00:00:00 2001 From: Ken Zangelin Date: Tue, 10 Dec 2024 21:59:41 +0100 Subject: [PATCH 1/2] Complex @contexts of subscriptions weren't persisted in mongodb --- CHANGES_NEXT_RELEASE | 2 +- .../contextCache/orionldContextCacheInit.cpp | 5 + src/lib/orionld/mhd/mhdConnectionTreat.cpp | 11 + ...xt_in_subscription_and_broker_restart.test | 367 ++++++++++++++++++ 4 files changed, 384 insertions(+), 1 deletion(-) create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_complex_context_in_subscription_and_broker_restart.test diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 41a22c7c04..4ba32f2d9c 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,5 +1,5 @@ ## Fixed Issues: - #xxxx: yyyy + #XXXX: Complex @contexts of subscriptions weren't persisted in mongodb ## New Features: * Support for ... diff --git a/src/lib/orionld/contextCache/orionldContextCacheInit.cpp b/src/lib/orionld/contextCache/orionldContextCacheInit.cpp index b6f755a866..2674cdeced 100644 --- a/src/lib/orionld/contextCache/orionldContextCacheInit.cpp +++ b/src/lib/orionld/contextCache/orionldContextCacheInit.cpp @@ -56,17 +56,20 @@ void dbContextToCache(KjNode* dbContextP, KjNode* atContextP, bool keyValues, bo KjNode* urlNodeP = kjLookup(dbContextP, "url"); KjNode* parentNodeP = kjLookup(dbContextP, "parent"); KjNode* originNodeP = kjLookup(dbContextP, "origin"); + KjNode* kindNodeP = kjLookup(dbContextP, "kind"); KjNode* createdAtNodeP = kjLookup(dbContextP, "createdAt"); if (idNodeP == NULL) LM_RVE(("Database Error (No 'id' node in cached context in DB)")); if (urlNodeP == NULL) LM_RVE(("Database Error (No 'url' node in cached context in DB)")); if (originNodeP == NULL) LM_RVE(("Database Error (No 'origin' node in cached context in DB)")); + if (kindNodeP == NULL) LM_RVE(("Database Error (No 'kind' node in cached context in DB)")); if (createdAtNodeP == NULL) LM_RVE(("Database Error (No 'createdAt' node in cached context in DB)")); char* id = idNodeP->value.s; char* url = urlNodeP->value.s; double createdAt = createdAtNodeP->value.f; OrionldContextOrigin origin = orionldOriginFromString(originNodeP->value.s); + OrionldContextKind kind = orionldKindFromString(kindNodeP->value.s); OrionldContext* contextP = orionldContextFromTree(url, origin, id, atContextP); if (contextP == NULL) @@ -74,6 +77,7 @@ void dbContextToCache(KjNode* dbContextP, KjNode* atContextP, bool keyValues, bo contextP->createdAt = createdAt; contextP->usedAt = 0; + contextP->kind = kind; if (coreContext == true) { @@ -125,6 +129,7 @@ void orionldContextCacheInit(void) { KjNode* contextNodeP = contextArray->value.firstChildP; KjNode* next; + while (contextNodeP != NULL) { next = contextNodeP->next; diff --git a/src/lib/orionld/mhd/mhdConnectionTreat.cpp b/src/lib/orionld/mhd/mhdConnectionTreat.cpp index 019225b384..4c8f4eec64 100644 --- a/src/lib/orionld/mhd/mhdConnectionTreat.cpp +++ b/src/lib/orionld/mhd/mhdConnectionTreat.cpp @@ -74,6 +74,7 @@ extern "C" #include "orionld/context/orionldContextUrlGenerate.h" // orionldContextUrlGenerate #include "orionld/context/orionldContextItemExpand.h" // orionldContextItemExpand #include "orionld/context/orionldAttributeExpand.h" // orionldAttributeExpand +#include "orionld/contextCache/orionldContextCachePersist.h" // orionldContextCachePersist #include "orionld/serviceRoutines/orionldPatchAttribute.h" // orionldPatchAttribute #include "orionld/serviceRoutines/orionldGetEntity.h" // orionldGetEntity #include "orionld/serviceRoutines/orionldGetEntities.h" // orionldGetEntities @@ -1234,6 +1235,16 @@ MHD_Result mhdConnectionTreat(void) if (pd.status >= 400) goto respond; + if (implicitlyCreated == true) + { + if (orionldState.payloadContextNode->type != KjString) + { + orionldState.contextP->origin = OrionldContextFromInline; + orionldState.contextP->kind = OrionldContextHosted; + orionldContextCachePersist(orionldState.contextP); + } + } + if ((orionldState.contextP != orionldCoreContextP) && (implicitlyCreated == true)) orionldState.contextP->kind = OrionldContextImplicit; // Too late - the context is already in mongo } diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_complex_context_in_subscription_and_broker_restart.test b/test/functionalTest/cases/0000_ngsild/ngsild_complex_context_in_subscription_and_broker_restart.test new file mode 100644 index 0000000000..57d96eee96 --- /dev/null +++ b/test/functionalTest/cases/0000_ngsild/ngsild_complex_context_in_subscription_and_broker_restart.test @@ -0,0 +1,367 @@ +# Copyright 2024 FIWARE Foundation e.V. +# +# This file is part of Orion-LD Context Broker. +# +# Orion-LD Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion-LD Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# orionld at fiware dot org + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Issue #1708 + +--SHELL-INIT-- +dbInit CB +orionldStart CB +accumulatorStart --pretty-print 127.0.0.1 ${LISTENER_PORT} + +--SHELL-- +# +# 01. Create an entity urn:ngsi-ld:test_entity_id, as from the issue #1708 +# 02. Create a matching subscription 'urn:s1' (with a complex @context), to notify the accumulator +# 03. Modify the entity, to provoke a notification to the accumulator +# 04. GET the subscription - see especially its jsonldContext +# 05. GET the jsonldContext of the subscription +# 06. Dump and reset the accumulator - see one notification +# 07. Restart the broker +# 08. GET the subscription 'urn:s1', make sure all is OK +# 09. GET the jsonldContext of the subscription +# 09. Modify the entity again, to provoke a notification to the accumulator +# 10. Dump and reset the accumulator - see one notification +# + +echo "01. Create an entity urn:ngsi-ld:test_entity_id, as from the issue #1708" +echo "========================================================================" +payload='{ + "id": "urn:ngsi-ld:test_entity_id", + "type": "test_entity_type", + "temperature": { + "type": "Property", + "value": 23.4 + }, + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.8.jsonld", + { + "temperature": "https://mysite.com/api/ngsi-ld-attributes/temperature" + } + ] +}' +orionCurl --url /ngsi-ld/v1/entities --payload "$payload" --in jsonld +echo +echo + + +echo "02. Create a matching subscription 'urn:s1' (with a complex @context), to notify the accumulator" +echo "================================================================================================" +payload='{ + "id": "urn:s1", + "description": "Notify QuantumLeap of changes to test_entity_type", + "type": "Subscription", + "entities": [ + { + "id": "urn:ngsi-ld:test_entity_id", + "type": "test_entity_type" + } + ], + "watchedAttributes": ["temperature"], + "notification": { + "attributes": ["temperature"], + "format": "normalized", + "endpoint": { + "uri": "http://127.0.0.1:'${LISTENER_PORT}'/notify", + "accept": "application/json", + "receiverInfo": [ + { + "key": "fiware-service", + "value": "openiot" + } + ] + } + }, + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.8.jsonld", + { + "temperature": "https://mysite.com/api/ngsi-ld-attributes/temperature" + } + ] +}' +orionCurl --url /ngsi-ld/v1/subscriptions --payload "$payload" --in jsonld +echo +echo + + +echo "03. Modify the entity, to provoke a notification to the accumulator" +echo "===================================================================" +payload='{ + "temperature": { + "type": "Property", + "value": 8.5 + }, + "@context": [ + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.8.jsonld", + { + "temperature": "https://mysite.com/api/ngsi-ld-attributes/temperature" + } + ] +}' +orionCurl --url /ngsi-ld/v1/entities/urn:ngsi-ld:test_entity_id/attrs -X POST --payload "$payload" --in jsonld +echo +echo + + +echo "04. GET the subscription - see especially its jsonldContext" +echo "===========================================================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:s1 +jsonldContext=$(echo $_response | awk -F\"jsonldContext\" '{ print $2 }' | awk -F\" '{ print $2 }') +echo +echo + + +echo "05. GET the jsonldContext of the subscription" +echo "=============================================" +curl $jsonldContext 2> /dev/null +echo +echo + + +echo "06. Dump and reset the accumulator - see one notification" +echo "=========================================================" +accumulatorDump +accumulatorReset +echo +echo + + +echo "06. See the jsonldContext in mongo" +echo "==================================" +contextId=$(echo $jsonldContext | awk -F jsonldContexts/ '{ print $2 }') +mongoCmd2 orionld 'db.contexts.findOne({"_id": "'$contextId'"})' +echo +echo + + +echo "07. Restart the broker" +echo "======================" +brokerStop CB +brokerStart CB +echo "The broker should be running again" +echo +echo + + +echo "08. GET the subscription 'urn:s1', make sure all is OK" +echo "======================================================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:s1 +echo +echo + + +echo "09. GET the jsonldContext of the subscription" +echo "=============================================" +curl $jsonldContext 2> /dev/null +echo +echo +echo + + +--REGEXPECT-- +01. Create an entity urn:ngsi-ld:test_entity_id, as from the issue #1708 +======================================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/entities/urn:ngsi-ld:test_entity_id + + + +02. Create a matching subscription 'urn:s1' (with a complex @context), to notify the accumulator +================================================================================================ +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:s1 + + + +03. Modify the entity, to provoke a notification to the accumulator +=================================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) + + + +04. GET the subscription - see especially its jsonldContext +=========================================================== +HTTP/1.1 200 OK +Content-Length: 750 +Content-Type: application/json +Date: REGEX(.*) +Link: Date: Wed, 11 Dec 2024 10:31:21 +0100 Subject: [PATCH 2/2] More REGEX needed for a functest. Added one more functest --- ...xt_in_subscription_and_broker_restart.test | 4 +- ...xt_in_subscription_and_broker_restart.test | 367 ++++++++++++++++++ 2 files changed, 369 insertions(+), 2 deletions(-) create mode 100644 test/functionalTest/cases/0000_ngsild/ngsild_new_complex_context_in_subscription_and_broker_restart.test diff --git a/test/functionalTest/cases/0000_ngsild/ngsild_complex_context_in_subscription_and_broker_restart.test b/test/functionalTest/cases/0000_ngsild/ngsild_complex_context_in_subscription_and_broker_restart.test index 57d96eee96..96eb199db1 100644 --- a/test/functionalTest/cases/0000_ngsild/ngsild_complex_context_in_subscription_and_broker_restart.test +++ b/test/functionalTest/cases/0000_ngsild/ngsild_complex_context_in_subscription_and_broker_restart.test @@ -206,7 +206,7 @@ Date: REGEX(.*) 04. GET the subscription - see especially its jsonldContext =========================================================== HTTP/1.1 200 OK -Content-Length: 750 +Content-Length: REGEX(.*) Content-Type: application/json Date: REGEX(.*) Link: /dev/null +echo +echo + + +echo "06. Dump and reset the accumulator - see one notification" +echo "=========================================================" +accumulatorDump +accumulatorReset +echo +echo + + +echo "06. See the jsonldContext in mongo" +echo "==================================" +contextId=$(echo $jsonldContext | awk -F jsonldContexts/ '{ print $2 }') +mongoCmd2 orionld 'db.contexts.findOne({"_id": "'$contextId'"})' +echo +echo + + +echo "07. Restart the broker" +echo "======================" +brokerStop CB +brokerStart CB +echo "The broker should be running again" +echo +echo + + +echo "08. GET the subscription 'urn:s1', make sure all is OK" +echo "======================================================" +orionCurl --url /ngsi-ld/v1/subscriptions/urn:s1 +echo +echo + + +echo "09. GET the jsonldContext of the subscription" +echo "=============================================" +curl $jsonldContext 2> /dev/null +echo +echo +echo + + +--REGEXPECT-- +01. Create an entity urn:ngsi-ld:test_entity_id, as from the issue #1708 +======================================================================== +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/entities/urn:ngsi-ld:test_entity_id + + + +02. Create a matching subscription 'urn:s1' (with a complex @context), to notify the accumulator +================================================================================================ +HTTP/1.1 201 Created +Content-Length: 0 +Date: REGEX(.*) +Location: /ngsi-ld/v1/subscriptions/urn:s1 + + + +03. Modify the entity, to provoke a notification to the accumulator +=================================================================== +HTTP/1.1 204 No Content +Date: REGEX(.*) + + + +04. GET the subscription - see especially its jsonldContext +=========================================================== +HTTP/1.1 200 OK +Content-Length: REGEX(.*) +Content-Type: application/json +Date: REGEX(.*) +Link: