[SCHEMATIC-212] Prevent traces from being combined (#1552)

* Set instance id in github CI run, uninstrument flask auto during integration test run

.github/workflows/test.yml

@@ -151,6 +151,7 @@ jobs:
           LOGGING_EXPORT_FORMAT: ${{ vars.LOGGING_EXPORT_FORMAT }}
           TRACING_SERVICE_NAME: ${{ vars.TRACING_SERVICE_NAME }}
           LOGGING_SERVICE_NAME: ${{ vars.LOGGING_SERVICE_NAME }}
+          SERVICE_INSTANCE_ID: ${{ github.head_ref || github.ref_name }} 
         run: >
           poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/
           -m "not (rule_benchmark or single_process_execution)" --reruns 4 -n 8 --ignore=tests/unit

env.example

@@ -11,6 +11,8 @@ SERVICE_ACCOUNT_CREDS='Provide service account creds'
 # LOGGING_EXPORT_FORMAT=otlp
 # TRACING_SERVICE_NAME=schematic-api
 # LOGGING_SERVICE_NAME=schematic-api
+## Instance ID is used during integration tests export to identify the git branch
+# SERVICE_INSTANCE_ID=schematic-1234
 ## Other examples: dev, staging, prod
 # DEPLOYMENT_ENVIRONMENT=local
 # OTEL_EXPORTER_OTLP_ENDPOINT=https://..../telemetry

schematic/__init__.py

@@ -12,6 +12,7 @@
 from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
 from opentelemetry.sdk.resources import (
     DEPLOYMENT_ENVIRONMENT,
+    SERVICE_INSTANCE_ID,
     SERVICE_NAME,
     SERVICE_VERSION,
     Resource,
@@ -97,10 +98,12 @@ def set_up_tracing(session: requests.Session) -> None:
         Synapse.enable_open_telemetry(True)
         tracing_service_name = os.environ.get("TRACING_SERVICE_NAME", "schematic-api")
         deployment_environment = os.environ.get("DEPLOYMENT_ENVIRONMENT", "")
+        service_instance_id = os.environ.get("SERVICE_INSTANCE_ID", "")
         trace.set_tracer_provider(
             TracerProvider(
                 resource=Resource(
                     attributes={
+                        SERVICE_INSTANCE_ID: service_instance_id,
                         SERVICE_NAME: tracing_service_name,
                         SERVICE_VERSION: __version__,
                         DEPLOYMENT_ENVIRONMENT: deployment_environment,
@@ -124,9 +127,11 @@ def set_up_logging(session: requests.Session) -> None:
     logging_export = os.environ.get("LOGGING_EXPORT_FORMAT", None)
     logging_service_name = os.environ.get("LOGGING_SERVICE_NAME", "schematic-api")
     deployment_environment = os.environ.get("DEPLOYMENT_ENVIRONMENT", "")
+    service_instance_id = os.environ.get("SERVICE_INSTANCE_ID", "")
     if logging_export == "otlp":
         resource = Resource.create(
             {
+                SERVICE_INSTANCE_ID: service_instance_id,
                 SERVICE_NAME: logging_service_name,
                 DEPLOYMENT_ENVIRONMENT: deployment_environment,
                 SERVICE_VERSION: __version__,

tests/conftest.py

@@ -13,6 +13,7 @@
 from dotenv import load_dotenv
 from flask.testing import FlaskClient
 from opentelemetry import trace
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
 from synapseclient.client import Synapse
 
 from schematic.configuration.configuration import CONFIG, Configuration
@@ -42,6 +43,8 @@
 TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(TESTS_DIR, "data")
 
+FlaskInstrumentor().uninstrument()
+
 
 @pytest.fixture(scope="session")
 def dataset_id():