add health check for hgweb to match api version

backend/LexBoxApi/LexBoxKernel.cs

@@ -5,6 +5,7 @@
 using LexCore.Config;
 using LexCore.ServiceInterfaces;
 using LexSyncReverseProxy;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Swashbuckle.AspNetCore.Swagger;
 
 namespace LexBoxApi;
@@ -22,9 +23,9 @@ public static void AddLexBoxApi(this IServiceCollection services,
             .ValidateDataAnnotations()
             .ValidateOnStart();
         // services.AddOptions<HasuraConfig>()
-            // .BindConfiguration("HasuraConfig")
-            // .ValidateDataAnnotations()
-            // .ValidateOnStart();
+        // .BindConfiguration("HasuraConfig")
+        // .ValidateDataAnnotations()
+        // .ValidateOnStart();
         services.AddOptions<CloudFlareConfig>()
             .BindConfiguration("CloudFlare")
             .ValidateDataAnnotations()
@@ -50,18 +51,20 @@ public static void AddLexBoxApi(this IServiceCollection services,
         services.AddScoped<TusService>();
         services.AddScoped<TurnstileService>();
         services.AddScoped<IHgService, HgService>();
+        services.AddTransient<HgWebHealthCheck>();
         services.AddScoped<ILexProxyService, LexProxyService>();
         services.AddSingleton<ISendReceiveService, SendReceiveService>();
         services.AddSingleton<LexboxLinkGenerator>();
         if (environment.IsDevelopment())
             services.AddHostedService<SwaggerValidationService>();
         services.AddScheduledTasks(configuration);
+        services.AddHealthChecks().AddCheck<HgWebHealthCheck>("hgweb", HealthStatus.Unhealthy, ["hg"], TimeSpan.FromSeconds(5));
         services.AddSyncProxy();
         AuthKernel.AddLexBoxAuth(services, configuration, environment);
         services.AddLexGraphQL(environment);
     }
 
-    private class SwaggerValidationService(IAsyncSwaggerProvider swaggerProvider): IHostedService
+    private class SwaggerValidationService(IAsyncSwaggerProvider swaggerProvider) : IHostedService
     {
         public async Task StartAsync(CancellationToken cancellationToken)
         {

backend/LexBoxApi/Services/HgService.cs

@@ -276,6 +276,13 @@ public async Task<HttpContent> ExecuteHgRecover(string code, CancellationToken t
         return int.TryParse(str, out int result) ? result : null;
     }
 
+    public async Task<string> HgCommandHealth()
+    {
+        var content = await ExecuteHgCommandServerCommand("health", "healthz", default);
+        var version = await content.ReadAsStringAsync();
+        return version.Trim();
+    }
+
     private async Task<HttpContent> ExecuteHgCommandServerCommand(string code, string command, CancellationToken token)
     {
         var httpClient = _hgClient.Value;

backend/LexBoxApi/Services/HgWebHealthCheck.cs

@@ -0,0 +1,23 @@
+using LexCore.ServiceInterfaces;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace LexBoxApi.Services;
+
+public class HgWebHealthCheck(IHgService hgService) : IHealthCheck
+{
+    public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context,
+        CancellationToken cancellationToken = new())
+    {
+        var version = await hgService.HgCommandHealth();
+        if (string.IsNullOrEmpty(version))
+        {
+            return HealthCheckResult.Unhealthy();
+        }
+        if (version != AppVersionService.Version)
+        {
+            return HealthCheckResult.Degraded(
+                $"api version: '{AppVersionService.Version}' hg version: '{version}' mismatch");
+        }
+        return HealthCheckResult.Healthy();
+    }
+}

backend/LexCore/ServiceInterfaces/IHgService.cs

@@ -19,4 +19,5 @@ public interface IHgService
     Task<string?> GetRepositoryIdentifier(Project project);
     Task<HttpContent> ExecuteHgRecover(string code, CancellationToken token);
     bool HasAbandonedTransactions(string projectCode);
+    Task<string> HgCommandHealth();
 }

deployment/base/app-config.yaml

@@ -4,6 +4,6 @@ metadata:
   name: app-config
 data:
   environment-name: "Development"
-  hg-otel-enabled: "ON"
+  hg-otel-disabled: "false" # "true" to disable OpenTelemetry
   hg-domain: "hg.localhost"
 

deployment/base/hg-deployment.yaml

@@ -66,11 +66,11 @@ spec:
             value: ".*"
           - name: OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE
             value: ".*"
-          - name: OTEL_ENABLED
+          - name: OTEL_SDK_DISABLED
             valueFrom:
               configMapKeyRef:
                 name: app-config
-                key: hg-otel-enabled
+                key: hg-otel-disabled
           - name: ENABLE_DEMAND_IMPORT
             value: "false"
         ports:

deployment/develop/app-config.yaml

@@ -4,6 +4,5 @@ metadata:
   name: app-config
 data:
   environment-name: "Development"
-  hg-otel-enabled: "ON"
   hg-domain: "hg-develop.lexbox.org"
 

deployment/local-dev/kustomization.yaml

@@ -30,7 +30,6 @@ patches:
     path: change-storage-class.patch.yaml
   - path: lexbox-deployment.patch.yaml
   - path: ui-deployment.patch.yaml
-  - path: hg-deployment-patch.yaml
   - path: hg-repos-pvc.patch.yaml
   - path: lexbox-service.patch.yaml
   - path: ingress-config.patch.yaml

deployment/staging/app-config.yaml

@@ -4,5 +4,4 @@ metadata:
   name: app-config
 data:
   environment-name: "Staging"
-  hg-otel-enabled: "ON"
   hg-domain: "hg-staging.languageforge.org"

hgweb/Dockerfile

@@ -48,4 +48,4 @@ VOLUME /var/hg/repos
 
 ARG APP_VERSION
 ENV APP_VERSION=$APP_VERSION
-ENV OTEL_ENABLED=ON
+ENV OTEL_RESOURCE_ATTRIBUTES="service.name=hgweb,service.version=${APP_VERSION}"

hgweb/command-runner.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Define the list of allowed commands
-allowed_commands=("verify" "tip" "lexentrycount" "recover")
+allowed_commands=("verify" "tip" "lexentrycount" "recover" "healthz")
 
 # Get the project code and command name from the URL
 IFS='/' read -ra PATH_SEGMENTS <<< "$PATH_INFO"
@@ -29,6 +29,15 @@ if [[ ! " ${allowed_commands[@]} " =~ " ${command_name} " ]]; then
     exit 1
 fi
 
+if [[ $command_name == "healthz" ]]; then
+    echo "lexbox-version: $APP_VERSION"
+    echo "Status: 200 OK"
+    echo "Content-type: text/plain"
+    echo ""
+    echo "$APP_VERSION"
+    exit 0
+fi
+
 # Start outputting the result right away so the HTTP connection won't be timed out
 echo "Content-type: text/plain"
 echo ""

hgweb/hg.conf

@@ -26,6 +26,7 @@ ServerName localhost
         </IfVersion>
     </Directory>
 
+    SetEnv APP_VERSION ${APP_VERSION}
     ScriptAliasMatch "^/command/(.*)" "/usr/local/www/commands/command-runner.sh/$1"
     <Directory /usr/local/www/commands/>
         Options +ExecCGI

hgweb/hgweb.wsgi

@@ -23,7 +23,6 @@ if os.getenv('ENABLE_DEMAND_IMPORT', 'false').lower() in ['1', 'true', 'yes']:
 else:
     demandimport.disable()
 
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.sdk.trace import TracerProvider
@@ -32,10 +31,7 @@ from opentelemetry.sdk.trace.export import (
     ConsoleSpanExporter,
 )
 
-resource = Resource(attributes={
-    SERVICE_NAME: "hgweb"
-})
-provider = TracerProvider(resource=resource)
+provider = TracerProvider()
 processor = BatchSpanProcessor(OTLPSpanExporter())
 provider.add_span_processor(processor)
 

hgweb/opentelemetry_module.conf

@@ -13,7 +13,6 @@ LoadFile /opt/opentelemetry-webserver-sdk/sdk_lib/lib/libopentelemetry_exporter_
 LoadFile /opt/opentelemetry-webserver-sdk/sdk_lib/lib/libopentelemetry_webserver_sdk.so
 #Load the Apache Module. In this example for Apache
 LoadModule otel_apache_module /opt/opentelemetry-webserver-sdk/WebServerModule/Apache/libmod_apache_otel.so
-ApacheModuleEnabled ${OTEL_ENABLED}
 
 #ApacheModule Otel Exporter details
 ApacheModuleOtelSpanExporter otlp
@@ -37,7 +36,6 @@ ApacheModuleServiceInstanceId ${APP_VERSION}
 
 ApacheModuleResolveBackends ON
 #https://github.com/open-telemetry/opentelemetry-cpp-contrib/blob/2a0db982f3d7ee91dfbe8150435e49e837bfb7ce/instrumentation/otel-webserver-module/src/apache/ApacheTracing.cpp#L90
-ApacheModuleTraceAsError ${OTEL_ENABLED}
 #ApacheModuleWebserverContext DemoService DemoServiceNamespace DemoInstanceId
 
 #ApacheModuleSegmentType custom

skaffold.yaml

@@ -30,6 +30,8 @@ build:
       context: hgweb
       docker:
         dockerfile: Dockerfile
+        buildArgs:
+          APP_VERSION: dockerDev
   local:
     useBuildkit: true
     concurrency: 2