Merge pull request #212 from GeoWerkstatt/fix-cors-model-preview

Fix cors error in model preview
GeoWerkstatt · Jul 18, 2024 · 8eca6d5 · 8eca6d5
2 parents c846f0d + 4bd98db
commit 8eca6d5
Show file tree

Hide file tree

Showing 16 changed files with 267 additions and 119 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -76,3 +76,6 @@
 
 # *nix shell scripts always use LF (see .editorconfig)
 *.sh eol=lf
+
+# INTERLIS files set to a fixed line ending to get consistent file hashes
+*.ili text eol=lf
diff --git a/src/ClientApp/src/components/Detail.jsx b/src/ClientApp/src/components/Detail.jsx
@@ -20,7 +20,6 @@ import { getAllModels } from "./Utils";
 export function Detail() {
   const [model, setModel] = useState();
   const [loading, setLoading] = useState();
-  const [modelText, setModelText] = useState("");
   const { t } = useTranslation("common");
 
   const location = useLocation();
@@ -51,39 +50,16 @@ export function Detail() {
   useEffect(() => {
     setLoading(true);
 
-    async function getModelPreview(model) {
-      // Use try catch block to avoid error when CORS prevents successful fetch.
-      try {
-        const response = await fetch(model.uri);
-        if (response?.ok) {
-          setModelText(await response.text());
-          setLoading(false);
-        } else {
-          setModelText(t("no-model-preview"));
-          setLoading(false);
-        }
-      } catch {
-        setModelText(t("no-model-preview"));
-        setLoading(false);
-      }
-    }
-
     async function getModel(md5, name) {
       const response = await fetch("/model/" + md5 + "/" + name);
 
-      if (response.ok) {
-        if (response.status === 204 /* No Content */) {
-          setModel();
-          setLoading(false);
-        } else {
-          const model = await response.json();
-          setModel(model);
-          getModelPreview(model);
-        }
+      if (response.ok && response.status !== 204 /* No Content */) {
+        const model = await response.json();
+        setModel(model);
       } else {
         setModel();
-        setLoading(false);
       }
+      setLoading(false);
     }
     getModel(md5, name);
   }, [md5, name, t]);
@@ -99,7 +75,7 @@ export function Detail() {
           {t("to-search")}
         </Button>
       )}
-      {(!model || !modelText) && loading && (
+      {!model && loading && (
         <Box mt={10}>
           <CircularProgress />
         </Box>
@@ -109,7 +85,7 @@ export function Detail() {
           {t("invalid-model-url")}
         </Typography>
       )}
-      {model && modelText && (
+      {model && (
         <>
           <Stack direction="row" alignItems="flex-end" flexWrap="wrap" sx={{ color: "text.secondary" }}>
             <Typography mt={5} variant="h4">
@@ -169,9 +145,7 @@ export function Detail() {
                       key={m}
                       label={m}
                       variant="outlined"
-                    >
-                      {m}
-                    </Chip>
+                    />
                   ))}
               </Box>
             )}
@@ -181,12 +155,13 @@ export function Detail() {
                 {t("catalogue-files")}:{" "}
                 {model.catalogueFiles &&
                   model.catalogueFiles
-                    .sort((a, b) => {
-                      const result = (a.match(/\//g) || []).length - (b.match(/\//g) || []).length;
-                      return result === 0 ? a.localeCompare(b, undefined, { sensitivity: "base" }) : result;
-                    })
+                    .sort(
+                      (a, b) =>
+                        (a.match(/\//g) || []).length - (b.match(/\//g) || []).length ||
+                        a.localeCompare(b, undefined, { sensitivity: "base" }),
+                    )
                     .map((f) => (
-                      <Box sx={{ ml: 4 }}>
+                      <Box key={f} sx={{ ml: 4 }}>
                         <Typography variant="body" sx={{ mr: 1, fontSize: 14 }}>
                           <a href={f} target="_blank" rel="noreferrer">
                             {f}
@@ -222,7 +197,7 @@ export function Detail() {
               inputProps={{ style: { fontSize: 12, fontFamily: "'Courier New', monospace" } }}
               InputLabelProps={{ style: { fontSize: 22 } }}
               InputProps={{ readOnly: true, style: { fontSize: 22 } }}
-              value={modelText}
+              value={model.fileContent?.content ?? t("no-model-preview")}
               focused={false}
             />
           </Stack>

diff --git a/src/Controllers/ModelController.cs b/src/Controllers/ModelController.cs
@@ -33,6 +33,7 @@ public ModelController(ILogger<ModelController> logger, RepoBrowserContext conte
 
         var model = context.Models
             .Include(m => m.ModelRepository)
+            .Include(m => m.FileContent)
             .Where(m => m.MD5 == md5 && m.Name == name)
             .AsNoTracking()
             .SingleOrDefault();

diff --git a/src/Crawler/IRepositoryCrawler.cs b/src/Crawler/IRepositoryCrawler.cs
@@ -14,4 +14,12 @@ public interface IRepositoryCrawler
     /// <param name="options">The <see cref="RepositoryCrawlerOptions"/> that contain the repository at the root of the model repository tree and other configurations.</param>
     /// <returns>Dictionary containing all repositories found in tree. Repository host is used as key. Repositories contain all found information. Root repository contains full tree. </returns>
     Task<IDictionary<string, Repository>> CrawlModelRepositories(RepositoryCrawlerOptions options);
+
+    /// <summary>
+    /// Fetches the INTERLIS files from the <paramref name="repositories"/>. Files are identified by their MD5 hash and only downloaded if not already contained in <paramref name="existingFiles"/>.
+    /// If a <see cref="Model"/> is missing the <see cref="Model.MD5"/> property, it is set according to the downloaded file.
+    /// </summary>
+    /// <param name="existingFiles">The <see cref="InterlisFile"/>s previously fetched.</param>
+    /// <param name="repositories">The repositories to fetch the files for.</param>
+    Task FetchInterlisFiles(IEnumerable<InterlisFile> existingFiles, IEnumerable<Repository> repositories);
 }
diff --git a/src/Crawler/RepositoryCrawler.cs b/src/Crawler/RepositoryCrawler.cs
@@ -18,6 +18,44 @@ public RepositoryCrawler(ILogger<RepositoryCrawler> logger, IHttpClientFactory h
         httpClient = httpClientFactory.CreateClient();
     }
 
+    /// <inheritdoc />
+    public async Task FetchInterlisFiles(IEnumerable<InterlisFile> existingFiles, IEnumerable<Repository> repositories)
+    {
+        var allFiles = existingFiles.ToDictionary(f => f.MD5, StringComparer.OrdinalIgnoreCase);
+        foreach (var repository in repositories)
+        {
+            foreach (var model in repository.Models)
+            {
+                InterlisFile? file;
+                if (!string.IsNullOrEmpty(model.MD5) && allFiles.TryGetValue(model.MD5, out file))
+                {
+                    model.FileContent = file;
+                    continue;
+                }
+
+                var modelFileUrl = model.ModelRepository.Uri.Append(model.File);
+                file = await FetchInterlisFile(modelFileUrl).ConfigureAwait(false);
+                if (file != null)
+                {
+                    if (!allFiles.TryAdd(file.MD5, file))
+                    {
+                        file = allFiles[file.MD5];
+                    }
+
+                    model.FileContent = file;
+                    if (string.IsNullOrEmpty(model.MD5))
+                    {
+                        model.MD5 = file.MD5;
+                    }
+                    else if (!model.MD5.Equals(file.MD5, StringComparison.OrdinalIgnoreCase))
+                    {
+                        logger.LogWarning("The MD5 Hash of Model <{Model}> ({MD5Model}) does not match that of the file <{URL}> ({MD5File}).", model.Name, model.MD5, modelFileUrl, file.MD5);
+                    }
+                }
+            }
+        }
+    }
+
     /// <inheritdoc />
     public async Task<IDictionary<string, Repository>> CrawlModelRepositories(RepositoryCrawlerOptions options)
     {
@@ -191,26 +229,6 @@ private async Task<ISet<Model>> CrawlIlimodels(Uri repositoryUri)
                 })
                 .ToHashSet();
 
-            foreach (var model in models)
-            {
-                if (string.IsNullOrEmpty(model.MD5))
-                {
-                    var modelFileUrl = repositoryUri.Append(model.File);
-                    logger.LogInformation("Calculate missing MD5 for Model <{Model}> in File <{URL}>.", model.Name, modelFileUrl);
-
-                    try
-                    {
-                        var stream = await GetStreamFromUrl(modelFileUrl).ConfigureAwait(false);
-                        var md5 = await GetMD5FromStream(stream).ConfigureAwait(false);
-                        model.MD5 = md5;
-                    }
-                    catch (Exception ex) when (ex is HttpRequestException || ex is OperationCanceledException)
-                    {
-                        logger.LogError(ex, "Failed to calculate missing MD5 for Model <{Model}> in File <{URL}>", model.Name, modelFileUrl);
-                    }
-                }
-            }
-
             return models;
         }
     }
@@ -224,6 +242,29 @@ private async Task<ISet<Model>> CrawlIlimodels(Uri repositoryUri)
         }
     }
 
+    private async Task<InterlisFile?> FetchInterlisFile(Uri fileUri)
+    {
+        logger.LogDebug("Download INTERLIS file <{URL}>", fileUri);
+        try
+        {
+            var stream = await GetStreamFromUrl(fileUri).ConfigureAwait(false);
+            var md5 = await GetMD5FromStream(stream).ConfigureAwait(false);
+            stream.Seek(0, SeekOrigin.Begin);
+            using var reader = new StreamReader(stream);
+            var content = reader.ReadToEnd();
+            return new InterlisFile
+            {
+                MD5 = md5,
+                Content = content,
+            };
+        }
+        catch (Exception ex) when (ex is HttpRequestException || ex is OperationCanceledException)
+        {
+            logger.LogError(ex, "Failed to download INTERLIS file <{URL}>", fileUri);
+            return null;
+        }
+    }
+
     private async Task<Stream> GetStreamFromUrl(Uri url)
     {
         var response = await httpClient.GetAsync(url).ConfigureAwait(false);

diff --git a/src/DbUpdateService.cs b/src/DbUpdateService.cs
@@ -39,37 +39,38 @@ private async Task UpdateModelRepoDatabase()
 
         try
         {
-            using (var scope = scopeFactory.CreateScope())
+            using var scope = scopeFactory.CreateScope();
+
+            var crawler = scope.ServiceProvider.GetRequiredService<IRepositoryCrawler>();
+            var repositories = await crawler.CrawlModelRepositories(crawlerOptions).ConfigureAwait(false);
+            using var context = scope.ServiceProvider.GetRequiredService<RepoBrowserContext>();
+
+            var knownParentRepositories = context.Repositories
+                .Where(r => r.SubsidiarySites.Any())
+                .Select(r => r.HostNameId)
+                .ToList();
+
+            var allParentRepositoriesCrawled = knownParentRepositories.All(repositories.ContainsKey);
+
+            if (repositories.Any() && allParentRepositoriesCrawled)
+            {
+                using var transaction = context.Database.BeginTransaction();
+
+                context.Catalogs.ExecuteDelete();
+                context.Models.ExecuteDelete();
+                context.Repositories.ExecuteDelete();
+                context.SaveChanges();
+
+                await crawler.FetchInterlisFiles(context.InterlisFiles, repositories.Values);
+                context.Repositories.AddRange(repositories.Values);
+                context.SaveChanges();
+
+                transaction.Commit();
+                logger.LogInformation("Updating ModelRepoDatabase complete. Inserted {RepositoryCount} repositories.", repositories.Count);
+            }
+            else
             {
-                var crawler = scope.ServiceProvider.GetRequiredService<IRepositoryCrawler>();
-                var repositories = await crawler.CrawlModelRepositories(crawlerOptions).ConfigureAwait(false);
-                using var context = scope.ServiceProvider.GetRequiredService<RepoBrowserContext>();
-
-                var knownParentRepositories = context.Repositories
-                    .Where(r => r.SubsidiarySites.Any())
-                    .Select(r => r.HostNameId)
-                    .ToList();
-
-                var allParentRepositoriesCrawled = knownParentRepositories.All(repositories.ContainsKey);
-
-                if (repositories.Any() && allParentRepositoriesCrawled)
-                {
-                    context.Database.BeginTransaction();
-                    context.Catalogs.RemoveRange(context.Catalogs);
-                    context.Models.RemoveRange(context.Models);
-                    context.Repositories.RemoveRange(context.Repositories);
-                    context.SaveChanges();
-
-                    context.Repositories.AddRange(repositories.Values);
-                    context.SaveChanges();
-
-                    context.Database.CommitTransaction();
-                    logger.LogInformation("Updating ModelRepoDatabase complete. Inserted {RepositoryCount} repositories.", repositories.Count);
-                }
-                else
-                {
-                    logger.LogError("Updating ModelRepoDatabase aborted. Crawler could not parse all required repositories.");
-                }
+                logger.LogError("Updating ModelRepoDatabase aborted. Crawler could not parse all required repositories.");
             }
 
             healthCheck.LastDbUpdateSuccessful = true;

diff --git a/src/Models/Catalog.cs b/src/Models/Catalog.cs
@@ -19,4 +19,6 @@ public class Catalog
     public string? Title { get; set; }
 
     public List<string> ReferencedModels { get; set; }
+
+    public Repository ModelRepository { get; set; }
 }
diff --git a/src/Models/InterlisFile.cs b/src/Models/InterlisFile.cs
@@ -0,0 +1,13 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace ModelRepoBrowser.Models;
+
+public class InterlisFile
+{
+    [Key]
+    public string MD5 { get; set; }
+
+    public string Content { get; set; }
+
+    public ICollection<Model> Models { get; set; }
+}
diff --git a/src/Models/Model.cs b/src/Models/Model.cs
@@ -7,6 +7,9 @@ public class Model
 {
     public int Id { get; set; }
 
+    /// <summary>
+    /// The MD5 Hash of the INTERLIS file that contains this model.
+    /// </summary>
     public string? MD5 { get; set; }
 
     public string Name { get; set; }
@@ -33,6 +36,11 @@ public class Model
 
     public string? FurtherInformation { get; set; }
 
+    /// <summary>
+    /// The actual content of the INTERLIS file.
+    /// </summary>
+    public InterlisFile FileContent { get; set; }
+
     [NotMapped]
     public bool? IsDependOnModelResult { get; set; } = false;
 

diff --git a/src/RepoBrowserContext.cs b/src/RepoBrowserContext.cs
@@ -17,4 +17,5 @@ public RepoBrowserContext(DbContextOptions options)
     public DbSet<Model> Models { get; set; }
     public DbSet<Catalog> Catalogs { get; set; }
     public DbSet<SearchQuery> SearchQueries { get; set; }
+    public DbSet<InterlisFile> InterlisFiles { get; set; }
 }
diff --git a/tests/Initialize.cs b/tests/Initialize.cs
@@ -1,9 +1,5 @@
-using Microsoft.VisualStudio.TestTools.UnitTesting;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 namespace ModelRepoBrowser;
 
@@ -20,10 +16,11 @@ public static void AssemplyInitialize(TestContext testContext)
         context.Database.EnsureCreated();
 
         // Clear database and fill it with test data
-        context.SearchQueries.RemoveRange(context.SearchQueries);
-        context.Catalogs.RemoveRange(context.Catalogs);
-        context.Models.RemoveRange(context.Models);
-        context.Repositories.RemoveRange(context.Repositories);
+        context.SearchQueries.ExecuteDelete();
+        context.Catalogs.ExecuteDelete();
+        context.Models.ExecuteDelete();
+        context.Repositories.ExecuteDelete();
+        context.InterlisFiles.ExecuteDelete();
         context.SaveChanges();
 
         context.SeedData();

diff --git a/tests/ModelRepoBrowser.Test.csproj b/tests/ModelRepoBrowser.Test.csproj
@@ -58,6 +58,9 @@
     <None Update="Testdata\models.multiparent.testdata\TestModel.ili">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Update="Testdata\models.multiparent.testdata\TwoModelsInOneFile.ili">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
 </Project>