From efb85abdc8282fd269dc8ff5070bae2c09bbccd7 Mon Sep 17 00:00:00 2001 From: Mark Otway Date: Thu, 19 Nov 2020 23:28:36 +0000 Subject: [PATCH] Various data and indexing fixes --- .DS_Store | Bin 10244 -> 10244 bytes .../20201119225531_ImproveIndexes.Designer.cs | 470 ++++++++++++++++++ .../20201119225531_ImproveIndexes.cs | 22 + .../Migrations/ImageContextModelSnapshot.cs | 2 + Damselfly.Core/Models/ImageContext.cs | 6 + Damselfly.Core/Services/SearchService.cs | 13 +- Damselfly.Desktop/package.json | 2 +- Damselfly.Web/Shared/DatePickerEx.razor | 2 +- Damselfly.Web/Startup.cs | 8 +- .../VirtualScroll/BlazorVirtualScrolling.js | 1 + Dockerfile | 21 +- VERSION | 2 +- 12 files changed, 527 insertions(+), 22 deletions(-) create mode 100644 Damselfly.Core/Migrations/20201119225531_ImproveIndexes.Designer.cs create mode 100644 Damselfly.Core/Migrations/20201119225531_ImproveIndexes.cs diff --git a/.DS_Store b/.DS_Store index e8407ac49fe8e4a5b35dfc44d05be9566bd4b666..e62a7aaff963d06966e3f1bfb18f674aab956c53 100644 GIT binary patch delta 56 zcmZn(XbITxPMq=NvX KUa*6eiUI&frW4En delta 29 kcmZn(XbITxPJD8Q;JL}k(sG+`O6an0W> +using System; +using Damselfly.Core.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace Damselfly.Core.Migrations +{ + [DbContext(typeof(ImageContext))] + [Migration("20201119225531_ImproveIndexes")] + partial class ImproveIndexes + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "5.0.0"); + + modelBuilder.Entity("Damselfly.Core.Models.Basket", b => + { + b.Property("BasketId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("DateAdded") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.HasKey("BasketId"); + + b.ToTable("Baskets"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.BasketEntry", b => + { + b.Property("BasketEntryId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("BasketId") + .HasColumnType("INTEGER"); + + b.Property("DateAdded") + .HasColumnType("TEXT"); + + b.Property("ImageId") + .HasColumnType("INTEGER"); + + b.HasKey("BasketEntryId"); + + b.HasIndex("BasketId"); + + b.HasIndex("ImageId") + .IsUnique(); + + b.ToTable("BasketEntries"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Camera", b => + { + b.Property("CameraId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Make") + .HasColumnType("TEXT"); + + b.Property("Model") + .HasColumnType("TEXT"); + + b.Property("Serial") + .HasColumnType("TEXT"); + + b.HasKey("CameraId"); + + b.ToTable("Cameras"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ConfigSetting", b => + { + b.Property("ConfigSettingId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("ConfigSettingId"); + + b.ToTable("ConfigSettings"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ExifOperation", b => + { + b.Property("ExifOperationId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ImageId") + .HasColumnType("INTEGER"); + + b.Property("Operation") + .HasColumnType("INTEGER"); + + b.Property("State") + .HasColumnType("INTEGER"); + + b.Property("Text") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TimeStamp") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.HasKey("ExifOperationId"); + + b.HasIndex("TimeStamp"); + + b.HasIndex("ImageId", "Text"); + + b.ToTable("KeywordOperations"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ExportConfig", b => + { + b.Property("ExportConfigId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("Size") + .HasColumnType("INTEGER"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.Property("WatermarkText") + .HasColumnType("TEXT"); + + b.HasKey("ExportConfigId"); + + b.ToTable("DownloadConfigs"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.FTSTag", b => + { + b.Property("FTSTagId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Keyword") + .HasColumnType("TEXT"); + + b.HasKey("FTSTagId"); + + b.ToTable("FTSTags"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Folder", b => + { + b.Property("FolderId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("FolderScanDate") + .HasColumnType("TEXT"); + + b.Property("ParentFolderId") + .HasColumnType("INTEGER"); + + b.Property("Path") + .HasColumnType("TEXT"); + + b.HasKey("FolderId"); + + b.HasIndex("FolderScanDate"); + + b.HasIndex("Path"); + + b.ToTable("Folders"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Image", b => + { + b.Property("ImageId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("FileCreationDate") + .HasColumnType("TEXT"); + + b.Property("FileLastModDate") + .HasColumnType("TEXT"); + + b.Property("FileName") + .HasColumnType("TEXT"); + + b.Property("FileSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("FolderId") + .HasColumnType("INTEGER"); + + b.Property("LastUpdated") + .HasColumnType("TEXT"); + + b.HasKey("ImageId"); + + b.HasIndex("FileLastModDate"); + + b.HasIndex("FileName"); + + b.HasIndex("FolderId"); + + b.HasIndex("LastUpdated"); + + b.ToTable("Images"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ImageMetaData", b => + { + b.Property("MetaDataId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CameraId") + .HasColumnType("INTEGER"); + + b.Property("Caption") + .HasColumnType("TEXT"); + + b.Property("DateTaken") + .HasColumnType("TEXT"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("Exposure") + .HasColumnType("TEXT"); + + b.Property("FNum") + .HasColumnType("TEXT"); + + b.Property("FlashFired") + .HasColumnType("INTEGER"); + + b.Property("Height") + .HasColumnType("INTEGER"); + + b.Property("ISO") + .HasColumnType("TEXT"); + + b.Property("ImageId") + .HasColumnType("INTEGER"); + + b.Property("LastUpdated") + .HasColumnType("TEXT"); + + b.Property("LensId") + .HasColumnType("INTEGER"); + + b.Property("Rating") + .HasColumnType("INTEGER"); + + b.Property("ThumbLastUpdated") + .HasColumnType("TEXT"); + + b.Property("Width") + .HasColumnType("INTEGER"); + + b.HasKey("MetaDataId"); + + b.HasIndex("CameraId"); + + b.HasIndex("DateTaken"); + + b.HasIndex("ImageId") + .IsUnique(); + + b.HasIndex("LensId"); + + b.HasIndex("ThumbLastUpdated"); + + b.ToTable("ImageMetaData"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ImageTag", b => + { + b.Property("ImageId") + .HasColumnType("INTEGER"); + + b.Property("TagId") + .HasColumnType("INTEGER"); + + b.HasKey("ImageId", "TagId"); + + b.HasIndex("TagId"); + + b.HasIndex("ImageId", "TagId") + .IsUnique(); + + b.ToTable("ImageTags"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Lens", b => + { + b.Property("LensId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Make") + .HasColumnType("TEXT"); + + b.Property("Model") + .HasColumnType("TEXT"); + + b.Property("Serial") + .HasColumnType("TEXT"); + + b.HasKey("LensId"); + + b.ToTable("Lenses"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Tag", b => + { + b.Property("TagId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Keyword") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TimeStamp") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("TEXT"); + + b.HasKey("TagId"); + + b.HasIndex("Keyword") + .IsUnique(); + + b.ToTable("Tags"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.BasketEntry", b => + { + b.HasOne("Damselfly.Core.Models.Basket", "Basket") + .WithMany("BasketEntries") + .HasForeignKey("BasketId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Damselfly.Core.Models.Image", "Image") + .WithOne("BasketEntry") + .HasForeignKey("Damselfly.Core.Models.BasketEntry", "ImageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Basket"); + + b.Navigation("Image"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ExifOperation", b => + { + b.HasOne("Damselfly.Core.Models.Image", "Image") + .WithMany() + .HasForeignKey("ImageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Image"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Image", b => + { + b.HasOne("Damselfly.Core.Models.Folder", "Folder") + .WithMany("Images") + .HasForeignKey("FolderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Folder"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ImageMetaData", b => + { + b.HasOne("Damselfly.Core.Models.Camera", "Camera") + .WithMany() + .HasForeignKey("CameraId"); + + b.HasOne("Damselfly.Core.Models.Image", "Image") + .WithOne("MetaData") + .HasForeignKey("Damselfly.Core.Models.ImageMetaData", "ImageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Damselfly.Core.Models.Lens", "Lens") + .WithMany() + .HasForeignKey("LensId"); + + b.Navigation("Camera"); + + b.Navigation("Image"); + + b.Navigation("Lens"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.ImageTag", b => + { + b.HasOne("Damselfly.Core.Models.Image", "Image") + .WithMany("ImageTags") + .HasForeignKey("ImageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Damselfly.Core.Models.Tag", "Tag") + .WithMany("ImageTags") + .HasForeignKey("TagId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Image"); + + b.Navigation("Tag"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Basket", b => + { + b.Navigation("BasketEntries"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Folder", b => + { + b.Navigation("Images"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Image", b => + { + b.Navigation("BasketEntry"); + + b.Navigation("ImageTags"); + + b.Navigation("MetaData"); + }); + + modelBuilder.Entity("Damselfly.Core.Models.Tag", b => + { + b.Navigation("ImageTags"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Damselfly.Core/Migrations/20201119225531_ImproveIndexes.cs b/Damselfly.Core/Migrations/20201119225531_ImproveIndexes.cs new file mode 100644 index 00000000..1033d17c --- /dev/null +++ b/Damselfly.Core/Migrations/20201119225531_ImproveIndexes.cs @@ -0,0 +1,22 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace Damselfly.Core.Migrations +{ + public partial class ImproveIndexes : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateIndex( + name: "IX_Images_FileName", + table: "Images", + column: "FileName"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_Images_FileName", + table: "Images"); + } + } +} diff --git a/Damselfly.Core/Migrations/ImageContextModelSnapshot.cs b/Damselfly.Core/Migrations/ImageContextModelSnapshot.cs index d7fca338..65c12d5a 100644 --- a/Damselfly.Core/Migrations/ImageContextModelSnapshot.cs +++ b/Damselfly.Core/Migrations/ImageContextModelSnapshot.cs @@ -218,6 +218,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("FileLastModDate"); + b.HasIndex("FileName"); + b.HasIndex("FolderId"); b.HasIndex("LastUpdated"); diff --git a/Damselfly.Core/Models/ImageContext.cs b/Damselfly.Core/Models/ImageContext.cs index b3a93788..7ee949e6 100644 --- a/Damselfly.Core/Models/ImageContext.cs +++ b/Damselfly.Core/Models/ImageContext.cs @@ -58,9 +58,15 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .WithOne(b => b.BasketEntry) .HasForeignKey(e => e.ImageId); + modelBuilder.Entity() + .HasOne(img => img.MetaData) + .WithOne(meta => meta.Image) + .HasForeignKey(i => i.ImageId); + modelBuilder.Entity().HasIndex(x => new { x.ImageId, x.TagId }).IsUnique(); modelBuilder.Entity().HasIndex(x => new { x.FolderId }); modelBuilder.Entity().HasIndex(x => x.LastUpdated); + modelBuilder.Entity().HasIndex(x => x.FileName); modelBuilder.Entity().HasIndex(x => x.FileLastModDate); modelBuilder.Entity().HasIndex(x => x.FolderScanDate); modelBuilder.Entity().HasIndex(x => x.Path); diff --git a/Damselfly.Core/Services/SearchService.cs b/Damselfly.Core/Services/SearchService.cs index b4c646de..0ff37906 100644 --- a/Damselfly.Core/Services/SearchService.cs +++ b/Damselfly.Core/Services/SearchService.cs @@ -128,11 +128,14 @@ private void LoadMoreData(int first, int count) images = images.Include(x => x.MetaData); - // Always filter by date - because if there's no filter - // set then they'll be set to min/max date. - images = images.Where(x => x.MetaData == null || - (x.MetaData.DateTaken >= query.MinDate && - x.MetaData.DateTaken <= query.MaxDate )); + if (query.MinDate > DateTime.MinValue || query.MaxDate < DateTime.MaxValue) + { + // Always filter by date - because if there's no filter + // set then they'll be set to min/max date. + images = images.Where(x => x.MetaData == null || + (x.MetaData.DateTaken >= query.MinDate && + x.MetaData.DateTaken <= query.MaxDate)); + } if( query.MinSizeKB > ulong.MinValue ) { diff --git a/Damselfly.Desktop/package.json b/Damselfly.Desktop/package.json index c392a5b1..58f62f5f 100644 --- a/Damselfly.Desktop/package.json +++ b/Damselfly.Desktop/package.json @@ -1,6 +1,6 @@ { "name": "Damselfly", - "version": "1.0.81", + "version": "1.0.82", "description": "Damselfy Desktop App", "main": "main.js", "scripts": { diff --git a/Damselfly.Web/Shared/DatePickerEx.razor b/Damselfly.Web/Shared/DatePickerEx.razor index bc8d7276..0807da5b 100644 --- a/Damselfly.Web/Shared/DatePickerEx.razor +++ b/Damselfly.Web/Shared/DatePickerEx.razor @@ -43,11 +43,11 @@ DateRanges.Add("This year", new DateRange { Start = Jan1ThisYear, End = DateTime.UtcNow }); DateRanges.Add("Last year", new DateRange { Start = Jan1ThisYear.AddYears(-1), End = Jan1ThisYear.AddSeconds(-1) }); - DateRanges.Add($"{Jan1ThisYear.AddYears(-1).Year}", new DateRange { Start = Jan1ThisYear.AddYears(-1), End = Jan1ThisYear.AddSeconds(-1) }); DateRanges.Add($"{Jan1ThisYear.AddYears(-2).Year}", new DateRange { Start = Jan1ThisYear.AddYears(-2), End = Jan1ThisYear.AddYears(-1).AddSeconds(-1) }); DateRanges.Add($"{Jan1ThisYear.AddYears(-3).Year}", new DateRange { Start = Jan1ThisYear.AddYears(-3), End = Jan1ThisYear.AddYears(-2).AddSeconds(-1) }); DateRanges.Add($"{Jan1ThisYear.AddYears(-4).Year}", new DateRange { Start = Jan1ThisYear.AddYears(-4), End = Jan1ThisYear.AddYears(-3).AddSeconds(-1) }); DateRanges.Add($"{Jan1ThisYear.AddYears(-5).Year}", new DateRange { Start = Jan1ThisYear.AddYears(-5), End = Jan1ThisYear.AddYears(-4).AddSeconds(-1) }); + DateRanges.Add($"{Jan1ThisYear.AddYears(-10).Year}", new DateRange { Start = Jan1ThisYear.AddYears(-10), End = Jan1ThisYear.AddYears(-9).AddSeconds(-1) }); } } diff --git a/Damselfly.Web/Startup.cs b/Damselfly.Web/Startup.cs index 4e9e4c45..683aefd2 100644 --- a/Damselfly.Web/Startup.cs +++ b/Damselfly.Web/Startup.cs @@ -118,11 +118,11 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) SearchService.Instance.PreLoadSearchData(); BasketService.Instance.Initialise(); - if (IndexingService.EnableIndexing) - IndexingService.Instance.StartService(); + //if (IndexingService.EnableIndexing) + // IndexingService.Instance.StartService(); - if (IndexingService.EnableThumbnailGeneration) - ThumbnailService.Instance.StartService(); + //if (IndexingService.EnableThumbnailGeneration) + // ThumbnailService.Instance.StartService(); Logging.Log("Preloading complete"); diff --git a/Damselfly.Web/wwwroot/VirtualScroll/BlazorVirtualScrolling.js b/Damselfly.Web/wwwroot/VirtualScroll/BlazorVirtualScrolling.js index 28b169c7..8cb55ded 100644 --- a/Damselfly.Web/wwwroot/VirtualScroll/BlazorVirtualScrolling.js +++ b/Damselfly.Web/wwwroot/VirtualScroll/BlazorVirtualScrolling.js @@ -9,6 +9,7 @@ var blazorVirtualScrolling = { (e) => { cmp.invokeMethodAsync("VirtualScrollingSetView", blazorVirtualScrolling.getScrollView(ref)); + }); } diff --git a/Dockerfile b/Dockerfile index 364226b5..9d86bcd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,6 @@ ARG RUNTIMEVERSION=5.0-alpine FROM mcr.microsoft.com/dotnet/aspnet:$RUNTIMEVERSION AS base WORKDIR /app EXPOSE 6363 -RUN mkdir -p ./Damselfly.Web/wwwroot/desktop -# Copy the mac desktop app into the image (this will be built outside docker) -COPY ./Damselfly.Web/wwwroot/desktop/damselfly-macos.zip /Damselfly.Web/wwwroot/desktop -# Copy the entrypoint script -COPY ./damselfly-entrypoint.sh / -RUN ["chmod", "+x", "/damselfly-entrypoint.sh"] -ADD VERSION . # First, build the Windows desktop app # FROM electronuserland/builder:wine as desktop @@ -23,7 +16,6 @@ ADD VERSION . # RUN yarn version --new-version ${DAMSELFLY_VERSION} # RUN yarn distwin # WORKDIR "/Damselfly.Desktop/dist" -# RUN ls # RUN zip /damselfly-win.zip *.* # Now build the app itself @@ -50,11 +42,20 @@ WORKDIR /app # This may re-copy the desktop/* apps, but who cares. COPY --from=publish /app/publish . +# Copy the mac desktop app into the image (this will be built outside docker) +# TODO: Maybe do this in the publish step? +RUN mkdir -p ./wwwroot/desktop +COPY ./Damselfly.Web/wwwroot/desktop/damselfly-macos.zip ./wwwroot/desktop +# Copy the entrypoint script +COPY ./damselfly-entrypoint.sh / +RUN ["chmod", "+x", "/damselfly-entrypoint.sh"] +ADD VERSION . + # Add Microsoft fonts that'll be used for watermarking -RUN apk add --no-cache msttcorefonts-installer fontconfig -RUN update-ms-fonts +RUN apk add --no-cache msttcorefonts-installer fontconfig && update-ms-fonts +# Add ExifTool RUN apk --update add exiftool && rm -rf /var/cache/apk/* # Make and install exiftool if we want a newer version diff --git a/VERSION b/VERSION index 818b8de7..224d004c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.81 +1.0.82