Experimental ability to cache References

Adds ability to cache `Reference` objects, avoiding round-trips to the backend database, beneficial for read heavy workloads.

Reference-caching can be enabled via two new configuration options: to define the expiration time for `Reference`s (holding the current HEAD/tip) and to define the expiration time for non-existing `Reference`s.

Looking up references happens via the name of the reference, usually without the reference type (aka whether it is a branch or a tag), so Nessie has to look up both types - the given name as a branch and the given name as a tag. This is where negative-caching comes into play, because that caches the existing entry and the non-existing "other" reference type. Hence, if you enable reference-caching, it is recommended to also enable negative reference-caching.

Operations that are about to change a reference (committing and reference create/assign/delete operations), always consult the backing database, implicitly refreshing the cache.

Mutliple Nessie (against the same repository) do not communicate with each other. If for example a commit happened against one Nessie instance, the other instances may or may not return the new commit. This is why this feature is still experimental and only useful for Nessie setups with a _single_ Nessie instance. This change will be later with another change to allow distributed cache-eviction, so that other Nessie instances accessing the same repository work fine.

servers/quarkus-common/src/main/java/org/projectnessie/quarkus/config/QuarkusStoreConfig.java

@@ -20,6 +20,8 @@
 import io.smallrye.config.WithConverter;
 import io.smallrye.config.WithDefault;
 import io.smallrye.config.WithName;
+import java.time.Duration;
+import java.util.Optional;
 import java.util.OptionalDouble;
 import java.util.OptionalInt;
 import org.projectnessie.versioned.storage.common.config.StoreConfig;
@@ -136,4 +138,12 @@ public interface QuarkusStoreConfig extends StoreConfig {
    */
   @WithName(CONFIG_CACHE_CAPACITY_FRACTION_ADJUST_MB)
   OptionalInt cacheCapacityFractionAdjustMB();
+
+  @WithName(CONFIG_CACHE_REFERENCE_TTL)
+  @Override
+  Optional<Duration> cacheReferenceTtl();
+
+  @WithName(CONFIG_CACHE_REFERENCE_NEGATIVE_TTL)
+  @Override
+  Optional<Duration> cacheReferenceNegativeTtl();
 }

servers/quarkus-common/src/main/java/org/projectnessie/quarkus/providers/storage/PersistProvider.java

@@ -146,12 +146,21 @@ public Persist producePersist(MeterRegistry meterRegistry) {
 
     String cacheInfo;
     if (effectiveCacheSizeMB > 0) {
-      CacheConfig cacheConfig =
-          CacheConfig.builder()
-              .capacityMb(effectiveCacheSizeMB)
-              .meterRegistry(meterRegistry)
-              .build();
-      CacheBackend cacheBackend = PersistCaches.newBackend(cacheConfig);
+      CacheConfig.Builder cacheConfig =
+          CacheConfig.builder().capacityMb(effectiveCacheSizeMB).meterRegistry(meterRegistry);
+
+      storeConfig
+          .cacheReferenceTtl()
+          .ifPresent(
+              refTtl -> {
+                LOGGER.warn(
+                    "Reference caching is an experimental feature but enabled with a TTL of {}",
+                    refTtl);
+                cacheConfig.referenceTtl(refTtl);
+              });
+      storeConfig.cacheReferenceNegativeTtl().ifPresent(cacheConfig::referenceNegativeTtl);
+
+      CacheBackend cacheBackend = PersistCaches.newBackend(cacheConfig.build());
       persist = cacheBackend.wrap(persist);
       cacheInfo = "with " + effectiveCacheSizeMB + " MB objects cache";
     } else {

site/in-dev/configuration.md

@@ -157,6 +157,11 @@ default is conservative, bumping the cache size is recommended.
 
 {% include './generated-docs/smallrye-nessie_version_store_persist.md' %}
 
+!!! warning
+    Reference caching, enabled via the two `nessie.version.store.persist.cache-reference-*` settings, is still
+    experimental and only usable when only one Nessie server instance accesses a Nessie repository. Using this
+    feature is not recommended.
+
 ### Authentication settings
 
 {% include './generated-docs/smallrye-nessie_server_authentication.md' %}

versioned/storage/batching/src/main/java/org/projectnessie/versioned/storage/batching/BatchingPersistImpl.java

@@ -404,13 +404,27 @@ public Reference fetchReference(@Nonnull @javax.annotation.Nonnull String name)
     return delegate().fetchReference(name);
   }
 
+  @Override
+  @Nullable
+  @javax.annotation.Nullable
+  public Reference fetchReferenceForUpdate(@Nonnull @javax.annotation.Nonnull String name) {
+    return delegate().fetchReferenceForUpdate(name);
+  }
+
   @Override
   @Nonnull
   @javax.annotation.Nonnull
   public Reference[] fetchReferences(@Nonnull @javax.annotation.Nonnull String[] names) {
     return delegate().fetchReferences(names);
   }
 
+  @Override
+  @Nonnull
+  @javax.annotation.Nonnull
+  public Reference[] fetchReferencesForUpdate(@Nonnull @javax.annotation.Nonnull String[] names) {
+    return delegate().fetchReferencesForUpdate(names);
+  }
+
   @Override
   @Nonnull
   @javax.annotation.Nonnull

versioned/storage/bigtable/src/main/java/org/projectnessie/versioned/storage/bigtable/BigTablePersist.java

@@ -182,7 +182,7 @@ public Reference addReference(@Nonnull Reference reference) throws RefAlreadyExi
                       .otherwise(mutation));
 
       if (success) {
-        throw new RefAlreadyExistsException(fetchReference(reference.name()));
+        throw new RefAlreadyExistsException(fetchReferenceForUpdate(reference.name()));
       }
 
       return reference;

versioned/storage/cache/src/main/java/org/projectnessie/versioned/storage/cache/CacheBackend.java

@@ -15,17 +15,24 @@
  */
 package org.projectnessie.versioned.storage.cache;
 
+import static org.projectnessie.versioned.storage.common.persist.ObjId.zeroLengthObjId;
+import static org.projectnessie.versioned.storage.common.persist.Reference.reference;
+
 import jakarta.annotation.Nonnull;
 import org.projectnessie.versioned.storage.common.persist.Backend;
 import org.projectnessie.versioned.storage.common.persist.Obj;
 import org.projectnessie.versioned.storage.common.persist.ObjId;
 import org.projectnessie.versioned.storage.common.persist.Persist;
+import org.projectnessie.versioned.storage.common.persist.Reference;
 
 /**
  * Provides the cache primitives for a caching {@link Persist} facade, suitable for multiple
  * repositories. It is adviseable to have one {@link CacheBackend} per {@link Backend}.
  */
 public interface CacheBackend {
+  Reference NON_EXISTENT_REFERENCE_SENTINEL =
+      reference("NON_EXISTENT", zeroLengthObjId(), false, -1L, null);
+
   Obj get(@Nonnull String repositoryId, @Nonnull ObjId id);
 
   void put(@Nonnull String repositoryId, @Nonnull Obj obj);
@@ -35,4 +42,12 @@ public interface CacheBackend {
   void clear(@Nonnull String repositoryId);
 
   Persist wrap(@Nonnull Persist perist);
+
+  Reference getReference(@Nonnull String repositoryId, @Nonnull String name);
+
+  void removeReference(@Nonnull String repositoryId, @Nonnull String name);
+
+  void putReference(@Nonnull String repositoryId, @Nonnull Reference r);
+
+  void putNegative(@Nonnull String repositoryId, @Nonnull String name);
 }

versioned/storage/cache/src/main/java/org/projectnessie/versioned/storage/cache/CacheConfig.java

@@ -15,8 +15,11 @@
  */
 package org.projectnessie.versioned.storage.cache;
 
+import static com.google.common.base.Preconditions.checkState;
+
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import io.micrometer.core.instrument.MeterRegistry;
+import java.time.Duration;
 import java.util.Optional;
 import java.util.function.LongSupplier;
 import org.immutables.value.Value;
@@ -27,6 +30,10 @@ public interface CacheConfig {
 
   Optional<MeterRegistry> meterRegistry();
 
+  Optional<Duration> referenceTtl();
+
+  Optional<Duration> referenceNegativeTtl();
+
   @Value.Default
   default LongSupplier clockNanos() {
     return System::nanoTime;
@@ -36,13 +43,35 @@ static Builder builder() {
     return ImmutableCacheConfig.builder();
   }
 
+  @Value.Check
+  default void check() {
+    referenceTtl()
+        .ifPresent(
+            ttl ->
+                checkState(
+                    ttl.compareTo(Duration.ZERO) > 0,
+                    "Cache reference-TTL must be positive, if present."));
+    referenceNegativeTtl()
+        .ifPresent(
+            ttl ->
+                checkState(
+                    referenceTtl().isPresent() && ttl.compareTo(Duration.ZERO) > 0,
+                    "Cache reference-negative-TTL must only be present, if reference-TTL is configured, and must only be positive."));
+  }
+
   interface Builder {
     @CanIgnoreReturnValue
     Builder capacityMb(long capacityMb);
 
     @CanIgnoreReturnValue
     Builder meterRegistry(MeterRegistry meterRegistry);
 
+    @CanIgnoreReturnValue
+    Builder referenceTtl(Duration referenceTtl);
+
+    @CanIgnoreReturnValue
+    Builder referenceNegativeTtl(Duration referenceNegativeTtl);
+
     @CanIgnoreReturnValue
     Builder clockNanos(LongSupplier clockNanos);
 

versioned/storage/cache/src/main/java/org/projectnessie/versioned/storage/cache/CachingPersistImpl.java

@@ -15,6 +15,8 @@
  */
 package org.projectnessie.versioned.storage.cache;
 
+import static org.projectnessie.versioned.storage.cache.CacheBackend.NON_EXISTENT_REFERENCE_SENTINEL;
+
 import jakarta.annotation.Nonnull;
 import java.util.Set;
 import org.projectnessie.versioned.storage.common.config.StoreConfig;
@@ -261,40 +263,148 @@ public String name() {
     return persist.name();
   }
 
+  // References
+
   @Override
   @Nonnull
   public Reference addReference(@Nonnull Reference reference) throws RefAlreadyExistsException {
-    return persist.addReference(reference);
+    Reference r = null;
+    try {
+      return r = persist.addReference(reference);
+    } finally {
+      if (r != null) {
+        cache.putReference(r);
+      } else {
+        cache.removeReference(reference.name());
+      }
+    }
   }
 
   @Override
   @Nonnull
   public Reference markReferenceAsDeleted(@Nonnull Reference reference)
       throws RefNotFoundException, RefConditionFailedException {
-    return persist.markReferenceAsDeleted(reference);
+    Reference r = null;
+    try {
+      return r = persist.markReferenceAsDeleted(reference);
+    } finally {
+      if (r != null) {
+        cache.putReference(r);
+      } else {
+        cache.removeReference(reference.name());
+      }
+    }
   }
 
   @Override
   public void purgeReference(@Nonnull Reference reference)
       throws RefNotFoundException, RefConditionFailedException {
-    persist.purgeReference(reference);
+    try {
+      persist.purgeReference(reference);
+    } finally {
+      cache.removeReference(reference.name());
+    }
   }
 
   @Override
   @Nonnull
   public Reference updateReferencePointer(@Nonnull Reference reference, @Nonnull ObjId newPointer)
       throws RefNotFoundException, RefConditionFailedException {
-    return persist.updateReferencePointer(reference, newPointer);
+    Reference r = null;
+    try {
+      return r = persist.updateReferencePointer(reference, newPointer);
+    } finally {
+      if (r != null) {
+        cache.putReference(r);
+      } else {
+        cache.removeReference(reference.name());
+      }
+    }
   }
 
   @Override
   public Reference fetchReference(@Nonnull String name) {
-    return persist.fetchReference(name);
+    return fetchReferenceInternal(name, false);
+  }
+
+  @Override
+  public Reference fetchReferenceForUpdate(@Nonnull String name) {
+    return fetchReferenceInternal(name, true);
+  }
+
+  private Reference fetchReferenceInternal(@Nonnull String name, boolean bypassCache) {
+    Reference r = null;
+    if (!bypassCache) {
+      r = cache.getReference(name);
+      if (r == NON_EXISTENT_REFERENCE_SENTINEL) {
+        return null;
+      }
+    }
+
+    if (r == null) {
+      r = persist.fetchReferenceForUpdate(name);
+      if (r == null) {
+        cache.putNegative(name);
+      } else {
+        cache.putReference(r);
+      }
+    }
+    return r;
   }
 
   @Override
   @Nonnull
   public Reference[] fetchReferences(@Nonnull String[] names) {
-    return persist.fetchReferences(names);
+    return fetchReferencesInternal(names, false);
+  }
+
+  @Override
+  @Nonnull
+  public Reference[] fetchReferencesForUpdate(@Nonnull String[] names) {
+    return fetchReferencesInternal(names, true);
+  }
+
+  private Reference[] fetchReferencesInternal(@Nonnull String[] names, boolean bypassCache) {
+    Reference[] r = new Reference[names.length];
+
+    String[] backend = null;
+    if (!bypassCache) {
+      for (int i = 0; i < names.length; i++) {
+        String name = names[i];
+        if (name != null) {
+          Reference cr = cache.getReference(name);
+          if (cr != null) {
+            if (cr != NON_EXISTENT_REFERENCE_SENTINEL) {
+              r[i] = cr;
+            }
+          } else {
+            if (backend == null) {
+              backend = new String[names.length];
+            }
+            backend[i] = name;
+          }
+        }
+      }
+    } else {
+      backend = names;
+    }
+
+    if (backend != null) {
+      Reference[] br = persist.fetchReferencesForUpdate(backend);
+      for (int i = 0; i < br.length; i++) {
+        String name = backend[i];
+        if (name != null) {
+          Reference ref = br[i];
+          if (ref != null) {
+            r[i] = ref;
+            cache.putReference(ref);
+          } else {
+            cache.putNegative(name);
+          }
+        }
+      }
+    }
+
+    return r;
   }
 }