Skip to content

Commit

Permalink
Storage: record previous HEADs of named references
Browse files Browse the repository at this point in the history
Persisted `Reference`s now keep a history of previous HEADs, limited in size and age, defaults to up to 20 elementa with a maximum age of 300 seconds.
  • Loading branch information
snazy committed Oct 10, 2023
1 parent 13bea7b commit d498667
Show file tree
Hide file tree
Showing 45 changed files with 1,225 additions and 61 deletions.
3 changes: 3 additions & 0 deletions api/model/src/main/java/org/projectnessie/api/v2/TreeApi.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.projectnessie.model.MergeResponse;
import org.projectnessie.model.Operations;
import org.projectnessie.model.Reference;
import org.projectnessie.model.ReferenceHistoryResponse;
import org.projectnessie.model.ReferencesResponse;
import org.projectnessie.model.SingleReferenceResponse;
import org.projectnessie.model.Validation;
Expand Down Expand Up @@ -109,6 +110,8 @@ SingleReferenceResponse getReferenceByName(
GetReferenceParams params)
throws NessieNotFoundException;

ReferenceHistoryResponse getReferenceHistory(String refName) throws NessieNotFoundException;

/**
* Retrieve objects for a ref, potentially truncated by the backend.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2023 Dremio
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.projectnessie.model;

public enum CommitConsistency {
/** Consistency of the commit was not checked. */
NOT_CHECKED,
/**
* The commit is inconsistent in a way that makes it impossible to access the commit, for example
* if the commit object itself or its index information is missing.
*/
COMMIT_INCONSISTENT,
/**
* The commit object is present and its index is accessible, but some content reachable from the
* commit is not present.
*/
COMMIT_CONTENT_INCONSISTENT,
/** The commit object, its index information and all reachable content is present. */
COMMIT_CONSISTENT
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Copyright (C) 2023 Dremio
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.projectnessie.model;

import org.immutables.value.Value;

@Value.Immutable
public interface ReferenceHistory {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Copyright (C) 2020 Dremio
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.projectnessie.model;

import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.util.List;
import javax.annotation.Nullable;
import javax.validation.constraints.NotNull;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import org.immutables.value.Value;

@Schema(type = SchemaType.OBJECT, title = "ReferenceHistoryResponse")
@Value.Immutable
@JsonSerialize(as = ImmutableReferenceHistoryResponse.class)
@JsonDeserialize(as = ImmutableReferenceHistoryResponse.class)
public interface ReferenceHistoryResponse {

static ImmutableReferenceHistoryResponse.Builder builder() {
return ImmutableReferenceHistoryResponse.builder();
}

@NotNull
@jakarta.validation.constraints.NotNull
Reference getReference();

@NotNull
@jakarta.validation.constraints.NotNull
ReferenceHistoryState current();

@NotNull
@jakarta.validation.constraints.NotNull
List<ReferenceHistoryState> previous();

@Schema(type = SchemaType.OBJECT, title = "ReferenceHistoryState")
@Value.Immutable
@JsonSerialize(as = ImmutableReferenceHistoryState.class)
@JsonDeserialize(as = ImmutableReferenceHistoryState.class)
interface ReferenceHistoryState {
@Value.Parameter(order = 1)
String pointer();

@Value.Parameter(order = 2)
CommitConsistency commitConsistency();

@Value.Parameter(order = 3)
@Nullable
@jakarta.annotation.Nullable
CommitMeta meta();

static ReferenceHistoryState referenceHistoryElement(
String pointer, CommitConsistency commitConsistency, CommitMeta meta) {
return ImmutableReferenceHistoryState.of(pointer, commitConsistency, meta);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,16 @@ public interface QuarkusStoreConfig extends StoreConfig {
@Override
boolean validateNamespaces();

@WithName(CONFIG_PREVIOUS_HEAD_COUNT)
@WithDefault("" + DEFAULT_PREVIOUS_HEAD_COUNT)
@Override
int referencePreviousHeadCount();

@WithName(CONFIG_PREVIOUS_HEAD_TIME_SPAN_SECONDS)
@WithDefault("" + DEFAULT_PREVIOUS_HEAD_TIME_SPAN_SECONDS)
@Override
long referencePreviousHeadTimeSpanSeconds();

String CONFIG_CACHE_CAPACITY_MB = "cache-capacity-mb";

@WithName(CONFIG_CACHE_CAPACITY_MB)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import org.projectnessie.model.MergeResponse;
import org.projectnessie.model.Operations;
import org.projectnessie.model.Reference;
import org.projectnessie.model.ReferenceHistoryResponse;
import org.projectnessie.model.ReferencesResponse;
import org.projectnessie.model.SingleReferenceResponse;
import org.projectnessie.model.ser.Views;
Expand Down Expand Up @@ -177,6 +178,14 @@ public SingleReferenceResponse getReferenceByName(GetReferenceParams params)
.build();
}

@JsonView(Views.V2.class)
@Override
public ReferenceHistoryResponse getReferenceHistory(String refName)
throws NessieNotFoundException {
ParsedReference reference = parseRefPathString(refName);
return tree().getReferenceHistory(reference.name());
}

@JsonView(Views.V2.class)
@Override
public EntriesResponse getEntries(String ref, EntriesParams params)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
import org.projectnessie.error.NessieReferenceConflictException;
import org.projectnessie.error.NessieReferenceNotFoundException;
import org.projectnessie.model.Branch;
import org.projectnessie.model.CommitConsistency;
import org.projectnessie.model.CommitMeta;
import org.projectnessie.model.CommitResponse;
import org.projectnessie.model.Content;
Expand All @@ -84,6 +85,7 @@
import org.projectnessie.model.ImmutableContentKeyDetails;
import org.projectnessie.model.ImmutableLogEntry;
import org.projectnessie.model.ImmutableMergeResponse;
import org.projectnessie.model.ImmutableReferenceHistoryResponse;
import org.projectnessie.model.ImmutableReferenceMetadata;
import org.projectnessie.model.LogResponse.LogEntry;
import org.projectnessie.model.MergeBehavior;
Expand All @@ -94,6 +96,7 @@
import org.projectnessie.model.Operations;
import org.projectnessie.model.Reference;
import org.projectnessie.model.Reference.ReferenceType;
import org.projectnessie.model.ReferenceHistoryResponse;
import org.projectnessie.model.ReferenceMetadata;
import org.projectnessie.model.Tag;
import org.projectnessie.model.Validation;
Expand Down Expand Up @@ -121,6 +124,7 @@
import org.projectnessie.versioned.Put;
import org.projectnessie.versioned.ReferenceAlreadyExistsException;
import org.projectnessie.versioned.ReferenceConflictException;
import org.projectnessie.versioned.ReferenceHistory;
import org.projectnessie.versioned.ReferenceInfo;
import org.projectnessie.versioned.ReferenceNotFoundException;
import org.projectnessie.versioned.TagName;
Expand Down Expand Up @@ -254,6 +258,36 @@ public Reference getReferenceByName(String refName, FetchOption fetchOption)
}
}

@Override
public ReferenceHistoryResponse getReferenceHistory(String refName)
throws NessieNotFoundException {
Reference ref;
ReferenceHistory history;
try {
ref = makeReference(getStore().getNamedRef(refName, getGetNamedRefsParams(false)), false);

startAccessCheck().canViewReference(RefUtil.toNamedRef(ref)).checkAndThrow();

history = getStore().getReferenceHistory(ref.getName(), null);
} catch (ReferenceNotFoundException e) {
throw new NessieReferenceNotFoundException(e.getMessage(), e);
}

ImmutableReferenceHistoryResponse.Builder response =
ReferenceHistoryResponse.builder().reference(ref);
response.current(convertStoreHistoryEntry(history.current()));
history.previous().stream().map(this::convertStoreHistoryEntry).forEach(response::addPrevious);
return response.build();
}

private ReferenceHistoryResponse.ReferenceHistoryState convertStoreHistoryEntry(
ReferenceHistory.ReferenceHistoryElement element) {
return ReferenceHistoryResponse.ReferenceHistoryState.referenceHistoryElement(
element.pointer().asString(),
CommitConsistency.valueOf(element.commitConsistency().name()),
element.meta());
}

@Override
public Reference createReference(
String refName, ReferenceType type, String targetHash, String sourceRefName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.projectnessie.model.Operations;
import org.projectnessie.model.Reference;
import org.projectnessie.model.Reference.ReferenceType;
import org.projectnessie.model.ReferenceHistoryResponse;
import org.projectnessie.versioned.NamedRef;
import org.projectnessie.versioned.WithHash;

Expand Down Expand Up @@ -77,6 +78,18 @@ Reference getReferenceByName(
FetchOption fetchOption)
throws NessieNotFoundException;

ReferenceHistoryResponse getReferenceHistory(
@Valid
@jakarta.validation.Valid
@NotNull
@jakarta.validation.constraints.NotNull
@Pattern(regexp = REF_NAME_REGEX, message = REF_NAME_MESSAGE)
@jakarta.validation.constraints.Pattern(
regexp = REF_NAME_REGEX,
message = REF_NAME_MESSAGE)
String refName)
throws NessieNotFoundException;

Reference createReference(
@Valid
@jakarta.validation.Valid
Expand Down
2 changes: 2 additions & 0 deletions site/docs/try/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ default is conservative, bumping the cache size is recommended.
| `nessie.version.store.persist.cache-capacity-fraction-of-heap` | see description | `double` | Fraction of Java's max heap size to use for cache objects, set to `0` to disable. Must not be used with fixed cache sizing. If neither this value nor a fixed size is configured, a default of `.7` (70%) is assumed. |
| `nessie.version.store.persist.cache-capacity-fraction-adjust-mb` | `256` | `int` | When using fractional cache sizing, this amount in MB of the heap will always be "kept free" when calculating the cache size. |
| `nessie.version.store.persist.cache-capacity-fraction-min-size-mb` | `64` | `int` | When using fractional cache sizing, this amount in MB is the minimum cache size. |
| `nessie.version.store.persist.ref-previous-head-count` | `20` | `int` | Named references keep a history of up to this amount of previous HEAD pointers, and up to the configured age. |
| `nessie.version.store.persist.ref-previous-head-time-span-seconds` | `300` | `int` | Named references keep a history of previous HEAD pointers with this age in _seconds_, and up to the configured amount. |

#### Legacy version store configuration

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@
import org.projectnessie.versioned.ReferenceConflictException;
import org.projectnessie.versioned.ReferenceCreatedResult;
import org.projectnessie.versioned.ReferenceDeletedResult;
import org.projectnessie.versioned.ReferenceHistory;
import org.projectnessie.versioned.ReferenceHistoryParams;
import org.projectnessie.versioned.ReferenceInfo;
import org.projectnessie.versioned.ReferenceNotFoundException;
import org.projectnessie.versioned.RelativeCommitSpec;
Expand Down Expand Up @@ -464,6 +466,11 @@ private CommitMeta deserializeMetadata(ByteString commitMeta) {
: null;
}

@Override
public ReferenceHistory getReferenceHistory(String refName, ReferenceHistoryParams params) {
throw new UnsupportedOperationException("Reference history not supported.");
}

@Override
public PaginationIterator<Commit> getCommits(Ref ref, boolean fetchAdditionalInfo)
throws ReferenceNotFoundException {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2023 Dremio
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.projectnessie.versioned;

public enum CommitConsistency {
/** Consistency of the commit was not checked. */
NOT_CHECKED,
/**
* The commit is inconsistent in a way that makes it impossible to access the commit, for example
* if the commit object itself or its index information is missing.
*/
COMMIT_INCONSISTENT,
/**
* The commit object is present and its index is accessible, but some content reachable from the
* commit is not present.
*/
COMMIT_CONTENT_INCONSISTENT,
/** The commit object, its index information and all reachable content is present. */
COMMIT_CONSISTENT
}
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,12 @@ public ReferenceInfo<CommitMeta> getNamedRef(String ref, GetNamedRefsParams para
return delegate.getNamedRef(ref, params);
}

@Override
public ReferenceHistory getReferenceHistory(String refName, ReferenceHistoryParams params)
throws ReferenceNotFoundException {
return delegate.getReferenceHistory(refName, params);
}

@Override
public PaginationIterator<ReferenceInfo<CommitMeta>> getNamedRefs(
GetNamedRefsParams params, String pagingToken) throws ReferenceNotFoundException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,15 @@ public ReferenceInfo<CommitMeta> getNamedRef(
return delegate.getNamedRef(ref, params);
}

@WithSpan
@Override
@Counted(PREFIX)
@Timed(value = PREFIX, histogram = true)
public ReferenceHistory getReferenceHistory(String refName, ReferenceHistoryParams params)
throws ReferenceNotFoundException {
return delegate.getReferenceHistory(refName, params);
}

@WithSpan
@Override
@Counted(PREFIX)
Expand Down
Loading

0 comments on commit d498667

Please sign in to comment.