Skip to content

Commit

Permalink
Update to Apache Lucene 10 for 3.0.0
Browse files Browse the repository at this point in the history
Signed-off-by: Andriy Redko <[email protected]>
  • Loading branch information
reta committed Oct 21, 2024
1 parent 2dfd519 commit 77dcc97
Show file tree
Hide file tree
Showing 100 changed files with 142 additions and 191 deletions.
17 changes: 3 additions & 14 deletions .github/workflows/assemble.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ 11, 17, 21 ]
java: [ 21, 23 ]
os: [ubuntu-latest, windows-latest, macos-13]
steps:
- uses: actions/checkout@v4
Expand All @@ -16,17 +16,6 @@ jobs:
with:
java-version: ${{ matrix.java }}
distribution: temurin
- name: Set up JDK 17
# See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
if: matrix.java == 11
uses: actions/setup-java@v4
with:
java-version: 17
distribution: temurin
- name: Set JAVA${{ matrix.java }}_HOME
shell: bash
run: |
echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
- name: Setup docker (missing on MacOS)
id: setup_docker
if: runner.os == 'macos'
Expand All @@ -47,8 +36,8 @@ jobs:
shell: bash
if: runner.os != 'macos'
run: |
./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}
./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
- name: Run Gradle (assemble)
if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
run: |
./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}
./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
15 changes: 2 additions & 13 deletions .github/workflows/precommit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ 11, 17, 21, 23 ]
java: [ 21, 23 ]
os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
steps:
- uses: actions/checkout@v4
Expand All @@ -17,18 +17,7 @@ jobs:
java-version: ${{ matrix.java }}
distribution: temurin
cache: gradle
- name: Set up JDK 17
# See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
if: matrix.java == 11
uses: actions/setup-java@v4
with:
java-version: 17
distribution: temurin
- name: Set JAVA${{ matrix.java }}_HOME
shell: bash
run: |
echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
- name: Run Gradle (precommit)
shell: bash
run: |
./gradlew javadoc precommit --parallel -Druntime.java=${{ matrix.java }}
./gradlew javadoc precommit --parallel
4 changes: 2 additions & 2 deletions buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,8 @@ if (project != rootProject) {

allprojects {
java {
targetCompatibility = JavaVersion.VERSION_11
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_21
sourceCompatibility = JavaVersion.VERSION_21
}
}

Expand Down
4 changes: 2 additions & 2 deletions buildSrc/reaper/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
apply plugin: 'java'

java {
targetCompatibility = JavaVersion.VERSION_11
sourceCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_21
sourceCompatibility = JavaVersion.VERSION_21
}

jar {
Expand Down
4 changes: 2 additions & 2 deletions buildSrc/src/testKit/thirdPartyAudit/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@ dependencies {
}

tasks.register("empty", ThirdPartyAuditTask) {
targetCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_21
signatureFile = file('third-party-audit-empty.txt')
}

tasks.register("absurd", ThirdPartyAuditTask) {
targetCompatibility = JavaVersion.VERSION_11
targetCompatibility = JavaVersion.VERSION_21
signatureFile = file('third-party-audit-absurd.txt')
}
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
opensearch = 3.0.0
lucene = 9.12.0
lucene = 10.0.0

bundled_jdk_vendor = adoptium
bundled_jdk = 23+37
Expand Down
1 change: 1 addition & 0 deletions libs/core/licenses/lucene-core-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5a9b3f728041df5b054aaaed3c3fd7ff0fed8ee7
1 change: 0 additions & 1 deletion libs/core/licenses/lucene-core-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public class LegacyESVersion extends Version {

public static final LegacyESVersion V_6_0_0 = new LegacyESVersion(6000099, org.apache.lucene.util.Version.fromBits(7, 0, 0));
public static final LegacyESVersion V_6_5_0 = new LegacyESVersion(6050099, org.apache.lucene.util.Version.fromBits(7, 0, 0));
public static final LegacyESVersion V_7_2_0 = new LegacyESVersion(7020099, org.apache.lucene.util.Version.LUCENE_8_0_0);
public static final LegacyESVersion V_7_2_0 = new LegacyESVersion(7020099, org.apache.lucene.util.Version.fromBits(8, 0, 0));

// todo move back to Version.java if retiring legacy version support
protected static final Map<Integer, Version> idToVersion;
Expand Down
2 changes: 1 addition & 1 deletion libs/core/src/main/java/org/opensearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1);
public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1);
public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_12_0);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_0_0);
public static final Version CURRENT = V_3_0_0;

public static Version fromId(int id) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
89b26348ec305598fc224cc9583939564b67b2cf

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
082c38c1335c069a73622c37ca3d39e64c1b2d33

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
26071742008630779523d08c0b46b2f371ef23a0

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
4a6ff02a1bd34a3c0165da05f714bb8188074bdc

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
4c488697df5038a78e5e65bb9b6da120af62d824

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1137b9846ec000b49c70c3fe5f8cd79b7129be22

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3fd86db5e9748063369db4bed84f1bd2ca62d387

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
4e6b940b3b934d6de174fedaaeaefd647698648d

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-analysis-common-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
13eb016bab14973158554a2e6cdf2abbc5c3eda1
1 change: 0 additions & 1 deletion server/licenses/lucene-analysis-common-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-backward-codecs-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8e21f708eb1bbb71ce79cbfea093b6ca913f4abf
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-core-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5a9b3f728041df5b054aaaed3c3fd7ff0fed8ee7
1 change: 0 additions & 1 deletion server/licenses/lucene-core-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-grouping-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
17145d786d31e7ecd68d149ccc3e7ab83270f282
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-highlighter-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b8324f1b859620912c186b27d9666215ce3d258b
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-join-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3a4c5bf84c855b011e740f30cb8a23f2ee85e1c1
1 change: 0 additions & 1 deletion server/licenses/lucene-join-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-memory-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
bc0f37a0a06b445555d07e5fe199d73436d51352
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-misc-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5f619b32c62bb9405e7af595cf7311113ed62e33
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-queries-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3d2f98787e27e8e2a65d994c86563edf16dd92f1
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-queryparser-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e11886c913058ef20378715dee715d942d04babc
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-sandbox-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1795ce5b066bda61483c375f6b8e358aaa4f6348
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-spatial-extras-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a701eb363cf0a75ebacd1844398314250abcf592
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-spatial3d-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d9e0a8a6084d7657a633c1aa94d750414f5288c4
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-9.12.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-suggest-10.0.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d79099abc148a1906e129abbabd5e1b18a20c117
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -192,10 +192,10 @@ public static CollapseTopFieldDocs merge(Sort sort, int start, int size, Collaps
final CollapseTopFieldDocs shard = shardHits[shardIDX];
// totalHits can be non-zero even if no hits were
// collected, when searchAfter was used:
totalHitCount += shard.totalHits.value;
totalHitCount += shard.totalHits.value();
// If any hit count is a lower bound then the merged
// total hit count is a lower bound as well
if (shard.totalHits.relation == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO) {
if (shard.totalHits.relation() == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO) {
totalHitsRelation = TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
}
if (CollectionUtils.isEmpty(shard.scoreDocs) == false) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException {
public boolean advanceExact(int target) throws IOException {
if (sorted.advanceExact(target)) {
ord = (int) sorted.nextOrd();
if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_DOCS) {
throw new IllegalStateException(
"failed to collapse " + target + ", the collapse field must be single valued"
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,7 @@ public static Reader getDirectReader(IndexInput in) throws IOException {
*/
public static Mutable getMutable(int valueCount, int bitsPerValue, float acceptableOverheadRatio) {
final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio);
return getMutable(valueCount, formatAndBits.bitsPerValue, formatAndBits.format);
return getMutable(valueCount, formatAndBits.bitsPerValue(), formatAndBits.format());
}

/**
Expand Down Expand Up @@ -629,7 +629,13 @@ public static Writer getWriter(DataOutput out, int valueCount, int bitsPerValue,
assert valueCount >= 0;

final FormatAndBits formatAndBits = fastestFormatAndBits(valueCount, bitsPerValue, acceptableOverheadRatio);
final XWriter writer = getWriterNoHeader(out, formatAndBits.format, valueCount, formatAndBits.bitsPerValue, DEFAULT_BUFFER_SIZE);
final XWriter writer = getWriterNoHeader(
out,
formatAndBits.format(),
valueCount,
formatAndBits.bitsPerValue(),
DEFAULT_BUFFER_SIZE
);
writer.writeHeader();
return writer;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ SearchSortValuesAndFormats getBottomSortValues() {
}

synchronized void consumeTopDocs(TopFieldDocs topDocs, DocValueFormat[] sortValuesFormat) {
totalHits += topDocs.totalHits.value;
totalHits += topDocs.totalHits.value();
if (validateShardSortFields(topDocs.fields) == false) {
return;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -802,8 +802,8 @@ TotalHits getTotalHits() {

void add(TopDocsAndMaxScore topDocs, boolean timedOut, Boolean terminatedEarly) {
if (trackTotalHitsUpTo != SearchContext.TRACK_TOTAL_HITS_DISABLED) {
totalHits += topDocs.topDocs.totalHits.value;
if (topDocs.topDocs.totalHits.relation == Relation.GREATER_THAN_OR_EQUAL_TO) {
totalHits += topDocs.topDocs.totalHits.value();
if (topDocs.topDocs.totalHits.relation() == Relation.GREATER_THAN_OR_EQUAL_TO) {
totalHitsRelation = TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -703,7 +703,7 @@ String jvmVendor() {
}

String javaVersion() {
return Constants.JAVA_VERSION;
return Runtime.version().toString();
}

}
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/opensearch/common/lucene/Lucene.java
Original file line number Diff line number Diff line change
Expand Up @@ -423,8 +423,8 @@ public static ScoreDoc readScoreDoc(StreamInput in) throws IOException {
private static final Class<?> GEO_DISTANCE_SORT_TYPE_CLASS = LatLonDocValuesField.newDistanceSort("some_geo_field", 0, 0).getClass();

public static void writeTotalHits(StreamOutput out, TotalHits totalHits) throws IOException {
out.writeVLong(totalHits.value);
out.writeEnum(totalHits.relation);
out.writeVLong(totalHits.value());
out.writeEnum(totalHits.relation());
}

public static void writeTopDocs(StreamOutput out, TopDocsAndMaxScore topDocs) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOBooleanSupplier;
import org.opensearch.common.Nullable;

import java.io.IOException;
Expand Down Expand Up @@ -256,4 +257,9 @@ public ImpactsEnum impacts(int flags) throws IOException {
public BytesRef next() throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
}

@Override
public IOBooleanSupplier prepareSeekExact(BytesRef text) throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.codecs.lucene912.Lucene912Codec.Mode;
import org.apache.lucene.codecs.lucene100.Lucene100Codec;
import org.apache.lucene.codecs.lucene100.Lucene100Codec.Mode;
import org.opensearch.common.Nullable;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.index.IndexSettings;
Expand Down Expand Up @@ -70,10 +70,10 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
assert null != indexSettings;
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene912Codec());
codecs.put(LZ4, new Lucene912Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene912Codec(Mode.BEST_COMPRESSION));
codecs.put(ZLIB, new Lucene912Codec(Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene100Codec());
codecs.put(LZ4, new Lucene100Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene100Codec(Mode.BEST_COMPRESSION));
codecs.put(ZLIB, new Lucene100Codec(Mode.BEST_COMPRESSION));
} else {
// CompositeCodec still delegates to PerFieldMappingPostingFormatCodec
// We can still support all the compression codecs when composite index is present
Expand Down
Loading

0 comments on commit 77dcc97

Please sign in to comment.