diff --git a/hdt-java-core/pom.xml b/hdt-java-core/pom.xml index 7ba140a7..2f8bd1e7 100644 --- a/hdt-java-core/pom.xml +++ b/hdt-java-core/pom.xml @@ -48,11 +48,6 @@ org.apache.jena jena-arq - - org.visnow - JLargeArrays - 1.7-20220624.150242-7 - pl.pragmatists JUnitParams diff --git a/hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/bitmap/Bitmap64Big.java b/hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/bitmap/Bitmap64Big.java index 9666ae1b..e9dc9ab7 100644 --- a/hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/bitmap/Bitmap64Big.java +++ b/hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/bitmap/Bitmap64Big.java @@ -34,7 +34,6 @@ import org.rdfhdt.hdt.util.disk.LongArrayDisk; import org.rdfhdt.hdt.util.io.Closer; import org.rdfhdt.hdt.util.io.IOUtil; -import org.visnow.jlargearrays.LongLargeArray; import java.io.Closeable; import java.io.IOException; diff --git a/hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/sequence/SequenceLog64Big.java b/hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/sequence/SequenceLog64Big.java index 43deec59..54a4cdde 100644 --- a/hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/sequence/SequenceLog64Big.java +++ b/hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/sequence/SequenceLog64Big.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.ArrayList; import java.util.Iterator; import org.rdfhdt.hdt.compact.integer.VByte; @@ -38,14 +37,13 @@ import org.rdfhdt.hdt.exceptions.IllegalFormatException; import org.rdfhdt.hdt.hdt.HDTVocabulary; import org.rdfhdt.hdt.listener.ProgressListener; +import org.rdfhdt.hdt.unsafe.UnsafeLongArray; import org.rdfhdt.hdt.util.BitUtil; import org.rdfhdt.hdt.util.crc.CRC32; import org.rdfhdt.hdt.util.crc.CRC8; import org.rdfhdt.hdt.util.crc.CRCInputStream; import org.rdfhdt.hdt.util.crc.CRCOutputStream; import org.rdfhdt.hdt.util.io.IOUtil; -import org.visnow.jlargearrays.LargeArrayUtils; -import org.visnow.jlargearrays.LongLargeArray; /** * @author mario.arias,Lyudmila Balakireva @@ -55,7 +53,7 @@ public class SequenceLog64Big implements DynamicSequence { private static final byte W = 64; private static final int INDEX = 1073741824; - LongLargeArray data; + UnsafeLongArray data; private int numbits; private long numentries; private long maxvalue; @@ -74,8 +72,7 @@ public SequenceLog64Big(int numbits, long capacity) { this.maxvalue = BitUtil.maxVal(numbits); long size = numWordsFor(numbits, capacity); - LongLargeArray.setMaxSizeOf32bitArray(SequenceLog64Big.INDEX); - + data = IOUtil.createLargeArray(Math.max(size,1)); } @@ -115,7 +112,7 @@ public static long numBytesFor(int bitsField, long total) { * @param bitsField Length in bits of each field * @param index Position to be retrieved */ - private static long getField(LongLargeArray data, int bitsField, long index) { + private static long getField(UnsafeLongArray data, int bitsField, long index) { if(bitsField==0) return 0; long bitPos = index*bitsField; @@ -137,7 +134,7 @@ private static long getField(LongLargeArray data, int bitsField, long index) { * @param index Position to store in * @param value Value to be stored */ - private static void setField(LongLargeArray data, int bitsField, long index, long value) { + private static void setField(UnsafeLongArray data, int bitsField, long index, long value) { if(bitsField==0) return; long bitPos = index*bitsField; @@ -145,7 +142,7 @@ private static void setField(LongLargeArray data, int bitsField, long index, lon long j= bitPos%W; long mask = ~(~0L << bitsField) << j; - data.set(i, (data.getLong(i) & ~mask) | (value << j)); + data.set(i, (data.get(i) & ~mask) | (value << j)); if((j+bitsField>W)) { mask = ~0L << (bitsField+j-W); @@ -157,8 +154,8 @@ private void resizeArray(long size) { //data = Arrays.copyOf(data, size); if(size > 0) { if (data.length() != size) { - LongLargeArray a = IOUtil.createLargeArray(size, false); - LargeArrayUtils.arraycopy(data, 0, a, 0, Math.min(size, data.length())); + UnsafeLongArray a = IOUtil.createLargeArray(size, false); + UnsafeLongArray.arraycopy(data, 0, a, 0, Math.min(size, data.length())); data = a; } }else{ @@ -287,7 +284,7 @@ public void resize(long numentries) { @Override public void clear() { - IOUtil.fillLargeArray(data, 0); + data.clear(); } /* (non-Javadoc) @@ -316,7 +313,7 @@ public void save(OutputStream output, ProgressListener listener) throws IOExcept long numwords = numWordsFor(numbits, numentries); for(long i=0;i0) { diff --git a/hdt-java-core/src/main/java/org/rdfhdt/hdt/unsafe/MemoryUtils.java b/hdt-java-core/src/main/java/org/rdfhdt/hdt/unsafe/MemoryUtils.java new file mode 100644 index 00000000..17d6e9da --- /dev/null +++ b/hdt-java-core/src/main/java/org/rdfhdt/hdt/unsafe/MemoryUtils.java @@ -0,0 +1,250 @@ +package org.rdfhdt.hdt.unsafe; + +import org.rdfhdt.hdt.util.concurrent.ExceptionThread; + +import java.lang.ref.Cleaner; +import java.lang.reflect.Field; +import java.util.stream.IntStream; + +/** + * Unsafe memory utilities + * + * @author Antoine Willerval + */ +public class MemoryUtils { + /** + * Unsafe object + */ + private static final sun.misc.Unsafe UNSAFE; + /** + * Cleaner + */ + private static final Cleaner CLEANER = Cleaner.create(); + /** + * pointer to a zero buffer of size {@link #BUFFER_SIZE} + */ + private static final long ZERO_BUFFER; + /** + * pointer to a minus one buffer of size {@link #BUFFER_SIZE} + */ + private static final long MINUS_ONE_BUFFER; + /** + * Object to clean the ZERO/MINUS buffers + */ + private static final Object BUFFER_PARENT = new Object() { + }; + /** + * number of bits in the buffer size + */ + private static final int BUFFER_SIZE_BITS = 12; // 2**12 = 4096 + /** + * number of bytes in the buffer for {@link #MINUS_ONE_BUFFER} and + * {@link #ZERO_BUFFER}, is equal to 2 to the power of + * {@link #BUFFER_SIZE_BITS}. + */ + private static final int BUFFER_SIZE; + /** + * max size of a Java array before switching to unsafe array, non-final for + * debug + */ + static int maxArraySize = Integer.MAX_VALUE >> 1; + /** + * threshold before switching to parallel set in memset, non-final for debug + */ + static long thresholdParallelSizeSet = (long) maxArraySize * 10; + + static { + try { + Field f = Class.forName("sun.misc.Unsafe").getDeclaredField("theUnsafe"); + f.setAccessible(true); + UNSAFE = (sun.misc.Unsafe) f.get(null); + if (UNSAFE == null) { + throw new NullPointerException("Unsafe value is null!"); + } + } catch (NullPointerException | ClassNotFoundException | IllegalAccessException | IllegalArgumentException + | NoSuchFieldException | SecurityException e) { + throw new Error("Can't get field value sun.misc.Unsafe", e); + } + + if (BUFFER_SIZE_BITS < 3) { + throw new Error("BUFFER_SIZE_BITS can't be lower than 3!"); + } + BUFFER_SIZE = 1 << BUFFER_SIZE_BITS; + + // allocate the buffers and directly bind them to their parents + + ZERO_BUFFER = UNSAFE.allocateMemory(BUFFER_SIZE); + bindPointerTo(ZERO_BUFFER, BUFFER_PARENT); + + MINUS_ONE_BUFFER = UNSAFE.allocateMemory(BUFFER_SIZE); + bindPointerTo(MINUS_ONE_BUFFER, BUFFER_PARENT); + + // now we can set the values + for (long i = 0; i < (BUFFER_SIZE >> 3); i++) { + UNSAFE.putLong(ZERO_BUFFER + i * 8, 0); + // we can use -1 because -1L = 0xFFFF_FFFF_FFFF_FFFF, so full of + // 0xFF bytes + UNSAFE.putLong(MINUS_ONE_BUFFER + i * 8, -1L); + } + } + + /** + * @return unsafe object, be careful and gentle with it + */ + public static sun.misc.Unsafe getUnsafe() { + return UNSAFE; + } + + /** + * @return cleaner object + */ + public static Cleaner getCleaner() { + return CLEANER; + } + + /** + * @return max size of an array + */ + public static int getMaxArraySize() { + return maxArraySize; + } + + /** + * allocate memory + * + * @param size number of elements + * @param sizeOf size of an element + * @return pointer + */ + public static long malloc(long size, int sizeOf) { + return malloc(size * sizeOf); + } + + /** + * allocate memory + * + * @param size size to allocate + * @return pointer + */ + public static long malloc(long size) { + return UNSAFE.allocateMemory(size); + } + + /** + * free a pointer from the memory + * + * @param ptr pointer + */ + public static void free(long ptr) { + UNSAFE.freeMemory(ptr); + } + + /** + * bind this pointer to an object + * + * @param pointer pointer + * @param parent parent object + */ + public static void bindPointerTo(long pointer, Object parent) { + CLEANER.register(parent, new CleanerObject(pointer)); + } + + private static void memorySet0(long ptr, long size, byte value) { + // fast version using buffer + if (value == -1) { + memorySetBuffer(ptr, size, MINUS_ONE_BUFFER); + return; + } + if (value == 0) { + memorySetBuffer(ptr, size, ZERO_BUFFER); + return; + } + int uv = value & 0xFF; + long lvalue = uv | (uv << 8L) | (uv << 16L) | ((long) uv << 24L) | ((long) uv << 32) | ((long) uv << 40) + | ((long) uv << 48) | ((long) uv << 56); + long addr = ptr; + long lend = ptr + (size & ~7) - 1; + while (addr < lend) { + // use put long to reduce call to putByte + UNSAFE.putLong(addr, lvalue); + addr += 8; + } + // end padding + while (addr < ptr + size) { + UNSAFE.putByte(addr, value); + addr++; + } + } + + private static void memorySetBuffer(long ptr, long size, long buffer) { + long addr = ptr; + long tow = size; + while (tow > 0) { + long len; + if (tow > BUFFER_SIZE) { + len = BUFFER_SIZE; + } else { + len = tow; + } + UNSAFE.copyMemory(buffer, addr, len); + tow -= len; + addr += len; + } + } + + /** + * copy a memory block from one part of the memory to another one + * + * @param dst destination block start + * @param src source block start + * @param size size to copy + */ + public static void memcpy(long dst, long src, long size) { + UNSAFE.copyMemory(src, dst, size); + } + + /** + * set all the bytes at an address + * + * @param ptr pointer + * @param size bytes to allocate + * @param value value + */ + public static void memset(long ptr, long size, byte value) { + if (size > thresholdParallelSizeSet) { + int processors = Runtime.getRuntime().availableProcessors(); + long blockSize = size / processors + 1; + try { + ExceptionThread.async("memorySetThread", IntStream.range(0, processors).mapToObj(processor -> { + long start = blockSize * processor; + long end = Math.min(size, blockSize * (processor + 1)); + return (ExceptionThread.ExceptionRunnable) () -> memorySet0(ptr + start, end - start, value); + }).toArray(ExceptionThread.ExceptionRunnable[]::new)).startAll().joinAndCrashIfRequired(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } else { + memorySet0(ptr, size, value); + } + } + + private MemoryUtils() { + throw new IllegalArgumentException("Can't init utils class"); + } + + private static class CleanerObject implements Runnable { + private long ptr; + + private CleanerObject(long ptr) { + this.ptr = ptr; + } + + @Override + public void run() { + if (ptr != 0) { + free(ptr); + ptr = 0; + } + } + } +} diff --git a/hdt-java-core/src/main/java/org/rdfhdt/hdt/unsafe/UnsafeLongArray.java b/hdt-java-core/src/main/java/org/rdfhdt/hdt/unsafe/UnsafeLongArray.java new file mode 100644 index 00000000..772bf77a --- /dev/null +++ b/hdt-java-core/src/main/java/org/rdfhdt/hdt/unsafe/UnsafeLongArray.java @@ -0,0 +1,217 @@ +package org.rdfhdt.hdt.unsafe; + +import java.util.Arrays; +import java.util.Objects; + +/** + * long array switching to memory allocation if the size is too important + * + * @author Antoine Willerval + */ +public class UnsafeLongArray { + private static final int SIZE_OF = Long.BYTES; + + /** + * copy a block in a source array into a destination array, same as + * {@link System#arraycopy(Object, int, Object, int, int)} with + * #UnsafeLongArray + * + * @param src source array + * @param srcPos source position + * @param dest destination array + * @param destPos destination position + * @param length length to copy + */ + public static void arraycopy(UnsafeLongArray src, long srcPos, UnsafeLongArray dest, long destPos, long length) { + if (length < 0) { + throw new IllegalArgumentException("Negative length"); + } + if (length == 0 || src == dest) { + return; + } + if (srcPos < 0 || srcPos + length > src.size()) { + throw new IllegalArgumentException("source block out of bound!"); + } + if (destPos < 0 || destPos + length > dest.size()) { + throw new IllegalArgumentException("destination block out of bound!"); + } + if (src.isUsingUnsafe() && dest.isUsingUnsafe()) { + MemoryUtils.memcpy(dest.pointer + destPos * dest.sizeOf(), src.pointer + srcPos * src.sizeOf(), length); + } else if (!src.isUsingUnsafe() && !dest.isUsingUnsafe()) { + System.arraycopy(src.javaArray, (int) srcPos, dest.javaArray, (int) destPos, (int) length); + } else { + for (long i = 0; i < length; i++) { + dest.set(destPos + i, src.get(srcPos + i)); + } + } + } + + /** + * wrap an array + * + * @param array array + * @return unsafe array + */ + public static UnsafeLongArray wrapper(long[] array) { + return new UnsafeLongArray(array, false); + } + + /** + * allocate and clone an array + * + * @param array array + * @return unsafe array + */ + public static UnsafeLongArray allocate(long[] array) { + return new UnsafeLongArray(array, true); + } + + /** + * allocate an array + * + * @param size size of the array + * @return unsafe array + */ + public static UnsafeLongArray allocate(long size) { + return new UnsafeLongArray(size); + } + + /** + * allocate an array + * + * @param size size of the array + * @param initArray init the array with 0 + * @return unsafe array + */ + public static UnsafeLongArray allocate(long size, boolean initArray) { + return new UnsafeLongArray(size, initArray); + } + + private final long pointer; + private final long[] javaArray; + private final long size; + + /** + * create an array filled with 0 of a particular size + * + * @param javaArray array + */ + private UnsafeLongArray(long[] javaArray, boolean copyArray) { + this.size = javaArray.length; + if (!copyArray) { + pointer = 0; + this.javaArray = Objects.requireNonNull(javaArray, "javaArray can't be null!"); + } else { + if (size >= MemoryUtils.getMaxArraySize()) { + // allocate the pointer + pointer = MemoryUtils.malloc(size, SIZE_OF); + // bind this pointer to this object + MemoryUtils.bindPointerTo(pointer, this); + this.javaArray = null; + arraycopy(wrapper(javaArray), 0, this, 0, size); + } else { + this.javaArray = new long[(int) size]; + // clone the array + System.arraycopy(javaArray, 0, this.javaArray, 0, javaArray.length); + pointer = 0; + } + } + } + + /** + * create an array filled with 0 of a particular size + * + * @param size size of the array + */ + private UnsafeLongArray(long size) { + this(size, true); + } + + /** + * create an array of a particular size + * + * @param size size of the array + * @param init initialize the array with 0s + */ + private UnsafeLongArray(long size, boolean init) { + this.size = size; + if (size >= MemoryUtils.getMaxArraySize()) { + // allocate the pointer + pointer = MemoryUtils.malloc(size, SIZE_OF); + // bind this pointer to this object + MemoryUtils.bindPointerTo(pointer, this); + javaArray = null; + if (init) { + clear(); + } + } else { + javaArray = new long[(int) size]; + pointer = 0; + } + } + + /** + * clear the array + */ + public void clear() { + if (javaArray == null) { + MemoryUtils.memset(pointer, size * sizeOf(), (byte) 0); + } else { + Arrays.fill(javaArray, 0); + } + } + + /** + * get a value from the array + * + * @param index index + * @return value + */ + public long get(long index) { + if (javaArray != null) { + return javaArray[(int) index]; + } + assert index >= 0 && index < size; + return MemoryUtils.getUnsafe().getLong(pointer + index * SIZE_OF); + } + + /** + * set a value in the array + * + * @param index index + * @param value value + */ + public void set(long index, long value) { + if (javaArray != null) { + javaArray[(int) index] = value; + return; + } + assert index >= 0 && index < size; + MemoryUtils.getUnsafe().putLong(pointer + index * SIZE_OF, value); + } + + public boolean isUsingUnsafe() { + return javaArray == null; + } + + /** + * @return size of the array + */ + public long size() { + return size; + } + + /** + * @return size of the array + */ + public long length() { + return size; + } + + /** + * @return the size of an element in the array + */ + public int sizeOf() { + return 8; + } +} diff --git a/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/disk/LargeLongArray.java b/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/disk/LargeLongArray.java index 804a7a9f..f10b42c7 100644 --- a/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/disk/LargeLongArray.java +++ b/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/disk/LargeLongArray.java @@ -1,8 +1,7 @@ package org.rdfhdt.hdt.util.disk; +import org.rdfhdt.hdt.unsafe.UnsafeLongArray; import org.rdfhdt.hdt.util.io.IOUtil; -import org.visnow.jlargearrays.LargeArrayUtils; -import org.visnow.jlargearrays.LongLargeArray; import java.io.IOException; @@ -12,23 +11,23 @@ * @author Antoine Willerval */ public class LargeLongArray implements LongArray { - private LongLargeArray array; + private UnsafeLongArray array; /** * @param array large array */ - public LargeLongArray(LongLargeArray array) { + public LargeLongArray(UnsafeLongArray array) { this.array = array; } @Override public long get(long index) { - return array.getLong(index); + return array.get(index); } @Override public void set(long index, long value) { - array.setLong(index, value); + array.set(index, value); } @Override @@ -38,15 +37,15 @@ public long length() { @Override public int sizeOf() { - return (int) array.getType().sizeOf() * 8; + return array.sizeOf(); } @Override public void resize(long newSize) throws IOException { if (newSize > 0) { if (array.length() != newSize) { - LongLargeArray a = IOUtil.createLargeArray(newSize, false); - LargeArrayUtils.arraycopy(array, 0, a, 0, Math.min(newSize, array.length())); + UnsafeLongArray a = IOUtil.createLargeArray(newSize, false); + UnsafeLongArray.arraycopy(array, 0, a, 0, Math.min(newSize, array.length())); array = a; } } @@ -54,6 +53,6 @@ public void resize(long newSize) throws IOException { @Override public void clear() { - IOUtil.fillLargeArray(array, 0); + array.clear(); } } diff --git a/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/disk/SimpleSplitLongArray.java b/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/disk/SimpleSplitLongArray.java index 416cc5c8..57893f7e 100644 --- a/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/disk/SimpleSplitLongArray.java +++ b/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/disk/SimpleSplitLongArray.java @@ -2,7 +2,6 @@ import org.rdfhdt.hdt.util.BitUtil; import org.rdfhdt.hdt.util.io.IOUtil; -import org.visnow.jlargearrays.LongLargeArray; import java.io.Closeable; import java.io.IOException; diff --git a/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/io/IOUtil.java b/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/io/IOUtil.java index 42de8942..34fc4bd5 100644 --- a/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/io/IOUtil.java +++ b/hdt-java-core/src/main/java/org/rdfhdt/hdt/util/io/IOUtil.java @@ -28,15 +28,13 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; -import org.apache.commons.math3.util.FastMath; import org.rdfhdt.hdt.compact.integer.VByte; import org.rdfhdt.hdt.enums.CompressionType; import org.rdfhdt.hdt.listener.ProgressListener; +import org.rdfhdt.hdt.unsafe.MemoryUtils; +import org.rdfhdt.hdt.unsafe.UnsafeLongArray; import org.rdfhdt.hdt.util.string.ByteString; import org.rdfhdt.hdt.util.string.ByteStringUtil; -import org.visnow.jlargearrays.ConcurrencyUtils; -import org.visnow.jlargearrays.LargeArrayUtils; -import org.visnow.jlargearrays.LongLargeArray; import java.io.BufferedInputStream; import java.io.BufferedReader; @@ -61,8 +59,6 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.zip.GZIPInputStream; import java.nio.ByteBuffer; @@ -84,7 +80,7 @@ public static void cleanBuffer(ByteBuffer buffer) { return; } - LargeArrayUtils.UNSAFE.invokeCleaner(buffer); + MemoryUtils.getUnsafe().invokeCleaner(buffer); } /** @@ -117,7 +113,7 @@ public static CloseMappedByteBuffer mapChannel(String filename, FileChannel ch, * @param size size * @return array */ - public static LongLargeArray createLargeArray(long size) { + public static UnsafeLongArray createLargeArray(long size) { return createLargeArray(size, true); } /** @@ -126,72 +122,10 @@ public static LongLargeArray createLargeArray(long size) { * @param init is the array filled with 0 or not * @return array */ - public static LongLargeArray createLargeArray(long size, boolean init) { - if (init) { - return createLargeArray(size, 0); - } - return new LongLargeArray(size, false); - } - - /** - * create a large array with an initial value - * @param size size - * @param initialValue initial value to fill the array - * @return array - */ - public static LongLargeArray createLargeArray(long size, long initialValue) { - LongLargeArray array = new LongLargeArray(size, false); - fillLargeArray(array, initialValue); - return array; - } - - /** - * Set long large array all values, faster than default implementation because there is a bug. - * - * @param array array - * @param initValue initialization value - */ - public static void fillLargeArray(LongLargeArray array, long initValue) { - fillLargeArray(array, 0, array.length(), initValue); + public static UnsafeLongArray createLargeArray(long size, boolean init) { + return UnsafeLongArray.allocate(size, init); } - /** - * Set long large array all values, faster than default implementation because there is a bug. - * - * @param array array - * @param start start (inclusive) - * @param end end index (exclusive) - * @param initValue initialization value - */ - public static void fillLargeArray(LongLargeArray array, long start, long end, long initValue) { - if (start >= end) { - return; - } - long length = end - start; - final int nthreads = (int) FastMath.min(length, ConcurrencyUtils.getNumberOfThreads()); - if (nthreads <= 2 || length < ConcurrencyUtils.getConcurrentThreshold() || !array.isLarge()) { - for (long k = 0; k < length; k++) { - array.setLong(k, initValue); - } - } else { - final long perThreadElem = length / nthreads; - final Future[] threads = new Future[nthreads]; - for (int thread = 0; thread < nthreads; thread++) { - final long firstIdx = start + thread * perThreadElem; - final long lastIdx = (thread == nthreads - 1) ? end : (firstIdx + perThreadElem); - threads[thread] = ConcurrencyUtils.submit(() -> { - for (long k1 = firstIdx; k1 < lastIdx; k1++) { - array.setLong(k1, initValue); - } - }); - } - try { - ConcurrencyUtils.waitForCompletion(threads); - } catch (InterruptedException | ExecutionException ex) { - throw new IllegalStateException(ex); - } - } - } /** * call all the close method and merge the exceptions by suppressing them (if multiple) * diff --git a/hdt-java-core/src/test/java/org/rdfhdt/hdt/compact/sequence/LargeArrayTest.java b/hdt-java-core/src/test/java/org/rdfhdt/hdt/compact/sequence/LargeArrayTest.java deleted file mode 100644 index 510bcbc2..00000000 --- a/hdt-java-core/src/test/java/org/rdfhdt/hdt/compact/sequence/LargeArrayTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.rdfhdt.hdt.compact.sequence; - -import org.junit.Test; -import org.rdfhdt.hdt.util.io.IOUtil; -import org.visnow.jlargearrays.LargeArray; -import org.visnow.jlargearrays.LongLargeArray; - -public class LargeArrayTest { - - @Test - public void allocationTest() { - int old = LargeArray.getMaxSizeOf32bitArray(); - try { - LargeArray.setMaxSizeOf32bitArray(100); - long size = LargeArray.getMaxSizeOf32bitArray() + 2L; - IOUtil.createLargeArray(size, false); - } finally { - LargeArray.setMaxSizeOf32bitArray(old); - } - } -} diff --git a/hdt-java-core/src/test/java/org/rdfhdt/hdt/unsafe/MemoryUtilsTest.java b/hdt-java-core/src/test/java/org/rdfhdt/hdt/unsafe/MemoryUtilsTest.java new file mode 100644 index 00000000..84cc3edb --- /dev/null +++ b/hdt-java-core/src/test/java/org/rdfhdt/hdt/unsafe/MemoryUtilsTest.java @@ -0,0 +1,7 @@ +package org.rdfhdt.hdt.unsafe; + +public class MemoryUtilsTest { + public static void setMaxArraySize(int maxSize) { + MemoryUtils.maxArraySize = maxSize; + } +} diff --git a/hdt-java-core/src/test/java/org/rdfhdt/hdt/unsafe/UnsafeLongArrayTest.java b/hdt-java-core/src/test/java/org/rdfhdt/hdt/unsafe/UnsafeLongArrayTest.java new file mode 100644 index 00000000..d0832ca8 --- /dev/null +++ b/hdt-java-core/src/test/java/org/rdfhdt/hdt/unsafe/UnsafeLongArrayTest.java @@ -0,0 +1,98 @@ +package org.rdfhdt.hdt.unsafe; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class UnsafeLongArrayTest { + private static final int MAX_ARRAY_SIZE = 100; + private int minMemory; + + @Before + public void setup() { + minMemory = MemoryUtils.getMaxArraySize(); + MemoryUtilsTest.setMaxArraySize(MAX_ARRAY_SIZE); + } + + @After + public void after() { + MemoryUtilsTest.setMaxArraySize(minMemory); + } + + @Test + public void getSetTest() { + int len = MAX_ARRAY_SIZE * 10; + UnsafeLongArray array = UnsafeLongArray.allocate(len); + assertTrue(array.isUsingUnsafe()); + assertEquals(len, array.length()); + long[] excepted = new long[len]; + + Random rnd = new Random(89); + for (int i = 0; i < len; i++) { + long l = rnd.nextLong(); + excepted[i] = l; + array.set(i, l); + } + + for (int i = 0; i < excepted.length; i++) { + assertEquals(excepted[i], array.get(i)); + } + } + + @Test + public void arrayWrapperTest() { + int len = MAX_ARRAY_SIZE * 10; + long[] excepted = new long[len]; + + Random rnd = new Random(89); + for (int i = 0; i < len; i++) { + long l = rnd.nextLong(); + excepted[i] = l; + } + + UnsafeLongArray array = UnsafeLongArray.wrapper(excepted); + assertFalse(array.isUsingUnsafe()); + assertEquals(len, array.length()); + + for (int i = 0; i < excepted.length; i++) { + assertEquals(excepted[i], array.get(i)); + } + } + + @Test + public void copyTest() { + int len = MAX_ARRAY_SIZE * 10; + long[] excepted = new long[len]; + + Random rnd = new Random(89); + for (int i = 0; i < len; i++) { + long l = rnd.nextLong(); + excepted[i] = l; + } + + UnsafeLongArray array = UnsafeLongArray.wrapper(excepted); + assertFalse(array.isUsingUnsafe()); + assertEquals(len, array.length()); + UnsafeLongArray array2 = UnsafeLongArray.allocate(len); + assertTrue(array2.isUsingUnsafe()); + assertEquals(len, array2.length()); + + UnsafeLongArray.arraycopy(array, 0, array2, 0, len); + + for (int i = 0; i < excepted.length; i++) { + assertEquals(excepted[i], array.get(i)); + } + array2.clear(); + + for (int i = 0; i < array2.length(); i++) { + assertEquals("arr[" + i + "/" + array2.length() + "]", 0, array2.get(i)); + } + } + +} diff --git a/hdt-java-core/src/test/java/org/rdfhdt/hdt/util/disk/SimpleSplitLongArrayTest.java b/hdt-java-core/src/test/java/org/rdfhdt/hdt/util/disk/SimpleSplitLongArrayTest.java index 6c70a97d..63cd91ce 100644 --- a/hdt-java-core/src/test/java/org/rdfhdt/hdt/util/disk/SimpleSplitLongArrayTest.java +++ b/hdt-java-core/src/test/java/org/rdfhdt/hdt/util/disk/SimpleSplitLongArrayTest.java @@ -4,7 +4,6 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.rdfhdt.hdt.util.io.IOUtil; -import org.visnow.jlargearrays.LongLargeArray; import java.io.IOException; import java.util.Collection; diff --git a/pom.xml b/pom.xml index 6c212401..87ff1b88 100644 --- a/pom.xml +++ b/pom.xml @@ -183,11 +183,6 @@ slf4j-api ${slf4j.version} - - pl.edu.icm - JLargeArrays - 1.6 - junit junit