diff --git a/.asf.yaml b/.asf.yaml index a149bf396..57e258daa 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -45,11 +45,11 @@ github: edit_comment_discussion: "Re: [D] {title} ({repository})" delete_comment_discussion: "Re: [D] {title} ({repository})" - notifications: - commits: commits@dataskethces.apache.org - issues: dev@dataskethces.apache.org - discussions: dev@dataskethces.apache.org - pullrequests_status: dev@dataskethces.apache.org - pullrequests_comment: dev@dataskethces.apache.org - # Send dependabot PRs to commits@ instead - pullrequests_bot_dependabot: commits@dataskethces.apache.org +notifications: + commits: commits@dataskethces.apache.org + issues: dev@dataskethces.apache.org + discussions: dev@dataskethces.apache.org + pullrequests_status: dev@dataskethces.apache.org + pullrequests_comment: dev@dataskethces.apache.org + # Send dependabot PRs to commits@ instead + pullrequests_bot_dependabot: commits@dataskethces.apache.org diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml index 0afbaf065..a556d3e82 100644 --- a/.github/workflows/auto-jdk-matrix.yml +++ b/.github/workflows/auto-jdk-matrix.yml @@ -1,13 +1,12 @@ name: Auto JDK Matrix Test & Install on: - push: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] - pull_request: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] - # The branches below must be a subset of the branches above - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] +# push: +# branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] +# pull_request: +# paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] +# # The branches below must be a subset of the branches above +# branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] workflow_dispatch: env: diff --git a/.github/workflows/auto-os-matrix.yml b/.github/workflows/auto-os-matrix.yml index 83ecc1ffe..413b7a957 100644 --- a/.github/workflows/auto-os-matrix.yml +++ b/.github/workflows/auto-os-matrix.yml @@ -1,13 +1,13 @@ name: Auto OS Matrix Test & Install on: - push: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] - pull_request: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] - # The branches below must be a subset of the branches above - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] +# push: +# paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] +# branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] +# pull_request: +# paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] +# # The branches below must be a subset of the branches above +# branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] workflow_dispatch: env: diff --git a/.github/workflows/check_cpp_files.yml b/.github/workflows/check_cpp_files.yml index 243eda985..778859d0d 100644 --- a/.github/workflows/check_cpp_files.yml +++ b/.github/workflows/check_cpp_files.yml @@ -1,13 +1,13 @@ name: CPP SerDe Compatibility Test on: - push: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] - pull_request: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] - # The branches below must be a subset of the branches above - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] +# push: +# paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] +# branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] +# pull_request: +# paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] +# # The branches below must be a subset of the branches above +# branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] workflow_dispatch: jobs: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bb42fe345..f3fde1de0 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,13 +1,13 @@ name: "CodeQL" on: - push: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] - pull_request: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] - # The branches below must be a subset of the branches above - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] +# push: +# paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] +# branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] +# pull_request: +# paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/*.xml', '**/*.yaml', '**/*.yml', '**/LICENSE', '**/NOTICE' ] +# # The branches below must be a subset of the branches above +# branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] workflow_dispatch: jobs: diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml index 4862d64e4..977c87b27 100644 --- a/.github/workflows/javadoc.yml +++ b/.github/workflows/javadoc.yml @@ -1,8 +1,8 @@ name: JavaDoc on: - push: - branches: main +# push: +# branches: main workflow_dispatch: jobs: diff --git a/pom.xml b/pom.xml index 84e032947..760acd58d 100644 --- a/pom.xml +++ b/pom.xml @@ -83,10 +83,10 @@ under the License. - 6.0.0 + 6.1.0-SNAPSHOT - 7.10.2 + 7.11.0 generate_java_files check_cpp_files @@ -94,11 +94,11 @@ under the License. 3.6.3 - 21 - --enable-preview + 24 + ${java.version} ${java.version} - -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 ${jvm-ffm-flag} + -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 ${jvm-arguments} UTF-8 ${charset.encoding} ${charset.encoding} @@ -114,9 +114,9 @@ under the License. 3.4.2 3.11.2 3.1.1 - 3.2.0 3.3.1 - 3.5.2 + + 3.5.2 3.2.0 4.9.10 @@ -126,7 +126,7 @@ under the License. 4.3.0 - 0.8.12 + 0.8.13 2.18.0 @@ -164,7 +164,7 @@ under the License. ${maven-compiler-plugin.version} - ${jvm-ffm-flag} + ${jvm-arguments} @@ -190,7 +190,7 @@ under the License. - [21,) + [22,) [${maven.version},4.0.0) @@ -239,7 +239,7 @@ under the License. public all,-missing - ${jvm-ffm-flag} + ${jvm-arguments} @@ -285,7 +285,7 @@ under the License. maven-surefire-plugin ${maven-surefire-failsafe-plugins.version} - ${jvm-ffm-flag} + ${jvm-arguments} false false true @@ -340,6 +340,7 @@ under the License. **/*.sk LICENSE NOTICE + **/*.code-workspace @@ -374,6 +375,7 @@ under the License. default-report + test report @@ -668,5 +670,6 @@ under the License. + diff --git a/src/main/java/org/apache/datasketches/common/MemorySegmentStatus.java b/src/main/java/org/apache/datasketches/common/MemorySegmentStatus.java new file mode 100644 index 000000000..889e8c0b8 --- /dev/null +++ b/src/main/java/org/apache/datasketches/common/MemorySegmentStatus.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.common; + +import java.lang.foreign.MemorySegment; + +/** + * Methods for inquiring the status of a backing MemorySegment. + */ +public interface MemorySegmentStatus { + + /** + * Returns true if this object's internal data is backed by a MemorySegment, + * which may be on-heap or off-heap. + * @return true if this object's internal data is backed by a MemorySegment. + */ + boolean hasMemorySegment(); + + /** + * Returns true if this object's internal data is backed by an off-heap (direct or native)) MemorySegment. + * @return true if this object's internal data is backed by an off-heap (direct or native)) MemorySegment. + */ + boolean isDirect(); + + /** + * Returns true if the backing MemorySegment of this object refers to the same MemorySegment of that. + * They can either have the same off-heap memory location and size, or refer to the same on-heap array object. + * + *

If both segment are off-heap, they both must have the same starting address and the same size.

+ * + *

For on-heap segments, both segments must be based on or derived from the same array object and neither segment + * can be read-only.

+ * + *

Returns false if either argument is null;

+ * + * @param that The given MemorySegment. + * @return true if the backing MemorySegment of this object hierarchy refers to the same MemorySegment of that. + */ + boolean isSameResource(final MemorySegment that); + +} diff --git a/src/main/java/org/apache/datasketches/thetacommon/QuickSelect.java b/src/main/java/org/apache/datasketches/common/QuickSelect.java similarity index 99% rename from src/main/java/org/apache/datasketches/thetacommon/QuickSelect.java rename to src/main/java/org/apache/datasketches/common/QuickSelect.java index e48c9b790..aa32af575 100644 --- a/src/main/java/org/apache/datasketches/thetacommon/QuickSelect.java +++ b/src/main/java/org/apache/datasketches/common/QuickSelect.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.datasketches.thetacommon; +package org.apache.datasketches.common; /** * QuickSelect algorithm improved from Sedgewick. Gets the kth order value diff --git a/src/main/java/org/apache/datasketches/common/Util.java b/src/main/java/org/apache/datasketches/common/Util.java index 19a8ee614..5a993591f 100644 --- a/src/main/java/org/apache/datasketches/common/Util.java +++ b/src/main/java/org/apache/datasketches/common/Util.java @@ -24,8 +24,10 @@ import static java.lang.Math.log; import static java.lang.Math.pow; import static java.lang.Math.round; -import static java.util.Arrays.fill; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static org.apache.datasketches.hash.MurmurHash3.hash; +import java.lang.foreign.MemorySegment; import java.util.Comparator; /** @@ -67,6 +69,28 @@ public final class Util { */ public static final double LONG_MAX_VALUE_AS_DOUBLE = Long.MAX_VALUE; + /** + * The seed 9001 used in the sketch update methods is a prime number that + * was chosen very early on in experimental testing. Choosing a seed is somewhat arbitrary, and + * the author cannot prove that this particular seed is somehow superior to other seeds. There + * was some early Internet discussion that a seed of 0 did not produce as clean avalanche diagrams + * as non-zero seeds, but this may have been more related to the MurmurHash2 release, which did + * have some issues. As far as the author can determine, MurmurHash3 does not have these problems. + * + *

In order to perform set operations on two sketches it is critical that the same hash + * function and seed are identical for both sketches, otherwise the assumed 1:1 relationship + * between the original source key value and the hashed bit string would be violated. Once + * you have developed a history of stored sketches you are stuck with it. + * + *

WARNING: This seed is used internally by library sketches in different + * packages and thus must be declared public. However, this seed value must not be used by library + * users with the MurmurHash3 function. It should be viewed as existing for exclusive, private + * use by the library. + * + *

See Default Update Seed + */ + public static final long DEFAULT_UPDATE_SEED = 9001L; + private Util() {} //Byte Conversions @@ -257,7 +281,7 @@ public static String characterPad(final String s, final int fieldLength, final c final int sLen = s.length(); if (sLen < fieldLength) { final char[] cArr = new char[fieldLength - sLen]; - fill(cArr, padChar); + java.util.Arrays.fill(cArr, padChar); final String addstr = String.valueOf(cArr); return (postpend) ? s.concat(addstr) : addstr.concat(s); } @@ -798,4 +822,183 @@ public static boolean le(final Object item1, final Object item2, final Compa return c.compare((T)item1, (T)item2) <= 0; } + //MemorySegment related + + /** + * Clears all bytes of this MemorySegment to zero. + * @param seg the given MemorySegment + */ + public static void clear(final MemorySegment seg) { + seg.fill((byte)0); + } + + /** + * Clears a portion of this MemorySegment to zero. + * @param seg the given MemorySegment + * @param offsetBytes offset bytes relative to this MemorySegment start + * @param lengthBytes the length in bytes + */ + public static void clear(final MemorySegment seg, final long offsetBytes, final long lengthBytes) { + final MemorySegment slice = seg.asSlice(offsetBytes, lengthBytes); + slice.fill((byte)0); + } + + /** + * Clears the bits defined by the bitMask + * @param seg the given MemorySegment + * @param offsetBytes offset bytes relative to this Memory start. + * @param bitMask the bits set to one will be cleared + */ + public static void clearBits(final MemorySegment seg, final long offsetBytes, final byte bitMask) { + final byte b = seg.get(JAVA_BYTE, offsetBytes); + seg.set(JAVA_BYTE, offsetBytes, (byte)(b & ~bitMask)); + } + + /** + * Returns true if both segments have the same contents and the same length. + * @param seg1 the given MemorySegment #1 + * @param seg2 the given MemorySegment #2 + * @return true if both segments have the same contents and the same length. + */ + public static boolean equalContents(final MemorySegment seg1, final MemorySegment seg2) { + if (seg1.byteSize() != seg2.byteSize()) { return false; } + return equalContents(seg1, 0, seg2, 0, seg1.byteSize()); + } + + /** + * Returns true if both segments have the same content for the specified region. + * @param seg1 the given MemorySegment #1 + * @param seg1offsetBytes the starting offset for MemorySegment #1 in bytes. + * @param seg2 the given MemorySegment #2 + * @param seg2offsetBytes the starting offset for MemorySegment #2 in bytes. + * @param lengthBytes the length of the region to be compared, in bytes. + * @return true, if both segments have the content for the specified region. + */ + public static boolean equalContents( + final MemorySegment seg1, + final long seg1offsetBytes, + final MemorySegment seg2, + final long seg2offsetBytes, + final long lengthBytes) { + if (seg1.equals(seg2) && (seg1.byteSize() == seg2.byteSize())) { return true; } //identical segments + final long seg1EndOff = seg1offsetBytes + lengthBytes; + final long seg2EndOff = seg2offsetBytes + lengthBytes; + return MemorySegment.mismatch(seg1, seg1offsetBytes, seg1EndOff, seg2, seg2offsetBytes, seg2EndOff) == -1; + } + + /** + * Fills a portion of this Memory region to the given byte value. + * @param seg the given MemorySegment + * @param offsetBytes offset bytes relative to this Memory start + * @param lengthBytes the length in bytes + * @param value the given byte value + */ + public static void fill(final MemorySegment seg, final long offsetBytes, final long lengthBytes, final byte value) { + final MemorySegment slice = seg.asSlice(offsetBytes, lengthBytes); + slice.fill(value); + } + + /** + * Returns true if the two given MemorySegments refer to the same backing resource, + * which is either an off-heap memory location and size, or the same on-heap array object. + * + *

If both segment are off-heap, they both must have the same starting address and the same size.

+ * + *

For on-heap segments, both segments must be based on or derived from the same array object and neither segment + * can be read-only.

+ * + *

Returns false if either argument is null;

+ * + * @param seg1 The first given MemorySegment + * @param seg2 The second given MemorySegment + * @return true if both MemorySegments are determined to be the same backing memory. + */ + public static boolean isSameResource(final MemorySegment seg1, final MemorySegment seg2) { + if ((seg1 == null) || (seg2 == null)) { return false; } + if (!seg1.scope().isAlive() || !seg2.scope().isAlive()) { + throw new IllegalArgumentException("Both arguments must be alive."); + } + final boolean seg1Native = seg1.isNative(); + final boolean seg2Native = seg2.isNative(); + if (seg1Native ^ seg2Native) { return false; } + if (seg1Native && seg2Native) { //both off heap + return (seg1.address() == seg2.address()) && (seg1.byteSize() == seg2.byteSize()); + } + //both on heap + if (seg1.isReadOnly() || seg2.isReadOnly()) { + throw new IllegalArgumentException("Cannot determine 'isSameBackingMemory(..)' on heap if either MemorySegment is Read-only."); + } + return (seg1.heapBase().orElse(null) == seg2.heapBase().orElse(null)); + } + + /** + * Request a new heap MemorySegment with the given capacityBytes and either 8-byte aligned or one byte aligned. + * + *

If aligned is true, the returned MemorySegment will be constructed from a long[] array, + * and, as a result, it will have a memory alignment of 8 bytes. + * If the requested capacity is not exactly divisible by eight, the returned size + * will be rolled up to the next multiple of eight bytes.

+ * + *

If aligned is false, the returned MemorySegment will be constructed from a byte[] array, + * and have a memory alignment of 1 byte. + * + * @param capacityBytes The new capacity being requested. It must not be negative and cannot exceed Integer.MAX_VALUE. + * @param aligned if true, the new heap segment will have an alignment of 8 bytes, otherwise the alignment will be 1 byte. + * @return a new MemorySegment with the requested capacity and alignment. + */ + public static MemorySegment alignedHeapSegment(final int capacityBytes, final boolean aligned) { + if (aligned) { + final int lenLongs = capacityBytes >>> 3; + final long[] array = ((capacityBytes & 0x7) == 0) + ? new long[lenLongs] + : new long[lenLongs + 1]; + return MemorySegment.ofArray(array); + } + return MemorySegment.ofArray(new byte[capacityBytes]); + } + + /** + * Sets the bits defined by the bitMask + * @param seg the given MemorySegment + * @param offsetBytes offset bytes relative to this MemorySegment start + * @param bitMask the bits set to one will be set + */ + public static void setBits(final MemorySegment seg, final long offsetBytes, final byte bitMask) { + final byte b = seg.get(JAVA_BYTE, offsetBytes); + seg.set(JAVA_BYTE, offsetBytes, (byte)(b | bitMask)); + } + + /** + * Computes and checks the 16-bit seed hash from the given long seed. + * The seed hash may not be zero in order to maintain compatibility with older serialized + * versions that did not have this concept. + * @param seed See Update Hash Seed + * @return the seed hash. + */ + public static short computeSeedHash(final long seed) { + final long[] seedArr = {seed}; + final short seedHash = (short)(hash(seedArr, 0L)[0] & 0xFFFFL); + if (seedHash == 0) { + throw new SketchesArgumentException( + "The given seed: " + seed + " produced a seedHash of zero. " + + "You must choose a different seed."); + } + return seedHash; + } + + /** + * Check if the two seed hashes are equal. If not, throw an SketchesArgumentException. + * @param seedHashA the seedHash A + * @param seedHashB the seedHash B + * @return seedHashA if they are equal + */ + public static short checkSeedHashes(final short seedHashA, final short seedHashB) { + if (seedHashA != seedHashB) { + throw new SketchesArgumentException( + "Incompatible Seed Hashes. " + Integer.toHexString(seedHashA & 0XFFFF) + + ", " + Integer.toHexString(seedHashB & 0XFFFF)); + } + return seedHashA; + } + } diff --git a/src/main/java/org/apache/datasketches/cpc/BitMatrix.java b/src/main/java/org/apache/datasketches/cpc/BitMatrix.java index 419640a4f..ecf27b70f 100644 --- a/src/main/java/org/apache/datasketches/cpc/BitMatrix.java +++ b/src/main/java/org/apache/datasketches/cpc/BitMatrix.java @@ -23,7 +23,7 @@ import java.util.Arrays; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; /** * Used only in test. @@ -38,7 +38,7 @@ class BitMatrix { private boolean numCouponsInvalid; //only used if we allowed merges BitMatrix(final int lgK) { - this(lgK, ThetaUtil.DEFAULT_UPDATE_SEED); + this(lgK, Util.DEFAULT_UPDATE_SEED); } BitMatrix(final int lgK, final long seed) { diff --git a/src/main/java/org/apache/datasketches/cpc/CompressedState.java b/src/main/java/org/apache/datasketches/cpc/CompressedState.java index 6ea6fde4b..b88a71812 100644 --- a/src/main/java/org/apache/datasketches/cpc/CompressedState.java +++ b/src/main/java/org/apache/datasketches/cpc/CompressedState.java @@ -47,9 +47,9 @@ import static org.apache.datasketches.cpc.PreambleUtil.putSparseHybridMerged; import static org.apache.datasketches.cpc.RuntimeAsserts.rtAssert; +import org.apache.datasketches.common.Util; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; /** * @author Lee Rhodes @@ -82,7 +82,7 @@ private CompressedState(final int lgK, final short seedHash) { } static CompressedState compress(final CpcSketch source) { - final short seedHash = ThetaUtil.computeSeedHash(source.seed); + final short seedHash = Util.computeSeedHash(source.seed); final CompressedState target = new CompressedState(source.lgK, seedHash); target.fiCol = source.fiCol; target.mergeFlag = source.mergeFlag; diff --git a/src/main/java/org/apache/datasketches/cpc/CompressionCharacterization.java b/src/main/java/org/apache/datasketches/cpc/CompressionCharacterization.java index 4f0a93352..f34054d48 100644 --- a/src/main/java/org/apache/datasketches/cpc/CompressionCharacterization.java +++ b/src/main/java/org/apache/datasketches/cpc/CompressionCharacterization.java @@ -31,9 +31,9 @@ import java.io.PrintWriter; import org.apache.datasketches.common.SuppressFBWarnings; +import org.apache.datasketches.common.Util; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; /** * This code is used both by unit tests, for short running tests, @@ -222,7 +222,7 @@ private void doTrialsAtLgKAtN(final int lgK, final long n, final int totalTrials for (int trial = 0; trial < trialsPerWave; trial++) { final CompressedState state = compressedStates2[trial]; CpcSketch uncSk = null; - uncSk = CpcSketch.uncompress(state, ThetaUtil.DEFAULT_UPDATE_SEED); + uncSk = CpcSketch.uncompress(state, Util.DEFAULT_UPDATE_SEED); unCompressedSketches[trial] = uncSk; } diff --git a/src/main/java/org/apache/datasketches/cpc/CpcSketch.java b/src/main/java/org/apache/datasketches/cpc/CpcSketch.java index 4ed89dd19..a87fd1448 100644 --- a/src/main/java/org/apache/datasketches/cpc/CpcSketch.java +++ b/src/main/java/org/apache/datasketches/cpc/CpcSketch.java @@ -34,9 +34,9 @@ import java.util.Arrays; import org.apache.datasketches.common.Family; +import org.apache.datasketches.common.Util; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; /** * This is a unique-counting sketch that implements the @@ -89,7 +89,7 @@ public final class CpcSketch { * Constructor with default log_base2 of k */ public CpcSketch() { - this(DEFAULT_LG_K, ThetaUtil.DEFAULT_UPDATE_SEED); + this(DEFAULT_LG_K, Util.DEFAULT_UPDATE_SEED); } /** @@ -97,7 +97,7 @@ public CpcSketch() { * @param lgK the given log_base2 of k */ public CpcSketch(final int lgK) { - this(lgK, ThetaUtil.DEFAULT_UPDATE_SEED); + this(lgK, Util.DEFAULT_UPDATE_SEED); } /** @@ -233,7 +233,7 @@ public double getUpperBound(final int kappa) { * @return the given Memory as a CpcSketch on the Java heap. */ public static CpcSketch heapify(final Memory mem) { - return heapify(mem, ThetaUtil.DEFAULT_UPDATE_SEED); + return heapify(mem, Util.DEFAULT_UPDATE_SEED); } /** @@ -242,7 +242,7 @@ public static CpcSketch heapify(final Memory mem) { * @return the given byte array as a CpcSketch on the Java heap. */ public static CpcSketch heapify(final byte[] byteArray) { - return heapify(byteArray, ThetaUtil.DEFAULT_UPDATE_SEED); + return heapify(byteArray, Util.DEFAULT_UPDATE_SEED); } /** @@ -662,7 +662,7 @@ else if (col < (sketch.windowOffset + 8)) { // track the 8 bits inside the windo //also used in test static CpcSketch uncompress(final CompressedState source, final long seed) { - ThetaUtil.checkSeedHashes(ThetaUtil.computeSeedHash(seed), source.seedHash); + Util.checkSeedHashes(Util.computeSeedHash(seed), source.seedHash); final CpcSketch sketch = new CpcSketch(source.lgK, seed); sketch.numCoupons = source.numCoupons; sketch.windowOffset = source.getWindowOffset(); @@ -723,7 +723,7 @@ public String toString() { */ public String toString(final boolean detail) { final int numPairs = (pairTable == null) ? 0 : pairTable.getNumPairs(); - final int seedHash = Short.toUnsignedInt(ThetaUtil.computeSeedHash(seed)); + final int seedHash = Short.toUnsignedInt(Util.computeSeedHash(seed)); final double errConst = mergeFlag ? log(2) : sqrt(log(2) / 2.0); final double rse = errConst / Math.sqrt(1 << lgK); final StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/datasketches/cpc/CpcUnion.java b/src/main/java/org/apache/datasketches/cpc/CpcUnion.java index 4b944c00f..8aeb48bbf 100644 --- a/src/main/java/org/apache/datasketches/cpc/CpcUnion.java +++ b/src/main/java/org/apache/datasketches/cpc/CpcUnion.java @@ -27,7 +27,7 @@ import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; /* * The merging logic is somewhat involved, so it will be summarized here. @@ -102,7 +102,7 @@ public class CpcUnion { * Construct this unioning object with the default LgK and the default update seed. */ public CpcUnion() { - this(CpcSketch.DEFAULT_LG_K, ThetaUtil.DEFAULT_UPDATE_SEED); + this(CpcSketch.DEFAULT_LG_K, Util.DEFAULT_UPDATE_SEED); } /** @@ -110,7 +110,7 @@ public CpcUnion() { * @param lgK The given log2 of K. */ public CpcUnion(final int lgK) { - this(lgK, ThetaUtil.DEFAULT_UPDATE_SEED); + this(lgK, Util.DEFAULT_UPDATE_SEED); } /** diff --git a/src/main/java/org/apache/datasketches/fdt/FdtSketch.java b/src/main/java/org/apache/datasketches/fdt/FdtSketch.java index e4cac32a1..9dc6aecca 100644 --- a/src/main/java/org/apache/datasketches/fdt/FdtSketch.java +++ b/src/main/java/org/apache/datasketches/fdt/FdtSketch.java @@ -19,10 +19,10 @@ package org.apache.datasketches.fdt; +import java.lang.foreign.MemorySegment; import java.util.List; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.thetacommon.ThetaUtil; import org.apache.datasketches.tuple.strings.ArrayOfStringsSketch; @@ -59,14 +59,14 @@ public FdtSketch(final int lgK) { /** * Used by deserialization. - * @param mem the image of a FdtSketch + * @param seg the image of a FdtSketch * @deprecated As of 3.0.0, heapifying an UpdatableSketch is deprecated. * This capability will be removed in a future release. * Heapifying a CompactSketch is not deprecated. */ @Deprecated - FdtSketch(final Memory mem) { - super(mem); + FdtSketch(final MemorySegment seg) { + super(seg); } /** diff --git a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java index 7c166a29d..a56a5eeef 100644 --- a/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java +++ b/src/main/java/org/apache/datasketches/filters/bloomfilter/BloomFilter.java @@ -26,11 +26,11 @@ import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; +import org.apache.datasketches.hash.XxHash; import org.apache.datasketches.memory.Buffer; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableBuffer; import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.memory.XxHash; /** * A Bloom filter is a data structure that can be used for probabilistic diff --git a/src/main/java/org/apache/datasketches/frequencies/ReversePurgeItemHashMap.java b/src/main/java/org/apache/datasketches/frequencies/ReversePurgeItemHashMap.java index 355724e2e..67e946b01 100644 --- a/src/main/java/org/apache/datasketches/frequencies/ReversePurgeItemHashMap.java +++ b/src/main/java/org/apache/datasketches/frequencies/ReversePurgeItemHashMap.java @@ -25,7 +25,7 @@ import java.lang.reflect.Array; -import org.apache.datasketches.thetacommon.QuickSelect; +import org.apache.datasketches.common.QuickSelect; /** * Implements a linear-probing based hash map of (key, value) pairs and is distinguished by a diff --git a/src/main/java/org/apache/datasketches/frequencies/ReversePurgeLongHashMap.java b/src/main/java/org/apache/datasketches/frequencies/ReversePurgeLongHashMap.java index dd33589d4..358f85bce 100644 --- a/src/main/java/org/apache/datasketches/frequencies/ReversePurgeLongHashMap.java +++ b/src/main/java/org/apache/datasketches/frequencies/ReversePurgeLongHashMap.java @@ -24,8 +24,8 @@ import static org.apache.datasketches.common.Util.INVERSE_GOLDEN; import static org.apache.datasketches.frequencies.Util.hash; +import org.apache.datasketches.common.QuickSelect; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.thetacommon.QuickSelect; /** * Implements a linear-probing based hash map of (key, value) pairs and is distinguished by a diff --git a/src/main/java/org/apache/datasketches/hash/XxHash.java b/src/main/java/org/apache/datasketches/hash/XxHash.java index a93d4d348..2185471a8 100644 --- a/src/main/java/org/apache/datasketches/hash/XxHash.java +++ b/src/main/java/org/apache/datasketches/hash/XxHash.java @@ -19,41 +19,187 @@ package org.apache.datasketches.hash; -import org.apache.datasketches.memory.Memory; +import static org.apache.datasketches.memory.internal.XxHash64.hash; +import static org.apache.datasketches.memory.internal.XxHash64.hashBytes; +import static org.apache.datasketches.memory.internal.XxHash64.hashChars; +import static org.apache.datasketches.memory.internal.XxHash64.hashDoubles; +import static org.apache.datasketches.memory.internal.XxHash64.hashFloats; +import static org.apache.datasketches.memory.internal.XxHash64.hashInts; +import static org.apache.datasketches.memory.internal.XxHash64.hashLongs; +import static org.apache.datasketches.memory.internal.XxHash64.hashShorts; + +import org.apache.datasketches.memory.internal.XxHash64; /** * The XxHash is a fast, non-cryptographic, 64-bit hash function that has * excellent avalanche and 2-way bit independence properties. + * This java version adapted the C++ version and the OpenHFT/Zero-Allocation-Hashing implementation + * referenced below as inspiration. + * + *

The C++ source repository: + * + * https://github.com/Cyan4973/xxHash. It has a BSD 2-Clause License: + * + * http://www.opensource.org/licenses/bsd-license.php. See LICENSE. * - *

This class wraps the - * Memory Component XxHash - * implementation. + *

Portions of this code were adapted from + * + * OpenHFT/Zero-Allocation-Hashing, which has an Apache 2 license as does this site. See LICENSE. * * @author Lee Rhodes */ -public class XxHash { +public final class XxHash { + + private XxHash() { /* singleton */ } + + /** + * Hash the given arr starting at the given offset and continuing for the given length using the + * given seed. + * @param arr the given array + * @param offsetBytes starting at this offset + * @param lengthBytes continuing for this length + * @param seed the given seed + * @return the hash + */ + public static long hashByteArr( + final byte[] arr, + final int offsetBytes, + final int lengthBytes, + final long seed) { + return hashBytes(arr, offsetBytes, lengthBytes, seed); + } + + /** + * Hash the given arr starting at the given offset and continuing for the given length using the + * given seed. + * @param arr the given array + * @param offsetShorts starting at this offset + * @param lengthShorts continuing for this length + * @param seed the given seed + * @return the hash + */ + public static long hashShortArr( + final short[] arr, + final int offsetShorts, + final int lengthShorts, + final long seed) { + return hashShorts(arr, offsetShorts, lengthShorts, seed); + } + + /** + * Hash the given arr starting at the given offset and continuing for the given length using the + * given seed. + * @param arr the given array + * @param offsetChars starting at this offset + * @param lengthChars continuing for this length + * @param seed the given seed + * @return the hash + */ + public static long hashCharArr( + final char[] arr, + final int offsetChars, + final int lengthChars, + final long seed) { + return hashChars(arr, offsetChars, lengthChars, seed); + } /** - * Compute the hash of the given Memory object. - * @param mem The given Memory object - * @param offsetBytes Starting at this offset in bytes - * @param lengthBytes Continuing for this number of bytes - * @param seed use this seed for the hash function - * @return return the resulting 64-bit hash value. + * Hash the given arr starting at the given offset and continuing for the given length using the + * given seed. + * @param arr the given array + * @param offsetInts starting at this offset + * @param lengthInts continuing for this length + * @param seed the given seed + * @return the hash */ - public static long hash(final Memory mem, final long offsetBytes, final long lengthBytes, + public static long hashIntArr( + final int[] arr, + final int offsetInts, + final int lengthInts, final long seed) { - return mem.xxHash64(offsetBytes, lengthBytes, seed); + return hashInts(arr, offsetInts, lengthInts, seed); } /** - * Returns a 64-bit hash. - * @param in a long + * Hash the given arr starting at the given offset and continuing for the given length using the + * given seed. + * @param arr the given array + * @param offsetLongs starting at this offset + * @param lengthLongs continuing for this length + * @param seed the given seed + * @return the hash + */ + public static long hashLongArr( + final long[] arr, + final int offsetLongs, + final int lengthLongs, + final long seed) { + return hashLongs(arr, offsetLongs, lengthLongs, seed); + } + + /** + * Returns a 64-bit hash from a single long. This method has been optimized for speed when only + * a single hash of a long is required. + * @param in A long. * @param seed A long valued seed. + * @return the hash. + */ + public static long hashLong( + final long in, + final long seed) { + return hash(in, seed); + } + + /** + * Hash the given arr starting at the given offset and continuing for the given length using the + * given seed. + * @param arr the given array + * @param offsetFloats starting at this offset + * @param lengthFloats continuing for this length + * @param seed the given seed * @return the hash */ - public static long hash(final long in, final long seed) { - return org.apache.datasketches.memory.XxHash.hashLong(in, seed); + public static long hashFloatArr( + final float[] arr, + final int offsetFloats, + final int lengthFloats, + final long seed) { + return hashFloats(arr, offsetFloats, lengthFloats, seed); + } + + /** + * Hash the given arr starting at the given offset and continuing for the given length using the + * given seed. + * @param arr the given array + * @param offsetDoubles starting at this offset + * @param lengthDoubles continuing for this length + * @param seed the given seed + * @return the hash + */ + public static long hashDoubleArr( + final double[] arr, + final int offsetDoubles, + final int lengthDoubles, + final long seed) { + return hashDoubles(arr, offsetDoubles, lengthDoubles, seed); + } + + /** + * Hash the given arr starting at the given offset and continuing for the given length using the + * given seed. + * @param str the given string + * @param offsetChars starting at this offset + * @param lengthChars continuing for this length + * @param seed the given seed + * @return the hash + */ + public static long hashString( + final String str, + final int offsetChars, + final int lengthChars, + final long seed) { + return XxHash64.hashString(str, offsetChars, lengthChars, seed); } } + diff --git a/src/main/java/org/apache/datasketches/hll/BaseHllSketch.java b/src/main/java/org/apache/datasketches/hll/BaseHllSketch.java index c602948a8..ad21fc5ae 100644 --- a/src/main/java/org/apache/datasketches/hll/BaseHllSketch.java +++ b/src/main/java/org/apache/datasketches/hll/BaseHllSketch.java @@ -28,8 +28,8 @@ import java.nio.ByteBuffer; +import org.apache.datasketches.common.Util; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.thetacommon.ThetaUtil; /** * Although this class is package-private, it provides a single place to define and document @@ -299,7 +299,7 @@ public abstract String toString(boolean summary, boolean detail, boolean auxDeta */ public void update(final long datum) { final long[] data = { datum }; - couponUpdate(coupon(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED))); + couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED))); } /** @@ -314,7 +314,7 @@ public void update(final long datum) { public void update(final double datum) { final double d = (datum == 0.0) ? 0.0 : datum; // canonicalize -0.0, 0.0 final long[] data = { Double.doubleToLongBits(d) };// canonicalize all NaN & +/- infinity forms - couponUpdate(coupon(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED))); + couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED))); } /** @@ -334,7 +334,7 @@ public void update(final double datum) { public void update(final String datum) { if ((datum == null) || datum.isEmpty()) { return; } final byte[] data = datum.getBytes(UTF_8); - couponUpdate(coupon(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED))); + couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED))); } /** @@ -351,7 +351,7 @@ public void update(final String datum) { */ public void update(final ByteBuffer data) { if ((data == null) || (data.remaining() == 0)) { return; } - couponUpdate(coupon(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED))); + couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED))); } /** @@ -362,7 +362,7 @@ public void update(final ByteBuffer data) { */ public void update(final byte[] data) { if ((data == null) || (data.length == 0)) { return; } - couponUpdate(coupon(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED))); + couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED))); } /** @@ -376,7 +376,7 @@ public void update(final byte[] data) { */ public void update(final char[] data) { if ((data == null) || (data.length == 0)) { return; } - couponUpdate(coupon(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED))); + couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED))); } /** @@ -387,7 +387,7 @@ public void update(final char[] data) { */ public void update(final int[] data) { if ((data == null) || (data.length == 0)) { return; } - couponUpdate(coupon(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED))); + couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED))); } /** @@ -398,7 +398,7 @@ public void update(final int[] data) { */ public void update(final long[] data) { if ((data == null) || (data.length == 0)) { return; } - couponUpdate(coupon(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED))); + couponUpdate(coupon(hash(data, Util.DEFAULT_UPDATE_SEED))); } private static final int coupon(final long[] hash) { diff --git a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java index 1427f6279..47bad1c67 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/DoublesSortedView.java @@ -60,7 +60,7 @@ public interface DoublesSortedView extends SortedView { * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getCDF(double[] splitPoints, QuantileSearchCriteria searchCrit) { + default double[] getCDF(final double[] splitPoints, final QuantileSearchCriteria searchCrit) { QuantilesUtil.checkDoublesSplitPointsOrder(splitPoints); final int len = splitPoints.length + 1; final double[] buckets = new double[len]; @@ -129,7 +129,7 @@ default double[] getCDF(double[] splitPoints, QuantileSearchCriteria searchCrit) * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getPMF(double[] splitPoints, QuantileSearchCriteria searchCrit) { + default double[] getPMF(final double[] splitPoints, final QuantileSearchCriteria searchCrit) { final double[] buckets = getCDF(splitPoints, searchCrit); final int len = buckets.length; for (int i = len; i-- > 1; ) { diff --git a/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java index eec699d94..0667a6748 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/FloatsSortedView.java @@ -60,7 +60,7 @@ public interface FloatsSortedView extends SortedView { * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getCDF(float[] splitPoints, QuantileSearchCriteria searchCrit) { + default double[] getCDF(final float[] splitPoints, final QuantileSearchCriteria searchCrit) { QuantilesUtil.checkFloatsSplitPointsOrder(splitPoints); final int len = splitPoints.length + 1; final double[] buckets = new double[len]; @@ -129,7 +129,7 @@ default double[] getCDF(float[] splitPoints, QuantileSearchCriteria searchCrit) * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getPMF(float[] splitPoints, QuantileSearchCriteria searchCrit) { + default double[] getPMF(final float[] splitPoints, final QuantileSearchCriteria searchCrit) { final double[] buckets = getCDF(splitPoints, searchCrit); final int len = buckets.length; for (int i = len; i-- > 1; ) { diff --git a/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java index e7e3521c7..eaeceeb92 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/LongsSortedView.java @@ -60,7 +60,7 @@ public interface LongsSortedView extends SortedView { * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getCDF(long[] splitPoints, QuantileSearchCriteria searchCrit) { + default double[] getCDF(final long[] splitPoints, final QuantileSearchCriteria searchCrit) { QuantilesUtil.checkLongsSplitPointsOrder(splitPoints); final int len = splitPoints.length + 1; final double[] buckets = new double[len]; @@ -129,7 +129,7 @@ default double[] getCDF(long[] splitPoints, QuantileSearchCriteria searchCrit) { * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getPMF(long[] splitPoints, QuantileSearchCriteria searchCrit) { + default double[] getPMF(final long[] splitPoints, final QuantileSearchCriteria searchCrit) { final double[] buckets = getCDF(splitPoints, searchCrit); final int len = buckets.length; for (int i = len; i-- > 1; ) { diff --git a/src/main/java/org/apache/datasketches/quantilescommon/PartitioningFeature.java b/src/main/java/org/apache/datasketches/quantilescommon/PartitioningFeature.java index 5672c2a02..82b293b3a 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/PartitioningFeature.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/PartitioningFeature.java @@ -53,7 +53,7 @@ public interface PartitioningFeature { * * @return an instance of {@link GenericPartitionBoundaries GenericPartitionBoundaries}. */ - default GenericPartitionBoundaries getPartitionBoundariesFromNumParts(int numEquallySizedParts) { + default GenericPartitionBoundaries getPartitionBoundariesFromNumParts(final int numEquallySizedParts) { return getPartitionBoundariesFromNumParts(numEquallySizedParts, INCLUSIVE); } @@ -106,7 +106,7 @@ GenericPartitionBoundaries getPartitionBoundariesFromNumParts( * * @return an instance of {@link GenericPartitionBoundaries GenericPartitionBoundaries}. */ - default GenericPartitionBoundaries getPartitionBoundariesFromPartSize(long nominalPartSizeItems) { + default GenericPartitionBoundaries getPartitionBoundariesFromPartSize(final long nominalPartSizeItems) { return getPartitionBoundariesFromPartSize(nominalPartSizeItems, INCLUSIVE); } diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java index 8c4f6620f..09cfd6071 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesDoublesAPI.java @@ -35,7 +35,7 @@ public interface QuantilesDoublesAPI extends QuantilesAPI { * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getCDF(double[] splitPoints) { + default double[] getCDF(final double[] splitPoints) { return getCDF(splitPoints, INCLUSIVE); } @@ -98,7 +98,7 @@ default double[] getCDF(double[] splitPoints) { * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getPMF(double[] splitPoints) { + default double[] getPMF(final double[] splitPoints) { return getPMF(splitPoints, INCLUSIVE); } @@ -150,7 +150,7 @@ default double[] getPMF(double[] splitPoints) { * @return the approximate quantile given the normalized rank. * @throws IllegalArgumentException if sketch is empty. */ - default double getQuantile(double rank) { + default double getQuantile(final double rank) { return getQuantile(rank, INCLUSIVE); } @@ -207,7 +207,7 @@ default double getQuantile(double rank) { * @return an array of quantiles corresponding to the given array of normalized ranks. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getQuantiles(double[] ranks) { + default double[] getQuantiles(final double[] ranks) { return getQuantiles(ranks, INCLUSIVE); } @@ -230,7 +230,7 @@ default double[] getQuantiles(double[] ranks) { * @return the normalized rank corresponding to the given quantile * @throws IllegalArgumentException if sketch is empty. */ - default double getRank(double quantile) { + default double getRank(final double quantile) { return getRank(quantile, INCLUSIVE); } @@ -251,7 +251,7 @@ default double getRank(double quantile) { * @return an array of normalized ranks corresponding to the given array of quantiles. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getRanks(double[] quantiles) { + default double[] getRanks(final double[] quantiles) { return getRanks(quantiles, INCLUSIVE); } diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java index 8b8a91bdd..2713e1b02 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesFloatsAPI.java @@ -34,7 +34,7 @@ public interface QuantilesFloatsAPI extends QuantilesAPI { * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getCDF(float[] splitPoints) { + default double[] getCDF(final float[] splitPoints) { return getCDF(splitPoints, INCLUSIVE); } @@ -97,7 +97,7 @@ default double[] getCDF(float[] splitPoints) { * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getPMF(float[] splitPoints) { + default double[] getPMF(final float[] splitPoints) { return getPMF(splitPoints, INCLUSIVE); } @@ -149,7 +149,7 @@ default double[] getPMF(float[] splitPoints) { * @return the approximate quantile given the normalized rank. * @throws IllegalArgumentException if sketch is empty. */ - default float getQuantile(double rank) { + default float getQuantile(final double rank) { return getQuantile(rank, INCLUSIVE); } @@ -206,7 +206,7 @@ default float getQuantile(double rank) { * @return an array of quantiles corresponding to the given array of normalized ranks. * @throws IllegalArgumentException if sketch is empty. */ - default float[] getQuantiles(double[] ranks) { + default float[] getQuantiles(final double[] ranks) { return getQuantiles(ranks, INCLUSIVE); } @@ -229,7 +229,7 @@ default float[] getQuantiles(double[] ranks) { * @return the normalized rank corresponding to the given quantile. * @throws IllegalArgumentException if sketch is empty. */ - default double getRank(float quantile) { + default double getRank(final float quantile) { return getRank(quantile, INCLUSIVE); } @@ -250,7 +250,7 @@ default double getRank(float quantile) { * @return an array of normalized ranks corresponding to the given array of quantiles. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getRanks(float[] quantiles) { + default double[] getRanks(final float[] quantiles) { return getRanks(quantiles, INCLUSIVE); } diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java index c6a05ffa1..d1592e244 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesGenericAPI.java @@ -38,7 +38,7 @@ public interface QuantilesGenericAPI extends QuantilesAPI, PartitioningFeatur * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getCDF(T[] splitPoints) { + default double[] getCDF(final T[] splitPoints) { return getCDF(splitPoints, INCLUSIVE); } @@ -118,7 +118,7 @@ default int getMaxPartitions() { * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getPMF(T[] splitPoints) { + default double[] getPMF(final T[] splitPoints) { return getPMF(splitPoints, INCLUSIVE); } @@ -170,7 +170,7 @@ default double[] getPMF(T[] splitPoints) { * @return the approximate quantile given the normalized rank. * @throws IllegalArgumentException if sketch is empty. */ - default T getQuantile(double rank) { + default T getQuantile(final double rank) { return getQuantile(rank, INCLUSIVE); } @@ -227,7 +227,7 @@ default T getQuantile(double rank) { * @return an array of quantiles corresponding to the given array of normalized ranks. * @throws IllegalArgumentException if sketch is empty. */ - default T[] getQuantiles(double[] ranks) { + default T[] getQuantiles(final double[] ranks) { return getQuantiles(ranks, INCLUSIVE); } @@ -250,7 +250,7 @@ default T[] getQuantiles(double[] ranks) { * @return the normalized rank corresponding to the given quantile. * @throws IllegalArgumentException if sketch is empty. */ - default double getRank(T quantile) { + default double getRank(final T quantile) { return getRank(quantile, INCLUSIVE); } @@ -271,7 +271,7 @@ default double getRank(T quantile) { * @return an array of normalized ranks corresponding to the given array of quantiles. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getRanks(T[] quantiles) { + default double[] getRanks(final T[] quantiles) { return getRanks(quantiles, INCLUSIVE); } diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java index fb1ca5817..51802df71 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesLongsAPI.java @@ -35,7 +35,7 @@ public interface QuantilesLongsAPI extends QuantilesAPI { * @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getCDF(long[] splitPoints) { + default double[] getCDF(final long[] splitPoints) { return getCDF(splitPoints, INCLUSIVE); } @@ -98,7 +98,7 @@ default double[] getCDF(long[] splitPoints) { * @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0]. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getPMF(long[] splitPoints) { + default double[] getPMF(final long[] splitPoints) { return getPMF(splitPoints, INCLUSIVE); } @@ -150,7 +150,7 @@ default double[] getPMF(long[] splitPoints) { * @return the approximate quantile given the normalized rank. * @throws IllegalArgumentException if sketch is empty. */ - default long getQuantile(double rank) { + default long getQuantile(final double rank) { return getQuantile(rank, INCLUSIVE); } @@ -207,7 +207,7 @@ default long getQuantile(double rank) { * @return an array of quantiles corresponding to the given array of normalized ranks. * @throws IllegalArgumentException if sketch is empty. */ - default long[] getQuantiles(double[] ranks) { + default long[] getQuantiles(final double[] ranks) { return getQuantiles(ranks, INCLUSIVE); } @@ -230,7 +230,7 @@ default long[] getQuantiles(double[] ranks) { * @return the normalized rank corresponding to the given quantile * @throws IllegalArgumentException if sketch is empty. */ - default double getRank(long quantile) { + default double getRank(final long quantile) { return getRank(quantile, INCLUSIVE); } @@ -251,7 +251,7 @@ default double getRank(long quantile) { * @return an array of normalized ranks corresponding to the given array of quantiles. * @throws IllegalArgumentException if sketch is empty. */ - default double[] getRanks(long[] quantiles) { + default double[] getRanks(final long[] quantiles) { return getRanks(quantiles, INCLUSIVE); } diff --git a/src/main/java/org/apache/datasketches/theta/AnotB.java b/src/main/java/org/apache/datasketches/theta/AnotB.java index d9ab37416..cb195acd1 100644 --- a/src/main/java/org/apache/datasketches/theta/AnotB.java +++ b/src/main/java/org/apache/datasketches/theta/AnotB.java @@ -19,8 +19,9 @@ package org.apache.datasketches.theta; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; -import org.apache.datasketches.memory.WritableMemory; /** * Computes a set difference, A-AND-NOT-B, of two theta sketches. @@ -132,15 +133,14 @@ public Family getFamily() { * @param dstOrdered If true, the result will be an ordered {@link CompactSketch}. * See Destination Ordered. * - * @param dstMem if not null the given Memory will be the target location of the result. - * See Destination Memory. + * @param dstSeg if not null the given MemorySegment will be the target location of the result. * * @param reset If true, clears this operator to the empty state after this result is * returned. Set this to false if you wish to obtain an intermediate result. * - * @return the result of this operation as a {@link CompactSketch} in the given dstMem. + * @return the result of this operation as a {@link CompactSketch} in the given dstSeg. */ - public abstract CompactSketch getResult(boolean dstOrdered, WritableMemory dstMem, boolean reset); + public abstract CompactSketch getResult(boolean dstOrdered, MemorySegment dstSeg, boolean reset); /** * Perform A-and-not-B set operation on the two given sketches and return the result as an @@ -149,7 +149,7 @@ public Family getFamily() { *

This a stateless operation and has no impact on the internal state of this operator. * Thus, this is not an accumulating update and does not interact with the {@link #setA(Sketch)}, * {@link #notB(Sketch)}, {@link #getResult(boolean)}, or - * {@link #getResult(boolean, WritableMemory, boolean)} methods.

+ * {@link #getResult(boolean, MemorySegment, boolean)} methods.

* *

If either argument is null an exception is thrown.

* @@ -177,7 +177,7 @@ public CompactSketch aNotB(final Sketch skA, final Sketch skB) { *

This a stateless operation and has no impact on the internal state of this operator. * Thus, this is not an accumulating update and does not interact with the {@link #setA(Sketch)}, * {@link #notB(Sketch)}, {@link #getResult(boolean)}, or - * {@link #getResult(boolean, WritableMemory, boolean)} methods.

+ * {@link #getResult(boolean, MemorySegment, boolean)} methods.

* *

If either argument is null an exception is thrown.

* @@ -194,11 +194,10 @@ public CompactSketch aNotB(final Sketch skA, final Sketch skB) { * @param skB The incoming sketch for the second argument. It must not be null. * @param dstOrdered * See Destination Ordered. - * @param dstMem - * See Destination Memory. + * @param dstSeg the destination MemorySegment * @return the result as a CompactSketch. */ public abstract CompactSketch aNotB(Sketch skA, Sketch skB, boolean dstOrdered, - WritableMemory dstMem); + MemorySegment dstSeg); } diff --git a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java index cc076fd85..07904f0ad 100644 --- a/src/main/java/org/apache/datasketches/theta/AnotBimpl.java +++ b/src/main/java/org/apache/datasketches/theta/AnotBimpl.java @@ -26,10 +26,11 @@ import static org.apache.datasketches.thetacommon.HashOperations.hashSearchOrInsert; import static org.apache.datasketches.thetacommon.HashOperations.minLgHashTableSize; +import java.lang.foreign.MemorySegment; import java.util.Arrays; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -50,7 +51,7 @@ final class AnotBimpl extends AnotB { * @param seed See seed */ AnotBimpl(final long seed) { - this(ThetaUtil.computeSeedHash(seed)); + this(Util.computeSeedHash(seed)); } /** @@ -74,7 +75,7 @@ public void setA(final Sketch skA) { return; } //skA is not empty - ThetaUtil.checkSeedHashes(seedHash_, skA.getSeedHash()); + Util.checkSeedHashes(seedHash_, skA.getSeedHash()); //process A hashArr_ = getHashArrA(skA); @@ -87,7 +88,7 @@ public void setA(final Sketch skA) { public void notB(final Sketch skB) { if (empty_ || skB == null || skB.isEmpty()) { return; } //local and skB is not empty - ThetaUtil.checkSeedHashes(seedHash_, skB.getSeedHash()); + Util.checkSeedHashes(seedHash_, skB.getSeedHash()); thetaLong_ = Math.min(thetaLong_, skB.getThetaLong()); @@ -103,17 +104,17 @@ public CompactSketch getResult(final boolean reset) { } @Override - public CompactSketch getResult(final boolean dstOrdered, final WritableMemory dstMem, + public CompactSketch getResult(final boolean dstOrdered, final MemorySegment dstSeg, final boolean reset) { final CompactSketch result = CompactOperations.componentsToCompact( - thetaLong_, curCount_, seedHash_, empty_, true, false, dstOrdered, dstMem, hashArr_.clone()); + thetaLong_, curCount_, seedHash_, empty_, true, false, dstOrdered, dstSeg, hashArr_.clone()); if (reset) { reset(); } return result; } @Override public CompactSketch aNotB(final Sketch skA, final Sketch skB, final boolean dstOrdered, - final WritableMemory dstMem) { + final MemorySegment dstSeg) { if (skA == null || skB == null) { throw new SketchesArgumentException("Neither argument may be null"); } @@ -121,14 +122,14 @@ public CompactSketch aNotB(final Sketch skA, final Sketch skB, final boolean dst final long minThetaLong = Math.min(skA.getThetaLong(), skB.getThetaLong()); - if (skA.isEmpty()) { return skA.compact(dstOrdered, dstMem); } + if (skA.isEmpty()) { return skA.compact(dstOrdered, dstSeg); } //A is not Empty - ThetaUtil.checkSeedHashes(skA.getSeedHash(), seedHash_); + Util.checkSeedHashes(skA.getSeedHash(), seedHash_); if (skB.isEmpty()) { - return skA.compact(dstOrdered, dstMem); + return skA.compact(dstOrdered, dstSeg); } - ThetaUtil.checkSeedHashes(skB.getSeedHash(), seedHash_); + Util.checkSeedHashes(skB.getSeedHash(), seedHash_); //Both skA & skB are not empty //process A @@ -141,7 +142,7 @@ public CompactSketch aNotB(final Sketch skA, final Sketch skB, final boolean dst final boolean empty = countOut == 0 && minThetaLong == Long.MAX_VALUE; final CompactSketch result = CompactOperations.componentsToCompact( - minThetaLong, countOut, seedHash_, empty, true, false, dstOrdered, dstMem, hashArrOut); + minThetaLong, countOut, seedHash_, empty, true, false, dstOrdered, dstSeg, hashArrOut); return result; } @@ -233,6 +234,15 @@ long getThetaLong() { return thetaLong_; } + @Override + public boolean hasMemorySegment() { return false; } + + @Override + public boolean isDirect() { return false; } + + @Override + public boolean isSameResource( final MemorySegment that) { return false; } + @Override boolean isEmpty() { return empty_; diff --git a/src/main/java/org/apache/datasketches/theta/BitPacking.java b/src/main/java/org/apache/datasketches/theta/BitPacking.java index 99bcfb105..cd7dfe1c9 100644 --- a/src/main/java/org/apache/datasketches/theta/BitPacking.java +++ b/src/main/java/org/apache/datasketches/theta/BitPacking.java @@ -24,7 +24,9 @@ /** * Used as part of Theta compression. */ -public class BitPacking { +public final class BitPacking { + + private BitPacking() { } /** * The bit packing operation diff --git a/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java b/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java index 97792da26..5b8e97be7 100644 --- a/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java +++ b/src/main/java/org/apache/datasketches/theta/BytesCompactCompressedHashIterator.java @@ -22,7 +22,7 @@ /* * This is to uncompress serial version 4 sketch incrementally */ -class BytesCompactCompressedHashIterator implements HashIterator { +final class BytesCompactCompressedHashIterator implements HashIterator { private byte[] bytes; private int offset; private int entryBits; diff --git a/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java b/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java index 20e21da11..cd880de17 100644 --- a/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java +++ b/src/main/java/org/apache/datasketches/theta/BytesCompactHashIterator.java @@ -24,7 +24,7 @@ /* * This is to iterate over serial version 3 sketch representation */ -class BytesCompactHashIterator implements HashIterator { +final class BytesCompactHashIterator implements HashIterator { final private byte[] bytes; final private int offset; final private int numEntries; diff --git a/src/main/java/org/apache/datasketches/theta/CompactOperations.java b/src/main/java/org/apache/datasketches/theta/CompactOperations.java index 2b52f59fa..4ab7fdb31 100644 --- a/src/main/java/org/apache/datasketches/theta/CompactOperations.java +++ b/src/main/java/org/apache/datasketches/theta/CompactOperations.java @@ -19,6 +19,9 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.LG_NOM_LONGS_BYTE; @@ -43,13 +46,12 @@ import static org.apache.datasketches.theta.PreambleUtil.insertSerVer; import static org.apache.datasketches.theta.PreambleUtil.insertThetaLong; +import java.lang.foreign.MemorySegment; import java.util.Arrays; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; /** * @author Lee Rhodes @@ -66,10 +68,10 @@ static CompactSketch componentsToCompact( //No error checking final boolean srcCompact, final boolean srcOrdered, final boolean dstOrdered, - final WritableMemory dstMem, + final MemorySegment dstWSeg, final long[] hashArr) //may not be compacted, ordered or unordered, may be null { - final boolean direct = dstMem != null; + final boolean direct = dstWSeg != null; final boolean empty = srcEmpty || ((curCount == 0) && (thetaLong == Long.MAX_VALUE)); final boolean single = (curCount == 1) && (thetaLong == Long.MAX_VALUE); final long[] hashArrOut; @@ -90,9 +92,9 @@ static CompactSketch componentsToCompact( //No error checking flags |= dstOrderedOut ? ORDERED_FLAG_MASK : 0; flags |= single ? SINGLEITEM_FLAG_MASK : 0; - final Memory mem = - loadCompactMemory(hashArrOut, seedHash, curCount, thetaLong, dstMem, (byte)flags, preLongs); - return new DirectCompactSketch(mem); + final MemorySegment seg = + loadCompactMemorySegment(hashArrOut, seedHash, curCount, thetaLong, dstWSeg, (byte)flags, preLongs); + return new DirectCompactSketch(seg); } else { //Heap if (empty) { @@ -106,26 +108,26 @@ static CompactSketch componentsToCompact( //No error checking } /** - * Heapify or convert a source Theta Sketch Memory image into a heap or target Memory CompactSketch. + * Heapify or convert a source Theta Sketch MemorySegment image into a heap or target MemorySegment CompactSketch. * This assumes hashSeed is OK; serVer = 3. - * @param srcMem the given input source Memory image + * @param srcSeg the given input source MemorySegment image. Can be Read Only. * @param dstOrdered the desired ordering of the resulting CompactSketch - * @param dstMem Used for the target CompactSketch if it is Memory-based. + * @param dstWSeg Used for the target CompactSketch if it is MemorySegment-based. Must be Writable. * @return a CompactSketch of the correct form. */ @SuppressWarnings("unused") - static CompactSketch memoryToCompact( - final Memory srcMem, + static CompactSketch segmentToCompact( + final MemorySegment srcSeg, final boolean dstOrdered, - final WritableMemory dstMem) + final MemorySegment dstWSeg) { //extract Pre0 fields and Flags from srcMem - final int srcPreLongs = extractPreLongs(srcMem); - final int srcSerVer = extractSerVer(srcMem); //not used - final int srcFamId = extractFamilyID(srcMem); - final int srcLgArrLongs = extractLgArrLongs(srcMem); - final int srcFlags = extractFlags(srcMem); - final short srcSeedHash = (short) extractSeedHash(srcMem); + final int srcPreLongs = extractPreLongs(srcSeg); + final int srcSerVer = extractSerVer(srcSeg); //not used + final int srcFamId = extractFamilyID(srcSeg); + final int srcLgArrLongs = extractLgArrLongs(srcSeg); + final int srcFlags = extractFlags(srcSeg); + final short srcSeedHash = (short) extractSeedHash(srcSeg); //srcFlags final boolean srcReadOnlyFlag = (srcFlags & READ_ONLY_FLAG_MASK) > 0; @@ -138,8 +140,8 @@ static CompactSketch memoryToCompact( || SingleItemSketch.otherCheckForSingleItem(srcPreLongs, srcSerVer, srcFamId, srcFlags); //extract pre1 and pre2 fields - final int curCount = single ? 1 : (srcPreLongs > 1) ? extractCurCount(srcMem) : 0; - final long thetaLong = (srcPreLongs > 2) ? extractThetaLong(srcMem) : Long.MAX_VALUE; + final int curCount = single ? 1 : (srcPreLongs > 1) ? extractCurCount(srcSeg) : 0; + final long thetaLong = (srcPreLongs > 2) ? extractThetaLong(srcSeg) : Long.MAX_VALUE; //do some basic checks ... if (srcEmptyFlag) { assert (curCount == 0) && (thetaLong == Long.MAX_VALUE); } @@ -150,19 +152,19 @@ static CompactSketch memoryToCompact( //Note: for empty and single we always output the ordered form. final boolean dstOrderedOut = (srcEmptyFlag || single) ? true : dstOrdered; if (srcEmptyFlag) { - if (dstMem != null) { - dstMem.putByteArray(0, EmptyCompactSketch.EMPTY_COMPACT_SKETCH_ARR, 0, 8); - return new DirectCompactSketch(dstMem); + if (dstWSeg != null) { + MemorySegment.copy(EmptyCompactSketch.EMPTY_COMPACT_SKETCH_ARR, 0, dstWSeg, JAVA_BYTE, 0, 8); + return new DirectCompactSketch(dstWSeg); } else { return EmptyCompactSketch.getInstance(); } } if (single) { - final long hash = srcMem.getLong(srcPreLongs << 3); + final long hash = srcSeg.get(JAVA_LONG_UNALIGNED, srcPreLongs << 3); final SingleItemSketch sis = new SingleItemSketch(hash, srcSeedHash); - if (dstMem != null) { - dstMem.putByteArray(0, sis.toByteArray(), 0, 16); - return new DirectCompactSketch(dstMem); + if (dstWSeg != null) { + MemorySegment.copy(sis.toByteArray(), 0, dstWSeg, JAVA_BYTE, 0, 16); + return new DirectCompactSketch(dstWSeg); } else { //heap return sis; } @@ -172,11 +174,11 @@ static CompactSketch memoryToCompact( final long[] hashArr; if (srcCompactFlag) { hashArr = new long[curCount]; - srcMem.getLongArray(srcPreLongs << 3, hashArr, 0, curCount); + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, srcPreLongs << 3, hashArr, 0, curCount); } else { //update sketch, thus hashTable form final int srcCacheLen = 1 << srcLgArrLongs; final long[] tempHashArr = new long[srcCacheLen]; - srcMem.getLongArray(srcPreLongs << 3, tempHashArr, 0, srcCacheLen); + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, srcPreLongs << 3, tempHashArr, 0, srcCacheLen); hashArr = compactCache(tempHashArr, curCount, thetaLong, dstOrderedOut); } @@ -184,10 +186,10 @@ static CompactSketch memoryToCompact( | ((dstOrderedOut) ? ORDERED_FLAG_MASK : 0); //load the destination. - if (dstMem != null) { - final Memory tgtMem = loadCompactMemory(hashArr, srcSeedHash, curCount, thetaLong, dstMem, + if (dstWSeg != null) { + final MemorySegment tgtSeg = loadCompactMemorySegment(hashArr, srcSeedHash, curCount, thetaLong, dstWSeg, (byte)flagsOut, srcPreLongs); - return new DirectCompactSketch(tgtMem); + return new DirectCompactSketch(tgtSeg); } else { //heap return new HeapCompactSketch(hashArr, srcEmptyFlag, srcSeedHash, curCount, thetaLong, dstOrderedOut); @@ -213,27 +215,28 @@ private static final void checkFamilyAndFlags( } //All arguments must be valid and correct including flags. - // Used as helper to create byte arrays as well as loading Memory for direct compact sketches - static final Memory loadCompactMemory( + // Used as helper to create byte arrays as well as loading MemorySegment for direct compact sketches + //Input must be writable, return can be Read Only + static final MemorySegment loadCompactMemorySegment( final long[] compactHashArr, final short seedHash, final int curCount, final long thetaLong, - final WritableMemory dstMem, + final MemorySegment dstWSeg, final byte flags, final int preLongs) { - assert (dstMem != null) && (compactHashArr != null); + assert (dstWSeg != null) && (compactHashArr != null); final int outLongs = preLongs + curCount; final int outBytes = outLongs << 3; - final int dstBytes = (int) dstMem.getCapacity(); + final int dstBytes = (int) dstWSeg.byteSize(); if (outBytes > dstBytes) { - throw new SketchesArgumentException("Insufficient Memory: " + dstBytes + throw new SketchesArgumentException("Insufficient Space in MemorySegment: " + dstBytes + ", Need: " + outBytes); } final byte famID = (byte) Family.COMPACT.getID(); - //Caution: The following loads directly into Memory without creating a heap byte[] first, + //Caution: The following loads directly into a MemorySegment without creating a heap byte[] first, // which would act as a pre-clearing, initialization mechanism. So it is important to make sure // that all fields are initialized, even those that are not used by the CompactSketch. // Otherwise, uninitialized fields could be filled with off-heap garbage, which could cause @@ -241,30 +244,31 @@ static final Memory loadCompactMemory( // As written below, all fields are initialized avoiding an extra copy. //The first 8 bytes (pre0) - insertPreLongs(dstMem, preLongs); //RF not used = 0 - insertSerVer(dstMem, SER_VER); - insertFamilyID(dstMem, famID); + insertPreLongs(dstWSeg, preLongs); //RF not used = 0 + insertSerVer(dstWSeg, SER_VER); + insertFamilyID(dstWSeg, famID); //The following initializes the lgNomLongs and lgArrLongs to 0. //They are not used in CompactSketches. - dstMem.putShort(LG_NOM_LONGS_BYTE, (short)0); - insertFlags(dstMem, flags); - insertSeedHash(dstMem, seedHash); + dstWSeg.set(JAVA_SHORT_UNALIGNED, LG_NOM_LONGS_BYTE, (short)0); + insertFlags(dstWSeg, flags); + insertSeedHash(dstWSeg, seedHash); if ((preLongs == 1) && (curCount == 1)) { //singleItem, theta = 1.0 - dstMem.putLong(8, compactHashArr[0]); - return dstMem; + dstWSeg.set(JAVA_LONG_UNALIGNED, 8, compactHashArr[0]); + return dstWSeg; } if (preLongs > 1) { - insertCurCount(dstMem, curCount); - insertP(dstMem, (float) 1.0); + insertCurCount(dstWSeg, curCount); + insertP(dstWSeg, (float) 1.0); } if (preLongs > 2) { - insertThetaLong(dstMem, thetaLong); + insertThetaLong(dstWSeg, thetaLong); } if (curCount > 0) { //theta could be < 1.0. - dstMem.putLongArray(preLongs << 3, compactHashArr, 0, curCount); + //dstWSeg.putLongArray(preLongs << 3, compactHashArr, 0, curCount); + MemorySegment.copy(compactHashArr, 0, dstWSeg, JAVA_LONG_UNALIGNED, preLongs << 3, curCount); } - return dstMem; //if prelongs == 3 & curCount == 0, theta could be < 1.0. + return dstWSeg; //if prelongs == 3 & curCount == 0, theta could be < 1.0. This can be RO } /** diff --git a/src/main/java/org/apache/datasketches/theta/CompactSketch.java b/src/main/java/org/apache/datasketches/theta/CompactSketch.java index 630b2e135..413023042 100644 --- a/src/main/java/org/apache/datasketches/theta/CompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/CompactSketch.java @@ -19,6 +19,9 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; import static org.apache.datasketches.common.ByteArrayUtil.getShortLE; import static org.apache.datasketches.common.Family.idToFamily; import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK; @@ -28,22 +31,22 @@ import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE; import static org.apache.datasketches.theta.PreambleUtil.READ_ONLY_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.SEED_HASH_SHORT; +import static org.apache.datasketches.theta.PreambleUtil.extractEntryBitsV4; import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID; import static org.apache.datasketches.theta.PreambleUtil.extractFlags; +import static org.apache.datasketches.theta.PreambleUtil.extractNumEntriesBytesV4; import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs; import static org.apache.datasketches.theta.PreambleUtil.extractSeedHash; import static org.apache.datasketches.theta.PreambleUtil.extractSerVer; -import static org.apache.datasketches.theta.PreambleUtil.extractEntryBitsV4; -import static org.apache.datasketches.theta.PreambleUtil.extractNumEntriesBytesV4; import static org.apache.datasketches.theta.PreambleUtil.extractThetaLongV4; import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits; import static org.apache.datasketches.theta.SingleItemSketch.otherCheckForSingleItem; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; /** * The parent class of all the CompactSketches. CompactSketches are never created directly. @@ -61,9 +64,9 @@ public abstract class CompactSketch extends Sketch { /** - * Heapify takes a CompactSketch image in Memory and instantiates an on-heap CompactSketch. + * Heapify takes a CompactSketch image in a MemorySegment and instantiates an on-heap CompactSketch. * - *

The resulting sketch will not retain any link to the source Memory and all of its data will be + *

The resulting sketch will not retain any link to the source MemorySegment and all of its data will be * copied to the heap CompactSketch.

* *

This method assumes that the sketch image was created with the correct hash seed, so it is not checked. @@ -71,65 +74,63 @@ public abstract class CompactSketch extends Sketch { * However, Serial Version 1 sketch images do not have a seedHash field, * so the resulting heapified CompactSketch will be given the hash of the DEFAULT_UPDATE_SEED.

* - * @param srcMem an image of a CompactSketch. - * See Memory. + * @param srcSeg an image of a CompactSketch. * @return a CompactSketch on the heap. */ - public static CompactSketch heapify(final Memory srcMem) { - return heapify(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED, false); + public static CompactSketch heapify(final MemorySegment srcSeg) { + return heapify(srcSeg, Util.DEFAULT_UPDATE_SEED, false); } /** - * Heapify takes a CompactSketch image in Memory and instantiates an on-heap CompactSketch. + * Heapify takes a CompactSketch image in a MemorySegment and instantiates an on-heap CompactSketch. * - *

The resulting sketch will not retain any link to the source Memory and all of its data will be + *

The resulting sketch will not retain any link to the source MemorySegment and all of its data will be * copied to the heap CompactSketch.

* - *

This method checks if the given expectedSeed was used to create the source Memory image. + *

This method checks if the given expectedSeed was used to create the source MemorySegment image. * However, SerialVersion 1 sketch images cannot be checked as they don't have a seedHash field, * so the resulting heapified CompactSketch will be given the hash of the expectedSeed.

* - * @param srcMem an image of a CompactSketch that was created using the given expectedSeed. - * See Memory. - * @param expectedSeed the seed used to validate the given Memory image. + * @param srcSeg an image of a CompactSketch that was created using the given expectedSeed. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. * @return a CompactSketch on the heap. */ - public static CompactSketch heapify(final Memory srcMem, final long expectedSeed) { - return heapify(srcMem, expectedSeed, true); + public static CompactSketch heapify(final MemorySegment srcSeg, final long expectedSeed) { + return heapify(srcSeg, expectedSeed, true); } - private static CompactSketch heapify(final Memory srcMem, final long seed, final boolean enforceSeed) { - final int serVer = extractSerVer(srcMem); - final int familyID = extractFamilyID(srcMem); + private static CompactSketch heapify(final MemorySegment srcSeg, final long seed, final boolean enforceSeed) { + final int serVer = extractSerVer(srcSeg); + final int familyID = extractFamilyID(srcSeg); final Family family = idToFamily(familyID); if (family != Family.COMPACT) { throw new IllegalArgumentException("Corrupted: " + family + " is not Compact!"); } if (serVer == 4) { - return heapifyV4(srcMem, seed, enforceSeed); + return heapifyV4(srcSeg, seed, enforceSeed); } if (serVer == 3) { - final int flags = extractFlags(srcMem); + final int flags = extractFlags(srcSeg); final boolean srcOrdered = (flags & ORDERED_FLAG_MASK) != 0; final boolean empty = (flags & EMPTY_FLAG_MASK) != 0; - if (enforceSeed && !empty) { PreambleUtil.checkMemorySeedHash(srcMem, seed); } - return CompactOperations.memoryToCompact(srcMem, srcOrdered, null); + if (enforceSeed && !empty) { PreambleUtil.checkSegmentSeedHash(srcSeg, seed); } + return CompactOperations.segmentToCompact(srcSeg, srcOrdered, null); } //not SerVer 3, assume compact stored form - final short seedHash = ThetaUtil.computeSeedHash(seed); + final short seedHash = Util.computeSeedHash(seed); if (serVer == 1) { - return ForwardCompatibility.heapify1to3(srcMem, seedHash); + return ForwardCompatibility.heapify1to3(srcSeg, seedHash); } if (serVer == 2) { - return ForwardCompatibility.heapify2to3(srcMem, - enforceSeed ? seedHash : (short) extractSeedHash(srcMem)); + return ForwardCompatibility.heapify2to3(srcSeg, + enforceSeed ? seedHash : (short) extractSeedHash(srcSeg)); } throw new SketchesArgumentException("Unknown Serialization Version: " + serVer); } /** - * Wrap takes the CompactSketch image in given Memory and refers to it directly. + * Wrap takes the CompactSketch image in given MemorySegment and refers to it directly. * There is no data copying onto the java heap. * The wrap operation enables fast read-only merging and access to all the public read-only API. * @@ -140,22 +141,21 @@ private static CompactSketch heapify(final Memory srcMem, final long seed, final * *

Wrapping any subclass of this class that is empty or contains only a single item will * result in heapified forms of empty and single item sketch respectively. - * This is actually faster and consumes less overall memory.

+ * This is actually faster and consumes less overall space.

* *

This method assumes that the sketch image was created with the correct hash seed, so it is not checked. * However, Serial Version 1 sketch images do not have a seedHash field, * so the resulting on-heap CompactSketch will be given the hash of the DEFAULT_UPDATE_SEED.

* - * @param srcMem an image of a Sketch. - * See Memory. - * @return a CompactSketch backed by the given Memory except as above. + * @param srcSeg an image of a Sketch. + * @return a CompactSketch backed by the given MemorySegment except as above. */ - public static CompactSketch wrap(final Memory srcMem) { - return wrap(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED, false); + public static CompactSketch wrap(final MemorySegment srcSeg) { + return wrap(srcSeg, Util.DEFAULT_UPDATE_SEED, false); } /** - * Wrap takes the sketch image in the given Memory and refers to it directly. + * Wrap takes the sketch image in the given MemorySegment and refers to it directly. * There is no data copying onto the java heap. * The wrap operation enables fast read-only merging and access to all the public read-only API. * @@ -166,44 +166,43 @@ public static CompactSketch wrap(final Memory srcMem) { * *

Wrapping any subclass of this class that is empty or contains only a single item will * result in heapified forms of empty and single item sketch respectively. - * This is actually faster and consumes less overall memory.

+ * This is actually faster and consumes less overall space.

* - *

This method checks if the given expectedSeed was used to create the source Memory image. + *

This method checks if the given expectedSeed was used to create the source MemorySegment image. * However, SerialVersion 1 sketches cannot be checked as they don't have a seedHash field, * so the resulting heapified CompactSketch will be given the hash of the expectedSeed.

* - * @param srcMem an image of a Sketch that was created using the given expectedSeed. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * @param srcSeg an image of a Sketch that was created using the given expectedSeed. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. - * @return a CompactSketch backed by the given Memory except as above. + * @return a CompactSketch backed by the given MemorySegment except as above. */ - public static CompactSketch wrap(final Memory srcMem, final long expectedSeed) { - return wrap(srcMem, expectedSeed, true); + public static CompactSketch wrap(final MemorySegment srcSeg, final long expectedSeed) { + return wrap(srcSeg, expectedSeed, true); } - private static CompactSketch wrap(final Memory srcMem, final long seed, final boolean enforceSeed) { - final int serVer = extractSerVer(srcMem); - final int familyID = extractFamilyID(srcMem); + private static CompactSketch wrap(final MemorySegment srcSeg, final long seed, final boolean enforceSeed) { + final int serVer = extractSerVer(srcSeg); + final int familyID = extractFamilyID(srcSeg); final Family family = Family.idToFamily(familyID); if (family != Family.COMPACT) { throw new IllegalArgumentException("Corrupted: " + family + " is not Compact!"); } - final short seedHash = ThetaUtil.computeSeedHash(seed); + final short seedHash = Util.computeSeedHash(seed); if (serVer == 4) { - return DirectCompactCompressedSketch.wrapInstance(srcMem, - enforceSeed ? seedHash : (short) extractSeedHash(srcMem)); + return DirectCompactCompressedSketch.wrapInstance(srcSeg, + enforceSeed ? seedHash : (short) extractSeedHash(srcSeg)); } else if (serVer == 3) { - if (PreambleUtil.isEmptyFlag(srcMem)) { - return EmptyCompactSketch.getHeapInstance(srcMem); + if (PreambleUtil.isEmptyFlag(srcSeg)) { + return EmptyCompactSketch.getHeapInstance(srcSeg); } - if (otherCheckForSingleItem(srcMem)) { - return SingleItemSketch.heapify(srcMem, enforceSeed ? seedHash : (short) extractSeedHash(srcMem)); + if (otherCheckForSingleItem(srcSeg)) { + return SingleItemSketch.heapify(srcSeg, enforceSeed ? seedHash : (short) extractSeedHash(srcSeg)); } //not empty & not singleItem - final int flags = extractFlags(srcMem); + final int flags = extractFlags(srcSeg); final boolean compactFlag = (flags & COMPACT_FLAG_MASK) > 0; if (!compactFlag) { throw new SketchesArgumentException( @@ -214,22 +213,22 @@ else if (serVer == 3) { throw new SketchesArgumentException( "Corrupted: COMPACT family sketch image must have Read-Only flag set"); } - return DirectCompactSketch.wrapInstance(srcMem, - enforceSeed ? seedHash : (short) extractSeedHash(srcMem)); + return DirectCompactSketch.wrapInstance(srcSeg, + enforceSeed ? seedHash : (short) extractSeedHash(srcSeg)); } //end of serVer 3 else if (serVer == 1) { - return ForwardCompatibility.heapify1to3(srcMem, seedHash); + return ForwardCompatibility.heapify1to3(srcSeg, seedHash); } else if (serVer == 2) { - return ForwardCompatibility.heapify2to3(srcMem, - enforceSeed ? seedHash : (short) extractSeedHash(srcMem)); + return ForwardCompatibility.heapify2to3(srcSeg, + enforceSeed ? seedHash : (short) extractSeedHash(srcSeg)); } throw new SketchesArgumentException( "Corrupted: Serialization Version " + serVer + " not recognized."); } /** - * Wrap takes the sketch image in the given Memory and refers to it directly. + * Wrap takes the sketch image in the given MemorySegment and refers to it directly. * There is no data copying onto the java heap. * The wrap operation enables fast read-only merging and access to all the public read-only API. * @@ -240,23 +239,22 @@ else if (serVer == 2) { * *

Wrapping any subclass of this class that is empty or contains only a single item will * result in heapified forms of empty and single item sketch respectively. - * This is actually faster and consumes less overall memory.

+ * This is actually faster and consumes less overall space.

* - *

This method checks if the DEFAULT_UPDATE_SEED was used to create the source Memory image. + *

This method checks if the DEFAULT_UPDATE_SEED was used to create the source MemorySegment image. * Note that SerialVersion 1 sketches cannot be checked as they don't have a seedHash field, * so the resulting heapified CompactSketch will be given the hash of DEFAULT_UPDATE_SEED.

* * @param bytes a byte array image of a Sketch that was created using the DEFAULT_UPDATE_SEED. - * See Memory * - * @return a CompactSketch backed by the given Memory except as above. + * @return a CompactSketch backed by the given MemorySegment except as above. */ public static CompactSketch wrap(final byte[] bytes) { - return wrap(bytes, ThetaUtil.DEFAULT_UPDATE_SEED, false); + return wrap(bytes, Util.DEFAULT_UPDATE_SEED, false); } /** - * Wrap takes the sketch image in the given Memory and refers to it directly. + * Wrap takes the sketch image in the given MemorySegment and refers to it directly. * There is no data copying onto the java heap. * The wrap operation enables fast read-only merging and access to all the public read-only API. * @@ -267,17 +265,16 @@ public static CompactSketch wrap(final byte[] bytes) { * *

Wrapping any subclass of this class that is empty or contains only a single item will * result in heapified forms of empty and single item sketch respectively. - * This is actually faster and consumes less overall memory.

+ * This is actually faster and consumes less overall space.

* - *

This method checks if the given expectedSeed was used to create the source Memory image. + *

This method checks if the given expectedSeed was used to create the source MemorySegment image. * Note that SerialVersion 1 sketches cannot be checked as they don't have a seedHash field, * so the resulting heapified CompactSketch will be given the hash of the expectedSeed.

* * @param bytes a byte array image of a Sketch that was created using the given expectedSeed. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. - * @return a CompactSketch backed by the given Memory except as above. + * @return a CompactSketch backed by the given MemorySegment except as above. */ public static CompactSketch wrap(final byte[] bytes, final long expectedSeed) { return wrap(bytes, expectedSeed, true); @@ -290,17 +287,17 @@ private static CompactSketch wrap(final byte[] bytes, final long seed, final boo if (family != Family.COMPACT) { throw new IllegalArgumentException("Corrupted: " + family + " is not Compact!"); } - final short seedHash = ThetaUtil.computeSeedHash(seed); + final short seedHash = Util.computeSeedHash(seed); if (serVer == 4) { return WrappedCompactCompressedSketch.wrapInstance(bytes, seedHash); } else if (serVer == 3) { final int flags = bytes[FLAGS_BYTE]; if ((flags & EMPTY_FLAG_MASK) > 0) { - return EmptyCompactSketch.getHeapInstance(Memory.wrap(bytes)); + return EmptyCompactSketch.getHeapInstance(MemorySegment.ofArray(bytes)); } final int preLongs = bytes[PREAMBLE_LONGS_BYTE]; if (otherCheckForSingleItem(preLongs, serVer, familyId, flags)) { - return SingleItemSketch.heapify(Memory.wrap(bytes), enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT)); + return SingleItemSketch.heapify(MemorySegment.ofArray(bytes), enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT)); } //not empty & not singleItem final boolean compactFlag = (flags & COMPACT_FLAG_MASK) > 0; @@ -316,9 +313,9 @@ private static CompactSketch wrap(final byte[] bytes, final long seed, final boo return WrappedCompactSketch.wrapInstance(bytes, enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT)); } else if (serVer == 1) { - return ForwardCompatibility.heapify1to3(Memory.wrap(bytes), seedHash); + return ForwardCompatibility.heapify1to3(MemorySegment.ofArray(bytes), seedHash); } else if (serVer == 2) { - return ForwardCompatibility.heapify2to3(Memory.wrap(bytes), + return ForwardCompatibility.heapify2to3(MemorySegment.ofArray(bytes), enforceSeed ? seedHash : getShortLE(bytes, SEED_HASH_SHORT)); } throw new SketchesArgumentException( @@ -328,7 +325,7 @@ private static CompactSketch wrap(final byte[] bytes, final long seed, final boo //Sketch Overrides @Override - public abstract CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem); + public abstract CompactSketch compact(final boolean dstOrdered, final MemorySegment dstSeg); @Override public int getCompactBytes() { @@ -345,11 +342,26 @@ public Family getFamily() { return Family.COMPACT; } + @Override + public boolean hasMemorySegment() { + return (this instanceof DirectCompactSketch && ((DirectCompactSketch)this).hasMemorySegment()); + } + @Override public boolean isCompact() { return true; } + @Override + public boolean isDirect() { + return (this instanceof DirectCompactSketch && ((DirectCompactSketch)this).isDirect()); + } + + @Override + public boolean isSameResource(final MemorySegment that) { + return (this instanceof DirectCompactSketch && ((DirectCompactSketch)this).isSameResource(that)); + } + @Override public double getEstimate() { return Sketch.estimate(getThetaLong(), getRetainedEntries()); @@ -390,23 +402,23 @@ private byte[] toByteArrayV4() { final int sizeBytes = preambleLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(compressedBits); final byte[] bytes = new byte[sizeBytes]; - final WritableMemory mem = WritableMemory.writableWrap(bytes); + final MemorySegment wseg = MemorySegment.ofArray(bytes); int offsetBytes = 0; - mem.putByte(offsetBytes++, (byte) preambleLongs); - mem.putByte(offsetBytes++, (byte) 4); // to do: add constant - mem.putByte(offsetBytes++, (byte) Family.COMPACT.getID()); - mem.putByte(offsetBytes++, (byte) entryBits); - mem.putByte(offsetBytes++, (byte) numEntriesBytes); - mem.putByte(offsetBytes++, (byte) (COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK | ORDERED_FLAG_MASK)); - mem.putShort(offsetBytes, getSeedHash()); + wseg.set(JAVA_BYTE, offsetBytes++, (byte) preambleLongs); + wseg.set(JAVA_BYTE, offsetBytes++, (byte) 4); // to do: add constant + wseg.set(JAVA_BYTE, offsetBytes++, (byte) Family.COMPACT.getID()); + wseg.set(JAVA_BYTE, offsetBytes++, (byte) entryBits); + wseg.set(JAVA_BYTE, offsetBytes++, (byte) numEntriesBytes); + wseg.set(JAVA_BYTE, offsetBytes++, (byte) (COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK | ORDERED_FLAG_MASK)); + wseg.set(JAVA_SHORT_UNALIGNED, offsetBytes, getSeedHash()); offsetBytes += Short.BYTES; if (isEstimationMode()) { - mem.putLong(offsetBytes, getThetaLong()); + wseg.set(JAVA_LONG_UNALIGNED, offsetBytes, getThetaLong()); offsetBytes += Long.BYTES; } int numEntries = getRetainedEntries(); for (int i = 0; i < numEntriesBytes; i++) { - mem.putByte(offsetBytes++, (byte) (numEntries & 0xff)); + wseg.set(JAVA_BYTE, offsetBytes++, (byte) (numEntries & 0xff)); numEntries >>>= 8; } long previous = 0; @@ -434,32 +446,32 @@ private byte[] toByteArrayV4() { return bytes; } - private static CompactSketch heapifyV4(final Memory srcMem, final long seed, final boolean enforceSeed) { - final int preLongs = extractPreLongs(srcMem); - final int entryBits = extractEntryBitsV4(srcMem); - final int numEntriesBytes = extractNumEntriesBytesV4(srcMem); - final short seedHash = (short) extractSeedHash(srcMem); - if (enforceSeed) { PreambleUtil.checkMemorySeedHash(srcMem, seed); } + private static CompactSketch heapifyV4(final MemorySegment srcSeg, final long seed, final boolean enforceSeed) { + final int preLongs = extractPreLongs(srcSeg); + final int entryBits = extractEntryBitsV4(srcSeg); + final int numEntriesBytes = extractNumEntriesBytesV4(srcSeg); + final short seedHash = (short) extractSeedHash(srcSeg); + if (enforceSeed) { PreambleUtil.checkSegmentSeedHash(srcSeg, seed); } int offsetBytes = 8; long theta = Long.MAX_VALUE; if (preLongs > 1) { - theta = extractThetaLongV4(srcMem); + theta = extractThetaLongV4(srcSeg); offsetBytes += Long.BYTES; } int numEntries = 0; for (int i = 0; i < numEntriesBytes; i++) { - numEntries |= Byte.toUnsignedInt(srcMem.getByte(offsetBytes++)) << (i << 3); + numEntries |= Byte.toUnsignedInt(srcSeg.get(JAVA_BYTE, offsetBytes++)) << (i << 3); } final long[] entries = new long[numEntries]; final byte[] bytes = new byte[entryBits]; // temporary buffer for unpacking int i; for (i = 0; i + 7 < numEntries; i += 8) { - srcMem.getByteArray(offsetBytes, bytes, 0, entryBits); + MemorySegment.copy(srcSeg, JAVA_BYTE, offsetBytes, bytes, 0, entryBits); BitPacking.unpackBitsBlock8(entries, i, bytes, 0, entryBits); offsetBytes += entryBits; } if (i < numEntries) { - srcMem.getByteArray(offsetBytes, bytes, 0, wholeBytesToHoldBits((numEntries - i) * entryBits)); + MemorySegment.copy(srcSeg, JAVA_BYTE, offsetBytes, bytes, 0, wholeBytesToHoldBits((numEntries - i) * entryBits)); int offsetBits = 0; offsetBytes = 0; for (; i < numEntries; i++) { diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentBackgroundThetaPropagation.java b/src/main/java/org/apache/datasketches/theta/ConcurrentBackgroundThetaPropagation.java index 9ae9c9c57..c3d8bfaa5 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentBackgroundThetaPropagation.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentBackgroundThetaPropagation.java @@ -29,7 +29,7 @@ * * @author eshcar */ -class ConcurrentBackgroundThetaPropagation implements Runnable { +final class ConcurrentBackgroundThetaPropagation implements Runnable { // Shared sketch to absorb the data private final ConcurrentSharedThetaSketch sharedThetaSketch; diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java index dbdedebd5..d151ffdf5 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java @@ -19,15 +19,16 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.theta.PreambleUtil.THETA_LONG; +import java.lang.foreign.MemorySegment; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SuppressFBWarnings; -import org.apache.datasketches.memory.WritableMemory; /** * A concurrent shared sketch that is based on DirectQuickSelectSketch. @@ -63,18 +64,18 @@ final class ConcurrentDirectQuickSelectSketch extends DirectQuickSelectSketch private volatile long epoch_; /** - * Construct a new sketch instance and initialize the given Memory as its backing store. + * Construct a new sketch instance and initialize the given MemorySegment as its backing store. * * @param lgNomLongs See lgNomLongs. * @param seed See Update Hash Seed. * @param maxConcurrencyError the max error value including error induced by concurrency. - * @param dstMem the given Memory object destination. It cannot be null. + * @param dstSeg the given MemorySegment object destination. It cannot be null. */ ConcurrentDirectQuickSelectSketch(final int lgNomLongs, final long seed, - final double maxConcurrencyError, final WritableMemory dstMem) { + final double maxConcurrencyError, final MemorySegment dstSeg) { super(lgNomLongs, seed, 1.0F, //p ResizeFactor.X1, //rf, - null, dstMem, false); //unionGadget + dstSeg, false); //unionGadget volatileThetaLong_ = Long.MAX_VALUE; volatileEstimate_ = 0; @@ -86,11 +87,10 @@ final class ConcurrentDirectQuickSelectSketch extends DirectQuickSelectSketch } ConcurrentDirectQuickSelectSketch(final UpdateSketch sketch, final long seed, - final double maxConcurrencyError, final WritableMemory dstMem) { + final double maxConcurrencyError, final MemorySegment dstSeg) { super(sketch.getLgNomLongs(), seed, 1.0F, //p ResizeFactor.X1, //rf, - null, //mem Req Svr - dstMem, + dstSeg, false); //unionGadget exactLimit_ = ConcurrentSharedThetaSketch.computeExactLimit(1L << getLgNomLongs(), @@ -101,7 +101,7 @@ final class ConcurrentDirectQuickSelectSketch extends DirectQuickSelectSketch for (final long hashIn : sketch.getCache()) { propagate(hashIn); } - wmem_.putLong(THETA_LONG, sketch.getThetaLong()); + wseg_.set(JAVA_LONG_UNALIGNED, THETA_LONG, sketch.getThetaLong()); updateVolatileTheta(); updateEstimationSnapshot(); } diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java index e7b3ddaac..d3b706aa8 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapThetaBuffer.java @@ -23,6 +23,7 @@ import static org.apache.datasketches.theta.UpdateReturnState.ConcurrentPropagated; import static org.apache.datasketches.theta.UpdateReturnState.RejectedOverTheta; +import java.lang.foreign.MemorySegment; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.datasketches.common.ResizeFactor; @@ -148,8 +149,8 @@ public double getUpperBound(final int numStdDev) { } @Override - public boolean hasMemory() { - return shared.hasMemory(); + public boolean hasMemorySegment() { + return shared.hasMemorySegment(); } @Override @@ -167,6 +168,11 @@ public boolean isEstimationMode() { return shared.isEstimationMode(); } + @Override + public boolean isSameResource(final MemorySegment that) { + return shared.isSameResource(that); + } + //End of proxies @Override diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java index cdc843f8b..3dda0c31f 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentSharedThetaSketch.java @@ -19,10 +19,10 @@ package org.apache.datasketches.theta; +import java.lang.foreign.MemorySegment; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.datasketches.common.MemoryStatus; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.MemorySegmentStatus; /** * An internal interface to define the API of a concurrent shared theta sketch. @@ -31,12 +31,12 @@ * * @author eshcar */ -interface ConcurrentSharedThetaSketch extends MemoryStatus { +interface ConcurrentSharedThetaSketch extends MemorySegmentStatus { long NOT_SINGLE_HASH = -1L; double MIN_ERROR = 0.0000001; - static long computeExactLimit(long k, double error) { + static long computeExactLimit(final long k, final double error) { return 2 * Math.min(k, (long) Math.ceil(1.0 / Math.pow(Math.max(error,MIN_ERROR), 2.0))); } @@ -150,7 +150,7 @@ boolean propagate(final AtomicBoolean localPropagationInProgress, final Sketch s CompactSketch compact(); - CompactSketch compact(boolean ordered, WritableMemory wmem); + CompactSketch compact(boolean ordered, MemorySegment wseg); UpdateSketch rebuild(); diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java index 64c0fafd4..2bf154215 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectCompactCompressedSketch.java @@ -19,6 +19,7 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static org.apache.datasketches.theta.PreambleUtil.extractEntryBitsV4; import static org.apache.datasketches.theta.PreambleUtil.extractNumEntriesBytesV4; import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs; @@ -26,57 +27,57 @@ import static org.apache.datasketches.theta.PreambleUtil.extractThetaLongV4; import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.Util; /** * An off-heap (Direct), compact, compressed, read-only sketch. It is not empty, not a single item and ordered. * *

This sketch can only be associated with a Serialization Version 4 format binary image.

* - *

This implementation uses data in a given Memory that is owned and managed by the caller. - * This Memory can be off-heap, which if managed properly will greatly reduce the need for + *

This implementation uses data in a given MemorySegment that is owned and managed by the caller. + * This MemorySegment can be off-heap, which if managed properly will greatly reduce the need for * the JVM to perform garbage collection.

*/ -class DirectCompactCompressedSketch extends DirectCompactSketch { +final class DirectCompactCompressedSketch extends DirectCompactSketch { /** - * Construct this sketch with the given memory. - * @param mem Read-only Memory object. + * Construct this sketch with the given MemorySegment. + * @param seg Read-only MemorySegment object. */ - DirectCompactCompressedSketch(final Memory mem) { - super(mem); + DirectCompactCompressedSketch(final MemorySegment seg) { + super(seg); } /** - * Wraps the given Memory, which must be a SerVer 4 compressed CompactSketch image. - * Must check the validity of the Memory before calling. - * @param srcMem See Memory + * Wraps the given MemorySegment, which must be a SerVer 4 compressed CompactSketch image. + * Must check the validity of the MemorySegment before calling. + * @param srcSeg The source MemorySegment * @param seedHash The update seedHash. * See Seed Hash. * @return this sketch */ - static DirectCompactCompressedSketch wrapInstance(final Memory srcMem, final short seedHash) { - ThetaUtil.checkSeedHashes((short) extractSeedHash(srcMem), seedHash); - return new DirectCompactCompressedSketch(srcMem); + static DirectCompactCompressedSketch wrapInstance(final MemorySegment srcSeg, final short seedHash) { + Util.checkSeedHashes((short) extractSeedHash(srcSeg), seedHash); + return new DirectCompactCompressedSketch(srcSeg); } //Sketch Overrides @Override - public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) { - if (dstMem != null) { - mem_.copyTo(0, dstMem, 0, getCurrentBytes()); - return new DirectCompactSketch(dstMem); + public CompactSketch compact(final boolean dstOrdered, final MemorySegment dstSeg) { + if (dstSeg != null) { + MemorySegment.copy(seg_, 0, dstSeg, 0, getCurrentBytes()); + return new DirectCompactSketch(dstSeg); } - return CompactSketch.heapify(mem_); + return CompactSketch.heapify(seg_); } @Override public int getCurrentBytes() { - final int preLongs = extractPreLongs(mem_); - final int entryBits = extractEntryBitsV4(mem_); - final int numEntriesBytes = extractNumEntriesBytesV4(mem_); + final int preLongs = extractPreLongs(seg_); + final int entryBits = extractEntryBitsV4(seg_); + final int numEntriesBytes = extractNumEntriesBytesV4(seg_); return preLongs * Long.BYTES + numEntriesBytes + wholeBytesToHoldBits(getRetainedEntries() * entryBits); } @@ -88,20 +89,20 @@ public int getRetainedEntries(final boolean valid) { //compact is always valid // number of entries is stored using variable length encoding // most significant bytes with all zeros are not stored // one byte in the preamble has the number of non-zero bytes used - final int preLongs = extractPreLongs(mem_); // if > 1 then the second long has theta - final int numEntriesBytes = extractNumEntriesBytesV4(mem_); + final int preLongs = extractPreLongs(seg_); // if > 1 then the second long has theta + final int numEntriesBytes = extractNumEntriesBytesV4(seg_); int offsetBytes = preLongs > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE; int numEntries = 0; for (int i = 0; i < numEntriesBytes; i++) { - numEntries |= Byte.toUnsignedInt(mem_.getByte(offsetBytes++)) << (i << 3); + numEntries |= Byte.toUnsignedInt(seg_.get(JAVA_BYTE, offsetBytes++)) << (i << 3); } return numEntries; } @Override public long getThetaLong() { - final int preLongs = extractPreLongs(mem_); - return (preLongs > 1) ? extractThetaLongV4(mem_) : Long.MAX_VALUE; + final int preLongs = extractPreLongs(seg_); + return (preLongs > 1) ? extractThetaLongV4(seg_) : Long.MAX_VALUE; } @Override @@ -116,11 +117,11 @@ public boolean isOrdered() { @Override public HashIterator iterator() { - return new MemoryCompactCompressedHashIterator( - mem_, - (extractPreLongs(mem_) > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE) - + extractNumEntriesBytesV4(mem_), - extractEntryBitsV4(mem_), + return new MemorySegmentCompactCompressedHashIterator( + seg_, + (extractPreLongs(seg_) > 1 ? START_PACKED_DATA_ESTIMATION_MODE : START_PACKED_DATA_EXACT_MODE) + + extractNumEntriesBytesV4(seg_), + extractEntryBitsV4(seg_), getRetainedEntries() ); } diff --git a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java index 15b03311b..693329110 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectCompactSketch.java @@ -19,8 +19,10 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.theta.CompactOperations.checkIllegalCurCountAndEmpty; -import static org.apache.datasketches.theta.CompactOperations.memoryToCompact; +import static org.apache.datasketches.theta.CompactOperations.segmentToCompact; import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.extractCurCount; import static org.apache.datasketches.theta.PreambleUtil.extractFlags; @@ -29,9 +31,9 @@ import static org.apache.datasketches.theta.PreambleUtil.extractThetaLong; import static org.apache.datasketches.theta.SingleItemSketch.otherCheckForSingleItem; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.Util; /** * An off-heap (Direct), compact, read-only sketch. The internal hash array can be either ordered @@ -39,78 +41,78 @@ * *

This sketch can only be associated with a Serialization Version 3 format binary image.

* - *

This implementation uses data in a given Memory that is owned and managed by the caller. - * This Memory can be off-heap, which if managed properly will greatly reduce the need for + *

This implementation uses data in a given MemorySegment that is owned and managed by the caller. + * This MemorySegment can be off-heap, which if managed properly will greatly reduce the need for * the JVM to perform garbage collection.

* * @author Lee Rhodes */ class DirectCompactSketch extends CompactSketch { - final Memory mem_; + final MemorySegment seg_; /** - * Construct this sketch with the given memory. - * @param mem Read-only Memory object with the order bit properly set. + * Construct this sketch with the given MemorySegment. + * @param seg Read-only MemorySegment object with the order bit properly set. */ - DirectCompactSketch(final Memory mem) { - mem_ = mem; + DirectCompactSketch(final MemorySegment seg) { + seg_ = seg; } /** - * Wraps the given Memory, which must be a SerVer 3, CompactSketch image. - * Must check the validity of the Memory before calling. The order bit must be set properly. - * @param srcMem See Memory + * Wraps the given MemorySegment, which must be a SerVer 3, CompactSketch image. + * Must check the validity of the MemorySegment before calling. The order bit must be set properly. + * @param srcSeg the given MemorySegment * @param seedHash The update seedHash. * See Seed Hash. * @return this sketch */ - static DirectCompactSketch wrapInstance(final Memory srcMem, final short seedHash) { - ThetaUtil.checkSeedHashes((short) extractSeedHash(srcMem), seedHash); - return new DirectCompactSketch(srcMem); + static DirectCompactSketch wrapInstance(final MemorySegment srcSeg, final short seedHash) { + Util.checkSeedHashes((short) extractSeedHash(srcSeg), seedHash); + return new DirectCompactSketch(srcSeg); } //Sketch Overrides @Override - public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) { - return memoryToCompact(mem_, dstOrdered, dstMem); + public CompactSketch compact(final boolean dstOrdered, final MemorySegment dstSeg) { + return segmentToCompact(seg_, dstOrdered, dstSeg); } @Override public int getCurrentBytes() { - if (otherCheckForSingleItem(mem_)) { return 16; } - final int preLongs = extractPreLongs(mem_); - final int curCount = (preLongs == 1) ? 0 : extractCurCount(mem_); + if (otherCheckForSingleItem(seg_)) { return 16; } + final int preLongs = extractPreLongs(seg_); + final int curCount = (preLongs == 1) ? 0 : extractCurCount(seg_); return (preLongs + curCount) << 3; } @Override public int getRetainedEntries(final boolean valid) { //compact is always valid - if (otherCheckForSingleItem(mem_)) { return 1; } - final int preLongs = extractPreLongs(mem_); - final int curCount = (preLongs == 1) ? 0 : extractCurCount(mem_); + if (otherCheckForSingleItem(seg_)) { return 1; } + final int preLongs = extractPreLongs(seg_); + final int curCount = (preLongs == 1) ? 0 : extractCurCount(seg_); return curCount; } @Override public long getThetaLong() { - final int preLongs = extractPreLongs(mem_); - return (preLongs > 2) ? extractThetaLong(mem_) : Long.MAX_VALUE; + final int preLongs = extractPreLongs(seg_); + return (preLongs > 2) ? extractThetaLong(seg_) : Long.MAX_VALUE; } @Override - public boolean hasMemory() { - return mem_ != null; + public boolean hasMemorySegment() { + return seg_ != null && seg_.scope().isAlive(); } @Override public boolean isDirect() { - return hasMemory() ? mem_.isDirect() : false; + return hasMemorySegment() && seg_.isNative(); } @Override public boolean isEmpty() { - final boolean emptyFlag = PreambleUtil.isEmptyFlag(mem_); + final boolean emptyFlag = PreambleUtil.isEmptyFlag(seg_); final long thetaLong = getThetaLong(); final int curCount = getRetainedEntries(true); return emptyFlag || ((curCount == 0) && (thetaLong == Long.MAX_VALUE)); @@ -118,17 +120,18 @@ public boolean isEmpty() { @Override public boolean isOrdered() { - return (extractFlags(mem_) & ORDERED_FLAG_MASK) > 0; + return (extractFlags(seg_) & ORDERED_FLAG_MASK) > 0; } @Override - public boolean isSameResource(final Memory that) { - return hasMemory() ? mem_.isSameResource(that) : false; + public boolean isSameResource(final MemorySegment that) { + return hasMemorySegment() && Util.isSameResource(seg_, that); + } @Override public HashIterator iterator() { - return new MemoryHashIterator(mem_, getRetainedEntries(true), getThetaLong()); + return new MemorySegmentHashIterator(seg_, getRetainedEntries(true), getThetaLong()); } @Override @@ -136,7 +139,7 @@ public byte[] toByteArray() { checkIllegalCurCountAndEmpty(isEmpty(), getRetainedEntries()); final int outBytes = getCurrentBytes(); final byte[] byteArrOut = new byte[outBytes]; - mem_.getByteArray(0, byteArrOut, 0, outBytes); + MemorySegment.copy(seg_, JAVA_BYTE, 0, byteArrOut, 0, outBytes); return byteArrOut; } @@ -144,12 +147,12 @@ public byte[] toByteArray() { @Override long[] getCache() { - if (otherCheckForSingleItem(mem_)) { return new long[] { mem_.getLong(8) }; } - final int preLongs = extractPreLongs(mem_); - final int curCount = (preLongs == 1) ? 0 : extractCurCount(mem_); + if (otherCheckForSingleItem(seg_)) { return new long[] { seg_.get(JAVA_LONG_UNALIGNED, 8) }; } + final int preLongs = extractPreLongs(seg_); + final int curCount = (preLongs == 1) ? 0 : extractCurCount(seg_); if (curCount > 0) { final long[] cache = new long[curCount]; - mem_.getLongArray(preLongs << 3, cache, 0, curCount); + MemorySegment.copy(seg_, JAVA_LONG_UNALIGNED, preLongs << 3, cache, 0, curCount); return cache; } return new long[0]; @@ -157,21 +160,21 @@ long[] getCache() { @Override int getCompactPreambleLongs() { - return extractPreLongs(mem_); + return extractPreLongs(seg_); } @Override int getCurrentPreambleLongs() { - return extractPreLongs(mem_); + return extractPreLongs(seg_); } @Override - Memory getMemory() { - return mem_; + MemorySegment getMemorySegment() { + return seg_; } @Override short getSeedHash() { - return (short) extractSeedHash(mem_); + return (short) extractSeedHash(seg_); } } diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java index af073a5ee..7356d4d5f 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java @@ -19,6 +19,10 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.common.Util.LONG_MAX_VALUE_AS_DOUBLE; import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE; @@ -30,7 +34,7 @@ import static org.apache.datasketches.theta.PreambleUtil.extractLgArrLongs; import static org.apache.datasketches.theta.PreambleUtil.extractLgNomLongs; import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs; -import static org.apache.datasketches.theta.PreambleUtil.getMemBytes; +import static org.apache.datasketches.theta.PreambleUtil.getSegBytes; import static org.apache.datasketches.theta.PreambleUtil.insertCurCount; import static org.apache.datasketches.theta.PreambleUtil.insertFamilyID; import static org.apache.datasketches.theta.PreambleUtil.insertFlags; @@ -53,12 +57,12 @@ import static org.apache.datasketches.theta.UpdateReturnState.RejectedDuplicate; import static org.apache.datasketches.theta.UpdateReturnState.RejectedOverTheta; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.MemoryRequestServer; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.HashOperations; import org.apache.datasketches.thetacommon.ThetaUtil; @@ -66,35 +70,31 @@ * The default Theta Sketch using the QuickSelect algorithm. * This subclass implements methods, which affect the state (update, rebuild, reset) * - *

This implementation uses data in a given Memory that is owned and managed by the caller. - * This Memory can be off-heap, which if managed properly will greatly reduce the need for + *

This implementation uses data in a given MemorySegment that is owned and managed by the caller. + * This MemorySegment can be off-heap, which if managed properly will greatly reduce the need for * the JVM to perform garbage collection.

* * @author Lee Rhodes * @author Kevin Lang */ class DirectQuickSelectSketch extends DirectQuickSelectSketchR { - MemoryRequestServer memReqSvr_ = null; //never serialized private DirectQuickSelectSketch( final long seed, - final WritableMemory wmem) { - super(seed, wmem); + final MemorySegment wseg) { + super(seed, wseg); } /** - * Construct a new sketch instance and initialize the given Memory as its backing store. + * Construct a new sketch instance and initialize the given MemorySegment as its backing store. * * @param lgNomLongs See lgNomLongs. * @param seed See Update Hash Seed. * @param p * See Sampling Probability, p - * @param rf Currently internally fixed at 2. Unless dstMem is not configured with a valid - * MemoryRequest, in which case the rf is effectively 1, which is no resizing at all and the - * dstMem must be large enough for a full sketch. + * @param rf Resize Factor * See Resize Factor - * @param memReqSvr the given MemoryRequestServer - * @param dstMem the given Memory object destination. It cannot be null. + * @param dstSeg the given MemorySegment object destination. It cannot be null. * It will be cleared prior to use. * @param unionGadget true if this sketch is implementing the Union gadget function. * Otherwise, it is behaving as a normal QuickSelectSketch. @@ -104,19 +104,17 @@ private DirectQuickSelectSketch( final long seed, final float p, final ResizeFactor rf, - final MemoryRequestServer memReqSvr, - final WritableMemory dstMem, + final MemorySegment dstSeg, final boolean unionGadget) { this( - checkMemSize(lgNomLongs, rf, dstMem, unionGadget), + checkSegSize(lgNomLongs, rf, dstSeg, unionGadget), //SpotBugs CT_CONSTRUCTOR_THROW is false positive. //this construction scheme is compliant with SEI CERT Oracle Coding Standard for Java / OBJ11-J lgNomLongs, seed, p, rf, - memReqSvr, - dstMem, + dstSeg, unionGadget); } @@ -126,10 +124,9 @@ private DirectQuickSelectSketch( final long seed, final float p, final ResizeFactor rf, - final MemoryRequestServer memReqSvr, - final WritableMemory dstMem, + final MemorySegment dstSeg, final boolean unionGadget) { - super(seed, dstMem); + super(seed, dstSeg); //Choose family, preambleLongs final Family family; final int preambleLongs; @@ -148,86 +145,83 @@ private DirectQuickSelectSketch( //@formatter:off //Build preamble - insertPreLongs(dstMem, preambleLongs); //byte 0 - insertLgResizeFactor(dstMem, lgRF); //byte 0 - insertSerVer(dstMem, SER_VER); //byte 1 - insertFamilyID(dstMem, family.getID()); //byte 2 - insertLgNomLongs(dstMem, lgNomLongs); //byte 3 - insertLgArrLongs(dstMem, lgArrLongs); //byte 4 + insertPreLongs(dstSeg, preambleLongs); //byte 0 + insertLgResizeFactor(dstSeg, lgRF); //byte 0 + insertSerVer(dstSeg, SER_VER); //byte 1 + insertFamilyID(dstSeg, family.getID()); //byte 2 + insertLgNomLongs(dstSeg, lgNomLongs); //byte 3 + insertLgArrLongs(dstSeg, lgArrLongs); //byte 4 //flags: bigEndian = readOnly = compact = ordered = false; empty = true : 00100 = 4 - insertFlags(dstMem, EMPTY_FLAG_MASK); //byte 5 - insertSeedHash(dstMem, ThetaUtil.computeSeedHash(seed)); //bytes 6,7 - insertCurCount(dstMem, 0); //bytes 8-11 - insertP(dstMem, p); //bytes 12-15 + insertFlags(dstSeg, EMPTY_FLAG_MASK); //byte 5 + insertSeedHash(dstSeg, Util.computeSeedHash(seed)); //bytes 6,7 + insertCurCount(dstSeg, 0); //bytes 8-11 + insertP(dstSeg, p); //bytes 12-15 final long thetaLong = (long)(p * LONG_MAX_VALUE_AS_DOUBLE); - insertThetaLong(dstMem, thetaLong); //bytes 16-23 + insertThetaLong(dstSeg, thetaLong); //bytes 16-23 if (unionGadget) { - insertUnionThetaLong(dstMem, thetaLong); + insertUnionThetaLong(dstSeg, thetaLong); } //@formatter:on //clear hash table area - dstMem.clear(preambleLongs << 3, 8 << lgArrLongs); + dstSeg.asSlice(preambleLongs << 3, Long.BYTES << lgArrLongs).fill((byte)0); hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); - memReqSvr_ = memReqSvr; } - private static final boolean checkMemSize( - final int lgNomLongs, final ResizeFactor rf, final Memory dstMem, final boolean unionGadget) { + private static final boolean checkSegSize( + final int lgNomLongs, final ResizeFactor rf, final MemorySegment dstSeg, final boolean unionGadget) { final int preambleLongs = (unionGadget) ? Family.UNION.getMinPreLongs() : Family.QUICKSELECT.getMinPreLongs(); final int lgRF = rf.lg(); final int lgArrLongs = (lgRF == 0) ? lgNomLongs + 1 : ThetaUtil.MIN_LG_ARR_LONGS; - final int minReqBytes = getMemBytes(lgArrLongs, preambleLongs); - final long curMemCapBytes = dstMem.getCapacity(); - if (curMemCapBytes < minReqBytes) { + final int minReqBytes = getSegBytes(lgArrLongs, preambleLongs); + final long curSegCapBytes = dstSeg.byteSize(); + if (curSegCapBytes < minReqBytes) { throw new SketchesArgumentException( - "Memory capacity is too small: " + curMemCapBytes + " < " + minReqBytes); + "MemorySegment capacity is too small: " + curSegCapBytes + " < " + minReqBytes); } return true; } /** - * Wrap a sketch around the given source Memory containing sketch data that originated from + * Wrap a sketch around the given source MemorySegment containing sketch data that originated from * this sketch. - * @param srcMem See Memory - * The given Memory object must be in hash table form and not read only. + * @param srcSeg The given MemorySegment object must be in hash table form and not read only. * @param seed See Update Hash Seed * @return instance of this sketch */ - static DirectQuickSelectSketch writableWrap(final WritableMemory srcMem, final long seed) { - final int preambleLongs = extractPreLongs(srcMem); //byte 0 - final int lgNomLongs = extractLgNomLongs(srcMem); //byte 3 - final int lgArrLongs = extractLgArrLongs(srcMem); //byte 4 + static DirectQuickSelectSketch writableWrap(final MemorySegment srcSeg, final long seed) { + final int preambleLongs = extractPreLongs(srcSeg); //byte 0 + final int lgNomLongs = extractLgNomLongs(srcSeg); //byte 3 + final int lgArrLongs = extractLgArrLongs(srcSeg); //byte 4 - UpdateSketch.checkUnionQuickSelectFamily(srcMem, preambleLongs, lgNomLongs); - checkMemIntegrity(srcMem, seed, preambleLongs, lgNomLongs, lgArrLongs); + UpdateSketch.checkUnionQuickSelectFamily(srcSeg, preambleLongs, lgNomLongs); + checkSegIntegrity(srcSeg, seed, preambleLongs, lgNomLongs, lgArrLongs); - if (isResizeFactorIncorrect(srcMem, lgNomLongs, lgArrLongs)) { + if (isResizeFactorIncorrect(srcSeg, lgNomLongs, lgArrLongs)) { //If incorrect it sets it to X2 which always works. - insertLgResizeFactor(srcMem, ResizeFactor.X2.lg()); + insertLgResizeFactor(srcSeg, ResizeFactor.X2.lg()); } final DirectQuickSelectSketch dqss = - new DirectQuickSelectSketch(seed, srcMem); + new DirectQuickSelectSketch(seed, srcSeg); dqss.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqss; } /** - * Fast-wrap a sketch around the given source Memory containing sketch data that originated from - * this sketch. This does NO validity checking of the given Memory. - * @param srcMem See Memory - * The given Memory object must be in hash table form and not read only. + * Fast-wrap a sketch around the given source MemorySegment containing sketch data that originated from + * this sketch. This does NO validity checking of the given MemorySegment. + * @param srcSeg The given MemorySegment must be in hash table form and not read only. * @param seed See Update Hash Seed * @return instance of this sketch */ - static DirectQuickSelectSketch fastWritableWrap(final WritableMemory srcMem, final long seed) { - final int lgNomLongs = extractLgNomLongs(srcMem); //byte 3 - final int lgArrLongs = extractLgArrLongs(srcMem); //byte 4 + static DirectQuickSelectSketch fastWritableWrap(final MemorySegment srcSeg, final long seed) { + final int lgNomLongs = extractLgNomLongs(srcSeg); //byte 3 + final int lgArrLongs = extractLgArrLongs(srcSeg); //byte 4 final DirectQuickSelectSketch dqss = - new DirectQuickSelectSketch(seed, srcMem); + new DirectQuickSelectSketch(seed, srcSeg); dqss.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqss; } @@ -239,9 +233,9 @@ static DirectQuickSelectSketch fastWritableWrap(final WritableMemory srcMem, fin @Override public UpdateSketch rebuild() { final int lgNomLongs = getLgNomLongs(); - final int preambleLongs = wmem_.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; + final int preambleLongs = wseg_.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; if (getRetainedEntries(true) > (1 << lgNomLongs)) { - quickSelectAndRebuild(wmem_, preambleLongs, lgNomLongs); + quickSelectAndRebuild(wseg_, preambleLongs, lgNomLongs); } return this; } @@ -253,15 +247,15 @@ public void reset() { //lgArrLongs stays the same //thetaLongs resets to p final int arrLongs = 1 << getLgArrLongs(); - final int preambleLongs = wmem_.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; + final int preambleLongs = wseg_.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; final int preBytes = preambleLongs << 3; - wmem_.clear(preBytes, arrLongs * 8L); //clear data array + wseg_.asSlice(preBytes, arrLongs * 8L).fill((byte)0); //flags: bigEndian = readOnly = compact = ordered = false; empty = true. - wmem_.putByte(FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); - wmem_.putInt(RETAINED_ENTRIES_INT, 0); - final float p = wmem_.getFloat(P_FLOAT); + wseg_.set(JAVA_BYTE, FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); + wseg_.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, 0); + final float p = wseg_.get(JAVA_FLOAT_UNALIGNED, P_FLOAT); final long thetaLong = (long) (p * LONG_MAX_VALUE_AS_DOUBLE); - wmem_.putLong(THETA_LONG, thetaLong); + wseg_.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong); } //restricted methods @@ -270,7 +264,7 @@ public void reset() { UpdateReturnState hashUpdate(final long hash) { HashOperations.checkHashCorruption(hash); - wmem_.putByte(FLAGS_BYTE, (byte) (wmem_.getByte(FLAGS_BYTE) & ~EMPTY_FLAG_MASK)); + wseg_.set(JAVA_BYTE, FLAGS_BYTE, (byte) (wseg_.get(JAVA_BYTE, FLAGS_BYTE) & ~EMPTY_FLAG_MASK)); final long thetaLong = getThetaLong(); final int lgNomLongs = getLgNomLongs(); //The over-theta test @@ -279,64 +273,54 @@ UpdateReturnState hashUpdate(final long hash) { } final int lgArrLongs = getLgArrLongs(); - final int preambleLongs = wmem_.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; + final int preambleLongs = wseg_.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; //The duplicate test final int index = - HashOperations.hashSearchOrInsertMemory(wmem_, lgArrLongs, hash, preambleLongs << 3); + HashOperations.hashSearchOrInsertMemorySegment(wseg_, lgArrLongs, hash, preambleLongs << 3); if (index >= 0) { return RejectedDuplicate; //Duplicate, not inserted } //insertion occurred, increment curCount final int curCount = getRetainedEntries(true) + 1; - wmem_.putInt(RETAINED_ENTRIES_INT, curCount); //update curCount + wseg_.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, curCount); //update curCount if (isOutOfSpace(curCount)) { //we need to do something, we are out of space if (lgArrLongs > lgNomLongs) { //at full size, rebuild //Assumes no dirty values, changes thetaLong, curCount_ - assert (lgArrLongs == (lgNomLongs + 1)) - : "lgArr: " + lgArrLongs + ", lgNom: " + lgNomLongs; + assert (lgArrLongs == (lgNomLongs + 1)) : "lgArr: " + lgArrLongs + ", lgNom: " + lgNomLongs; //rebuild, refresh curCount based on # values in the hashtable. - quickSelectAndRebuild(wmem_, preambleLongs, lgNomLongs); + quickSelectAndRebuild(wseg_, preambleLongs, lgNomLongs); return InsertedCountIncrementedRebuilt; } //end of rebuild, exit - else { //Not at full size, resize. Should not get here if lgRF = 0 and memCap is too small. + else { //Not at full size, resize. Should not get here if lgRF = 0 and segCap is too small. final int lgRF = getLgRF(); - final int actLgRF = actLgResizeFactor(wmem_.getCapacity(), lgArrLongs, preambleLongs, lgRF); + final int actLgRF = actLgResizeFactor(wseg_.byteSize(), lgArrLongs, preambleLongs, lgRF); int tgtLgArrLongs = Math.min(lgArrLongs + actLgRF, lgNomLongs + 1); - if (actLgRF > 0) { //Expand in current Memory + if (actLgRF > 0) { //Expand in current MemorySegment //lgArrLongs will change; thetaLong, curCount will not - resize(wmem_, preambleLongs, lgArrLongs, tgtLgArrLongs); + resize(wseg_, preambleLongs, lgArrLongs, tgtLgArrLongs); hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, tgtLgArrLongs); return InsertedCountIncrementedResized; - } //end of Expand in current memory, exit. + } //end of Expand in current MemorySegment, exit. else { - //Request more memory, then resize. lgArrLongs will change; thetaLong, curCount will not + //Request more space, then resize. lgArrLongs will change; thetaLong, curCount will not final int preBytes = preambleLongs << 3; tgtLgArrLongs = Math.min(lgArrLongs + lgRF, lgNomLongs + 1); final int tgtArrBytes = 8 << tgtLgArrLongs; final int reqBytes = tgtArrBytes + preBytes; + final MemorySegment newDstSeg = MemorySegment.ofArray(new byte[reqBytes]); - memReqSvr_ = (memReqSvr_ == null) ? wmem_.getMemoryRequestServer() : memReqSvr_; - - if (memReqSvr_ == null) { //in case the MRS is not enabled or null. - throw new SketchesArgumentException("Out of Memory, MemoryRequestServer is null, cannot expand."); - } - - final WritableMemory newDstMem = memReqSvr_.request(wmem_, reqBytes); - - moveAndResize(wmem_, preambleLongs, lgArrLongs, newDstMem, tgtLgArrLongs, thetaLong); - - memReqSvr_.requestClose(wmem_); + moveAndResize(wseg_, preambleLongs, lgArrLongs, newDstSeg, tgtLgArrLongs, thetaLong); + wseg_ = newDstSeg; - wmem_ = newDstMem; hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, tgtLgArrLongs); return InsertedCountIncrementedResized; - } //end of Request more memory to resize + } //end of Request more space to resize } //end of resize } //end of isOutOfSpace return InsertedCountIncremented; diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java index fb2aed2a5..2c25ee618 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketchR.java @@ -19,6 +19,10 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.theta.CompactOperations.checkIllegalCurCountAndEmpty; import static org.apache.datasketches.theta.CompactOperations.computeCompactPreLongs; import static org.apache.datasketches.theta.CompactOperations.correctThetaOnCompact; @@ -37,20 +41,21 @@ import static org.apache.datasketches.theta.PreambleUtil.extractThetaLong; import static org.apache.datasketches.theta.PreambleUtil.insertThetaLong; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesReadOnlyException; import org.apache.datasketches.common.SuppressFBWarnings; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.ThetaUtil; /** * The default Theta Sketch using the QuickSelect algorithm. * This is the read-only implementation with non-functional methods, which affect the state. * - *

This implementation uses data in a given Memory that is owned and managed by the caller. - * This Memory can be off-heap, which if managed properly will greatly reduce the need for + *

This implementation uses data in a given MemorySegment that is owned and managed by the caller. + * This MemorySegment can be off-heap, which if managed properly will greatly reduce the need for * the JVM to perform garbage collection.

* * @author Lee Rhodes @@ -60,50 +65,48 @@ class DirectQuickSelectSketchR extends UpdateSketch { static final double DQS_RESIZE_THRESHOLD = 15.0 / 16.0; //tuned for space final long seed_; //provided, kept only on heap, never serialized. int hashTableThreshold_; //computed, kept only on heap, never serialized. - WritableMemory wmem_; //A WritableMemory for child class, but no write methods here + MemorySegment wseg_; //A MemorySegment for child class, but no write methods here //only called by DirectQuickSelectSketch and below - DirectQuickSelectSketchR(final long seed, final WritableMemory wmem) { + DirectQuickSelectSketchR(final long seed, final MemorySegment wseg) { seed_ = seed; - wmem_ = wmem; + wseg_ = wseg; } /** - * Wrap a sketch around the given source Memory containing sketch data that originated from + * Wrap a sketch around the given source MemorySegment containing sketch data that originated from * this sketch. - * @param srcMem See Memory - * The given Memory object must be in hash table form and not read only. + * @param srcSeg the source MemorySegment. + * The given MemorySegment object must be in hash table form and not read only. * @param seed See Update Hash Seed * @return instance of this sketch */ - static DirectQuickSelectSketchR readOnlyWrap(final Memory srcMem, final long seed) { - final int preambleLongs = extractPreLongs(srcMem); //byte 0 - final int lgNomLongs = extractLgNomLongs(srcMem); //byte 3 - final int lgArrLongs = extractLgArrLongs(srcMem); //byte 4 + static DirectQuickSelectSketchR readOnlyWrap(final MemorySegment srcSeg, final long seed) { + final int preambleLongs = extractPreLongs(srcSeg); //byte 0 + final int lgNomLongs = extractLgNomLongs(srcSeg); //byte 3 + final int lgArrLongs = extractLgArrLongs(srcSeg); //byte 4 - UpdateSketch.checkUnionQuickSelectFamily(srcMem, preambleLongs, lgNomLongs); - checkMemIntegrity(srcMem, seed, preambleLongs, lgNomLongs, lgArrLongs); + UpdateSketch.checkUnionQuickSelectFamily(srcSeg, preambleLongs, lgNomLongs); + checkSegIntegrity(srcSeg, seed, preambleLongs, lgNomLongs, lgArrLongs); final DirectQuickSelectSketchR dqssr = - new DirectQuickSelectSketchR(seed, (WritableMemory) srcMem); + new DirectQuickSelectSketchR(seed, srcSeg); dqssr.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqssr; } /** - * Fast-wrap a sketch around the given source Memory containing sketch data that originated from - * this sketch. This does NO validity checking of the given Memory. - * @param srcMem See Memory - * The given Memory object must be in hash table form and not read only. + * Fast-wrap a sketch around the given source MemorySegment containing sketch data that originated from + * this sketch. This does NO validity checking of the given MemorySegment. + * @param srcSeg The given MemorySegment object must be in hash table form and not read only. * @param seed See Update Hash Seed * @return instance of this sketch */ - static DirectQuickSelectSketchR fastReadOnlyWrap(final Memory srcMem, final long seed) { - final int lgNomLongs = srcMem.getByte(LG_NOM_LONGS_BYTE) & 0XFF; - final int lgArrLongs = srcMem.getByte(LG_ARR_LONGS_BYTE) & 0XFF; + static DirectQuickSelectSketchR fastReadOnlyWrap(final MemorySegment srcSeg, final long seed) { + final int lgNomLongs = srcSeg.get(JAVA_BYTE, LG_NOM_LONGS_BYTE) & 0XFF; + final int lgArrLongs = srcSeg.get(JAVA_BYTE, LG_ARR_LONGS_BYTE) & 0XFF; - final DirectQuickSelectSketchR dqss = - new DirectQuickSelectSketchR(seed, (WritableMemory) srcMem); + final DirectQuickSelectSketchR dqss = new DirectQuickSelectSketchR(seed, srcSeg); dqss.hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, lgArrLongs); return dqss; } @@ -113,70 +116,70 @@ static DirectQuickSelectSketchR fastReadOnlyWrap(final Memory srcMem, final long @Override public int getCurrentBytes() { //not compact - final byte lgArrLongs = wmem_.getByte(LG_ARR_LONGS_BYTE); - final int preLongs = wmem_.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; + final byte lgArrLongs = wseg_.get(JAVA_BYTE, LG_ARR_LONGS_BYTE); + final int preLongs = wseg_.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; final int lengthBytes = (preLongs + (1 << lgArrLongs)) << 3; return lengthBytes; } @Override public double getEstimate() { - final int curCount = extractCurCount(wmem_); - final long thetaLong = extractThetaLong(wmem_); + final int curCount = extractCurCount(wseg_); + final long thetaLong = extractThetaLong(wseg_); return Sketch.estimate(thetaLong, curCount); } @Override public Family getFamily() { - final int familyID = wmem_.getByte(FAMILY_BYTE) & 0XFF; + final int familyID = wseg_.get(JAVA_BYTE, FAMILY_BYTE) & 0XFF; return Family.idToFamily(familyID); } @Override public int getRetainedEntries(final boolean valid) { //always valid - return wmem_.getInt(RETAINED_ENTRIES_INT); + return wseg_.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT); } @Override public long getThetaLong() { - return isEmpty() ? Long.MAX_VALUE : wmem_.getLong(THETA_LONG); + return isEmpty() ? Long.MAX_VALUE : wseg_.get(JAVA_LONG_UNALIGNED, THETA_LONG); } @Override - public boolean hasMemory() { - return wmem_ != null; + public boolean hasMemorySegment() { + return wseg_ != null && wseg_.scope().isAlive(); } @Override public boolean isDirect() { - return hasMemory() ? wmem_.isDirect() : false; + return hasMemorySegment() && wseg_.isNative(); } @Override public boolean isEmpty() { - return PreambleUtil.isEmptyFlag(wmem_); + return PreambleUtil.isEmptyFlag(wseg_); } @Override - public boolean isSameResource(final Memory that) { - return hasMemory() ? wmem_.isSameResource(that) : false; + public boolean isSameResource(final MemorySegment that) { + return hasMemorySegment() && Util.isSameResource(wseg_, that); } @Override public HashIterator iterator() { - return new MemoryHashIterator(wmem_, 1 << getLgArrLongs(), getThetaLong()); + return new MemorySegmentHashIterator(wseg_, 1 << getLgArrLongs(), getThetaLong()); } @Override - public byte[] toByteArray() { //MY_FAMILY is stored in wmem_ - checkIllegalCurCountAndEmpty(isEmpty(), extractCurCount(wmem_)); + public byte[] toByteArray() { //MY_FAMILY is stored in wseg_ + checkIllegalCurCountAndEmpty(isEmpty(), extractCurCount(wseg_)); final int lengthBytes = getCurrentBytes(); final byte[] byteArray = new byte[lengthBytes]; - final WritableMemory mem = WritableMemory.writableWrap(byteArray); - wmem_.copyTo(0, mem, 0, lengthBytes); + final MemorySegment seg = MemorySegment.ofArray(byteArray); + MemorySegment.copy(wseg_, 0, seg, 0, lengthBytes); final long thetaLong = - correctThetaOnCompact(isEmpty(), extractCurCount(wmem_), extractThetaLong(wmem_)); - insertThetaLong(wmem_, thetaLong); + correctThetaOnCompact(isEmpty(), extractCurCount(wseg_), extractThetaLong(wseg_)); + insertThetaLong(wseg_, thetaLong); return byteArray; } @@ -184,12 +187,12 @@ public byte[] toByteArray() { //MY_FAMILY is stored in wmem_ @Override public final int getLgNomLongs() { - return PreambleUtil.extractLgNomLongs(wmem_); + return PreambleUtil.extractLgNomLongs(wseg_); } @Override float getP() { - return wmem_.getFloat(P_FLOAT); + return wseg_.get(JAVA_FLOAT_UNALIGNED, P_FLOAT); } @Override @@ -216,11 +219,10 @@ public void reset() { @Override long[] getCache() { - final long lgArrLongs = wmem_.getByte(LG_ARR_LONGS_BYTE) & 0XFF; - final int preambleLongs = wmem_.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; + final long lgArrLongs = wseg_.get(JAVA_BYTE, LG_ARR_LONGS_BYTE) & 0XFF; + final int preambleLongs = wseg_.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; final long[] cacheArr = new long[1 << lgArrLongs]; - final WritableMemory mem = WritableMemory.writableWrap(cacheArr); - wmem_.copyTo(preambleLongs << 3, mem, 0, 8 << lgArrLongs); + MemorySegment.copy(wseg_, JAVA_LONG_UNALIGNED, preambleLongs << 3, cacheArr, 0, 1 << lgArrLongs); return cacheArr; } @@ -231,17 +233,17 @@ int getCompactPreambleLongs() { @Override int getCurrentPreambleLongs() { - return PreambleUtil.extractPreLongs(wmem_); + return PreambleUtil.extractPreLongs(wseg_); } @Override - WritableMemory getMemory() { - return wmem_; + MemorySegment getMemorySegment() { + return wseg_; } @Override short getSeedHash() { - return (short) PreambleUtil.extractSeedHash(wmem_); + return (short) PreambleUtil.extractSeedHash(wseg_); } @Override @@ -256,11 +258,11 @@ boolean isOutOfSpace(final int numEntries) { @Override int getLgArrLongs() { - return wmem_.getByte(LG_ARR_LONGS_BYTE) & 0XFF; + return wseg_.get(JAVA_BYTE, LG_ARR_LONGS_BYTE) & 0XFF; } int getLgRF() { //only Direct needs this - return (wmem_.getByte(PREAMBLE_LONGS_BYTE) >>> LG_RESIZE_FACTOR_BIT) & 0X3; + return (wseg_.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) >>> LG_RESIZE_FACTOR_BIT) & 0X3; } @Override diff --git a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java index 8f6e4972a..45a17d40d 100644 --- a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java @@ -19,9 +19,12 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; + +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; /** * Singleton empty CompactSketch. @@ -46,14 +49,14 @@ static synchronized EmptyCompactSketch getInstance() { } //This should be a heapify - static synchronized EmptyCompactSketch getHeapInstance(final Memory srcMem) { - final long pre0 = srcMem.getLong(0); + static synchronized EmptyCompactSketch getHeapInstance(final MemorySegment srcSeg) { + final long pre0 = srcSeg.get(JAVA_LONG_UNALIGNED, 0); if (testCandidatePre0(pre0)) { return EMPTY_COMPACT_SKETCH; } final long maskedPre0 = pre0 & EMPTY_SKETCH_MASK; - throw new SketchesArgumentException("Input Memory does not match required Preamble. " - + "Memory Pre0: " + Long.toHexString(maskedPre0) + throw new SketchesArgumentException("Input MemorySegment does not match required Preamble. " + + "MemorySegment Pre0: " + Long.toHexString(maskedPre0) + ", required Pre0: " + Long.toHexString(EMPTY_SKETCH_TEST)); } @@ -61,10 +64,11 @@ static synchronized EmptyCompactSketch getHeapInstance(final Memory srcMem) { // This returns with ordered flag = true independent of dstOrdered. // This is required for fast detection. // The hashSeed is ignored and set == 0. - public CompactSketch compact(final boolean dstOrdered, final WritableMemory wmem) { - if (wmem == null) { return EmptyCompactSketch.getInstance(); } - wmem.putByteArray(0, EMPTY_COMPACT_SKETCH_ARR, 0, 8); - return new DirectCompactSketch(wmem); + public CompactSketch compact(final boolean dstOrdered, final MemorySegment dstWSeg) { + if (dstWSeg == null) { return EmptyCompactSketch.getInstance(); } + //dstWSeg.putByteArray(0, EMPTY_COMPACT_SKETCH_ARR, 0, 8); + MemorySegment.copy(EMPTY_COMPACT_SKETCH_ARR, 0, dstWSeg, JAVA_BYTE, 0, 8); + return new DirectCompactSketch(dstWSeg); } //static @@ -131,11 +135,6 @@ int getCurrentPreambleLongs() { return 1; } - @Override - Memory getMemory() { - return null; - } - @Override short getSeedHash() { return 0; diff --git a/src/main/java/org/apache/datasketches/theta/ForwardCompatibility.java b/src/main/java/org/apache/datasketches/theta/ForwardCompatibility.java index 16172d247..723a8b651 100644 --- a/src/main/java/org/apache/datasketches/theta/ForwardCompatibility.java +++ b/src/main/java/org/apache/datasketches/theta/ForwardCompatibility.java @@ -19,13 +19,15 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.theta.PreambleUtil.extractCurCount; import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID; import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs; import static org.apache.datasketches.theta.PreambleUtil.extractThetaLong; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; /** * Used to convert older serialization versions 1 and 2 to version 3. The Serialization @@ -36,13 +38,15 @@ */ final class ForwardCompatibility { + private ForwardCompatibility() { } + /** * Convert a serialization version (SerVer) 1 sketch (~Feb 2014) to a SerVer 3 sketch. * Note: SerVer 1 sketches always have (metadata) preamble-longs of 3 and are always stored * in a compact ordered form, but with 3 different sketch types. All SerVer 1 sketches will * be converted to a SerVer 3 sketches. There is no concept of p-sampling, no empty bit. * - * @param srcMem the image of a SerVer 1 sketch + * @param srcSeg the image of a SerVer 1 sketch * * @param seedHash See Seed Hash. * The seedHash that matches the seedHash of the original seed used to construct the sketch. @@ -50,35 +54,35 @@ final class ForwardCompatibility { * MUST be derived from the actual seed that was used when the SerVer 1 sketches were built. * @return a SerVer 3 {@link CompactSketch}. */ - static final CompactSketch heapify1to3(final Memory srcMem, final short seedHash) { - final int memCap = (int) srcMem.getCapacity(); - final int preLongs = extractPreLongs(srcMem); //always 3 for serVer 1 + static final CompactSketch heapify1to3(final MemorySegment srcSeg, final short seedHash) { + final int segCap = (int) srcSeg.byteSize(); + final int preLongs = extractPreLongs(srcSeg); //always 3 for serVer 1 if (preLongs != 3) { throw new SketchesArgumentException("PreLongs must be 3 for SerVer 1: " + preLongs); } - final int familyId = extractFamilyID(srcMem); //1,2,3 + final int familyId = extractFamilyID(srcSeg); //1,2,3 if ((familyId < 1) || (familyId > 3)) { throw new SketchesArgumentException("Family ID (Sketch Type) must be 1 to 3: " + familyId); } - final int curCount = extractCurCount(srcMem); - final long thetaLong = extractThetaLong(srcMem); + final int curCount = extractCurCount(srcSeg); + final long thetaLong = extractThetaLong(srcSeg); final boolean empty = (curCount == 0) && (thetaLong == Long.MAX_VALUE); - if (empty || (memCap <= 24)) { //return empty + if (empty || (segCap <= 24)) { //return empty return EmptyCompactSketch.getInstance(); } final int reqCap = (curCount + preLongs) << 3; - validateInputSize(reqCap, memCap); + validateInputSize(reqCap, segCap); if ((thetaLong == Long.MAX_VALUE) && (curCount == 1)) { - final long hash = srcMem.getLong(preLongs << 3); + final long hash = srcSeg.get(JAVA_LONG_UNALIGNED, preLongs << 3); return new SingleItemSketch(hash, seedHash); } //theta < 1.0 and/or curCount > 1 final long[] compactOrderedCache = new long[curCount]; - srcMem.getLongArray(preLongs << 3, compactOrderedCache, 0, curCount); + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preLongs << 3, compactOrderedCache, 0, curCount); return new HeapCompactSketch(compactOrderedCache, false, seedHash, curCount, thetaLong, true); } @@ -86,15 +90,15 @@ static final CompactSketch heapify1to3(final Memory srcMem, final short seedHash * Convert a serialization version (SerVer) 2 sketch to a SerVer 3 HeapCompactOrderedSketch. * Note: SerVer 2 sketches can have metadata-longs of 1,2 or 3 and are always stored * in a compact ordered form (not as a hash table), but with 4 different sketch types. - * @param srcMem the image of a SerVer 2 sketch + * @param srcSeg the image of a SerVer 2 sketch * @param seedHash See Seed Hash. * The seed used for building the sketch image in srcMem * @return a SerVer 3 HeapCompactOrderedSketch */ - static final CompactSketch heapify2to3(final Memory srcMem, final short seedHash) { - final int memCap = (int) srcMem.getCapacity(); - final int preLongs = extractPreLongs(srcMem); //1,2 or 3 - final int familyId = extractFamilyID(srcMem); //1,2,3,4 + static final CompactSketch heapify2to3(final MemorySegment srcSeg, final short seedHash) { + final int segCap = (int) srcSeg.byteSize(); + final int preLongs = extractPreLongs(srcSeg); //1,2 or 3 + final int familyId = extractFamilyID(srcSeg); //1,2,3,4 if ((familyId < 1) || (familyId > 4)) { throw new SketchesArgumentException("Family (Sketch Type) must be 1 to 4: " + familyId); } @@ -103,58 +107,59 @@ static final CompactSketch heapify2to3(final Memory srcMem, final short seedHash long thetaLong = Long.MAX_VALUE; if (preLongs == 1) { reqBytesIn = 8; - validateInputSize(reqBytesIn, memCap); + validateInputSize(reqBytesIn, segCap); return EmptyCompactSketch.getInstance(); } if (preLongs == 2) { //includes pre0 + count, no theta (== 1.0) reqBytesIn = preLongs << 3; - validateInputSize(reqBytesIn, memCap); - curCount = extractCurCount(srcMem); + validateInputSize(reqBytesIn, segCap); + curCount = extractCurCount(srcSeg); if (curCount == 0) { return EmptyCompactSketch.getInstance(); } if (curCount == 1) { reqBytesIn = (preLongs + 1) << 3; - validateInputSize(reqBytesIn, memCap); - final long hash = srcMem.getLong(preLongs << 3); + validateInputSize(reqBytesIn, segCap); + final long hash = srcSeg.get(JAVA_LONG_UNALIGNED, preLongs << 3); return new SingleItemSketch(hash, seedHash); } //curCount > 1 reqBytesIn = (curCount + preLongs) << 3; - validateInputSize(reqBytesIn, memCap); + validateInputSize(reqBytesIn, segCap); final long[] compactOrderedCache = new long[curCount]; - srcMem.getLongArray(preLongs << 3, compactOrderedCache, 0, curCount); + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preLongs << 3, compactOrderedCache, 0, curCount); return new HeapCompactSketch(compactOrderedCache, false, seedHash, curCount, thetaLong,true); } if (preLongs == 3) { //pre0 + count + theta reqBytesIn = (preLongs) << 3; // - validateInputSize(reqBytesIn, memCap); - curCount = extractCurCount(srcMem); - thetaLong = extractThetaLong(srcMem); + validateInputSize(reqBytesIn, segCap); + curCount = extractCurCount(srcSeg); + thetaLong = extractThetaLong(srcSeg); if ((curCount == 0) && (thetaLong == Long.MAX_VALUE)) { return EmptyCompactSketch.getInstance(); } if ((curCount == 1) && (thetaLong == Long.MAX_VALUE)) { reqBytesIn = (preLongs + 1) << 3; - validateInputSize(reqBytesIn, memCap); - final long hash = srcMem.getLong(preLongs << 3); + validateInputSize(reqBytesIn, segCap); + final long hash = srcSeg.get(JAVA_LONG_UNALIGNED, preLongs << 3); return new SingleItemSketch(hash, seedHash); } //curCount > 1 and/or theta < 1.0 reqBytesIn = (curCount + preLongs) << 3; - validateInputSize(reqBytesIn, memCap); + validateInputSize(reqBytesIn, segCap); final long[] compactOrderedCache = new long[curCount]; - srcMem.getLongArray(preLongs << 3, compactOrderedCache, 0, curCount); + //srcSeg.getLongArray(preLongs << 3, compactOrderedCache, 0, curCount); + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preLongs << 3, compactOrderedCache, 0, curCount); return new HeapCompactSketch(compactOrderedCache, false, seedHash, curCount, thetaLong, true); } throw new SketchesArgumentException("PreLongs must be 1,2, or 3: " + preLongs); } - private static final void validateInputSize(final int reqBytesIn, final int memCap) { - if (reqBytesIn > memCap) { + private static final void validateInputSize(final int reqBytesIn, final int segCap) { + if (reqBytesIn > segCap) { throw new SketchesArgumentException( - "Input Memory or byte[] size is too small: Required Bytes: " + reqBytesIn - + ", bytesIn: " + memCap); + "Input MemorySegment or byte[] size is too small: Required Bytes: " + reqBytesIn + + ", bytesIn: " + segCap); } } diff --git a/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java b/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java index 3ee7c4880..b4b7848c5 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java @@ -22,6 +22,7 @@ import static java.lang.Math.max; import static java.lang.Math.min; import static java.lang.Math.sqrt; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.common.Util.LONG_MAX_VALUE_AS_DOUBLE; import static org.apache.datasketches.common.Util.checkBounds; import static org.apache.datasketches.theta.PreambleUtil.extractCurCount; @@ -38,13 +39,12 @@ import static org.apache.datasketches.theta.UpdateReturnState.RejectedOverTheta; import static org.apache.datasketches.thetacommon.HashOperations.STRIDE_MASK; +import java.lang.foreign.MemorySegment; import java.util.Objects; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.HashOperations; import org.apache.datasketches.thetacommon.ThetaUtil; @@ -113,44 +113,43 @@ static HeapAlphaSketch newHeapInstance(final int lgNomLongs, final long seed, fi } /** - * Heapify a sketch from a Memory object containing sketch data. - * @param srcMem The source Memory object. - * See Memory + * Heapify a sketch from a MemorySegment object containing sketch data. + * @param srcSeg The source MemorySegment object. * It must have a size of at least 24 bytes. - * @param expectedSeed the seed used to validate the given Memory image. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See seed * @return instance of this sketch */ - static HeapAlphaSketch heapifyInstance(final Memory srcMem, final long expectedSeed) { - Objects.requireNonNull(srcMem, "Source Memory must not be null"); - checkBounds(0, 24, srcMem.getCapacity()); - final int preambleLongs = extractPreLongs(srcMem); //byte 0 - final int lgNomLongs = extractLgNomLongs(srcMem); //byte 3 - final int lgArrLongs = extractLgArrLongs(srcMem); //byte 4 + static HeapAlphaSketch heapifyInstance(final MemorySegment srcSeg, final long expectedSeed) { + Objects.requireNonNull(srcSeg, "Source MemorySegment must not be null"); + checkBounds(0, 24, srcSeg.byteSize()); + final int preambleLongs = extractPreLongs(srcSeg); //byte 0 + final int lgNomLongs = extractLgNomLongs(srcSeg); //byte 3 + final int lgArrLongs = extractLgArrLongs(srcSeg); //byte 4 - checkAlphaFamily(srcMem, preambleLongs, lgNomLongs); - checkMemIntegrity(srcMem, expectedSeed, preambleLongs, lgNomLongs, lgArrLongs); + checkAlphaFamily(srcSeg, preambleLongs, lgNomLongs); + checkSegIntegrity(srcSeg, expectedSeed, preambleLongs, lgNomLongs, lgArrLongs); - final float p = extractP(srcMem); //bytes 12-15 - final int memlgRF = extractLgResizeFactor(srcMem); //byte 0 - ResizeFactor memRF = ResizeFactor.getRF(memlgRF); + final float p = extractP(srcSeg); //bytes 12-15 + final int seglgRF = extractLgResizeFactor(srcSeg); //byte 0 + ResizeFactor segRF = ResizeFactor.getRF(seglgRF); final double nomLongs = (1L << lgNomLongs); final double alpha = nomLongs / (nomLongs + 1.0); final long split1 = (long) (((p * (alpha + 1.0)) / 2.0) * LONG_MAX_VALUE_AS_DOUBLE); - if (isResizeFactorIncorrect(srcMem, lgNomLongs, lgArrLongs)) { - memRF = ResizeFactor.X2; //X2 always works. + if (isResizeFactorIncorrect(srcSeg, lgNomLongs, lgArrLongs)) { + segRF = ResizeFactor.X2; //X2 always works. } - final HeapAlphaSketch has = new HeapAlphaSketch(lgNomLongs, expectedSeed, p, memRF, alpha, split1); + final HeapAlphaSketch has = new HeapAlphaSketch(lgNomLongs, expectedSeed, p, segRF, alpha, split1); has.lgArrLongs_ = lgArrLongs; has.hashTableThreshold_ = setHashTableThreshold(lgNomLongs, lgArrLongs); - has.curCount_ = extractCurCount(srcMem); - has.thetaLong_ = extractThetaLong(srcMem); - has.empty_ = PreambleUtil.isEmptyFlag(srcMem); + has.curCount_ = extractCurCount(srcSeg); + has.thetaLong_ = extractThetaLong(srcSeg); + has.empty_ = PreambleUtil.isEmptyFlag(srcSeg); has.cache_ = new long[1 << lgArrLongs]; - srcMem.getLongArray(preambleLongs << 3, has.cache_, 0, 1 << lgArrLongs); //read in as hash table + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preambleLongs << 3, has.cache_, 0, 1 << lgArrLongs); //read in as hash table return has; } @@ -294,11 +293,6 @@ int getCurrentPreambleLongs() { return Family.ALPHA.getMinPreLongs(); } - @Override - WritableMemory getMemory() { - return null; - } - @Override long[] getCache() { return cache_; @@ -576,9 +570,9 @@ private static final int setHashTableThreshold(final int lgNomLongs, final int l return (int) Math.floor(fraction * (1 << lgArrLongs)); } - static void checkAlphaFamily(final Memory mem, final int preambleLongs, final int lgNomLongs) { + static void checkAlphaFamily(final MemorySegment seg, final int preambleLongs, final int lgNomLongs) { //Check Family - final int familyID = extractFamilyID(mem); //byte 2 + final int familyID = extractFamilyID(seg); //byte 2 final Family family = Family.idToFamily(familyID); if (family.equals(Family.ALPHA)) { if (preambleLongs != Family.ALPHA.getMinPreLongs()) { diff --git a/src/main/java/org/apache/datasketches/theta/HeapCompactHashIterator.java b/src/main/java/org/apache/datasketches/theta/HeapCompactHashIterator.java index d5a6289c7..e8e5d8305 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapCompactHashIterator.java +++ b/src/main/java/org/apache/datasketches/theta/HeapCompactHashIterator.java @@ -19,7 +19,7 @@ package org.apache.datasketches.theta; -class HeapCompactHashIterator implements HashIterator { +final class HeapCompactHashIterator implements HashIterator { private long[] cache; private int index; diff --git a/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java b/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java index 2572ce5d5..6b5708901 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapCompactSketch.java @@ -24,22 +24,21 @@ import static org.apache.datasketches.theta.CompactOperations.computeCompactPreLongs; import static org.apache.datasketches.theta.CompactOperations.correctThetaOnCompact; import static org.apache.datasketches.theta.CompactOperations.isSingleItem; -import static org.apache.datasketches.theta.CompactOperations.loadCompactMemory; +import static org.apache.datasketches.theta.CompactOperations.loadCompactMemorySegment; import static org.apache.datasketches.theta.PreambleUtil.COMPACT_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.READ_ONLY_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.SINGLEITEM_FLAG_MASK; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import java.lang.foreign.MemorySegment; /** * Parent class of the Heap Compact Sketches. * * @author Lee Rhodes */ -class HeapCompactSketch extends CompactSketch { +final class HeapCompactSketch extends CompactSketch { private final long thetaLong_; //computed private final int curCount_; private final int preLongs_; //computed @@ -76,10 +75,10 @@ class HeapCompactSketch extends CompactSketch { //Sketch @Override - public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) { - if (dstMem == null && (dstOrdered == false || this.ordered_ == dstOrdered)) { return this; } + public CompactSketch compact(final boolean dstOrdered, final MemorySegment dstSeg) { + if (dstSeg == null && (dstOrdered == false || this.ordered_ == dstOrdered)) { return this; } return componentsToCompact(getThetaLong(), getRetainedEntries(true), getSeedHash(), isEmpty(), - true, ordered_, dstOrdered, dstMem, getCache().clone()); + true, ordered_, dstOrdered, dstSeg, getCache().clone()); } @Override @@ -129,30 +128,25 @@ int getCompactPreambleLongs() { return preLongs_; } - @Override - Memory getMemory() { - return null; - } - @Override short getSeedHash() { return seedHash_; } - //use of Memory is convenient. The byteArray and Memory are loaded simultaneously. + //use of a MemorySegment is convenient. The byteArray and MemorySegment are loaded simultaneously. @Override public byte[] toByteArray() { final int bytes = getCurrentBytes(); final byte[] byteArray = new byte[bytes]; - final WritableMemory dstMem = WritableMemory.writableWrap(byteArray); + final MemorySegment dstSeg = MemorySegment.ofArray(byteArray); final int emptyBit = isEmpty() ? EMPTY_FLAG_MASK : 0; final int orderedBit = ordered_ ? ORDERED_FLAG_MASK : 0; final int singleItemBit = singleItem_ ? SINGLEITEM_FLAG_MASK : 0; final byte flags = (byte) (emptyBit | READ_ONLY_FLAG_MASK | COMPACT_FLAG_MASK | orderedBit | singleItemBit); final int preLongs = getCompactPreambleLongs(); - loadCompactMemory(getCache(), getSeedHash(), getRetainedEntries(true), getThetaLong(), - dstMem, flags, preLongs); + loadCompactMemorySegment(getCache(), getSeedHash(), getRetainedEntries(true), getThetaLong(), + dstSeg, flags, preLongs); return byteArray; } diff --git a/src/main/java/org/apache/datasketches/theta/HeapHashIterator.java b/src/main/java/org/apache/datasketches/theta/HeapHashIterator.java index 9562da7d0..d689b912e 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapHashIterator.java +++ b/src/main/java/org/apache/datasketches/theta/HeapHashIterator.java @@ -22,7 +22,7 @@ /** * @author Lee Rhodes */ -class HeapHashIterator implements HashIterator { +final class HeapHashIterator implements HashIterator { private long[] cache; private long thetaLong; private int index; diff --git a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java index b9d4dc9e1..46e469004 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapQuickSelectSketch.java @@ -21,6 +21,8 @@ import static java.lang.Math.max; import static java.lang.Math.min; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static org.apache.datasketches.common.QuickSelect.selectExcludingZeros; import static org.apache.datasketches.common.Util.LONG_MAX_VALUE_AS_DOUBLE; import static org.apache.datasketches.theta.PreambleUtil.extractCurCount; import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID; @@ -35,12 +37,11 @@ import static org.apache.datasketches.theta.UpdateReturnState.InsertedCountIncrementedResized; import static org.apache.datasketches.theta.UpdateReturnState.RejectedDuplicate; import static org.apache.datasketches.theta.UpdateReturnState.RejectedOverTheta; -import static org.apache.datasketches.thetacommon.QuickSelect.selectExcludingZeros; + +import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.HashOperations; import org.apache.datasketches.thetacommon.ThetaUtil; @@ -100,40 +101,39 @@ private HeapQuickSelectSketch(final int lgNomLongs, final long seed, final float } /** - * Heapify a sketch from a Memory UpdateSketch or Union object + * Heapify a sketch from a MemorySegment UpdateSketch or Union object * containing sketch data. - * @param srcMem The source Memory object. - * See Memory + * @param srcSeg The source MemorySegment object. * @param seed See seed * @return instance of this sketch */ - static HeapQuickSelectSketch heapifyInstance(final Memory srcMem, final long seed) { - final int preambleLongs = extractPreLongs(srcMem); //byte 0 - final int lgNomLongs = extractLgNomLongs(srcMem); //byte 3 - final int lgArrLongs = extractLgArrLongs(srcMem); //byte 4 - - checkUnionQuickSelectFamily(srcMem, preambleLongs, lgNomLongs); - checkMemIntegrity(srcMem, seed, preambleLongs, lgNomLongs, lgArrLongs); - - final float p = extractP(srcMem); //bytes 12-15 - final int memlgRF = extractLgResizeFactor(srcMem); //byte 0 - ResizeFactor memRF = ResizeFactor.getRF(memlgRF); - final int familyID = extractFamilyID(srcMem); + static HeapQuickSelectSketch heapifyInstance(final MemorySegment srcSeg, final long seed) { + final int preambleLongs = extractPreLongs(srcSeg); //byte 0 + final int lgNomLongs = extractLgNomLongs(srcSeg); //byte 3 + final int lgArrLongs = extractLgArrLongs(srcSeg); //byte 4 + + checkUnionQuickSelectFamily(srcSeg, preambleLongs, lgNomLongs); + checkSegIntegrity(srcSeg, seed, preambleLongs, lgNomLongs, lgArrLongs); + + final float p = extractP(srcSeg); //bytes 12-15 + final int seglgRF = extractLgResizeFactor(srcSeg); //byte 0 + ResizeFactor segRF = ResizeFactor.getRF(seglgRF); + final int familyID = extractFamilyID(srcSeg); final Family family = Family.idToFamily(familyID); - if (isResizeFactorIncorrect(srcMem, lgNomLongs, lgArrLongs)) { - memRF = ResizeFactor.X2; //X2 always works. + if (isResizeFactorIncorrect(srcSeg, lgNomLongs, lgArrLongs)) { + segRF = ResizeFactor.X2; //X2 always works. } - final HeapQuickSelectSketch hqss = new HeapQuickSelectSketch(lgNomLongs, seed, p, memRF, + final HeapQuickSelectSketch hqss = new HeapQuickSelectSketch(lgNomLongs, seed, p, segRF, preambleLongs, family); hqss.lgArrLongs_ = lgArrLongs; hqss.hashTableThreshold_ = getHashTableThreshold(lgNomLongs, lgArrLongs); - hqss.curCount_ = extractCurCount(srcMem); - hqss.thetaLong_ = extractThetaLong(srcMem); - hqss.empty_ = PreambleUtil.isEmptyFlag(srcMem); + hqss.curCount_ = extractCurCount(srcSeg); + hqss.thetaLong_ = extractThetaLong(srcSeg); + hqss.empty_ = PreambleUtil.isEmptyFlag(srcSeg); hqss.cache_ = new long[1 << lgArrLongs]; - srcMem.getLongArray(preambleLongs << 3, hqss.cache_, 0, 1 << lgArrLongs); //read in as hash table + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preambleLongs << 3, hqss.cache_, 0, 1 << lgArrLongs); //read in as hash table return hqss; } @@ -230,11 +230,6 @@ int getLgArrLongs() { return lgArrLongs_; } - @Override - WritableMemory getMemory() { - return null; - } - @Override UpdateReturnState hashUpdate(final long hash) { HashOperations.checkHashCorruption(hash); diff --git a/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java b/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java index 49734a9e8..87e1892b8 100644 --- a/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java +++ b/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java @@ -19,6 +19,7 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.theta.CompactOperations.checkIllegalCurCountAndEmpty; import static org.apache.datasketches.theta.CompactOperations.correctThetaOnCompact; import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK; @@ -35,8 +36,10 @@ import static org.apache.datasketches.theta.PreambleUtil.insertSerVer; import static org.apache.datasketches.theta.PreambleUtil.insertThetaLong; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ResizeFactor; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -92,7 +95,7 @@ long getSeed() { @Override short getSeedHash() { - return ThetaUtil.computeSeedHash(getSeed()); + return Util.computeSeedHash(getSeed()); } //Used by HeapAlphaSketch and HeapQuickSelectSketch / Theta UpdateSketch @@ -102,33 +105,35 @@ byte[] toByteArray(final int preLongs, final byte familyID) { final int preBytes = (preLongs << 3) & 0X3F; //24 bytes final int dataBytes = getCurrentDataLongs() << 3; final byte[] byteArrOut = new byte[preBytes + dataBytes]; - final WritableMemory memOut = WritableMemory.writableWrap(byteArrOut); + + final MemorySegment segOut = MemorySegment.ofArray(byteArrOut); //preamble first 8 bytes. Note: only compact can be reduced to 8 bytes. final int lgRf = getResizeFactor().lg() & 0x3; - insertPreLongs(memOut, preLongs); //byte 0 low 6 bits - insertLgResizeFactor(memOut, lgRf); //byte 0 high 2 bits - insertSerVer(memOut, SER_VER); //byte 1 - insertFamilyID(memOut, familyID); //byte 2 - insertLgNomLongs(memOut, getLgNomLongs()); //byte 3 - insertLgArrLongs(memOut, getLgArrLongs()); //byte 4 - insertSeedHash(memOut, getSeedHash()); //bytes 6 & 7 - - insertCurCount(memOut, this.getRetainedEntries(true)); - insertP(memOut, getP()); + insertPreLongs(segOut, preLongs); //byte 0 low 6 bits + insertLgResizeFactor(segOut, lgRf); //byte 0 high 2 bits + insertSerVer(segOut, SER_VER); //byte 1 + insertFamilyID(segOut, familyID); //byte 2 + insertLgNomLongs(segOut, getLgNomLongs()); //byte 3 + insertLgArrLongs(segOut, getLgArrLongs()); //byte 4 + insertSeedHash(segOut, getSeedHash()); //bytes 6 & 7 + + insertCurCount(segOut, this.getRetainedEntries(true)); + insertP(segOut, getP()); final long thetaLong = correctThetaOnCompact(isEmpty(), getRetainedEntries(true), getThetaLong()); - insertThetaLong(memOut, thetaLong); + insertThetaLong(segOut, thetaLong); //Flags: BigEnd=0, ReadOnly=0, Empty=X, compact=0, ordered=0 final byte flags = isEmpty() ? (byte) EMPTY_FLAG_MASK : 0; - insertFlags(memOut, flags); + insertFlags(segOut, flags); //Data final int arrLongs = 1 << getLgArrLongs(); final long[] cache = getCache(); - memOut.putLongArray(preBytes, cache, 0, arrLongs); //load byteArrOut + //segOut.putLongArray(preBytes, cache, 0, arrLongs); //load byteArrOut + MemorySegment.copy(cache, 0, segOut, JAVA_LONG_UNALIGNED, preBytes, arrLongs); return byteArrOut; } diff --git a/src/main/java/org/apache/datasketches/theta/Intersection.java b/src/main/java/org/apache/datasketches/theta/Intersection.java index 111a9ce55..a31dc3ef9 100644 --- a/src/main/java/org/apache/datasketches/theta/Intersection.java +++ b/src/main/java/org/apache/datasketches/theta/Intersection.java @@ -19,21 +19,23 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static org.apache.datasketches.common.Util.floorPowerOf2; import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.SER_VER; +import static org.apache.datasketches.theta.PreambleUtil.SER_VER_BYTE; import static org.apache.datasketches.theta.PreambleUtil.extractCurCount; import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID; import static org.apache.datasketches.theta.PreambleUtil.extractFlags; import static org.apache.datasketches.theta.PreambleUtil.extractPreLongs; import static org.apache.datasketches.theta.PreambleUtil.extractSerVer; +import java.lang.foreign.MemorySegment; import java.util.Arrays; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -61,7 +63,7 @@ public CompactSketch getResult() { } /** - * Gets the result of this operation as a CompactSketch in the given dstMem. + * Gets the result of this operation as a CompactSketch in the given dstSeg. * This does not disturb the underlying data structure of this intersection. * The {@link #intersect(Sketch)} method must have been called at least once, otherwise an * exception will be thrown. This is because a virgin Intersection object represents the @@ -77,13 +79,12 @@ public CompactSketch getResult() { * @param dstOrdered * See Destination Ordered * - * @param dstMem - * See Destination Memory. + * @param dstSeg the destination MemorySegment. * - * @return the result of this operation as a CompactSketch stored in the given dstMem, + * @return the result of this operation as a CompactSketch stored in the given dstSeg, * which can be either on or off-heap.. */ - public abstract CompactSketch getResult(boolean dstOrdered, WritableMemory dstMem); + public abstract CompactSketch getResult(boolean dstOrdered, MemorySegment dstSeg); /** * Returns true if there is a valid intersection result available @@ -131,32 +132,57 @@ public CompactSketch intersect(final Sketch a, final Sketch b) { * @param b The second sketch argument * @param dstOrdered * See Destination Ordered. - * @param dstMem - * See Destination Memory. + * @param dstSeg the destination MemorySegment. * @return the result as a CompactSketch. */ public abstract CompactSketch intersect(Sketch a, Sketch b, boolean dstOrdered, - WritableMemory dstMem); + MemorySegment dstSeg); + + /** + * Factory: Wrap an Intersection target around the given source MemorySegment containing intersection data. + * This method assumes the Default Update Seed. + * If the given source MemorySegment is read-only, the returned object will also be read-only. + * @param srcSeg The source MemorySegment image. + * @return an Intersection that wraps a source MemorySegment that contains an Intersection image + */ + public static Intersection wrap(final MemorySegment srcSeg) { + return wrap(srcSeg, Util.DEFAULT_UPDATE_SEED); + } + + /** + * Factory: Wrap an Intersection target around the given source MemorySegment containing intersection data. + * If the given source MemorySegment is read-only, the returned object will also be read-only. + * @param srcSeg The source MemorySegment image. + * @param expectedSeed See seed + * @return an Intersection that wraps a source MemorySegment that contains an Intersection image + */ + public static Intersection wrap(final MemorySegment srcSeg, final long expectedSeed) { + final int serVer = srcSeg.get(JAVA_BYTE, SER_VER_BYTE); + if (serVer != 3) { + throw new SketchesArgumentException("SerVer must be 3: " + serVer); + } + return IntersectionImpl.wrapInstance(srcSeg, expectedSeed, srcSeg.isReadOnly() ); + } // Restricted /** - * Returns the maximum lgArrLongs given the capacity of the Memory. - * @param dstMem the given Memory - * @return the maximum lgArrLongs given the capacity of the Memory + * Returns the maximum lgArrLongs given the capacity of the MemorySegment. + * @param dstSeg the given MemorySegment + * @return the maximum lgArrLongs given the capacity of the MemorySegment */ - protected static int getMaxLgArrLongs(final Memory dstMem) { + protected static int getMaxLgArrLongs(final MemorySegment dstSeg) { final int preBytes = CONST_PREAMBLE_LONGS << 3; - final long cap = dstMem.getCapacity(); + final long cap = dstSeg.byteSize(); return Integer.numberOfTrailingZeros(floorPowerOf2((int)(cap - preBytes)) >>> 3); } - protected static void checkMinSizeMemory(final Memory mem) { + protected static void checkMinSizeMemorySegment(final MemorySegment seg) { final int minBytes = (CONST_PREAMBLE_LONGS << 3) + (8 << ThetaUtil.MIN_LG_ARR_LONGS);//280 - final long cap = mem.getCapacity(); + final long cap = seg.byteSize(); if (cap < minBytes) { throw new SketchesArgumentException( - "Memory must be at least " + minBytes + " bytes. Actual capacity: " + cap); + "MemorySegment must be at least " + minBytes + " bytes. Actual capacity: " + cap); } } @@ -191,19 +217,19 @@ static final long[] compactCachePart(final long[] srcCache, final int lgArrLongs return cacheOut; } - protected static void memChecks(final Memory srcMem) { + protected static void segChecks(final MemorySegment srcSeg) { //Get Preamble //Note: Intersection does not use lgNomLongs (or k), per se. //seedHash loaded and checked in private constructor - final int preLongs = extractPreLongs(srcMem); - final int serVer = extractSerVer(srcMem); - final int famID = extractFamilyID(srcMem); - final boolean empty = (extractFlags(srcMem) & EMPTY_FLAG_MASK) > 0; - final int curCount = extractCurCount(srcMem); + final int preLongs = extractPreLongs(srcSeg); + final int serVer = extractSerVer(srcSeg); + final int famID = extractFamilyID(srcSeg); + final boolean empty = (extractFlags(srcSeg) & EMPTY_FLAG_MASK) > 0; + final int curCount = extractCurCount(srcSeg); //Checks if (preLongs != CONST_PREAMBLE_LONGS) { throw new SketchesArgumentException( - "Memory PreambleLongs must equal " + CONST_PREAMBLE_LONGS + ": " + preLongs); + "MemorySegment PreambleLongs must equal " + CONST_PREAMBLE_LONGS + ": " + preLongs); } if (serVer != SER_VER) { throw new SketchesArgumentException("Serialization Version must equal " + SER_VER); @@ -212,7 +238,7 @@ protected static void memChecks(final Memory srcMem) { if (empty) { if (curCount != 0) { throw new SketchesArgumentException( - "srcMem empty state inconsistent with curCount: " + empty + "," + curCount); + "srcSeg empty state inconsistent with curCount: " + empty + "," + curCount); } //empty = true AND curCount_ = 0: OK } //else empty = false, curCount could be anything diff --git a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java index 772480fea..6bf922be0 100644 --- a/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java +++ b/src/main/java/org/apache/datasketches/theta/IntersectionImpl.java @@ -20,6 +20,13 @@ package org.apache.datasketches.theta; import static java.lang.Math.min; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; +import static org.apache.datasketches.common.Util.clearBits; +import static org.apache.datasketches.common.Util.setBits; import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.FAMILY_BYTE; import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE; @@ -47,36 +54,36 @@ import static org.apache.datasketches.theta.PreambleUtil.setEmpty; import static org.apache.datasketches.thetacommon.HashOperations.continueCondition; import static org.apache.datasketches.thetacommon.HashOperations.hashInsertOnly; -import static org.apache.datasketches.thetacommon.HashOperations.hashInsertOnlyMemory; +import static org.apache.datasketches.thetacommon.HashOperations.hashInsertOnlyMemorySegment; import static org.apache.datasketches.thetacommon.HashOperations.hashSearch; import static org.apache.datasketches.thetacommon.HashOperations.minLgHashTableSize; +import java.lang.foreign.MemorySegment; import java.util.Arrays; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesReadOnlyException; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.ThetaUtil; /** * Intersection operation for Theta Sketches. * - *

This implementation uses data either on-heap or off-heap in a given Memory + *

This implementation uses data either on-heap or off-heap in a given MemorySegment * that is owned and managed by the caller. - * The off-heap Memory, which if managed properly, will greatly reduce the need for + * The off-heap MemorySegment, which if managed properly, will greatly reduce the need for * the JVM to perform garbage collection.

* * @author Lee Rhodes * @author Kevin Lang */ -class IntersectionImpl extends Intersection { +final class IntersectionImpl extends Intersection { protected final short seedHash_; protected final boolean readOnly_; //True if this sketch is to be treated as read only - protected final WritableMemory wmem_; - protected final int maxLgArrLongs_; //only used with WritableMemory, not serialized + protected final MemorySegment wseg_; + protected final int maxLgArrLongs_; //only used with MemorySegment, not serialized //Note: Intersection does not use lgNomLongs or k, per se. protected int lgArrLongs_; //current size of hash table @@ -87,30 +94,30 @@ class IntersectionImpl extends Intersection { /** * Constructor: Sets the class finals and computes, sets and checks the seedHash. - * @param wmem Can be either a Source(e.g. wrap) or Destination (new Direct) WritableMemory. + * @param wseg Can be either a Source(e.g. wrap) or Destination (new offHeap) MemorySegment. * @param seed Used to validate incoming sketch arguments. - * @param dstMemFlag The given memory is a Destination (new Direct) WritableMemory. - * @param readOnly True if memory is to be treated as read only. + * @param dstSegFlag The given MemorySegment is a Destination (new offHeap) MemorySegment. + * @param readOnly True if MemorySegment is to be treated as read only. */ - protected IntersectionImpl(final WritableMemory wmem, final long seed, final boolean dstMemFlag, + protected IntersectionImpl(final MemorySegment wseg, final long seed, final boolean dstSegFlag, final boolean readOnly) { readOnly_ = readOnly; - if (wmem != null) { - wmem_ = wmem; - if (dstMemFlag) { //DstMem: compute & store seedHash, no seedhash checking - checkMinSizeMemory(wmem); - maxLgArrLongs_ = !readOnly ? getMaxLgArrLongs(wmem) : 0; //Only Off Heap - seedHash_ = ThetaUtil.computeSeedHash(seed); - wmem_.putShort(SEED_HASH_SHORT, seedHash_); - } else { //SrcMem:gets and stores the seedHash, checks mem_seedHash against the seed - seedHash_ = wmem_.getShort(SEED_HASH_SHORT); - ThetaUtil.checkSeedHashes(seedHash_, ThetaUtil.computeSeedHash(seed)); //check for seed hash conflict + if (wseg != null) { + wseg_ = wseg; + if (dstSegFlag) { //DstSeg: compute & store seedHash, no seedHash checking + checkMinSizeMemorySegment(wseg); + maxLgArrLongs_ = !readOnly ? getMaxLgArrLongs(wseg) : 0; //Only Off Heap + seedHash_ = Util.computeSeedHash(seed); + wseg_.set(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT, seedHash_); + } else { //SrcSeg:gets and stores the seedHash, checks seg_seedHash against the seed + seedHash_ = wseg_.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT); + Util.checkSeedHashes(seedHash_, Util.computeSeedHash(seed)); //check for seed hash conflict maxLgArrLongs_ = 0; } } else { //compute & store seedHash - wmem_ = null; + wseg_ = null; maxLgArrLongs_ = 0; - seedHash_ = ThetaUtil.computeSeedHash(seed); + seedHash_ = Util.computeSeedHash(seed); } } @@ -122,105 +129,101 @@ protected IntersectionImpl(final WritableMemory wmem, final long seed, final boo * @return a new IntersectionImpl on the Java heap */ static IntersectionImpl initNewHeapInstance(final long seed) { - final boolean dstMemFlag = false; + final boolean dstSegFlag = false; final boolean readOnly = false; - final IntersectionImpl impl = new IntersectionImpl(null, seed, dstMemFlag, readOnly); + final IntersectionImpl impl = new IntersectionImpl(null, seed, dstSegFlag, readOnly); impl.hardReset(); return impl; } /** - * Factory: Construct a new Intersection target direct to the given destination Memory. + * Factory: Construct a new Intersection target direct to the given destination MemorySegment. * Called by SetOperationBuilder, test. * * @param seed See Seed - * @param dstMem destination Memory - * See Memory + * @param dstSeg destination MemorySegment * @return a new IntersectionImpl that may be off-heap */ - static IntersectionImpl initNewDirectInstance(final long seed, final WritableMemory dstMem) { + static IntersectionImpl initNewDirectInstance(final long seed, final MemorySegment dstSeg) { //Load Preamble //Pre0 - dstMem.clear(0, CONST_PREAMBLE_LONGS << 3); - insertPreLongs(dstMem, CONST_PREAMBLE_LONGS); //RF not used = 0 - insertSerVer(dstMem, SER_VER); - insertFamilyID(dstMem, Family.INTERSECTION.getID()); + dstSeg.asSlice(0, CONST_PREAMBLE_LONGS << 3).fill((byte)0); + insertPreLongs(dstSeg, CONST_PREAMBLE_LONGS); //RF not used = 0 + insertSerVer(dstSeg, SER_VER); + insertFamilyID(dstSeg, Family.INTERSECTION.getID()); //lgNomLongs not used by Intersection //lgArrLongs set by hardReset //flags are already 0: bigEndian = readOnly = compact = ordered = empty = false; //seedHash loaded and checked in IntersectionImpl constructor //Pre1 //CurCount set by hardReset - insertP(dstMem, (float) 1.0); //not used by intersection + insertP(dstSeg, (float) 1.0); //not used by intersection //Pre2 //thetaLong set by hardReset //Initialize - final boolean dstMemFlag = true; + final boolean dstSegFlag = true; final boolean readOnly = false; - final IntersectionImpl impl = new IntersectionImpl(dstMem, seed, dstMemFlag, readOnly); + final IntersectionImpl impl = new IntersectionImpl(dstSeg, seed, dstSegFlag, readOnly); impl.hardReset(); return impl; } /** - * Factory: Heapify an intersection target from a Memory image containing data. - * @param srcMem The source Memory object. - * See Memory + * Factory: Heapify an intersection target from a MemorySegment image containing data. + * @param srcSeg The source MemorySegment object. * @param seed See seed * @return a IntersectionImpl instance on the Java heap */ - static IntersectionImpl heapifyInstance(final Memory srcMem, final long seed) { - final boolean dstMemFlag = false; + static IntersectionImpl heapifyInstance(final MemorySegment srcSeg, final long seed) { + final boolean dstSegFlag = false; final boolean readOnly = false; - final IntersectionImpl impl = new IntersectionImpl(null, seed, dstMemFlag, readOnly); - memChecks(srcMem); + final IntersectionImpl impl = new IntersectionImpl(null, seed, dstSegFlag, readOnly); + segChecks(srcSeg); //Initialize - impl.lgArrLongs_ = extractLgArrLongs(srcMem); - impl.curCount_ = extractCurCount(srcMem); - impl.thetaLong_ = extractThetaLong(srcMem); - impl.empty_ = (extractFlags(srcMem) & EMPTY_FLAG_MASK) > 0; + impl.lgArrLongs_ = extractLgArrLongs(srcSeg); + impl.curCount_ = extractCurCount(srcSeg); + impl.thetaLong_ = extractThetaLong(srcSeg); + impl.empty_ = (extractFlags(srcSeg) & EMPTY_FLAG_MASK) > 0; if (!impl.empty_) { if (impl.curCount_ > 0) { impl.hashTable_ = new long[1 << impl.lgArrLongs_]; - srcMem.getLongArray(CONST_PREAMBLE_LONGS << 3, impl.hashTable_, 0, 1 << impl.lgArrLongs_); + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, CONST_PREAMBLE_LONGS << 3, impl.hashTable_, 0, 1 << impl.lgArrLongs_); } } return impl; } /** - * Factory: Wrap an Intersection target around the given source WritableMemory containing - * intersection data. - * @param srcMem The source WritableMemory image. - * See Memory + * Factory: Wrap an Intersection target around the given source MemorySegment containing intersection data. + * If the given source MemorySegment is read-only, the returned object will also be read-only. + * @param srcSeg The source MemorySegment image. * @param seed See seed - * @param readOnly True if memory is to be treated as read only - * @return a IntersectionImpl that wraps a source WritableMemory that contains an Intersection image + * @param readOnly True if MemorySegment is to be treated as read only + * @return a IntersectionImpl that wraps a source MemorySegment that contains an Intersection image */ static IntersectionImpl wrapInstance( - final WritableMemory srcMem, + final MemorySegment srcSeg, final long seed, final boolean readOnly) { - final boolean dstMemFlag = false; - final IntersectionImpl impl = new IntersectionImpl(srcMem, seed, dstMemFlag, readOnly); - memChecks(srcMem); - impl.lgArrLongs_ = extractLgArrLongs(srcMem); - impl.curCount_ = extractCurCount(srcMem); - impl.thetaLong_ = extractThetaLong(srcMem); - impl.empty_ = (extractFlags(srcMem) & EMPTY_FLAG_MASK) > 0; + final boolean dstSegFlag = false; + final IntersectionImpl impl = new IntersectionImpl(srcSeg, seed, dstSegFlag, readOnly); + segChecks(srcSeg); + impl.lgArrLongs_ = extractLgArrLongs(srcSeg); + impl.curCount_ = extractCurCount(srcSeg); + impl.thetaLong_ = extractThetaLong(srcSeg); + impl.empty_ = (extractFlags(srcSeg) & EMPTY_FLAG_MASK) > 0; return impl; } @Override - public CompactSketch intersect(final Sketch a, final Sketch b, final boolean dstOrdered, - final WritableMemory dstMem) { - if (wmem_ != null && readOnly_) { throw new SketchesReadOnlyException(); } + public CompactSketch intersect(final Sketch a, final Sketch b, final boolean dstOrdered, final MemorySegment dstSeg) { + if (wseg_ != null && readOnly_) { throw new SketchesReadOnlyException(); } hardReset(); intersect(a); intersect(b); - final CompactSketch csk = getResult(dstOrdered, dstMem); + final CompactSketch csk = getResult(dstOrdered, dstSeg); hardReset(); return csk; } @@ -230,20 +233,20 @@ public void intersect(final Sketch sketchIn) { if (sketchIn == null) { throw new SketchesArgumentException("Intersection argument must not be null."); } - if (wmem_ != null && readOnly_) { throw new SketchesReadOnlyException(); } + if (wseg_ != null && readOnly_) { throw new SketchesReadOnlyException(); } if (empty_ || sketchIn.isEmpty()) { //empty rule //Because of the def of null above and the Empty Rule (which is OR), empty_ must be true. //Whatever the current internal state, we make our local empty. resetToEmpty(); return; } - ThetaUtil.checkSeedHashes(seedHash_, sketchIn.getSeedHash()); + Util.checkSeedHashes(seedHash_, sketchIn.getSeedHash()); //Set minTheta thetaLong_ = min(thetaLong_, sketchIn.getThetaLong()); //Theta rule empty_ = false; - if (wmem_ != null) { - insertThetaLong(wmem_, thetaLong_); - clearEmpty(wmem_); //false + if (wseg_ != null) { + insertThetaLong(wseg_, thetaLong_); + clearEmpty(wseg_); //false } // The truth table for the following state machine. MinTheta is set above. @@ -261,8 +264,8 @@ public void intersect(final Sketch sketchIn) { //states 1,2,3,6 if (curCount_ == 0 || sketchInEntries == 0) { curCount_ = 0; - if (wmem_ != null) { insertCurCount(wmem_, 0); } - hashTable_ = null; //No need for a HT. Don't bother clearing mem if valid + if (wseg_ != null) { insertCurCount(wseg_, 0); } + hashTable_ = null; //No need for a HT. Don't bother clearing seg if valid } //end of states 1,2,3,6 // state 5 @@ -272,17 +275,17 @@ else if (curCount_ < 0 && sketchInEntries > 0) { final int priorLgArrLongs = lgArrLongs_; //prior only used in error message lgArrLongs_ = requiredLgArrLongs; - if (wmem_ != null) { //Off heap, check if current dstMem is large enough - insertCurCount(wmem_, curCount_); - insertLgArrLongs(wmem_, lgArrLongs_); + if (wseg_ != null) { //Off heap, check if current dstSeg is large enough + insertCurCount(wseg_, curCount_); + insertLgArrLongs(wseg_, lgArrLongs_); if (requiredLgArrLongs <= maxLgArrLongs_) { - wmem_.clear(CONST_PREAMBLE_LONGS << 3, 8 << lgArrLongs_); //clear only what required + wseg_.asSlice(CONST_PREAMBLE_LONGS << 3, 8 << lgArrLongs_).fill((byte)0); } - else { //not enough space in dstMem + else { //not enough space in dstSeg final int requiredBytes = (8 << requiredLgArrLongs) + 24; final int givenBytes = (8 << priorLgArrLongs) + 24; throw new SketchesArgumentException( - "Insufficient internal Memory space: " + requiredBytes + " > " + givenBytes); + "Insufficient internal MemorySegment space: " + requiredBytes + " > " + givenBytes); } } else { //On the heap, allocate a HT @@ -303,7 +306,10 @@ else if (curCount_ > 0 && sketchInEntries > 0) { } @Override - public CompactSketch getResult(final boolean dstOrdered, final WritableMemory dstMem) { + MemorySegment getMemorySegment() { return wseg_; } + + @Override + public CompactSketch getResult(final boolean dstOrdered, final MemorySegment dstSeg) { if (curCount_ < 0) { throw new SketchesStateException( "Calling getResult() with no intervening intersections would represent the infinite set, " @@ -314,17 +320,17 @@ public CompactSketch getResult(final boolean dstOrdered, final WritableMemory ds if (curCount_ == 0) { compactCache = new long[0]; srcCompact = true; - srcOrdered = false; //hashTable, even tho empty + srcOrdered = false; //hashTable, even though empty return CompactOperations.componentsToCompact( thetaLong_, curCount_, seedHash_, empty_, srcCompact, srcOrdered, dstOrdered, - dstMem, compactCache); + dstSeg, compactCache); } //else curCount > 0 final long[] hashTable; - if (wmem_ != null) { + if (wseg_ != null) { final int htLen = 1 << lgArrLongs_; hashTable = new long[htLen]; - wmem_.getLongArray(CONST_PREAMBLE_LONGS << 3, hashTable, 0, htLen); + MemorySegment.copy(wseg_, JAVA_LONG_UNALIGNED, CONST_PREAMBLE_LONGS << 3, hashTable, 0, htLen); } else { hashTable = hashTable_; } @@ -333,27 +339,27 @@ public CompactSketch getResult(final boolean dstOrdered, final WritableMemory ds srcOrdered = dstOrdered; return CompactOperations.componentsToCompact( thetaLong_, curCount_, seedHash_, empty_, srcCompact, srcOrdered, dstOrdered, - dstMem, compactCache); + dstSeg, compactCache); } @Override - public boolean hasMemory() { - return wmem_ != null; + public boolean hasMemorySegment() { + return wseg_ != null && wseg_.scope().isAlive(); } @Override public boolean hasResult() { - return hasMemory() ? wmem_.getInt(RETAINED_ENTRIES_INT) >= 0 : curCount_ >= 0; + return hasMemorySegment() ? wseg_.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT) >= 0 : curCount_ >= 0; } @Override public boolean isDirect() { - return hasMemory() ? wmem_.isDirect() : false; + return hasMemorySegment() && wseg_.isNative(); } @Override - public boolean isSameResource(final Memory that) { - return hasMemory() ? wmem_.isSameResource(that) : false; + public boolean isSameResource(final MemorySegment that) { + return hasMemorySegment() && Util.isSameResource(wseg_, that); } @Override @@ -366,28 +372,28 @@ public byte[] toByteArray() { final int preBytes = CONST_PREAMBLE_LONGS << 3; final int dataBytes = curCount_ > 0 ? 8 << lgArrLongs_ : 0; final byte[] byteArrOut = new byte[preBytes + dataBytes]; - if (wmem_ != null) { - wmem_.getByteArray(0, byteArrOut, 0, preBytes + dataBytes); + if (wseg_ != null) { + MemorySegment.copy(wseg_, JAVA_BYTE, 0, byteArrOut, 0, preBytes + dataBytes); } else { - final WritableMemory memOut = WritableMemory.writableWrap(byteArrOut); + final MemorySegment segOut = MemorySegment.ofArray(byteArrOut); //preamble - memOut.putByte(PREAMBLE_LONGS_BYTE, (byte) CONST_PREAMBLE_LONGS); //RF not used = 0 - memOut.putByte(SER_VER_BYTE, (byte) SER_VER); - memOut.putByte(FAMILY_BYTE, (byte) Family.INTERSECTION.getID()); - memOut.putByte(LG_NOM_LONGS_BYTE, (byte) 0); //not used - memOut.putByte(LG_ARR_LONGS_BYTE, (byte) lgArrLongs_); - if (empty_) { memOut.setBits(FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); } - else { memOut.clearBits(FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); } - memOut.putShort(SEED_HASH_SHORT, seedHash_); - memOut.putInt(RETAINED_ENTRIES_INT, curCount_); - memOut.putFloat(P_FLOAT, (float) 1.0); - memOut.putLong(THETA_LONG, thetaLong_); + segOut.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, (byte) CONST_PREAMBLE_LONGS); //RF not used = 0 + segOut.set(JAVA_BYTE, SER_VER_BYTE, (byte) SER_VER); + segOut.set(JAVA_BYTE, FAMILY_BYTE, (byte) Family.INTERSECTION.getID()); + segOut.set(JAVA_BYTE, LG_NOM_LONGS_BYTE, (byte) 0); //not used + segOut.set(JAVA_BYTE, LG_ARR_LONGS_BYTE, (byte) lgArrLongs_); + if (empty_) { setBits(segOut, FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); } + else { clearBits(segOut, FLAGS_BYTE, (byte) EMPTY_FLAG_MASK); } + segOut.set(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT, seedHash_); + segOut.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, curCount_); + segOut.set(JAVA_FLOAT_UNALIGNED, P_FLOAT, (float) 1.0); + segOut.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong_); //data if (curCount_ > 0) { - memOut.putLongArray(preBytes, hashTable_, 0, 1 << lgArrLongs_); + MemorySegment.copy(hashTable_, 0, segOut, JAVA_LONG_UNALIGNED, preBytes, 1 << lgArrLongs_); } } return byteArrOut; @@ -411,13 +417,13 @@ boolean isEmpty() { @Override long[] getCache() { - if (wmem_ == null) { + if (wseg_ == null) { return hashTable_ != null ? hashTable_ : new long[0]; } - //Direct + //offHeap final int arrLongs = 1 << lgArrLongs_; final long[] outArr = new long[arrLongs]; - wmem_.getLongArray(CONST_PREAMBLE_LONGS << 3, outArr, 0, arrLongs); + MemorySegment.copy(wseg_, JAVA_LONG_UNALIGNED, CONST_PREAMBLE_LONGS << 3, outArr, 0, arrLongs); return outArr; } @@ -435,10 +441,10 @@ private void performIntersect(final Sketch sketchIn) { // curCount and input data are nonzero, match against HT assert curCount_ > 0 && !empty_; final long[] hashTable; - if (wmem_ != null) { + if (wseg_ != null) { final int htLen = 1 << lgArrLongs_; hashTable = new long[htLen]; - wmem_.getLongArray(CONST_PREAMBLE_LONGS << 3, hashTable, 0, htLen); + MemorySegment.copy(wseg_, JAVA_LONG_UNALIGNED, CONST_PREAMBLE_LONGS << 3, hashTable, 0, htLen); } else { hashTable = hashTable_; } @@ -462,10 +468,10 @@ private void performIntersect(final Sketch sketchIn) { //reduce effective array size to minimum curCount_ = matchSetCount; lgArrLongs_ = minLgHashTableSize(matchSetCount, ThetaUtil.REBUILD_THRESHOLD); - if (wmem_ != null) { - insertCurCount(wmem_, matchSetCount); - insertLgArrLongs(wmem_, lgArrLongs_); - wmem_.clear(CONST_PREAMBLE_LONGS << 3, 8 << lgArrLongs_); //clear for rebuild + if (wseg_ != null) { + insertCurCount(wseg_, matchSetCount); + insertLgArrLongs(wseg_, lgArrLongs_); + wseg_.asSlice(CONST_PREAMBLE_LONGS << 3, 8 << lgArrLongs_).fill((byte)0); //clear for rebuild } else { Arrays.fill(hashTable_, 0, 1 << lgArrLongs_, 0L); //clear for rebuild } @@ -482,14 +488,14 @@ private void performIntersect(final Sketch sketchIn) { private void moveDataToTgt(final long[] arr, final int count) { final int arrLongsIn = arr.length; int tmpCnt = 0; - if (wmem_ != null) { //Off Heap puts directly into mem + if (wseg_ != null) { //Off Heap puts directly into mem final int preBytes = CONST_PREAMBLE_LONGS << 3; final int lgArrLongs = lgArrLongs_; final long thetaLong = thetaLong_; for (int i = 0; i < arrLongsIn; i++ ) { final long hashIn = arr[i]; if (continueCondition(thetaLong, hashIn)) { continue; } - hashInsertOnlyMemory(wmem_, lgArrLongs, hashIn, preBytes); + hashInsertOnlyMemorySegment(wseg_, lgArrLongs, hashIn, preBytes); tmpCnt++; } } else { //On Heap. Assumes HT exists and is large enough @@ -506,7 +512,7 @@ private void moveDataToTgt(final long[] arr, final int count) { private void moveDataToTgt(final Sketch sketch) { final int count = sketch.getRetainedEntries(); int tmpCnt = 0; - if (wmem_ != null) { //Off Heap puts directly into mem + if (wseg_ != null) { //Off Heap puts directly into mem final int preBytes = CONST_PREAMBLE_LONGS << 3; final int lgArrLongs = lgArrLongs_; final long thetaLong = thetaLong_; @@ -514,7 +520,7 @@ private void moveDataToTgt(final Sketch sketch) { while (it.next()) { final long hash = it.get(); if (continueCondition(thetaLong, hash)) { continue; } - hashInsertOnlyMemory(wmem_, lgArrLongs, hash, preBytes); + hashInsertOnlyMemorySegment(wseg_, lgArrLongs, hash, preBytes); tmpCnt++; } } else { //On Heap. Assumes HT exists and is large enough @@ -531,9 +537,9 @@ private void moveDataToTgt(final Sketch sketch) { private void hardReset() { resetCommon(); - if (wmem_ != null) { - insertCurCount(wmem_, -1); //Universal Set - clearEmpty(wmem_); //false + if (wseg_ != null) { + insertCurCount(wseg_, -1); //Universal Set + clearEmpty(wseg_); //false } curCount_ = -1; //Universal Set empty_ = false; @@ -541,20 +547,20 @@ private void hardReset() { private void resetToEmpty() { resetCommon(); - if (wmem_ != null) { - insertCurCount(wmem_, 0); - setEmpty(wmem_); //true + if (wseg_ != null) { + insertCurCount(wseg_, 0); + setEmpty(wseg_); //true } curCount_ = 0; empty_ = true; } private void resetCommon() { - if (wmem_ != null) { + if (wseg_ != null) { if (readOnly_) { throw new SketchesReadOnlyException(); } - wmem_.clear(CONST_PREAMBLE_LONGS << 3, 8 << ThetaUtil.MIN_LG_ARR_LONGS); - insertLgArrLongs(wmem_, ThetaUtil.MIN_LG_ARR_LONGS); - insertThetaLong(wmem_, Long.MAX_VALUE); + wseg_.asSlice(CONST_PREAMBLE_LONGS << 3, 8 << ThetaUtil.MIN_LG_ARR_LONGS).fill((byte)0); + insertLgArrLongs(wseg_, ThetaUtil.MIN_LG_ARR_LONGS); + insertThetaLong(wseg_, Long.MAX_VALUE); } lgArrLongs_ = ThetaUtil.MIN_LG_ARR_LONGS; thetaLong_ = Long.MAX_VALUE; diff --git a/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java b/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java index 1f9e710c0..a058f649a 100644 --- a/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java +++ b/src/main/java/org/apache/datasketches/theta/JaccardSimilarity.java @@ -37,6 +37,8 @@ public final class JaccardSimilarity { private static final double[] ZEROS = {0.0, 0.0, 0.0}; // LB, Estimate, UB private static final double[] ONES = {1.0, 1.0, 1.0}; + private JaccardSimilarity() { } + /** * Computes the Jaccard similarity index with upper and lower bounds. The Jaccard similarity index * J(A,B) = (A ^ B)/(A U B) is used to measure how similar the two sketches are to each diff --git a/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java b/src/main/java/org/apache/datasketches/theta/MemorySegmentCompactCompressedHashIterator.java similarity index 72% rename from src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java rename to src/main/java/org/apache/datasketches/theta/MemorySegmentCompactCompressedHashIterator.java index d5f37de96..031cee687 100644 --- a/src/main/java/org/apache/datasketches/theta/MemoryCompactCompressedHashIterator.java +++ b/src/main/java/org/apache/datasketches/theta/MemorySegmentCompactCompressedHashIterator.java @@ -19,15 +19,19 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits; -import org.apache.datasketches.memory.Memory; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.MemorySegmentStatus; +import org.apache.datasketches.common.Util; /* * This is to uncompress serial version 4 sketch incrementally */ -class MemoryCompactCompressedHashIterator implements HashIterator { - private Memory mem; +final class MemorySegmentCompactCompressedHashIterator implements HashIterator, MemorySegmentStatus { + private MemorySegment seg; private int offset; private int entryBits; private int numEntries; @@ -39,13 +43,12 @@ class MemoryCompactCompressedHashIterator implements HashIterator { private boolean isBlockMode; private boolean isFirstUnpack1; - MemoryCompactCompressedHashIterator( - final Memory mem, + MemorySegmentCompactCompressedHashIterator( + final MemorySegment srcSeg, final int offset, final int entryBits, - final int numEntries - ) { - this.mem = mem; + final int numEntries) { + this.seg = srcSeg; this.offset = offset; this.entryBits = entryBits; this.numEntries = numEntries; @@ -63,6 +66,21 @@ public long get() { return buffer[index & 7]; } + @Override + public boolean hasMemorySegment() { + return seg != null && seg.scope().isAlive(); + } + + @Override + public boolean isDirect() { + return hasMemorySegment() && seg.isNative(); + } + + @Override + public boolean isSameResource(final MemorySegment that) { + return hasMemorySegment() && Util.isSameResource(seg, that); + } + @Override public boolean next() { if (++index == numEntries) { return false; } @@ -83,7 +101,7 @@ public boolean next() { private void unpack1() { if (isFirstUnpack1) { - mem.getByteArray(offset, bytes, 0, wholeBytesToHoldBits((numEntries - index) * entryBits)); + MemorySegment.copy(seg, JAVA_BYTE, offset, bytes, 0, wholeBytesToHoldBits((numEntries - index) * entryBits)); offset = 0; isFirstUnpack1 = false; } @@ -96,7 +114,7 @@ private void unpack1() { } private void unpack8() { - mem.getByteArray(offset, bytes, 0, entryBits); + MemorySegment.copy(seg, JAVA_BYTE, offset, bytes, 0, entryBits); BitPacking.unpackBitsBlock8(buffer, 0, bytes, 0, entryBits); offset += entryBits; for (int i = 0; i < 8; i++) { diff --git a/src/main/java/org/apache/datasketches/theta/MemoryHashIterator.java b/src/main/java/org/apache/datasketches/theta/MemorySegmentHashIterator.java similarity index 74% rename from src/main/java/org/apache/datasketches/theta/MemoryHashIterator.java rename to src/main/java/org/apache/datasketches/theta/MemorySegmentHashIterator.java index 926d0ad9e..548c79ef3 100644 --- a/src/main/java/org/apache/datasketches/theta/MemoryHashIterator.java +++ b/src/main/java/org/apache/datasketches/theta/MemorySegmentHashIterator.java @@ -19,24 +19,26 @@ package org.apache.datasketches.theta; -import org.apache.datasketches.memory.Memory; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; + +import java.lang.foreign.MemorySegment; /** * @author Lee Rhodes */ -class MemoryHashIterator implements HashIterator { - private Memory mem; +final class MemorySegmentHashIterator implements HashIterator { + private MemorySegment seg; private int arrLongs; private long thetaLong; private long offsetBytes; private int index; private long hash; - MemoryHashIterator(final Memory mem, final int arrLongs, final long thetaLong) { - this.mem = mem; + MemorySegmentHashIterator(final MemorySegment srcSeg, final int arrLongs, final long thetaLong) { + this.seg = srcSeg; this.arrLongs = arrLongs; this.thetaLong = thetaLong; - offsetBytes = PreambleUtil.extractPreLongs(mem) << 3; + offsetBytes = PreambleUtil.extractPreLongs(srcSeg) << 3; index = -1; hash = 0; } @@ -49,7 +51,7 @@ public long get() { @Override public boolean next() { while (++index < arrLongs) { - hash = mem.getLong(offsetBytes + (index << 3)); + hash = seg.get(JAVA_LONG_UNALIGNED, offsetBytes + (index << 3)); if ((hash != 0) && (hash < thetaLong)) { return true; } diff --git a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java index ec0bc1268..ed368bd8b 100644 --- a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java @@ -19,18 +19,21 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; import static org.apache.datasketches.common.Util.LS; import static org.apache.datasketches.common.Util.zeroPad; +import java.lang.foreign.MemorySegment; import java.nio.ByteOrder; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.Util; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; //@formatter:off @@ -198,14 +201,13 @@ private PreambleUtil() {} (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN); /** - * Computes the number of bytes required for a non-full sized sketch in hash-table form. - * This can be used to compute current storage size for heap sketches, or current off-heap memory - * required for off-heap (direct) sketches. This does not apply for compact sketches. + * Computes the number of bytes required for an updatable sketch using a hash-table cache. + * This does not apply for compact sketches. * @param lgArrLongs log2(current hash-table size) * @param preambleLongs current preamble size * @return the size in bytes */ - static final int getMemBytes(final int lgArrLongs, final int preambleLongs) { + static final int getSegBytes(final int lgArrLongs, final int preambleLongs) { return (8 << lgArrLongs) + (preambleLongs << 3); } @@ -219,31 +221,31 @@ static final int getMemBytes(final int lgArrLongs, final int preambleLongs) { * @return the summary preamble string. */ static String preambleToString(final byte[] byteArr) { - final Memory mem = Memory.wrap(byteArr); - return preambleToString(mem); + final MemorySegment seg = MemorySegment.ofArray(byteArr); + return preambleToString(seg); } /** - * Returns a human readable string summary of the preamble state of the given Memory. - * Note: other than making sure that the given Memory size is large + * Returns a human readable string summary of the preamble state of the given MemorySegment. + * Note: other than making sure that the given MemorySegment size is large * enough for just the preamble, this does not do much value checking of the contents of the * preamble as this is primarily a tool for debugging the preamble visually. * - * @param mem the given Memory. + * @param seg the given MemorySegment. * @return the summary preamble string. */ - static String preambleToString(final Memory mem) { - final int preLongs = getAndCheckPreLongs(mem); - final int rfId = extractLgResizeFactor(mem); + static String preambleToString(final MemorySegment seg) { + final int preLongs = getAndCheckPreLongs(seg); + final int rfId = extractLgResizeFactor(seg); final ResizeFactor rf = ResizeFactor.getRF(rfId); - final int serVer = extractSerVer(mem); - final int familyId = extractFamilyID(mem); + final int serVer = extractSerVer(seg); + final int familyId = extractFamilyID(seg); final Family family = Family.idToFamily(familyId); - final int lgNomLongs = extractLgNomLongs(mem); - final int lgArrLongs = extractLgArrLongs(mem); + final int lgNomLongs = extractLgNomLongs(seg); + final int lgArrLongs = extractLgArrLongs(seg); //Flags - final int flags = extractFlags(mem); + final int flags = extractFlags(seg); final String flagsStr = (flags) + ", 0x" + (Integer.toHexString(flags)) + ", " + zeroPad(Integer.toBinaryString(flags), 8); final String nativeOrder = ByteOrder.nativeOrder().toString(); @@ -254,7 +256,7 @@ static String preambleToString(final Memory mem) { final boolean ordered = (flags & ORDERED_FLAG_MASK) > 0; final boolean singleItem = (flags & SINGLEITEM_FLAG_MASK) > 0; //!empty && (preLongs == 1); - final int seedHash = extractSeedHash(mem); + final int seedHash = extractSeedHash(seg); //assumes preLongs == 1; empty or singleItem int curCount = singleItem ? 1 : 0; @@ -263,20 +265,20 @@ static String preambleToString(final Memory mem) { long thetaULong = thetaLong; //preLongs 1, 2 or 3 if (preLongs == 2) { //exact (non-estimating) CompactSketch - curCount = extractCurCount(mem); - p = extractP(mem); + curCount = extractCurCount(seg); + p = extractP(seg); } else if (preLongs == 3) { //Update Sketch - curCount = extractCurCount(mem); - p = extractP(mem); - thetaLong = extractThetaLong(mem); + curCount = extractCurCount(seg); + p = extractP(seg); + thetaLong = extractThetaLong(seg); thetaULong = thetaLong; } else if (preLongs == 4) { //Union - curCount = extractCurCount(mem); - p = extractP(mem); - thetaLong = extractThetaLong(mem); - thetaULong = extractUnionThetaLong(mem); + curCount = extractCurCount(seg); + p = extractP(seg); + thetaLong = extractThetaLong(seg); + thetaULong = extractUnionThetaLong(seg); } //else the same as an empty sketch or singleItem @@ -340,171 +342,171 @@ else if (preLongs == 3) { sb.append( "Preamble Bytes : ").append(preLongs * 8).append(LS); sb.append( "Data Bytes : ").append(curCount * 8).append(LS); sb.append( "TOTAL Sketch Bytes : ").append((preLongs + curCount) * 8).append(LS); - sb.append( "TOTAL Capacity Bytes : ").append(mem.getCapacity()).append(LS); + sb.append( "TOTAL Capacity Bytes : ").append(seg.byteSize()).append(LS); sb.append("### END SKETCH PREAMBLE SUMMARY").append(LS); return sb.toString(); } //@formatter:on - static int extractPreLongs(final Memory mem) { - return mem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; + static int extractPreLongs(final MemorySegment seg) { + return seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; } - static int extractLgResizeFactor(final Memory mem) { - return (mem.getByte(PREAMBLE_LONGS_BYTE) >>> LG_RESIZE_FACTOR_BIT) & 0X3; + static int extractLgResizeFactor(final MemorySegment seg) { + return (seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) >>> LG_RESIZE_FACTOR_BIT) & 0X3; } - static int extractLgResizeRatioV1(final Memory mem) { - return mem.getByte(LG_RESIZE_RATIO_BYTE_V1) & 0X3; + static int extractLgResizeRatioV1(final MemorySegment seg) { + return seg.get(JAVA_BYTE, LG_RESIZE_RATIO_BYTE_V1) & 0X3; } - static int extractSerVer(final Memory mem) { - return mem.getByte(SER_VER_BYTE) & 0XFF; + static int extractSerVer(final MemorySegment seg) { + return seg.get(JAVA_BYTE, SER_VER_BYTE) & 0XFF; } - static int extractFamilyID(final Memory mem) { - return mem.getByte(FAMILY_BYTE) & 0XFF; + static int extractFamilyID(final MemorySegment seg) { + return seg.get(JAVA_BYTE, FAMILY_BYTE) & 0XFF; } - static int extractLgNomLongs(final Memory mem) { - return mem.getByte(LG_NOM_LONGS_BYTE) & 0XFF; + static int extractLgNomLongs(final MemorySegment seg) { + return seg.get(JAVA_BYTE, LG_NOM_LONGS_BYTE) & 0XFF; } - static int extractLgArrLongs(final Memory mem) { - return mem.getByte(LG_ARR_LONGS_BYTE) & 0XFF; + static int extractLgArrLongs(final MemorySegment seg) { + return seg.get(JAVA_BYTE, LG_ARR_LONGS_BYTE) & 0XFF; } - static int extractFlags(final Memory mem) { - return mem.getByte(FLAGS_BYTE) & 0XFF; + static int extractFlags(final MemorySegment seg) { + return seg.get(JAVA_BYTE, FLAGS_BYTE) & 0XFF; } - static int extractFlagsV1(final Memory mem) { - return mem.getByte(FLAGS_BYTE_V1) & 0XFF; + static int extractFlagsV1(final MemorySegment seg) { + return seg.get(JAVA_BYTE, FLAGS_BYTE_V1) & 0XFF; } - static int extractSeedHash(final Memory mem) { - return mem.getShort(SEED_HASH_SHORT) & 0XFFFF; + static int extractSeedHash(final MemorySegment seg) { + return seg.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT) & 0XFFFF; } - static int extractCurCount(final Memory mem) { - return mem.getInt(RETAINED_ENTRIES_INT); + static int extractCurCount(final MemorySegment seg) { + return seg.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT); } - static float extractP(final Memory mem) { - return mem.getFloat(P_FLOAT); + static float extractP(final MemorySegment seg) { + return seg.get(JAVA_FLOAT_UNALIGNED, P_FLOAT); } - static long extractThetaLong(final Memory mem) { - return mem.getLong(THETA_LONG); + static long extractThetaLong(final MemorySegment seg) { + return seg.get(JAVA_LONG_UNALIGNED, THETA_LONG); } - static long extractUnionThetaLong(final Memory mem) { - return mem.getLong(UNION_THETA_LONG); + static long extractUnionThetaLong(final MemorySegment seg) { + return seg.get(JAVA_LONG_UNALIGNED, UNION_THETA_LONG); } - static int extractEntryBitsV4(final Memory mem) { - return mem.getByte(ENTRY_BITS_BYTE_V4) & 0XFF; + static int extractEntryBitsV4(final MemorySegment seg) { + return seg.get(JAVA_BYTE, ENTRY_BITS_BYTE_V4) & 0XFF; } - static int extractNumEntriesBytesV4(final Memory mem) { - return mem.getByte(NUM_ENTRIES_BYTES_BYTE_V4) & 0XFF; + static int extractNumEntriesBytesV4(final MemorySegment seg) { + return seg.get(JAVA_BYTE, NUM_ENTRIES_BYTES_BYTE_V4) & 0XFF; } - static long extractThetaLongV4(final Memory mem) { - return mem.getLong(THETA_LONG_V4); + static long extractThetaLongV4(final MemorySegment seg) { + return seg.get(JAVA_LONG_UNALIGNED, THETA_LONG_V4); } /** * Sets PreLongs in the low 6 bits and sets LgRF in the upper 2 bits = 0. - * @param wmem the target WritableMemory + * @param seg the target MemorySegment * @param preLongs the given number of preamble longs */ - static void insertPreLongs(final WritableMemory wmem, final int preLongs) { - wmem.putByte(PREAMBLE_LONGS_BYTE, (byte) (preLongs & 0X3F)); + static void insertPreLongs(final MemorySegment seg, final int preLongs) { + seg.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, (byte) (preLongs & 0X3F)); } /** * Sets the top 2 lgRF bits and does not affect the lower 6 bits (PreLongs). * To work properly, this should be called after insertPreLongs(). - * @param wmem the target WritableMemory + * @param seg the target MemorySegment * @param rf the given lgRF bits */ - static void insertLgResizeFactor(final WritableMemory wmem, final int rf) { - final int curByte = wmem.getByte(PREAMBLE_LONGS_BYTE) & 0xFF; + static void insertLgResizeFactor(final MemorySegment seg, final int rf) { + final int curByte = seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0xFF; final int shift = LG_RESIZE_FACTOR_BIT; // shift in bits final int mask = 3; final byte newByte = (byte) (((rf & mask) << shift) | (~(mask << shift) & curByte)); - wmem.putByte(PREAMBLE_LONGS_BYTE, newByte); + seg.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, newByte); } - static void insertSerVer(final WritableMemory wmem, final int serVer) { - wmem.putByte(SER_VER_BYTE, (byte) serVer); + static void insertSerVer(final MemorySegment seg, final int serVer) { + seg.set(JAVA_BYTE, SER_VER_BYTE, (byte) serVer); } - static void insertFamilyID(final WritableMemory wmem, final int famId) { - wmem.putByte(FAMILY_BYTE, (byte) famId); + static void insertFamilyID(final MemorySegment seg, final int famId) { + seg.set(JAVA_BYTE, FAMILY_BYTE, (byte) famId); } - static void insertLgNomLongs(final WritableMemory wmem, final int lgNomLongs) { - wmem.putByte(LG_NOM_LONGS_BYTE, (byte) lgNomLongs); + static void insertLgNomLongs(final MemorySegment seg, final int lgNomLongs) { + seg.set(JAVA_BYTE, LG_NOM_LONGS_BYTE, (byte) lgNomLongs); } - static void insertLgArrLongs(final WritableMemory wmem, final int lgArrLongs) { - wmem.putByte(LG_ARR_LONGS_BYTE, (byte) lgArrLongs); + static void insertLgArrLongs(final MemorySegment seg, final int lgArrLongs) { + seg.set(JAVA_BYTE, LG_ARR_LONGS_BYTE, (byte) lgArrLongs); } - static void insertFlags(final WritableMemory wmem, final int flags) { - wmem.putByte(FLAGS_BYTE, (byte) flags); + static void insertFlags(final MemorySegment seg, final int flags) { + seg.set(JAVA_BYTE, FLAGS_BYTE, (byte) flags); } - static void insertSeedHash(final WritableMemory wmem, final int seedHash) { - wmem.putShort(SEED_HASH_SHORT, (short) seedHash); + static void insertSeedHash(final MemorySegment seg, final int seedHash) { + seg.set(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT, (short) seedHash); } - static void insertCurCount(final WritableMemory wmem, final int curCount) { - wmem.putInt(RETAINED_ENTRIES_INT, curCount); + static void insertCurCount(final MemorySegment seg, final int curCount) { + seg.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, curCount); } - static void insertP(final WritableMemory wmem, final float p) { - wmem.putFloat(P_FLOAT, p); + static void insertP(final MemorySegment seg, final float p) { + seg.set(JAVA_FLOAT_UNALIGNED, P_FLOAT, p); } - static void insertThetaLong(final WritableMemory wmem, final long thetaLong) { - wmem.putLong(THETA_LONG, thetaLong); + static void insertThetaLong(final MemorySegment seg, final long thetaLong) { + seg.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong); } - static void insertUnionThetaLong(final WritableMemory wmem, final long unionThetaLong) { - wmem.putLong(UNION_THETA_LONG, unionThetaLong); + static void insertUnionThetaLong(final MemorySegment seg, final long unionThetaLong) { + seg.set(JAVA_LONG_UNALIGNED, UNION_THETA_LONG, unionThetaLong); } - static void setEmpty(final WritableMemory wmem) { - int flags = wmem.getByte(FLAGS_BYTE) & 0XFF; + static void setEmpty(final MemorySegment seg) { + int flags = seg.get(JAVA_BYTE, FLAGS_BYTE) & 0XFF; flags |= EMPTY_FLAG_MASK; - wmem.putByte(FLAGS_BYTE, (byte) flags); + seg.set(JAVA_BYTE, FLAGS_BYTE, (byte) flags); } - static void clearEmpty(final WritableMemory wmem) { - int flags = wmem.getByte(FLAGS_BYTE) & 0XFF; + static void clearEmpty(final MemorySegment seg) { + int flags = seg.get(JAVA_BYTE, FLAGS_BYTE) & 0XFF; flags &= ~EMPTY_FLAG_MASK; - wmem.putByte(FLAGS_BYTE, (byte) flags); + seg.set(JAVA_BYTE, FLAGS_BYTE, (byte) flags); } - static boolean isEmptyFlag(final Memory mem) { - return ((extractFlags(mem) & EMPTY_FLAG_MASK) > 0); + static boolean isEmptyFlag(final MemorySegment seg) { + return ((extractFlags(seg) & EMPTY_FLAG_MASK) > 0); } /** - * Checks Memory for capacity to hold the preamble and returns the extracted preLongs. - * @param mem the given Memory + * Checks MemorySegment for capacity to hold the preamble and returns the extracted preLongs. + * @param seg the given MemorySegment * @return the extracted prelongs value. */ - static int getAndCheckPreLongs(final Memory mem) { - final long cap = mem.getCapacity(); + static int getAndCheckPreLongs(final MemorySegment seg) { + final long cap = seg.byteSize(); if (cap < 8) { throwNotBigEnough(cap, 8); } - final int preLongs = extractPreLongs(mem); + final int preLongs = extractPreLongs(seg); final int required = Math.max(preLongs << 3, 8); if (cap < required) { throwNotBigEnough(cap, required); @@ -512,15 +514,15 @@ static int getAndCheckPreLongs(final Memory mem) { return preLongs; } - static final short checkMemorySeedHash(final Memory mem, final long seed) { - final short seedHashMem = (short) extractSeedHash(mem); - ThetaUtil.checkSeedHashes(seedHashMem, ThetaUtil.computeSeedHash(seed)); //throws if bad seedHash - return seedHashMem; + static final short checkSegmentSeedHash(final MemorySegment seg, final long seed) { + final short seedHashSeg = (short) extractSeedHash(seg); + Util.checkSeedHashes(seedHashSeg, Util.computeSeedHash(seed)); //throws if bad seedHash + return seedHashSeg; } private static void throwNotBigEnough(final long cap, final int required) { throw new SketchesArgumentException( - "Possible Corruption: Size of byte array or Memory not large enough: Size: " + cap + "Possible Corruption: Size of byte array or MemorySegment not large enough: Size: " + cap + ", Required: " + required); } diff --git a/src/main/java/org/apache/datasketches/theta/Rebuilder.java b/src/main/java/org/apache/datasketches/theta/Rebuilder.java index b6e3de342..54b1e190b 100644 --- a/src/main/java/org/apache/datasketches/theta/Rebuilder.java +++ b/src/main/java/org/apache/datasketches/theta/Rebuilder.java @@ -19,6 +19,9 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static org.apache.datasketches.common.QuickSelect.selectExcludingZeros; import static org.apache.datasketches.theta.PreambleUtil.LG_ARR_LONGS_BYTE; import static org.apache.datasketches.theta.PreambleUtil.extractCurCount; import static org.apache.datasketches.theta.PreambleUtil.extractLgArrLongs; @@ -26,16 +29,19 @@ import static org.apache.datasketches.theta.PreambleUtil.insertCurCount; import static org.apache.datasketches.theta.PreambleUtil.insertLgArrLongs; import static org.apache.datasketches.theta.PreambleUtil.insertThetaLong; -import static org.apache.datasketches.thetacommon.QuickSelect.selectExcludingZeros; + +import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.Util; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.HashOperations; /** - * Handles common resize, rebuild and move operations. - * The Memory based operations assume a specific data structure that is unique to the theta sketches. + * This class performs resize, rebuild and move operations where the input and output are Theta sketch images in MemorySegments. + * + *

NOTE: These operations copy data from the input MemorySegment into local arrays, perform the required operations on the + * arrays, and then copies the result to the destination MemorySegment. Attempting to perform these operations directly on the + * MemorySegments would be slower due to MemorySegment internal checks. Meanwhile, he bulk copies performed by the MemorySegments are + * vectorized at the machine level and are quite fast. Measurements reveal that this is a good tradeoff.

* * @author Lee Rhodes */ @@ -44,73 +50,63 @@ final class Rebuilder { private Rebuilder() {} /** - * Rebuild the hashTable in the given Memory at its current size. Changes theta and thus count. - * This assumes a Memory preamble of standard form with correct values of curCount and thetaLong. + * Rebuild the hashTable in the given MemorySegment at its current size. Changes theta and thus count. + * This assumes a MemorySegment preamble of standard form with correct values of curCount and thetaLong. * ThetaLong and curCount will change. - * Afterwards, caller must update local class members curCount and thetaLong from Memory. + * Afterwards, caller must update local class members curCount and thetaLong from MemorySegment. * - * @param mem the Memory the given Memory + * @param seg the given MemorySegment * @param preambleLongs size of preamble in longs * @param lgNomLongs the log_base2 of k, the configuration parameter of the sketch */ - static final void quickSelectAndRebuild(final WritableMemory mem, final int preambleLongs, - final int lgNomLongs) { - //Note: This copies the Memory data onto the heap and then at the end copies the result - // back to Memory. Even if we tried to do this directly into Memory it would require pre-clearing, - // and the internal loops would be slower. The bulk copies are performed at a low level and - // are quite fast. Measurements reveal that we are not paying much of a penalty. - - //Pull data into tmp arr for QS algo - final int lgArrLongs = extractLgArrLongs(mem); - final int curCount = extractCurCount(mem); + static final void quickSelectAndRebuild(final MemorySegment seg, final int preambleLongs, final int lgNomLongs) { + + //Copy data from input segment into local buffer array for QS algorithm + final int lgArrLongs = extractLgArrLongs(seg); final int arrLongs = 1 << lgArrLongs; final long[] tmpArr = new long[arrLongs]; final int preBytes = preambleLongs << 3; - mem.getLongArray(preBytes, tmpArr, 0, arrLongs); //copy mem data to tmpArr + MemorySegment.copy(seg, JAVA_LONG_UNALIGNED, preBytes, tmpArr, 0, arrLongs); //Do the QuickSelect on a tmp arr to create new thetaLong final int pivot = (1 << lgNomLongs) + 1; // (K+1) pivot for QS - final long newThetaLong = selectExcludingZeros(tmpArr, curCount, pivot); - insertThetaLong(mem, newThetaLong); //UPDATE thetalong + final long newThetaLong = selectExcludingZeros(tmpArr, extractCurCount(seg), pivot); + insertThetaLong(seg, newThetaLong); //UPDATE thetaLong //Rebuild to clean up dirty data, update count final long[] tgtArr = new long[arrLongs]; final int newCurCount = HashOperations.hashArrayInsert(tmpArr, tgtArr, lgArrLongs, newThetaLong); - insertCurCount(mem, newCurCount); //UPDATE curCount + insertCurCount(seg, newCurCount); //UPDATE curCount - //put the rebuilt array back into memory - mem.putLongArray(preBytes, tgtArr, 0, arrLongs); + //put the rebuilt array back into MemorySegment + MemorySegment.copy(tgtArr, 0, seg, JAVA_LONG_UNALIGNED, preBytes, arrLongs); } /** - * Moves me (the entire updatable sketch) to a new larger Memory location and rebuilds the hash table. - * This assumes a Memory preamble of standard form with the correct value of thetaLong. - * Afterwards, the caller must update the local Memory reference, lgArrLongs - * and hashTableThreshold from the dstMemory and free the source Memory. + * Moves me (the entire updatable sketch) to a new larger MemorySegment location and rebuilds the hash table. + * This assumes a MemorySegment preamble of standard form with the correct value of thetaLong. + * Afterwards, the caller must update the local MemorySegment reference, lgArrLongs + * and hashTableThreshold from the destination MemorySegment and free the source MemorySegment. * - * @param srcMem the source Memory + * @param srcSeg the source MemorySegment * @param preambleLongs size of preamble in longs * @param srcLgArrLongs size (log_base2) of source hash table - * @param dstMem the destination Memory, which may be garbage + * @param dstSeg the destination MemorySegment, which may be garbage * @param dstLgArrLongs the destination hash table target size * @param thetaLong theta as a long */ - static final void moveAndResize(final Memory srcMem, final int preambleLongs, - final int srcLgArrLongs, final WritableMemory dstMem, final int dstLgArrLongs, final long thetaLong) { - //Note: This copies the Memory data onto the heap and then at the end copies the result - // back to Memory. Even if we tried to do this directly into Memory it would require pre-clearing, - // and the internal loops would be slower. The bulk copies are performed at a low level and - // are quite fast. Measurements reveal that we are not paying much of a penalty. - - //Move Preamble to destination memory + static final void moveAndResize(final MemorySegment srcSeg, final int preambleLongs, + final int srcLgArrLongs, final MemorySegment dstSeg, final int dstLgArrLongs, final long thetaLong) { + + //Move Preamble to destination MemorySegment final int preBytes = preambleLongs << 3; - srcMem.copyTo(0, dstMem, 0, preBytes); //copy the preamble + MemorySegment.copy(srcSeg, 0, dstSeg, 0, preBytes); - //Bulk copy source to on-heap buffer + //Bulk copy source Hash Table to local buffer array final int srcHTLen = 1 << srcLgArrLongs; final long[] srcHTArr = new long[srcHTLen]; - srcMem.getLongArray(preBytes, srcHTArr, 0, srcHTLen); + MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preBytes, srcHTArr, 0, srcHTLen); //Create destination buffer final int dstHTLen = 1 << dstLgArrLongs; @@ -119,51 +115,51 @@ static final void moveAndResize(final Memory srcMem, final int preambleLongs, //Rebuild hash table in destination buffer HashOperations.hashArrayInsert(srcHTArr, dstHTArr, dstLgArrLongs, thetaLong); - //Bulk copy to destination memory - dstMem.putLongArray(preBytes, dstHTArr, 0, dstHTLen); - dstMem.putByte(LG_ARR_LONGS_BYTE, (byte)dstLgArrLongs); //update in dstMem + //Bulk copy to destination MemorySegment + MemorySegment.copy(dstHTArr, 0, dstSeg, JAVA_LONG_UNALIGNED, preBytes, dstHTLen); + dstSeg.set(JAVA_BYTE, LG_ARR_LONGS_BYTE, (byte)dstLgArrLongs); //update lgArrLongs in dstSeg } /** - * Resizes existing hash array into a larger one within a single Memory assuming enough space. - * This assumes a Memory preamble of standard form with the correct value of thetaLong. - * The Memory lgArrLongs will change. - * Afterwards, the caller must update local copies of lgArrLongs and hashTableThreshold from - * Memory. + * Resizes existing hash array into a larger one within a single MemorySegment, assuming enough space. + * This assumes a preamble of standard form with the correct value of thetaLong. + * The lgArrLongs will change. + * Afterwards, the caller must update the caller's local copies of lgArrLongs and hashTableThreshold + * from the returned MemorySegment. * - * @param mem the Memory + * @param seg the source and destination MemorySegment * @param preambleLongs the size of the preamble in longs * @param srcLgArrLongs the size of the source hash table * @param tgtLgArrLongs the LgArrLongs value for the new hash table */ - static final void resize(final WritableMemory mem, final int preambleLongs, + static final void resize(final MemorySegment seg, final int preambleLongs, final int srcLgArrLongs, final int tgtLgArrLongs) { - //Note: This copies the Memory data onto the heap and then at the end copies the result - // back to Memory. Even if we tried to do this directly into Memory it would require pre-clearing, - // and the internal loops would be slower. The bulk copies are performed at a low level and - // are quite fast. Measurements reveal that we are not paying much of a penalty. //Preamble stays in place final int preBytes = preambleLongs << 3; + //Bulk copy source to on-heap buffer final int srcHTLen = 1 << srcLgArrLongs; //current value final long[] srcHTArr = new long[srcHTLen]; //on-heap src buffer - mem.getLongArray(preBytes, srcHTArr, 0, srcHTLen); + //seg.getLongArray(preBytes, srcHTArr, 0, srcHTLen); + MemorySegment.copy(seg, JAVA_LONG_UNALIGNED, preBytes, srcHTArr, 0, srcHTLen); + //Create destination on-heap buffer final int dstHTLen = 1 << tgtLgArrLongs; final long[] dstHTArr = new long[dstHTLen]; //on-heap dst buffer + //Rebuild hash table in destination buffer - final long thetaLong = extractThetaLong(mem); - HashOperations.hashArrayInsert(srcHTArr, dstHTArr, tgtLgArrLongs, thetaLong); - //Bulk copy to destination memory - mem.putLongArray(preBytes, dstHTArr, 0, dstHTLen); //put it back, no need to clear - insertLgArrLongs(mem, tgtLgArrLongs); //update in mem + HashOperations.hashArrayInsert(srcHTArr, dstHTArr, tgtLgArrLongs, extractThetaLong(seg)); + + //Bulk copy to destination segment + MemorySegment.copy(dstHTArr, 0, seg, JAVA_LONG_UNALIGNED, preBytes, dstHTLen); + insertLgArrLongs(seg, tgtLgArrLongs); //update in mem } /** * Returns the actual log2 Resize Factor that can be used to grow the hash table. This will be * an integer value between zero and the given lgRF, inclusive; - * @param capBytes the current memory capacity in bytes + * @param capBytes the current MemorySegment capacity in bytes * @param lgArrLongs the current lg hash table size in longs * @param preLongs the current preamble size in longs * @param lgRF the configured lg Resize Factor diff --git a/src/main/java/org/apache/datasketches/theta/SetOperation.java b/src/main/java/org/apache/datasketches/theta/SetOperation.java index c198dceb7..126a9298a 100644 --- a/src/main/java/org/apache/datasketches/theta/SetOperation.java +++ b/src/main/java/org/apache/datasketches/theta/SetOperation.java @@ -19,24 +19,25 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static org.apache.datasketches.common.Family.idToFamily; import static org.apache.datasketches.common.Util.ceilingPowerOf2; import static org.apache.datasketches.theta.PreambleUtil.FAMILY_BYTE; import static org.apache.datasketches.theta.PreambleUtil.SER_VER_BYTE; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; -import org.apache.datasketches.common.MemoryStatus; +import org.apache.datasketches.common.MemorySegmentStatus; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; /** * The parent API for all Set Operations * * @author Lee Rhodes */ -public abstract class SetOperation implements MemoryStatus { +public abstract class SetOperation implements MemorySegmentStatus { static final int CONST_PREAMBLE_LONGS = 3; /** @@ -54,45 +55,43 @@ public static final SetOperationBuilder builder() { } /** - * Heapify takes the SetOperations image in Memory and instantiates an on-heap + * Heapify takes the SetOperations image in MemorySegment and instantiates an on-heap * SetOperation using the * Default Update Seed. - * The resulting SetOperation will not retain any link to the source Memory. + * The resulting SetOperation will not retain any link to the source MemorySegment. * *

Note: Only certain set operators during stateful operations can be serialized and thus * heapified.

* - * @param srcMem an image of a SetOperation where the image seed hash matches the default seed hash. - * See Memory - * @return a Heap-based SetOperation from the given Memory + * @param srcSeg an image of a SetOperation where the image seed hash matches the default seed hash. + * @return a Heap-based SetOperation from the given MemorySegment */ - public static SetOperation heapify(final Memory srcMem) { - return heapify(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static SetOperation heapify(final MemorySegment srcSeg) { + return heapify(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Heapify takes the SetOperation image in Memory and instantiates an on-heap + * Heapify takes the SetOperation image in MemorySegment and instantiates an on-heap * SetOperation using the given expectedSeed. - * The resulting SetOperation will not retain any link to the source Memory. + * The resulting SetOperation will not retain any link to the source MemorySegment. * *

Note: Only certain set operators during stateful operations can be serialized and thus * heapified.

* - * @param srcMem an image of a SetOperation where the hash of the given expectedSeed matches the image seed hash. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * @param srcSeg an image of a SetOperation where the hash of the given expectedSeed matches the image seed hash. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. - * @return a Heap-based SetOperation from the given Memory + * @return a Heap-based SetOperation from the given MemorySegment */ - public static SetOperation heapify(final Memory srcMem, final long expectedSeed) { - final byte famID = srcMem.getByte(FAMILY_BYTE); + public static SetOperation heapify(final MemorySegment srcSeg, final long expectedSeed) { + final byte famID = srcSeg.get(JAVA_BYTE, FAMILY_BYTE); final Family family = idToFamily(famID); switch (family) { case UNION : { - return UnionImpl.heapifyInstance(srcMem, expectedSeed); + return UnionImpl.heapifyInstance(srcSeg, expectedSeed); } case INTERSECTION : { - return IntersectionImpl.heapifyInstance(srcMem, expectedSeed); + return IntersectionImpl.heapifyInstance(srcSeg, expectedSeed); } default: { throw new SketchesArgumentException("SetOperation cannot heapify family: " @@ -102,104 +101,51 @@ public static SetOperation heapify(final Memory srcMem, final long expectedSeed) } /** - * Wrap takes the SetOperation image in Memory and refers to it directly. + * Wrap takes the SetOperation image in MemorySegment and refers to it directly. * There is no data copying onto the java heap. - * This method assumes the - * Default Update Seed. + * This method assumes the Default Update Seed. + * If the given source MemorySegment is read-only, the returned object will also be read-only. * - *

Note: Only certain set operators during stateful operations can be serialized and thus - * wrapped.

+ *

Note: Only certain set operators during stateful operations can be serialized and thus wrapped.

* - * @param srcMem an image of a SetOperation where the image seed hash matches the default seed hash. - * See Memory - * @return a SetOperation backed by the given Memory + * @param srcSeg an image of a SetOperation where the image seed hash matches the default seed hash. + * @return a SetOperation backed by the given MemorySegment */ - public static SetOperation wrap(final Memory srcMem) { - return wrap(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static SetOperation wrap(final MemorySegment srcSeg) { + return wrap(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Wrap takes the SetOperation image in Memory and refers to it directly. + * Wrap takes the SetOperation image in MemorySegment and refers to it directly. * There is no data copying onto the java heap. + * If the given source MemorySegment is read-only, the returned object will also be read-only. * - *

Note: Only certain set operators during stateful operations can be serialized and thus - * wrapped.

+ *

Note: Only certain set operators during stateful operations can be serialized and thus wrapped.

* - * @param srcMem an image of a SetOperation where the hash of the given expectedSeed matches the image seed hash. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * @param srcSeg an image of a SetOperation where the hash of the given expectedSeed matches the image seed hash. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. - * @return a SetOperation backed by the given Memory + * @return a SetOperation backed by the given MemorySegment */ - public static SetOperation wrap(final Memory srcMem, final long expectedSeed) { - final byte famID = srcMem.getByte(FAMILY_BYTE); + public static SetOperation wrap(final MemorySegment srcSeg, final long expectedSeed) { + final byte famID = srcSeg.get(JAVA_BYTE, FAMILY_BYTE); final Family family = idToFamily(famID); - final int serVer = srcMem.getByte(SER_VER_BYTE); + final int serVer = srcSeg.get(JAVA_BYTE, SER_VER_BYTE); if (serVer != 3) { throw new SketchesArgumentException("SerVer must be 3: " + serVer); } switch (family) { case UNION : { - return UnionImpl.wrapInstance(srcMem, expectedSeed); + return UnionImpl.wrapInstance(srcSeg, expectedSeed); } case INTERSECTION : { - return IntersectionImpl.wrapInstance((WritableMemory)srcMem, expectedSeed, true); + return IntersectionImpl.wrapInstance(srcSeg, expectedSeed, srcSeg.isReadOnly() ); } default: throw new SketchesArgumentException("SetOperation cannot wrap family: " + family.toString()); } } - /** - * Wrap takes the SetOperation image in Memory and refers to it directly. - * There is no data copying onto the java heap. - * This method assumes the - * Default Update Seed. - * - *

Note: Only certain set operators during stateful operations can be serialized and thus - * wrapped.

- * - * @param srcMem an image of a SetOperation where the image seed hash matches the default seed hash. - * See Memory - * @return a SetOperation backed by the given Memory - */ - public static SetOperation wrap(final WritableMemory srcMem) { - return wrap(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); - } - - /** - * Wrap takes the SetOperation image in Memory and refers to it directly. - * There is no data copying onto the java heap. - * - *

Note: Only certain set operators during stateful operations can be serialized and thus - * wrapped.

- * - * @param srcMem an image of a SetOperation where the hash of the given expectedSeed matches the image seed hash. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. - * See Update Hash Seed. - * @return a SetOperation backed by the given Memory - */ - public static SetOperation wrap(final WritableMemory srcMem, final long expectedSeed) { - final byte famID = srcMem.getByte(FAMILY_BYTE); - final Family family = idToFamily(famID); - final int serVer = srcMem.getByte(SER_VER_BYTE); - if (serVer != 3) { - throw new SketchesArgumentException("SerVer must be 3: " + serVer); - } - switch (family) { - case UNION : { - return UnionImpl.wrapInstance(srcMem, expectedSeed); - } - case INTERSECTION : { - return IntersectionImpl.wrapInstance(srcMem, expectedSeed, false); - } - default: - throw new SketchesArgumentException("SetOperation cannot wrap family: " - + family.toString()); - } - } - /** * Returns the maximum required storage bytes given a nomEntries parameter for Union operations * @param nomEntries Nominal Entries @@ -251,6 +197,12 @@ public static int getMaxAnotBResultBytes(final int nomEntries) { */ abstract long[] getCache(); + /** + * Returns the backing MemorySegment object if it exists, otherwise null. + * @return the backing MemorySegment object if it exists, otherwise null. + */ + MemorySegment getMemorySegment() { return null; } + /** * Gets the current count of retained entries. * This is only useful during stateful operations. @@ -274,6 +226,12 @@ public static int getMaxAnotBResultBytes(final int nomEntries) { */ abstract long getThetaLong(); + @Override + public abstract boolean hasMemorySegment(); + + @Override + public abstract boolean isDirect(); + /** * Returns true if this set operator is empty. * Only useful during stateful operations. @@ -282,4 +240,7 @@ public static int getMaxAnotBResultBytes(final int nomEntries) { */ abstract boolean isEmpty(); + @Override + public abstract boolean isSameResource(final MemorySegment seg); + } diff --git a/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java b/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java index 4a35cf67d..d4af63681 100644 --- a/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java +++ b/src/main/java/org/apache/datasketches/theta/SetOperationBuilder.java @@ -23,12 +23,12 @@ import static org.apache.datasketches.common.Util.TAB; import static org.apache.datasketches.common.Util.ceilingPowerOf2; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.DefaultMemoryRequestServer; -import org.apache.datasketches.memory.MemoryRequestServer; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -36,30 +36,28 @@ * * @author Lee Rhodes */ -public class SetOperationBuilder { +public final class SetOperationBuilder { private int bLgNomLongs; private long bSeed; private ResizeFactor bRF; private float bP; - private MemoryRequestServer bMemReqSvr; /** * Constructor for building a new SetOperation. The default configuration is *
    *
  • Max Nominal Entries (max K): * {@value org.apache.datasketches.thetacommon.ThetaUtil#DEFAULT_NOMINAL_ENTRIES}
    • - *
  • Seed: {@value org.apache.datasketches.thetacommon.ThetaUtil#DEFAULT_UPDATE_SEED}
    • + *
  • Seed: {@value org.apache.datasketches.common.Util#DEFAULT_UPDATE_SEED}
    • *
  • {@link ResizeFactor#X8}
    • *
  • Input Sampling Probability: 1.0
    • - *
  • Memory: null
    • + *
  • MemorySegment: null
    • *
*/ public SetOperationBuilder() { bLgNomLongs = Integer.numberOfTrailingZeros(ThetaUtil.DEFAULT_NOMINAL_ENTRIES); - bSeed = ThetaUtil.DEFAULT_UPDATE_SEED; + bSeed = Util.DEFAULT_UPDATE_SEED; bP = (float) 1.0; bRF = ResizeFactor.X8; - bMemReqSvr = new DefaultMemoryRequestServer(); } /** @@ -160,24 +158,6 @@ public ResizeFactor getResizeFactor() { return bRF; } - /** - * Set the MemoryRequestServer - * @param memReqSvr the given MemoryRequestServer - * @return this SetOperationBuilder - */ - public SetOperationBuilder setMemoryRequestServer(final MemoryRequestServer memReqSvr) { - bMemReqSvr = memReqSvr; - return this; - } - - /** - * Returns the MemoryRequestServer - * @return the MemoryRequestServer - */ - public MemoryRequestServer getMemoryRequestServer() { - return bMemReqSvr; - } - /** * Returns a SetOperation with the current configuration of this Builder and the given Family. * @param family the chosen SetOperation family @@ -189,34 +169,34 @@ public SetOperation build(final Family family) { /** * Returns a SetOperation with the current configuration of this Builder, the given Family - * and the given destination memory. Note that the destination memory cannot be used with AnotB. + * and the given destination MemorySegment. Note that the destination MemorySegment cannot be used with AnotB. * @param family the chosen SetOperation family - * @param dstMem The destination Memory. + * @param dstSeg The destination MemorySegment. * @return a SetOperation */ - public SetOperation build(final Family family, final WritableMemory dstMem) { + public SetOperation build(final Family family, final MemorySegment dstSeg) { SetOperation setOp = null; switch (family) { case UNION: { - if (dstMem == null) { + if (dstSeg == null) { setOp = UnionImpl.initNewHeapInstance(bLgNomLongs, bSeed, bP, bRF); } else { - setOp = UnionImpl.initNewDirectInstance(bLgNomLongs, bSeed, bP, bRF, bMemReqSvr, dstMem); + setOp = UnionImpl.initNewDirectInstance(bLgNomLongs, bSeed, bP, bRF, dstSeg); } break; } case INTERSECTION: { - if (dstMem == null) { + if (dstSeg == null) { setOp = IntersectionImpl.initNewHeapInstance(bSeed); } else { - setOp = IntersectionImpl.initNewDirectInstance(bSeed, dstMem); + setOp = IntersectionImpl.initNewDirectInstance(bSeed, dstSeg); } break; } case A_NOT_B: { - if (dstMem == null) { + if (dstSeg == null) { setOp = new AnotBimpl(bSeed); } else { @@ -244,12 +224,12 @@ public Union buildUnion() { /** * Convenience method, returns a configured SetOperation Union with * Default Nominal Entries - * and the given destination memory. - * @param dstMem The destination Memory. + * and the given destination MemorySegment. + * @param dstSeg The destination MemorySegment. * @return a Union object */ - public Union buildUnion(final WritableMemory dstMem) { - return (Union) build(Family.UNION, dstMem); + public Union buildUnion(final MemorySegment dstSeg) { + return (Union) build(Family.UNION, dstSeg); } /** @@ -264,12 +244,12 @@ public Intersection buildIntersection() { /** * Convenience method, returns a configured SetOperation Intersection with * Default Nominal Entries - * and the given destination memory. - * @param dstMem The destination Memory. + * and the given destination MemorySegment. + * @param dstSeg The destination MemorySegment. * @return an Intersection object */ - public Intersection buildIntersection(final WritableMemory dstMem) { - return (Intersection) build(Family.INTERSECTION, dstMem); + public Intersection buildIntersection(final MemorySegment dstSeg) { + return (Intersection) build(Family.INTERSECTION, dstSeg); } /** @@ -290,8 +270,6 @@ public String toString() { sb.append("Seed:").append(TAB).append(bSeed).append(LS); sb.append("p:").append(TAB).append(bP).append(LS); sb.append("ResizeFactor:").append(TAB).append(bRF).append(LS); - final String mrsStr = bMemReqSvr.getClass().getSimpleName(); - sb.append("MemoryRequestServer:").append(TAB).append(mrsStr).append(LS); return sb.toString(); } diff --git a/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java b/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java index 3cfc13b1e..c6ed7ec52 100644 --- a/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java +++ b/src/main/java/org/apache/datasketches/theta/SingleItemSketch.java @@ -19,6 +19,7 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.datasketches.common.ByteArrayUtil.putLongLE; import static org.apache.datasketches.hash.MurmurHash3.hash; @@ -29,11 +30,11 @@ import static org.apache.datasketches.theta.PreambleUtil.extractSeedHash; import static org.apache.datasketches.theta.PreambleUtil.extractSerVer; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; /** * A CompactSketch that holds only one item hash. @@ -41,7 +42,7 @@ * @author Lee Rhodes */ final class SingleItemSketch extends CompactSketch { - private static final long DEFAULT_SEED_HASH = ThetaUtil.computeSeedHash(ThetaUtil.DEFAULT_UPDATE_SEED) & 0xFFFFL; + private static final long DEFAULT_SEED_HASH = Util.computeSeedHash(Util.DEFAULT_UPDATE_SEED) & 0xFFFFL; // For backward compatibility, a candidate pre0_ long must have: // Flags (byte 5): Ordered, Compact, NOT Empty, Read Only, LittleEndian = 11010 = 0x1A. @@ -62,7 +63,7 @@ private SingleItemSketch(final long hash) { //All checking & hashing has been done, given the relevant seed SingleItemSketch(final long hash, final long seed) { - final long seedHash = ThetaUtil.computeSeedHash(seed) & 0xFFFFL; + final long seedHash = Util.computeSeedHash(seed) & 0xFFFFL; pre0_ = (seedHash << 48) | PRE0_LO6_SI; hash_ = hash; } @@ -75,26 +76,26 @@ private SingleItemSketch(final long hash) { } /** - * Creates a SingleItemSketch on the heap given a SingleItemSketch Memory image and a seedHash. - * Checks the seed hash of the given Memory against the given seedHash. - * @param srcMem the Memory to be heapified. - * @param expectedSeedHash the given seedHash to be checked against the srcMem seedHash + * Creates a SingleItemSketch on the heap given a SingleItemSketch MemorySegment image and a seedHash. + * Checks the seed hash of the given MemorySegment against the given seedHash. + * @param srcSeg the MemorySegment to be heapified. + * @param expectedSeedHash the given seedHash to be checked against the srcSeg seedHash * @return a SingleItemSketch */ //does not override Sketch - static SingleItemSketch heapify(final Memory srcMem, final short expectedSeedHash) { - ThetaUtil.checkSeedHashes((short) extractSeedHash(srcMem), expectedSeedHash); - final boolean singleItem = otherCheckForSingleItem(srcMem); - if (singleItem) { return new SingleItemSketch(srcMem.getLong(8), expectedSeedHash); } - throw new SketchesArgumentException("Input Memory is not a SingleItemSketch."); + static SingleItemSketch heapify(final MemorySegment srcSeg, final short expectedSeedHash) { + Util.checkSeedHashes((short) extractSeedHash(srcSeg), expectedSeedHash); + final boolean singleItem = otherCheckForSingleItem(srcSeg); + if (singleItem) { return new SingleItemSketch(srcSeg.get(JAVA_LONG_UNALIGNED, 8), expectedSeedHash); } + throw new SketchesArgumentException("Input MemorySegment is not a SingleItemSketch."); } @Override - public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) { - if (dstMem == null) { return this; } + public CompactSketch compact(final boolean dstOrdered, final MemorySegment dstSeg) { + if (dstSeg == null) { return this; } else { - dstMem.putLong(0, pre0_); - dstMem.putLong(8, hash_); - return new DirectCompactSketch(dstMem); + dstSeg.set(JAVA_LONG_UNALIGNED, 0, pre0_); + dstSeg.set(JAVA_LONG_UNALIGNED, 8, hash_); + return new DirectCompactSketch(dstSeg); } } @@ -108,7 +109,7 @@ public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstM */ static SingleItemSketch create(final long datum) { final long[] data = { datum }; - return new SingleItemSketch(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1); + return new SingleItemSketch(hash(data, Util.DEFAULT_UPDATE_SEED)[0] >>> 1); } /** @@ -124,7 +125,7 @@ static SingleItemSketch create(final long datum) { static SingleItemSketch create(final double datum) { final double d = (datum == 0.0) ? 0.0 : datum; // canonicalize -0.0, 0.0 final long[] data = { Double.doubleToLongBits(d) };// canonicalize all NaN forms - return new SingleItemSketch(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1); + return new SingleItemSketch(hash(data, Util.DEFAULT_UPDATE_SEED)[0] >>> 1); } /** @@ -142,7 +143,7 @@ static SingleItemSketch create(final double datum) { static SingleItemSketch create(final String datum) { if ((datum == null) || datum.isEmpty()) { return null; } final byte[] data = datum.getBytes(UTF_8); - return new SingleItemSketch(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1); + return new SingleItemSketch(hash(data, Util.DEFAULT_UPDATE_SEED)[0] >>> 1); } /** @@ -154,7 +155,7 @@ static SingleItemSketch create(final String datum) { */ static SingleItemSketch create(final byte[] data) { if ((data == null) || (data.length == 0)) { return null; } - return new SingleItemSketch(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1); + return new SingleItemSketch(hash(data, Util.DEFAULT_UPDATE_SEED)[0] >>> 1); } /** @@ -169,7 +170,7 @@ static SingleItemSketch create(final byte[] data) { */ static SingleItemSketch create(final char[] data) { if ((data == null) || (data.length == 0)) { return null; } - return new SingleItemSketch(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1); + return new SingleItemSketch(hash(data, Util.DEFAULT_UPDATE_SEED)[0] >>> 1); } /** @@ -181,7 +182,7 @@ static SingleItemSketch create(final char[] data) { */ static SingleItemSketch create(final int[] data) { if ((data == null) || (data.length == 0)) { return null; } - return new SingleItemSketch(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1); + return new SingleItemSketch(hash(data, Util.DEFAULT_UPDATE_SEED)[0] >>> 1); } /** @@ -193,7 +194,7 @@ static SingleItemSketch create(final int[] data) { */ static SingleItemSketch create(final long[] data) { if ((data == null) || (data.length == 0)) { return null; } - return new SingleItemSketch(hash(data, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1); + return new SingleItemSketch(hash(data, Util.DEFAULT_UPDATE_SEED)[0] >>> 1); } //Updates with a user specified seed @@ -378,19 +379,14 @@ int getCurrentPreambleLongs() { return 1; } - @Override - Memory getMemory() { - return null; - } - @Override short getSeedHash() { return (short) (pre0_ >>> 48); } - static final boolean otherCheckForSingleItem(final Memory mem) { - return otherCheckForSingleItem(extractPreLongs(mem), extractSerVer(mem), - extractFamilyID(mem), extractFlags(mem) ); + static final boolean otherCheckForSingleItem(final MemorySegment seg) { + return otherCheckForSingleItem(extractPreLongs(seg), extractSerVer(seg), + extractFamilyID(seg), extractFlags(seg) ); } static final boolean otherCheckForSingleItem(final int preLongs, final int serVer, diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java index cb202a189..6310d82c4 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketch.java +++ b/src/main/java/org/apache/datasketches/theta/Sketch.java @@ -19,6 +19,7 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static org.apache.datasketches.common.Family.idToFamily; import static org.apache.datasketches.common.Util.LONG_MAX_VALUE_AS_DOUBLE; import static org.apache.datasketches.common.Util.LS; @@ -31,11 +32,12 @@ import static org.apache.datasketches.theta.PreambleUtil.SER_VER_BYTE; import static org.apache.datasketches.thetacommon.HashOperations.count; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; -import org.apache.datasketches.common.MemoryStatus; +import org.apache.datasketches.common.MemorySegmentStatus; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.BinomialBoundsN; import org.apache.datasketches.thetacommon.ThetaUtil; @@ -45,64 +47,62 @@ * * @author Lee Rhodes */ -public abstract class Sketch implements MemoryStatus { - static final int DEFAULT_LG_RESIZE_FACTOR = 3; //Unique to Heap +public abstract class Sketch implements MemorySegmentStatus { Sketch() {} //public static factory constructor-type methods /** - * Heapify takes the sketch image in Memory and instantiates an on-heap Sketch. + * Heapify takes the sketch image in MemorySegment and instantiates an on-heap Sketch. * - *

The resulting sketch will not retain any link to the source Memory.

+ *

The resulting sketch will not retain any link to the source MemorySegment.

* *

For Update Sketches this method checks if the * Default Update Seed

- * was used to create the source Memory image. + * was used to create the source MemorySegment image. * *

For Compact Sketches this method assumes that the sketch image was created with the * correct hash seed, so it is not checked.

* - * @param srcMem an image of a Sketch. - * See Memory. + * @param srcSeg an image of a Sketch. + * * @return a Sketch on the heap. */ - public static Sketch heapify(final Memory srcMem) { - final byte familyID = srcMem.getByte(FAMILY_BYTE); + public static Sketch heapify(final MemorySegment srcSeg) { + final byte familyID = srcSeg.get(JAVA_BYTE, FAMILY_BYTE); final Family family = idToFamily(familyID); if (family == Family.COMPACT) { - return CompactSketch.heapify(srcMem); + return CompactSketch.heapify(srcSeg); } - return heapifyUpdateFromMemory(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + return heapifyUpdateFromMemorySegment(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Heapify takes the sketch image in Memory and instantiates an on-heap Sketch. + * Heapify takes the sketch image in MemorySegment and instantiates an on-heap Sketch. * - *

The resulting sketch will not retain any link to the source Memory.

+ *

The resulting sketch will not retain any link to the source MemorySegment.

* *

For Update and Compact Sketches this method checks if the given expectedSeed was used to - * create the source Memory image. However, SerialVersion 1 sketches cannot be checked.

+ * create the source MemorySegment image. However, SerialVersion 1 sketches cannot be checked.

* - * @param srcMem an image of a Sketch that was created using the given expectedSeed. - * See Memory. - * @param expectedSeed the seed used to validate the given Memory image. + * @param srcSeg an image of a Sketch that was created using the given expectedSeed. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. * Compact sketches store a 16-bit hash of the seed, but not the seed itself. * @return a Sketch on the heap. */ - public static Sketch heapify(final Memory srcMem, final long expectedSeed) { - final byte familyID = srcMem.getByte(FAMILY_BYTE); + public static Sketch heapify(final MemorySegment srcSeg, final long expectedSeed) { + final byte familyID = srcSeg.get(JAVA_BYTE, FAMILY_BYTE); final Family family = idToFamily(familyID); if (family == Family.COMPACT) { - return CompactSketch.heapify(srcMem, expectedSeed); + return CompactSketch.heapify(srcSeg, expectedSeed); } - return heapifyUpdateFromMemory(srcMem, expectedSeed); + return heapifyUpdateFromMemorySegment(srcSeg, expectedSeed); } /** - * Wrap takes the sketch image in the given Memory and refers to it directly. + * Wrap takes the sketch image in the given MemorySegment and refers to it directly. * There is no data copying onto the java heap. * The wrap operation enables fast read-only merging and access to all the public read-only API. * @@ -114,41 +114,40 @@ public static Sketch heapify(final Memory srcMem, final long expectedSeed) { * *

Wrapping any subclass of this class that is empty or contains only a single item will * result in on-heap equivalent forms of empty and single item sketch respectively. - * This is actually faster and consumes less overall memory.

+ * This is actually faster and consumes less overall space.

* *

For Update Sketches this method checks if the * Default Update Seed

- * was used to create the source Memory image. + * was used to create the source MemorySegment image. * *

For Compact Sketches this method assumes that the sketch image was created with the * correct hash seed, so it is not checked.

* - * @param srcMem an image of a Sketch. - * See Memory. - * @return a Sketch backed by the given Memory - */ - public static Sketch wrap(final Memory srcMem) { - final int preLongs = srcMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; - final int serVer = srcMem.getByte(SER_VER_BYTE) & 0XFF; - final int familyID = srcMem.getByte(FAMILY_BYTE) & 0XFF; + * @param srcSeg an image of a Sketch. + * @return a Sketch backed by the given MemorySegment + */ + public static Sketch wrap(final MemorySegment srcSeg) { + final int preLongs = srcSeg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; + final int serVer = srcSeg.get(JAVA_BYTE, SER_VER_BYTE) & 0XFF; + final int familyID = srcSeg.get(JAVA_BYTE, FAMILY_BYTE) & 0XFF; final Family family = Family.idToFamily(familyID); if (family == Family.QUICKSELECT) { if (serVer == 3 && preLongs == 3) { - return DirectQuickSelectSketchR.readOnlyWrap(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + return DirectQuickSelectSketchR.readOnlyWrap(srcSeg, Util.DEFAULT_UPDATE_SEED); } else { throw new SketchesArgumentException( "Corrupted: " + family + " family image: must have SerVer = 3 and preLongs = 3"); } } if (family == Family.COMPACT) { - return CompactSketch.wrap(srcMem); + return CompactSketch.wrap(srcSeg); } throw new SketchesArgumentException( "Cannot wrap family: " + family + " as a Sketch"); } /** - * Wrap takes the sketch image in the given Memory and refers to it directly. + * Wrap takes the sketch image in the given MemorySegment and refers to it directly. * There is no data copying onto the java heap. * The wrap operation enables fast read-only merging and access to all the public read-only API. * @@ -160,32 +159,31 @@ public static Sketch wrap(final Memory srcMem) { * *

Wrapping any subclass of this class that is empty or contains only a single item will * result in on-heap equivalent forms of empty and single item sketch respectively. - * This is actually faster and consumes less overall memory.

+ * This is actually faster and consumes less overall space.

* *

For Update and Compact Sketches this method checks if the given expectedSeed was used to - * create the source Memory image. However, SerialVersion 1 sketches cannot be checked.

+ * create the source MemorySegment image. However, SerialVersion 1 sketches cannot be checked.

* - * @param srcMem an image of a Sketch. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * @param srcSeg a MemorySegment with an image of a Sketch. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. - * @return a UpdateSketch backed by the given Memory except as above. + * @return a UpdateSketch backed by the given MemorySegment except as above. */ - public static Sketch wrap(final Memory srcMem, final long expectedSeed) { - final int preLongs = srcMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; - final int serVer = srcMem.getByte(SER_VER_BYTE) & 0XFF; - final int familyID = srcMem.getByte(FAMILY_BYTE) & 0XFF; + public static Sketch wrap(final MemorySegment srcSeg, final long expectedSeed) { + final int preLongs = srcSeg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; + final int serVer = srcSeg.get(JAVA_BYTE, SER_VER_BYTE) & 0XFF; + final int familyID = srcSeg.get(JAVA_BYTE, FAMILY_BYTE) & 0XFF; final Family family = Family.idToFamily(familyID); if (family == Family.QUICKSELECT) { if (serVer == 3 && preLongs == 3) { - return DirectQuickSelectSketchR.readOnlyWrap(srcMem, expectedSeed); + return DirectQuickSelectSketchR.readOnlyWrap(srcSeg, expectedSeed); } else { throw new SketchesArgumentException( "Corrupted: " + family + " family image: must have SerVer = 3 and preLongs = 3"); } } if (family == Family.COMPACT) { - return CompactSketch.wrap(srcMem, expectedSeed); + return CompactSketch.wrap(srcSeg, expectedSeed); } throw new SketchesArgumentException( "Cannot wrap family: " + family + " as a Sketch"); @@ -198,7 +196,7 @@ public static Sketch wrap(final Memory srcMem, final long expectedSeed) { * *

If this.isCompact() == true this method returns this, * otherwise, this method is equivalent to - * {@link #compact(boolean, WritableMemory) compact(true, null)}. + * {@link #compact(boolean, MemorySegment) compact(true, null)}. * *

A CompactSketch is always immutable.

* @@ -223,9 +221,9 @@ public CompactSketch compact() { *

A CompactSketch is always immutable.

* *

A new CompactSketch object is created:

- *
  • if dstMem != null
    • - *
  • if dstMem == null and this.hasMemory() == true
    • - *
  • if dstMem == null and this has more than 1 item and this.isOrdered() == false + *
    • if dstSeg!= null
      • + *
    • if dstSeg == null and this.hasMemorySegment() == true
      • + *
    • if dstSeg == null and this has more than 1 item and this.isOrdered() == false * and dstOrdered == true.
      • *
    * @@ -234,12 +232,12 @@ public CompactSketch compact() { * @param dstOrdered assumed true if this sketch is empty or has only one value * See Destination Ordered * - * @param dstMem - * See Destination Memory. + * @param dstSeg + * See Destination MemorySegment. * * @return this sketch as a CompactSketch. */ - public abstract CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem); + public abstract CompactSketch compact(final boolean dstOrdered, final MemorySegment dstSeg); /** * Returns the number of storage bytes required for this Sketch if its current state were @@ -348,12 +346,12 @@ public int getRetainedEntries() { public abstract int getRetainedEntries(boolean valid); /** - * Returns the serialization version from the given Memory - * @param mem the sketch Memory - * @return the serialization version from the Memory + * Returns the serialization version from the given MemorySegment + * @param seg the sketch MemorySegment + * @return the serialization version from the MemorySegment */ - public static int getSerializationVersion(final Memory mem) { - return mem.getByte(SER_VER_BYTE); + public static int getSerializationVersion(final MemorySegment seg) { + return seg.get(JAVA_BYTE, SER_VER_BYTE); } /** @@ -537,19 +535,19 @@ public static String toString(final byte[] byteArr) { } /** - * Returns a human readable string of the preamble of a Memory image of a Theta Sketch. - * @param mem the given Memory object - * @return a human readable string of the preamble of a Memory image of a Theta Sketch. + * Returns a human readable string of the preamble of a MemorySegment image of a Theta Sketch. + * @param seg the given MemorySegment object + * @return a human readable string of the preamble of a MemorySegment image of a Theta Sketch. */ - public static String toString(final Memory mem) { - return PreambleUtil.preambleToString(mem); + public static String toString(final MemorySegment seg) { + return PreambleUtil.preambleToString(seg); } //Restricted methods /** * Gets the internal cache array. For on-heap sketches this will return a reference to the actual - * cache array. For Memory-based sketches this returns a copy. + * cache array. For MemorySegment-based sketches this returns a copy. * @return the internal cache array. */ abstract long[] getCache(); @@ -574,10 +572,11 @@ public static String toString(final Memory mem) { abstract int getCurrentPreambleLongs(); /** - * Returns the backing Memory object if it exists, otherwise null. - * @return the backing Memory object if it exists, otherwise null. + * Returns the backing MemorySegment object if it exists, otherwise null. + * This is overridden where relevant. + * @return the backing MemorySegment object if it exists, otherwise null. */ - abstract Memory getMemory(); + MemorySegment getMemorySegment() { return null; } /** * Gets the 16-bit seed hash @@ -597,20 +596,20 @@ static final boolean isValidSketchID(final int id) { } /** - * Checks Ordered and Compact flags for integrity between sketch and Memory + * Checks Ordered and Compact flags for integrity between sketch and a MemorySegment * @param sketch the given sketch */ - static final void checkSketchAndMemoryFlags(final Sketch sketch) { - final Memory mem = sketch.getMemory(); - if (mem == null) { return; } - final int flags = PreambleUtil.extractFlags(mem); + static final void checkSketchAndMemorySegmentFlags(final Sketch sketch) { + final MemorySegment seg = sketch.getMemorySegment(); + if (seg == null) { return; } + final int flags = PreambleUtil.extractFlags(seg); if ((flags & COMPACT_FLAG_MASK) > 0 ^ sketch.isCompact()) { throw new SketchesArgumentException("Possible corruption: " - + "Memory Compact Flag inconsistent with Sketch"); + + "MemorySegment Compact Flag inconsistent with Sketch"); } if ((flags & ORDERED_FLAG_MASK) > 0 ^ sketch.isOrdered()) { throw new SketchesArgumentException("Possible corruption: " - + "Memory Ordered Flag inconsistent with Sketch"); + + "MemorySegment Ordered Flag inconsistent with Sketch"); } } @@ -635,32 +634,32 @@ private static final boolean estMode(final long thetaLong, final boolean empty) } /** - * Instantiates a Heap Update Sketch from Memory. Only SerVer3. SerVer 1 & 2 already handled. - * @param srcMem See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * Instantiates a Heap Update Sketch from MemorySegment. Only SerVer3. SerVer 1 & 2 already handled. + * @param srcSeg the source MemorySegment + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. * @return a Sketch */ - private static final Sketch heapifyUpdateFromMemory(final Memory srcMem, final long expectedSeed) { - final long cap = srcMem.getCapacity(); + private static final Sketch heapifyUpdateFromMemorySegment(final MemorySegment srcSeg, final long expectedSeed) { + final long cap = srcSeg.byteSize(); if (cap < 8) { throw new SketchesArgumentException( "Corrupted: valid sketch must be at least 8 bytes."); } - final byte familyID = srcMem.getByte(FAMILY_BYTE); + final byte familyID = srcSeg.get(JAVA_BYTE, FAMILY_BYTE); final Family family = idToFamily(familyID); if (family == Family.ALPHA) { - final int flags = PreambleUtil.extractFlags(srcMem); + final int flags = PreambleUtil.extractFlags(srcSeg); final boolean compactFlag = (flags & COMPACT_FLAG_MASK) != 0; if (compactFlag) { throw new SketchesArgumentException( "Corrupted: ALPHA family image: cannot be compact"); } - return HeapAlphaSketch.heapifyInstance(srcMem, expectedSeed); + return HeapAlphaSketch.heapifyInstance(srcSeg, expectedSeed); } if (family == Family.QUICKSELECT) { - return HeapQuickSelectSketch.heapifyInstance(srcMem, expectedSeed); + return HeapQuickSelectSketch.heapifyInstance(srcSeg, expectedSeed); } throw new SketchesArgumentException( "Sketch cannot heapify family: " + family + " as a Sketch"); diff --git a/src/main/java/org/apache/datasketches/theta/Sketches.java b/src/main/java/org/apache/datasketches/theta/Sketches.java index 2e7fa0915..40c7ccf86 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketches.java +++ b/src/main/java/org/apache/datasketches/theta/Sketches.java @@ -19,6 +19,9 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.FAMILY_BYTE; import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE; @@ -27,11 +30,11 @@ import static org.apache.datasketches.theta.PreambleUtil.SER_VER_BYTE; import static org.apache.datasketches.theta.PreambleUtil.THETA_LONG; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; /** * This class brings together the common sketch and set operation creation methods and @@ -44,31 +47,31 @@ public final class Sketches { private Sketches() {} /** - * Gets the unique count estimate from a valid memory image of a Sketch - * @param srcMem See Memory + * Gets the unique count estimate from a valid MemorySegment image of a Sketch + * @param srcSeg the source MemorySegment * @return the sketch's best estimate of the cardinality of the input stream. */ - public static double getEstimate(final Memory srcMem) { - checkIfValidThetaSketch(srcMem); - return Sketch.estimate(getThetaLong(srcMem), getRetainedEntries(srcMem)); + public static double getEstimate(final MemorySegment srcSeg) { + checkIfValidThetaSketch(srcSeg); + return Sketch.estimate(getThetaLong(srcSeg), getRetainedEntries(srcSeg)); } /** - * Gets the approximate lower error bound from a valid memory image of a Sketch + * Gets the approximate lower error bound from a valid MemorySegment image of a Sketch * given the specified number of Standard Deviations. * This will return getEstimate() if isEmpty() is true. * * @param numStdDev * See Number of Standard Deviations - * @param srcMem See Memory + * @param srcSeg the source MemorySegment * @return the lower bound. */ - public static double getLowerBound(final int numStdDev, final Memory srcMem) { - return Sketch.lowerBound(getRetainedEntries(srcMem), getThetaLong(srcMem), numStdDev, getEmpty(srcMem)); + public static double getLowerBound(final int numStdDev, final MemorySegment srcSeg) { + return Sketch.lowerBound(getRetainedEntries(srcSeg), getThetaLong(srcSeg), numStdDev, getEmpty(srcSeg)); } /** - * Ref: {@link SetOperation#getMaxAnotBResultBytes(int)}. + * Convenience method, ref: {@link SetOperation#getMaxAnotBResultBytes(int)}. * Returns the maximum number of bytes for the returned CompactSketch, given the maximum * value of nomEntries of the first sketch A of AnotB. * @param maxNomEntries the given value @@ -102,7 +105,7 @@ public static int getCompactSketchMaxBytes(final int lgNomEntries) { } /** - * Ref: {@link SetOperation#getMaxIntersectionBytes(int)} + * Convenience method, ref: {@link SetOperation#getMaxIntersectionBytes(int)} * @param nomEntries Ref: {@link SetOperation#getMaxIntersectionBytes(int)}, {@code nomEntries} * @return Ref: {@link SetOperation#getMaxIntersectionBytes(int)} */ @@ -111,7 +114,7 @@ public static int getMaxIntersectionBytes(final int nomEntries) { } /** - * Ref: {@link SetOperation#getMaxUnionBytes(int)} + * Convenience method, ref: {@link SetOperation#getMaxUnionBytes(int)} * @param nomEntries Ref: {@link SetOperation#getMaxUnionBytes(int)}, {@code nomEntries} * @return Ref: {@link SetOperation#getMaxUnionBytes(int)} */ @@ -120,7 +123,7 @@ public static int getMaxUnionBytes(final int nomEntries) { } /** - * Ref: {@link Sketch#getMaxUpdateSketchBytes(int)} + * Convenience method, ref: {@link Sketch#getMaxUpdateSketchBytes(int)} * @param nomEntries Ref: {@link Sketch#getMaxUpdateSketchBytes(int)}, {@code nomEntries} * @return Ref: {@link Sketch#getMaxUpdateSketchBytes(int)} */ @@ -129,133 +132,132 @@ public static int getMaxUpdateSketchBytes(final int nomEntries) { } /** - * Ref: {@link Sketch#getSerializationVersion(Memory)} - * @param srcMem Ref: {@link Sketch#getSerializationVersion(Memory)}, {@code srcMem} - * @return Ref: {@link Sketch#getSerializationVersion(Memory)} + * Convenience method, ref: {@link Sketch#getSerializationVersion(MemorySegment)} + * @param srcSeg Ref: {@link Sketch#getSerializationVersion(MemorySegment)}, {@code srcSeg} + * @return Ref: {@link Sketch#getSerializationVersion(MemorySegment)} */ - public static int getSerializationVersion(final Memory srcMem) { - return Sketch.getSerializationVersion(srcMem); + public static int getSerializationVersion(final MemorySegment srcSeg) { + return Sketch.getSerializationVersion(srcSeg); } /** - * Gets the approximate upper error bound from a valid memory image of a Sketch + * Gets the approximate upper error bound from a valid MemorySegment image of a Sketch * given the specified number of Standard Deviations. * This will return getEstimate() if isEmpty() is true. * * @param numStdDev * See Number of Standard Deviations - * @param srcMem - * See Memory + * @param srcSeg the source MemorySegment * @return the upper bound. */ - public static double getUpperBound(final int numStdDev, final Memory srcMem) { - return Sketch.upperBound(getRetainedEntries(srcMem), getThetaLong(srcMem), numStdDev, getEmpty(srcMem)); + public static double getUpperBound(final int numStdDev, final MemorySegment srcSeg) { + return Sketch.upperBound(getRetainedEntries(srcSeg), getThetaLong(srcSeg), numStdDev, getEmpty(srcSeg)); } //Heapify Operations /** - * Ref: {@link CompactSketch#heapify(Memory) CompactSketch.heapify(Memory)} - * @param srcMem Ref: {@link CompactSketch#heapify(Memory) CompactSketch.heapify(Memory)}, {@code srcMem} + * Convenience method, ref: {@link CompactSketch#heapify(MemorySegment) CompactSketch.heapify(MemorySegment)} + * @param srcSeg Ref: {@link CompactSketch#heapify(MemorySegment) CompactSketch.heapify(MemorySegment)}, {@code srcSeg} * @return {@link CompactSketch CompactSketch} */ - public static CompactSketch heapifyCompactSketch(final Memory srcMem) { - return CompactSketch.heapify(srcMem); + public static CompactSketch heapifyCompactSketch(final MemorySegment srcSeg) { + return CompactSketch.heapify(srcSeg); } /** - * Ref: {@link CompactSketch#heapify(Memory, long) CompactSketch.heapify(Memory, long)} - * @param srcMem Ref: {@link CompactSketch#heapify(Memory, long) CompactSketch.heapify(Memory, long)}, {@code srcMem} - * @param expectedSeed Ref: {@link CompactSketch#heapify(Memory, long) CompactSketch.heapify(Memory, long)}, + * Convenience method, ref: {@link CompactSketch#heapify(MemorySegment, long) CompactSketch.heapify(MemorySegment, long)} + * @param srcSeg Ref: {@link CompactSketch#heapify(MemorySegment, long) CompactSketch.heapify(MemorySegment, long)}, {@code srcSeg} + * @param expectedSeed Ref: {@link CompactSketch#heapify(MemorySegment, long) CompactSketch.heapify(MemorySegment, long)}, * {@code expectedSeed} * @return {@link CompactSketch CompactSketch} */ - public static CompactSketch heapifyCompactSketch(final Memory srcMem, final long expectedSeed) { - return CompactSketch.heapify(srcMem, expectedSeed); + public static CompactSketch heapifyCompactSketch(final MemorySegment srcSeg, final long expectedSeed) { + return CompactSketch.heapify(srcSeg, expectedSeed); } /** - * Ref: {@link CompactSketch#wrap(Memory) CompactSketch.wrap(Memory)} - * @param srcMem Ref: {@link CompactSketch#wrap(Memory) CompactSketch.wrap(Memory)}, {@code srcMem} + * Convenience method, ref: {@link CompactSketch#wrap(MemorySegment) CompactSketch.wrap(MemorySegment)} + * @param srcSeg Ref: {@link CompactSketch#wrap(MemorySegment) CompactSketch.wrap(MemorySegment)}, {@code srcSeg} * @return {@link CompactSketch CompactSketch} */ - public static CompactSketch wrapCompactSketch(final Memory srcMem) { - return CompactSketch.wrap(srcMem); + public static CompactSketch wrapCompactSketch(final MemorySegment srcSeg) { + return CompactSketch.wrap(srcSeg); } /** - * Ref: {@link CompactSketch#wrap(Memory, long) CompactSketch.wrap(Memory, long)} - * @param srcMem Ref: {@link CompactSketch#wrap(Memory, long) CompactSketch.wrap(Memory, long)}, {@code srcMem} - * @param expectedSeed Ref: {@link CompactSketch#wrap(Memory, long) CompactSketch.wrap(Memory, long)}, + * Convenience method, ref: {@link CompactSketch#wrap(MemorySegment, long) CompactSketch.wrap(MemorySegment, long)} + * @param srcSeg Ref: {@link CompactSketch#wrap(MemorySegment, long) CompactSketch.wrap(MemorySegment, long)}, {@code srcSeg} + * @param expectedSeed Ref: {@link CompactSketch#wrap(MemorySegment, long) CompactSketch.wrap(MemorySegment, long)}, * {@code expectedSeed} * @return {@link CompactSketch CompactSketch} */ - public static CompactSketch wrapCompactSketch(final Memory srcMem, final long expectedSeed) { - return CompactSketch.wrap(srcMem, expectedSeed); + public static CompactSketch wrapCompactSketch(final MemorySegment srcSeg, final long expectedSeed) { + return CompactSketch.wrap(srcSeg, expectedSeed); } /** - * Ref: {@link SetOperation#heapify(Memory) SetOperation.heapify(Memory)} - * @param srcMem Ref: {@link SetOperation#heapify(Memory) SetOperation.heapify(Memory)}, {@code srcMem} + * Convenience method, ref: {@link SetOperation#heapify(MemorySegment) SetOperation.heapify(MemorySegment)} + * @param srcSeg Ref: {@link SetOperation#heapify(MemorySegment) SetOperation.heapify(MemorySegment)}, {@code srcSeg} * @return {@link SetOperation SetOperation} */ - public static SetOperation heapifySetOperation(final Memory srcMem) { - return SetOperation.heapify(srcMem); + public static SetOperation heapifySetOperation(final MemorySegment srcSeg) { + return SetOperation.heapify(srcSeg); } /** - * Ref: {@link SetOperation#heapify(Memory, long) SetOperation.heapify(Memory, long)} - * @param srcMem Ref: {@link SetOperation#heapify(Memory, long) SetOperation.heapify(Memory, long)}, - * {@code srcMem} - * @param expectedSeed the seed used to validate the given Memory image. - * Ref: {@link SetOperation#heapify(Memory, long) SetOperation.heapify(Memory, long)}, + * Convenience method, ref: {@link SetOperation#heapify(MemorySegment, long) SetOperation.heapify(MemorySegment, long)} + * @param srcSeg Ref: {@link SetOperation#heapify(MemorySegment, long) SetOperation.heapify(MemorySegment, long)}, + * {@code srcSeg} + * @param expectedSeed the seed used to validate the given MemorySegment image. + * Ref: {@link SetOperation#heapify(MemorySegment, long) SetOperation.heapify(MemorySegment, long)}, * {@code expectedSeed} * @return {@link SetOperation SetOperation} */ - public static SetOperation heapifySetOperation(final Memory srcMem, final long expectedSeed) { - return SetOperation.heapify(srcMem, expectedSeed); + public static SetOperation heapifySetOperation(final MemorySegment srcSeg, final long expectedSeed) { + return SetOperation.heapify(srcSeg, expectedSeed); } /** - * Ref: {@link Sketch#heapify(Memory) Sketch.heapify(Memory)} - * @param srcMem Ref: {@link Sketch#heapify(Memory) Sketch.heapify(Memory)}, {@code srcMem} + * Convenience method, ref: {@link Sketch#heapify(MemorySegment) Sketch.heapify(MemorySegment)} + * @param srcSeg Ref: {@link Sketch#heapify(MemorySegment) Sketch.heapify(MemorySegment)}, {@code srcSeg} * @return {@link Sketch Sketch} */ - public static Sketch heapifySketch(final Memory srcMem) { - return Sketch.heapify(srcMem); + public static Sketch heapifySketch(final MemorySegment srcSeg) { + return Sketch.heapify(srcSeg); } /** - * Ref: {@link Sketch#heapify(Memory, long) Sketch.heapify(Memory, long)} - * @param srcMem Ref: {@link Sketch#heapify(Memory, long) Sketch.heapify(Memory, long)}, {@code srcMem} - * @param expectedSeed the seed used to validate the given Memory image. - * Ref: {@link Sketch#heapify(Memory, long) Sketch.heapify(Memory, long)}, {@code expectedSeed} + * Convenience method, ref: {@link Sketch#heapify(MemorySegment, long) Sketch.heapify(MemorySegment, long)} + * @param srcSeg Ref: {@link Sketch#heapify(MemorySegment, long) Sketch.heapify(MemorySegment, long)}, {@code srcSeg} + * @param expectedSeed the seed used to validate the given MemorySegment image. + * Ref: {@link Sketch#heapify(MemorySegment, long) Sketch.heapify(MemorySegment, long)}, {@code expectedSeed} * @return {@link Sketch Sketch} */ - public static Sketch heapifySketch(final Memory srcMem, final long expectedSeed) { - return Sketch.heapify(srcMem, expectedSeed); + public static Sketch heapifySketch(final MemorySegment srcSeg, final long expectedSeed) { + return Sketch.heapify(srcSeg, expectedSeed); } /** - * Ref: {@link UpdateSketch#heapify(Memory) UpdateSketch.heapify(Memory)} - * @param srcMem Ref: {@link UpdateSketch#heapify(Memory) UpdateSketch.heapify(Memory)}, {@code srcMem} + * Convenience method, ref: {@link UpdateSketch#heapify(MemorySegment) UpdateSketch.heapify(MemorySegment)} + * @param srcSeg Ref: {@link UpdateSketch#heapify(MemorySegment) UpdateSketch.heapify(MemorySegment)}, {@code srcSeg} * @return {@link UpdateSketch UpdateSketch} */ - public static UpdateSketch heapifyUpdateSketch(final Memory srcMem) { - return UpdateSketch.heapify(srcMem); + public static UpdateSketch heapifyUpdateSketch(final MemorySegment srcSeg) { + return UpdateSketch.heapify(srcSeg); } /** - * Ref: {@link UpdateSketch#heapify(Memory, long) UpdateSketch.heapify(Memory, long)} - * @param srcMem Ref: {@link UpdateSketch#heapify(Memory, long) UpdateSketch.heapify(Memory, long)}, - * {@code srcMem} - * @param expectedSeed the seed used to validate the given Memory image. - * Ref: {@link UpdateSketch#heapify(Memory, long) UpdateSketch.heapify(Memory, long)}, + * Convenience method, ref: {@link UpdateSketch#heapify(MemorySegment, long) UpdateSketch.heapify(MemorySegment, long)} + * @param srcSeg Ref: {@link UpdateSketch#heapify(MemorySegment, long) UpdateSketch.heapify(MemorySegment, long)}, + * {@code srcSeg} + * @param expectedSeed the seed used to validate the given MemorySegment image. + * Ref: {@link UpdateSketch#heapify(MemorySegment, long) UpdateSketch.heapify(MemorySegment, long)}, * {@code expectedSeed} * @return {@link UpdateSketch UpdateSketch} */ - public static UpdateSketch heapifyUpdateSketch(final Memory srcMem, final long expectedSeed) { - return UpdateSketch.heapify(srcMem, expectedSeed); + public static UpdateSketch heapifyUpdateSketch(final MemorySegment srcSeg, final long expectedSeed) { + return UpdateSketch.heapify(srcSeg, expectedSeed); } //Builders @@ -279,168 +281,126 @@ public static UpdateSketchBuilder updateSketchBuilder() { //Wrap operations /** - * Convenience method, calls {@link SetOperation#wrap(Memory)} and casts the result to a Intersection - * @param srcMem Ref: {@link SetOperation#wrap(Memory)}, {@code srcMem} - * @return a Intersection backed by the given Memory - */ - public static Intersection wrapIntersection(final Memory srcMem) { - return (Intersection) SetOperation.wrap(srcMem); - } - - /** - * Convenience method, calls {@link SetOperation#wrap(Memory)} and casts the result to a Intersection - * @param srcMem Ref: {@link SetOperation#wrap(Memory)}, {@code srcMem} - * @return a Intersection backed by the given Memory + * Convenience method, ref: {@link SetOperation#wrap(MemorySegment)} + * @param srcSeg Ref: {@link SetOperation#wrap(MemorySegment)}, {@code srcSeg} + * @return a Intersection backed by the given MemorySegment */ - public static Intersection wrapIntersection(final WritableMemory srcMem) { - return (Intersection) SetOperation.wrap(srcMem); + public static Intersection wrapIntersection(final MemorySegment srcSeg) { + return (Intersection) SetOperation.wrap(srcSeg); } /** - * Ref: {@link SetOperation#wrap(Memory) SetOperation.wrap(Memory)} - * @param srcMem Ref: {@link SetOperation#wrap(Memory) SetOperation.wrap(Memory)}, {@code srcMem} - * @return {@link SetOperation SetOperation} - */ - public static SetOperation wrapSetOperation(final Memory srcMem) { - return wrapSetOperation(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); - } - - /** - * Ref: {@link SetOperation#wrap(Memory, long) SetOperation.wrap(Memory, long)} - * @param srcMem Ref: {@link SetOperation#wrap(Memory, long) SetOperation.wrap(Memory, long)}, - * {@code srcMem} - * @param expectedSeed the seed used to validate the given Memory image. - * Ref: {@link SetOperation#wrap(Memory, long) SetOperation.wrap(Memory, long)}, - * {@code expectedSeed} + * Convenience method, ref: {@link SetOperation#wrap(MemorySegment) SetOperation.wrap(MemorySegment)} + * @param srcSeg Ref: {@link SetOperation#wrap(MemorySegment) SetOperation.wrap(MemorySegment)}, {@code srcSeg} * @return {@link SetOperation SetOperation} */ - public static SetOperation wrapSetOperation(final Memory srcMem, final long expectedSeed) { - return SetOperation.wrap(srcMem, expectedSeed); + public static SetOperation wrapSetOperation(final MemorySegment srcSeg) { + return wrapSetOperation(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Ref: {@link SetOperation#wrap(Memory) SetOperation.wrap(Memory)} - * @param srcMem Ref: {@link SetOperation#wrap(Memory) SetOperation.wrap(Memory)}, {@code srcMem} + * Convenience method, ref: {@link SetOperation#wrap(MemorySegment, long) SetOperation.wrap(MemorySegment, long)} + * @param srcSeg Ref: {@link SetOperation#wrap(MemorySegment, long) SetOperation.wrap(MemorySegment, long)}, {@code srcSeg} + * @param expectedSeed the seed used to validate the given MemorySegment image. + * Ref: {@link SetOperation#wrap(MemorySegment, long) SetOperation.wrap(MemorySegment, long)}, {@code expectedSeed} * @return {@link SetOperation SetOperation} */ - public static SetOperation wrapSetOperation(final WritableMemory srcMem) { - return wrapSetOperation(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static SetOperation wrapSetOperation(final MemorySegment srcSeg, final long expectedSeed) { + return SetOperation.wrap(srcSeg, expectedSeed); } /** - * Ref: {@link SetOperation#wrap(Memory, long) SetOperation.wrap(Memory, long)} - * @param srcMem Ref: {@link SetOperation#wrap(Memory, long) SetOperation.wrap(Memory, long)}, - * {@code srcMem} - * @param expectedSeed the seed used to validate the given Memory image. - * Ref: {@link SetOperation#wrap(Memory, long) SetOperation.wrap(Memory, long)}, - * {@code expectedSeed} - * @return {@link SetOperation SetOperation} - */ - public static SetOperation wrapSetOperation(final WritableMemory srcMem, final long expectedSeed) { - return SetOperation.wrap(srcMem, expectedSeed); - } - - /** - * Ref: {@link Sketch#wrap(Memory) Sketch.wrap(Memory)} - * @param srcMem Ref: {@link Sketch#wrap(Memory) Sketch.wrap(Memory)}, {@code srcMem} + * Convenience method, ref: {@link Sketch#wrap(MemorySegment) Sketch.wrap(MemorySegment)} + * @param srcSeg Ref: {@link Sketch#wrap(MemorySegment) Sketch.wrap(MemorySegment)}, {@code srcSeg} * @return {@link Sketch Sketch} */ - public static Sketch wrapSketch(final Memory srcMem) { - return Sketch.wrap(srcMem); + public static Sketch wrapSketch(final MemorySegment srcSeg) { + return Sketch.wrap(srcSeg); } /** - * Ref: {@link Sketch#wrap(Memory, long) Sketch.wrap(Memory, long)} - * @param srcMem Ref: {@link Sketch#wrap(Memory, long) Sketch.wrap(Memory, long)}, {@code srcMem} - * @param expectedSeed the expectedSeed used to validate the given Memory image. - * Ref: {@link Sketch#wrap(Memory, long) Sketch.wrap(Memory, long)}, {@code expectedSeed} + * Convenience method, ref: {@link Sketch#wrap(MemorySegment, long) Sketch.wrap(MemorySegment, long)} + * @param srcSeg Ref: {@link Sketch#wrap(MemorySegment, long) Sketch.wrap(MemorySegment, long)}, {@code srcSeg} + * @param expectedSeed the expectedSeed used to validate the given MemorySegment image. + * Ref: {@link Sketch#wrap(MemorySegment, long) Sketch.wrap(MemorySegment, long)}, {@code expectedSeed} * @return {@link Sketch Sketch} */ - public static Sketch wrapSketch(final Memory srcMem, final long expectedSeed) { - return Sketch.wrap(srcMem, expectedSeed); - } - - /** - * Convenience method, calls {@link SetOperation#wrap(Memory)} and casts the result to a Union - * @param srcMem Ref: {@link SetOperation#wrap(Memory)}, {@code srcMem} - * @return a Union backed by the given Memory - */ - public static Union wrapUnion(final Memory srcMem) { - return (Union) SetOperation.wrap(srcMem); + public static Sketch wrapSketch(final MemorySegment srcSeg, final long expectedSeed) { + return Sketch.wrap(srcSeg, expectedSeed); } /** - * Convenience method, calls {@link SetOperation#wrap(Memory)} and casts the result to a Union - * @param srcMem Ref: {@link SetOperation#wrap(Memory)}, {@code srcMem} - * @return a Union backed by the given Memory + * Convenience method, ref: {@link SetOperation#wrap(MemorySegment)} and casts the result to a Union + * @param srcSeg Ref: {@link SetOperation#wrap(MemorySegment)}, {@code srcSeg} + * @return a Union backed by the given MemorySegment. */ - public static Union wrapUnion(final WritableMemory srcMem) { - return (Union) SetOperation.wrap(srcMem); + public static Union wrapUnion(final MemorySegment srcSeg) { + return (Union) SetOperation.wrap(srcSeg); } /** - * Ref: {@link UpdateSketch#wrap(Memory) UpdateSketch.wrap(Memory)} - * @param srcMem Ref: {@link UpdateSketch#wrap(Memory) UpdateSketch.wrap(Memory)}, {@code srcMem} + * Convenience method, ref: {@link UpdateSketch#wrap(MemorySegment) UpdateSketch.wrap(MemorySegment)} + * @param srcSeg Ref: {@link UpdateSketch#wrap(MemorySegment) UpdateSketch.wrap(MemorySegment)}, {@code srcSeg} * @return {@link UpdateSketch UpdateSketch} */ - public static UpdateSketch wrapUpdateSketch(final WritableMemory srcMem) { - return wrapUpdateSketch(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static UpdateSketch wrapUpdateSketch(final MemorySegment srcSeg) { + return wrapUpdateSketch(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Ref: {@link UpdateSketch#wrap(Memory, long) UpdateSketch.wrap(Memory, long)} - * @param srcMem Ref: {@link UpdateSketch#wrap(Memory, long) UpdateSketch.wrap(Memory, long)}, {@code srcMem} - * @param expectedSeed the seed used to validate the given Memory image. - * Ref: {@link UpdateSketch#wrap(Memory, long) UpdateSketch.wrap(Memory, long)}, {@code expectedSeed} + * Convenience method, ref: {@link UpdateSketch#wrap(MemorySegment, long) UpdateSketch.wrap(MemorySegment, long)} + * @param srcSeg Ref: {@link UpdateSketch#wrap(MemorySegment, long) UpdateSketch.wrap(MemorySegment, long)}, {@code srcSeg} + * @param expectedSeed the seed used to validate the given MemorySegment image. + * Ref: {@link UpdateSketch#wrap(MemorySegment, long) UpdateSketch.wrap(MemorySegment, long)}, {@code expectedSeed} * @return {@link UpdateSketch UpdateSketch} */ - public static UpdateSketch wrapUpdateSketch(final WritableMemory srcMem, final long expectedSeed) { - return UpdateSketch.wrap(srcMem, expectedSeed); + public static UpdateSketch wrapUpdateSketch(final MemorySegment srcSeg, final long expectedSeed) { + return UpdateSketch.wrap(srcSeg, expectedSeed); } //Restricted static methods - static void checkIfValidThetaSketch(final Memory srcMem) { - final int fam = srcMem.getByte(FAMILY_BYTE); + static void checkIfValidThetaSketch(final MemorySegment srcSeg) { + final int fam = srcSeg.get(JAVA_BYTE, FAMILY_BYTE); if (!Sketch.isValidSketchID(fam)) { - throw new SketchesArgumentException("Source Memory not a valid Sketch. Family: " + throw new SketchesArgumentException("Source MemorySegment not a valid Sketch. Family: " + Family.idToFamily(fam).toString()); } } - static boolean getEmpty(final Memory srcMem) { - final int serVer = srcMem.getByte(SER_VER_BYTE); + static boolean getEmpty(final MemorySegment srcSeg) { + final int serVer = srcSeg.get(JAVA_BYTE, SER_VER_BYTE); if (serVer == 1) { - return ((getThetaLong(srcMem) == Long.MAX_VALUE) && (getRetainedEntries(srcMem) == 0)); + return ((getThetaLong(srcSeg) == Long.MAX_VALUE) && (getRetainedEntries(srcSeg) == 0)); } - return (srcMem.getByte(FLAGS_BYTE) & EMPTY_FLAG_MASK) != 0; //for SerVer 2 & 3 + return (srcSeg.get(JAVA_BYTE, FLAGS_BYTE) & EMPTY_FLAG_MASK) != 0; //for SerVer 2 & 3 } - static int getPreambleLongs(final Memory srcMem) { - return srcMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; //for SerVer 1,2,3 + static int getPreambleLongs(final MemorySegment srcSeg) { + return srcSeg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; //for SerVer 1,2,3 } - static int getRetainedEntries(final Memory srcMem) { - final int serVer = srcMem.getByte(SER_VER_BYTE); + static int getRetainedEntries(final MemorySegment srcSeg) { + final int serVer = srcSeg.get(JAVA_BYTE, SER_VER_BYTE); if (serVer == 1) { - final int entries = srcMem.getInt(RETAINED_ENTRIES_INT); - if ((getThetaLong(srcMem) == Long.MAX_VALUE) && (entries == 0)) { + final int entries = srcSeg.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT); + if ((getThetaLong(srcSeg) == Long.MAX_VALUE) && (entries == 0)) { return 0; } return entries; } //SerVer 2 or 3 - final int preLongs = getPreambleLongs(srcMem); - final boolean empty = (srcMem.getByte(FLAGS_BYTE) & EMPTY_FLAG_MASK) != 0; //for SerVer 2 & 3 + final int preLongs = getPreambleLongs(srcSeg); + final boolean empty = (srcSeg.get(JAVA_BYTE, FLAGS_BYTE) & EMPTY_FLAG_MASK) != 0; //for SerVer 2 & 3 if (preLongs == 1) { return empty ? 0 : 1; } //preLongs > 1 - return srcMem.getInt(RETAINED_ENTRIES_INT); //for SerVer 1,2,3 + return srcSeg.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT); //for SerVer 1,2,3 } - static long getThetaLong(final Memory srcMem) { - final int preLongs = getPreambleLongs(srcMem); - return (preLongs < 3) ? Long.MAX_VALUE : srcMem.getLong(THETA_LONG); //for SerVer 1,2,3 + static long getThetaLong(final MemorySegment srcSeg) { + final int preLongs = getPreambleLongs(srcSeg); + return (preLongs < 3) ? Long.MAX_VALUE : srcSeg.get(JAVA_LONG_UNALIGNED, THETA_LONG); //for SerVer 1,2,3 } } diff --git a/src/main/java/org/apache/datasketches/theta/Union.java b/src/main/java/org/apache/datasketches/theta/Union.java index 7a2ff9c5b..7503729cb 100644 --- a/src/main/java/org/apache/datasketches/theta/Union.java +++ b/src/main/java/org/apache/datasketches/theta/Union.java @@ -19,11 +19,15 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static org.apache.datasketches.theta.PreambleUtil.SER_VER_BYTE; + +import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import org.apache.datasketches.common.Family; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.common.Util; /** * Compute the union of two or more theta sketches. @@ -33,6 +37,62 @@ */ public abstract class Union extends SetOperation { + /** + * Wrap a Union object around a Union MemorySegment object containing data. + * This method assumes the Default Update Seed. + * This does NO validity checking of the given MemorySegment. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg The source MemorySegment object. + * @return this class + */ + public static Union fastWrap(final MemorySegment srcSeg) { + return fastWrap(srcSeg, Util.DEFAULT_UPDATE_SEED); + } + + /** + * Wrap a Union object around a Union MemorySegment object containing data. + * This does NO validity checking of the given MemorySegment. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg The source MemorySegment object. + * @param expectedSeed the seed used to validate the given MemorySegment image. + * See seed + * @return this class + */ + public static Union fastWrap(final MemorySegment srcSeg, final long expectedSeed) { + final int serVer = srcSeg.get(JAVA_BYTE, SER_VER_BYTE); + if (serVer != 3) { + throw new SketchesArgumentException("SerVer must be 3: " + serVer); + } + return UnionImpl.fastWrapInstance(srcSeg, expectedSeed); + } + + /** + * Wrap a Union object around a Union MemorySegment object containing data. + * This method assumes the Default Update Seed. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg The source MemorySegment object. + * @return this class + */ + public static Union wrap(final MemorySegment srcSeg) { + return wrap(srcSeg, Util.DEFAULT_UPDATE_SEED); + } + + /** + * Wrap a Union object around a Union MemorySegment object containing data. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg The source MemorySegment object. + * @param expectedSeed the seed used to validate the given MemorySegment image. + * See seed + * @return this class + */ + public static Union wrap(final MemorySegment srcSeg, final long expectedSeed) { + final int serVer = srcSeg.get(JAVA_BYTE, SER_VER_BYTE); + if (serVer != 3) { + throw new SketchesArgumentException("SerVer must be 3: " + serVer); + } + return UnionImpl.wrapInstance(srcSeg, expectedSeed); + } + /** * Returns the number of storage bytes required for this union in its current state. * @@ -67,12 +127,11 @@ public Family getFamily() { * @param dstOrdered * See Destination Ordered * - * @param dstMem - * See Destination Memory. + * @param dstSeg destination MemorySegment * * @return the result of this operation as a CompactSketch of the chosen form */ - public abstract CompactSketch getResult(boolean dstOrdered, WritableMemory dstMem); + public abstract CompactSketch getResult(boolean dstOrdered, MemorySegment dstSeg); /** * Resets this Union. The seed remains intact, everything else reverts back to its virgin state. @@ -108,11 +167,11 @@ public CompactSketch union(final Sketch sketchA, final Sketch sketchB) { * @param sketchA The first argument * @param sketchB The second argument * @param dstOrdered If true, the returned CompactSketch will be ordered. - * @param dstMem If not null, the returned CompactSketch will be placed in this WritableMemory. + * @param dstSeg If not null, the returned CompactSketch will be placed in this MemorySegment. * @return the result CompactSketch. */ public abstract CompactSketch union(Sketch sketchA, Sketch sketchB, boolean dstOrdered, - WritableMemory dstMem); + MemorySegment dstSeg); /** * Perform a Union operation with this union and the given on-heap sketch of the Theta Family. @@ -127,7 +186,7 @@ public abstract CompactSketch union(Sketch sketchA, Sketch sketchB, boolean dstO public abstract void union(Sketch sketchIn); /** - * Perform a Union operation with this union and the given Memory image of any sketch of the + * Perform a Union operation with this union and the given MemorySegment image of any sketch of the * Theta Family. The input image may be from earlier versions of the Theta Compact Sketch, * called the SetSketch (circa 2014), which was prior to Open Source and are compact and ordered. * @@ -135,9 +194,9 @@ public abstract CompactSketch union(Sketch sketchA, Sketch sketchB, boolean dstO * *

    Nulls and empty sketches are ignored.

    * - * @param mem Memory image of sketch to be merged + * @param seg MemorySegment image of sketch to be merged */ - public abstract void union(Memory mem); + public abstract void union(MemorySegment seg); /** * Update this union with the given long data item. diff --git a/src/main/java/org/apache/datasketches/theta/UnionImpl.java b/src/main/java/org/apache/datasketches/theta/UnionImpl.java index 8c5b2f8f0..387ee3455 100644 --- a/src/main/java/org/apache/datasketches/theta/UnionImpl.java +++ b/src/main/java/org/apache/datasketches/theta/UnionImpl.java @@ -20,22 +20,22 @@ package org.apache.datasketches.theta; import static java.lang.Math.min; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static org.apache.datasketches.common.QuickSelect.selectExcludingZeros; import static org.apache.datasketches.theta.PreambleUtil.UNION_THETA_LONG; import static org.apache.datasketches.theta.PreambleUtil.clearEmpty; import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID; import static org.apache.datasketches.theta.PreambleUtil.extractUnionThetaLong; import static org.apache.datasketches.theta.PreambleUtil.insertUnionThetaLong; -import static org.apache.datasketches.thetacommon.QuickSelect.selectExcludingZeros; +import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; +import java.util.Objects; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.MemoryRequestServer; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.HashOperations; -import org.apache.datasketches.thetacommon.ThetaUtil; /** * Shared code for the HeapUnion and DirectUnion implementations. @@ -61,7 +61,7 @@ final class UnionImpl extends Union { private UnionImpl(final UpdateSketch gadget, final long seed) { gadget_ = gadget; - expectedSeedHash_ = ThetaUtil.computeSeedHash(seed); + expectedSeedHash_ = Util.computeSeedHash(seed); } /** @@ -88,15 +88,14 @@ static UnionImpl initNewHeapInstance( } /** - * Construct a new Direct Union in the off-heap destination Memory. + * Construct a new Direct Union in the destination MemorySegment. * Called by SetOperationBuilder. * * @param lgNomLongs See lgNomLongs. * @param seed See seed * @param p See Sampling Probability, p * @param rf See Resize Factor - * @param memReqSvr a given instance of a MemoryRequestServer - * @param dstMem the given Memory object destination. It will be cleared prior to use. + * @param dstSeg the given MemorySegment object destination. It will be cleared prior to use. * @return this class */ static UnionImpl initNewDirectInstance( @@ -104,10 +103,9 @@ static UnionImpl initNewDirectInstance( final long seed, final float p, final ResizeFactor rf, - final MemoryRequestServer memReqSvr, - final WritableMemory dstMem) { + final MemorySegment dstSeg) { final UpdateSketch gadget = //create with UNION family - new DirectQuickSelectSketch(lgNomLongs, seed, p, rf, memReqSvr, dstMem, true); + new DirectQuickSelectSketch(lgNomLongs, seed, p, rf, dstSeg, true); final UnionImpl unionImpl = new UnionImpl(gadget, seed); unionImpl.unionThetaLong_ = gadget.getThetaLong(); unionImpl.unionEmpty_ = gadget.isEmpty(); @@ -115,92 +113,58 @@ static UnionImpl initNewDirectInstance( } /** - * Heapify a Union from a Memory Union object containing data. + * Heapify a Union from a MemorySegment Union object containing data. * Called by SetOperation. - * @param srcMem The source Memory Union object. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * @param srcSeg The source MemorySegment Union object. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See seed * @return this class */ - static UnionImpl heapifyInstance(final Memory srcMem, final long expectedSeed) { - Family.UNION.checkFamilyID(extractFamilyID(srcMem)); - final UpdateSketch gadget = HeapQuickSelectSketch.heapifyInstance(srcMem, expectedSeed); + static UnionImpl heapifyInstance(final MemorySegment srcSeg, final long expectedSeed) { + final MemorySegment srcSegRO = srcSeg.asReadOnly(); + Family.UNION.checkFamilyID(extractFamilyID(srcSegRO)); + final UpdateSketch gadget = HeapQuickSelectSketch.heapifyInstance(srcSegRO, expectedSeed); final UnionImpl unionImpl = new UnionImpl(gadget, expectedSeed); - unionImpl.unionThetaLong_ = extractUnionThetaLong(srcMem); - unionImpl.unionEmpty_ = PreambleUtil.isEmptyFlag(srcMem); + unionImpl.unionThetaLong_ = extractUnionThetaLong(srcSegRO); + unionImpl.unionEmpty_ = PreambleUtil.isEmptyFlag(srcSegRO); return unionImpl; } /** - * Fast-wrap a Union object around a Union Memory object containing data. - * This does NO validity checking of the given Memory. - * @param srcMem The source Memory object. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * Fast-wrap a Union object around a Union MemorySegment object containing data. + * This does NO validity checking of the given MemorySegment. + * @param srcSeg The source MemorySegment object. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See seed * @return this class */ - static UnionImpl fastWrap(final Memory srcMem, final long expectedSeed) { - Family.UNION.checkFamilyID(extractFamilyID(srcMem)); - final UpdateSketch gadget = DirectQuickSelectSketchR.fastReadOnlyWrap(srcMem, expectedSeed); + static UnionImpl fastWrapInstance(final MemorySegment srcSeg, final long expectedSeed) { + Family.UNION.checkFamilyID(extractFamilyID(srcSeg)); + final UpdateSketch gadget = srcSeg.isReadOnly() + ? DirectQuickSelectSketchR.fastReadOnlyWrap(srcSeg, expectedSeed) + : DirectQuickSelectSketch.fastWritableWrap(srcSeg, expectedSeed); final UnionImpl unionImpl = new UnionImpl(gadget, expectedSeed); - unionImpl.unionThetaLong_ = extractUnionThetaLong(srcMem); - unionImpl.unionEmpty_ = PreambleUtil.isEmptyFlag(srcMem); + unionImpl.unionThetaLong_ = extractUnionThetaLong(srcSeg); + unionImpl.unionEmpty_ = PreambleUtil.isEmptyFlag(srcSeg); return unionImpl; } /** - * Fast-wrap a Union object around a Union WritableMemory object containing data. - * This does NO validity checking of the given Memory. - * @param srcMem The source Memory object. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. - * See seed - * @return this class - */ - static UnionImpl fastWrap(final WritableMemory srcMem, final long expectedSeed) { - Family.UNION.checkFamilyID(extractFamilyID(srcMem)); - final UpdateSketch gadget = DirectQuickSelectSketch.fastWritableWrap(srcMem, expectedSeed); - final UnionImpl unionImpl = new UnionImpl(gadget, expectedSeed); - unionImpl.unionThetaLong_ = extractUnionThetaLong(srcMem); - unionImpl.unionEmpty_ = PreambleUtil.isEmptyFlag(srcMem); - return unionImpl; - } - - /** - * Wrap a Union object around a Union Memory object containing data. + * Wrap a Union object around a Union MemorySegment object containing data. * Called by SetOperation. - * @param srcMem The source Memory object. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * @param srcSeg The source MemorySegment object. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See seed * @return this class */ - static UnionImpl wrapInstance(final Memory srcMem, final long expectedSeed) { - Family.UNION.checkFamilyID(extractFamilyID(srcMem)); - final UpdateSketch gadget = DirectQuickSelectSketchR.readOnlyWrap(srcMem, expectedSeed); + static UnionImpl wrapInstance(final MemorySegment srcSeg, final long expectedSeed) { + Family.UNION.checkFamilyID(extractFamilyID(srcSeg)); + final UpdateSketch gadget = srcSeg.isReadOnly() + ? DirectQuickSelectSketchR.readOnlyWrap(srcSeg, expectedSeed) + : DirectQuickSelectSketch.writableWrap(srcSeg, expectedSeed); final UnionImpl unionImpl = new UnionImpl(gadget, expectedSeed); - unionImpl.unionThetaLong_ = extractUnionThetaLong(srcMem); - unionImpl.unionEmpty_ = PreambleUtil.isEmptyFlag(srcMem); - return unionImpl; - } - - /** - * Wrap a Union object around a Union WritableMemory object containing data. - * Called by SetOperation. - * @param srcMem The source Memory object. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. - * See seed - * @return this class - */ - static UnionImpl wrapInstance(final WritableMemory srcMem, final long expectedSeed) { - Family.UNION.checkFamilyID(extractFamilyID(srcMem)); - final UpdateSketch gadget = DirectQuickSelectSketch.writableWrap(srcMem, expectedSeed); - final UnionImpl unionImpl = new UnionImpl(gadget, expectedSeed); - unionImpl.unionThetaLong_ = extractUnionThetaLong(srcMem); - unionImpl.unionEmpty_ = PreambleUtil.isEmptyFlag(srcMem); + unionImpl.unionThetaLong_ = extractUnionThetaLong(srcSeg); + unionImpl.unionEmpty_ = PreambleUtil.isEmptyFlag(srcSeg); return unionImpl; } @@ -215,17 +179,22 @@ public int getMaxUnionBytes() { return (16 << lgK) + (Family.UNION.getMaxPreLongs() << 3); } + @Override + MemorySegment getMemorySegment() { + return hasMemorySegment() ? gadget_.getMemorySegment() : null; + } + @Override public CompactSketch getResult() { return getResult(true, null); } @Override - public CompactSketch getResult(final boolean dstOrdered, final WritableMemory dstMem) { + public CompactSketch getResult(final boolean dstOrdered, final MemorySegment dstSeg) { final int gadgetCurCount = gadget_.getRetainedEntries(true); final int k = 1 << gadget_.getLgNomLongs(); final long[] gadgetCacheCopy = - gadget_.hasMemory() ? gadget_.getCache() : gadget_.getCache().clone(); + gadget_.hasMemorySegment() ? gadget_.getCache() : gadget_.getCache().clone(); //Pull back to k final long curGadgetThetaLong = gadget_.getThetaLong(); @@ -233,8 +202,9 @@ public CompactSketch getResult(final boolean dstOrdered, final WritableMemory ds ? selectExcludingZeros(gadgetCacheCopy, gadgetCurCount, k + 1) : curGadgetThetaLong; //Finalize Theta and curCount - final long unionThetaLong = gadget_.hasMemory() - ? gadget_.getMemory().getLong(UNION_THETA_LONG) : unionThetaLong_; + final long unionThetaLong = gadget_.hasMemorySegment() + ? gadget_.getMemorySegment().get(JAVA_LONG_UNALIGNED, UNION_THETA_LONG) + : unionThetaLong_; final long minThetaLong = min(min(curGadgetThetaLong, adjGadgetThetaLong), unionThetaLong); final int curCountOut = minThetaLong < curGadgetThetaLong @@ -247,25 +217,22 @@ public CompactSketch getResult(final boolean dstOrdered, final WritableMemory ds final boolean empty = gadget_.isEmpty() && unionEmpty_; final short seedHash = gadget_.getSeedHash(); return CompactOperations.componentsToCompact( - minThetaLong, curCountOut, seedHash, empty, true, dstOrdered, dstOrdered, dstMem, compactCacheOut); + minThetaLong, curCountOut, seedHash, empty, true, dstOrdered, dstOrdered, dstSeg, compactCacheOut); } @Override - public boolean hasMemory() { - return gadget_ instanceof DirectQuickSelectSketchR - ? gadget_.hasMemory() : false; + public boolean hasMemorySegment() { + return gadget_.hasMemorySegment(); } @Override public boolean isDirect() { - return gadget_ instanceof DirectQuickSelectSketchR - ? gadget_.isDirect() : false; + return gadget_.isDirect(); } @Override - public boolean isSameResource(final Memory that) { - return gadget_ instanceof DirectQuickSelectSketchR - ? gadget_.isSameResource(that) : false; + public boolean isSameResource(final MemorySegment that) { + return gadget_.isSameResource(that); } @Override @@ -278,10 +245,10 @@ public void reset() { @Override public byte[] toByteArray() { final byte[] gadgetByteArr = gadget_.toByteArray(); - final WritableMemory mem = WritableMemory.writableWrap(gadgetByteArr); - insertUnionThetaLong(mem, unionThetaLong_); + final MemorySegment seg = MemorySegment.ofArray(gadgetByteArr); + insertUnionThetaLong(seg, unionThetaLong_); if (gadget_.isEmpty() != unionEmpty_) { - clearEmpty(mem); + clearEmpty(seg); unionEmpty_ = false; } return gadgetByteArr; @@ -289,11 +256,11 @@ public byte[] toByteArray() { @Override //Stateless Union public CompactSketch union(final Sketch sketchA, final Sketch sketchB, final boolean dstOrdered, - final WritableMemory dstMem) { + final MemorySegment dstSeg) { reset(); union(sketchA); union(sketchB); - final CompactSketch csk = getResult(dstOrdered, dstMem); + final CompactSketch csk = getResult(dstOrdered, dstSeg); reset(); return csk; } @@ -307,12 +274,12 @@ public void union(final Sketch sketchIn) { return; } //sketchIn is valid and not empty - ThetaUtil.checkSeedHashes(expectedSeedHash_, sketchIn.getSeedHash()); + Util.checkSeedHashes(expectedSeedHash_, sketchIn.getSeedHash()); if (sketchIn instanceof SingleItemSketch) { gadget_.hashUpdate(sketchIn.getCache()[0]); return; } - Sketch.checkSketchAndMemoryFlags(sketchIn); + Sketch.checkSketchAndMemorySegmentFlags(sketchIn); unionThetaLong_ = min(min(unionThetaLong_, sketchIn.getThetaLong()), gadget_.getThetaLong()); //Theta rule unionEmpty_ = false; @@ -327,18 +294,17 @@ public void union(final Sketch sketchIn) { } } unionThetaLong_ = min(unionThetaLong_, gadget_.getThetaLong()); //Theta rule with gadget - if (gadget_.hasMemory()) { - final WritableMemory wmem = (WritableMemory)gadget_.getMemory(); - PreambleUtil.insertUnionThetaLong(wmem, unionThetaLong_); - PreambleUtil.clearEmpty(wmem); + if (gadget_.hasMemorySegment()) { + final MemorySegment wseg = gadget_.getMemorySegment(); + PreambleUtil.insertUnionThetaLong(wseg, unionThetaLong_); + PreambleUtil.clearEmpty(wseg); } } @Override - public void union(final Memory skMem) { - if (skMem != null) { - union(Sketch.wrap(skMem)); - } + public void union(final MemorySegment seg) { + Objects.requireNonNull(seg, "MemorySegment must be non-null"); + union(Sketch.wrap(seg.asReadOnly())); } @Override diff --git a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java index cb6854b02..30d3b6f49 100644 --- a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java +++ b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java @@ -19,6 +19,7 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.datasketches.common.Util.LONG_MAX_VALUE_AS_DOUBLE; import static org.apache.datasketches.common.Util.checkBounds; @@ -32,24 +33,24 @@ import static org.apache.datasketches.theta.PreambleUtil.READ_ONLY_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.SER_VER; import static org.apache.datasketches.theta.PreambleUtil.SER_VER_BYTE; -import static org.apache.datasketches.theta.PreambleUtil.checkMemorySeedHash; +import static org.apache.datasketches.theta.PreambleUtil.checkSegmentSeedHash; import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID; import static org.apache.datasketches.theta.PreambleUtil.extractFlags; import static org.apache.datasketches.theta.PreambleUtil.extractLgResizeFactor; import static org.apache.datasketches.theta.PreambleUtil.extractP; import static org.apache.datasketches.theta.PreambleUtil.extractSerVer; import static org.apache.datasketches.theta.PreambleUtil.extractThetaLong; -import static org.apache.datasketches.theta.PreambleUtil.getMemBytes; +import static org.apache.datasketches.theta.PreambleUtil.getSegBytes; import static org.apache.datasketches.theta.UpdateReturnState.RejectedNullOrEmpty; +import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import java.util.Objects; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -64,47 +65,45 @@ public abstract class UpdateSketch extends Sketch { UpdateSketch() {} /** - * Wrap takes the sketch image in Memory and refers to it directly. There is no data copying onto + * Wrap takes the writable sketch image in MemorySegment and refers to it directly. There is no data copying onto * the java heap. Only "Direct" Serialization Version 3 (i.e, OpenSource) sketches that have - * been explicitly stored as direct objects can be wrapped. This method assumes the - * {@link org.apache.datasketches.thetacommon.ThetaUtil#DEFAULT_UPDATE_SEED}. + * been explicitly stored as writable, direct objects can be wrapped. This method assumes the + * {@link org.apache.datasketches.common.Util#DEFAULT_UPDATE_SEED}. * Default Update Seed. - * @param srcMem an image of a Sketch where the image seed hash matches the default seed hash. + * @param srcWSeg an image of a writable sketch where the image seed hash matches the default seed hash. * It must have a size of at least 24 bytes. - * See Memory - * @return a Sketch backed by the given Memory + * @return an UpdateSketch backed by the given MemorySegment */ - public static UpdateSketch wrap(final WritableMemory srcMem) { - return wrap(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static UpdateSketch wrap(final MemorySegment srcWSeg) { + return wrap(srcWSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Wrap takes the sketch image in Memory and refers to it directly. There is no data copying onto + * Wrap takes the sketch image in MemorySegment and refers to it directly. There is no data copying onto * the java heap. Only "Direct" Serialization Version 3 (i.e, OpenSource) sketches that have - * been explicitly stored as direct objects can be wrapped. + * been explicitly stored as writable direct objects can be wrapped. * An attempt to "wrap" earlier version sketches will result in a "heapified", normal * Java Heap version of the sketch where all data will be copied to the heap. - * @param srcMem an image of a Sketch where the image seed hash matches the given seed hash. + * @param srcWSeg an image of a writable sketch where the image seed hash matches the given seed hash. * It must have a size of at least 24 bytes. - * See Memory - * @param expectedSeed the seed used to validate the given Memory image. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. * Compact sketches store a 16-bit hash of the seed, but not the seed itself. - * @return a UpdateSketch backed by the given Memory + * @return a UpdateSketch backed by the given MemorySegment */ - public static UpdateSketch wrap(final WritableMemory srcMem, final long expectedSeed) { - Objects.requireNonNull(srcMem, "Source Memory must not be null"); - checkBounds(0, 24, srcMem.getCapacity()); //need min 24 bytes - final int preLongs = srcMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F; - final int serVer = srcMem.getByte(SER_VER_BYTE) & 0XFF; - final int familyID = srcMem.getByte(FAMILY_BYTE) & 0XFF; + public static UpdateSketch wrap(final MemorySegment srcWSeg, final long expectedSeed) { + Objects.requireNonNull(srcWSeg, "Source MemorySeg e t must not be null"); + checkBounds(0, 24, srcWSeg.byteSize()); //need min 24 bytes + final int preLongs = srcWSeg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE) & 0X3F; + final int serVer = srcWSeg.get(JAVA_BYTE, SER_VER_BYTE) & 0XFF; + final int familyID = srcWSeg.get(JAVA_BYTE, FAMILY_BYTE) & 0XFF; final Family family = Family.idToFamily(familyID); if (family != Family.QUICKSELECT) { throw new SketchesArgumentException( "A " + family + " sketch cannot be wrapped as an UpdateSketch."); } if ((serVer == 3) && (preLongs == 3)) { - return DirectQuickSelectSketch.writableWrap(srcMem, expectedSeed); + return DirectQuickSelectSketch.writableWrap(srcWSeg, expectedSeed); } else { throw new SketchesArgumentException( "Corrupted: An UpdateSketch image must have SerVer = 3 and preLongs = 3"); @@ -112,40 +111,40 @@ public static UpdateSketch wrap(final WritableMemory srcMem, final long expected } /** - * Instantiates an on-heap UpdateSketch from Memory. This method assumes the - * {@link org.apache.datasketches.thetacommon.ThetaUtil#DEFAULT_UPDATE_SEED}. - * @param srcMem See Memory + * Instantiates an on-heap UpdateSketch from a MemorySegment. This method assumes the + * {@link org.apache.datasketches.common.Util#DEFAULT_UPDATE_SEED}. + * @param srcSeg the given MemorySegment with a sketch image. * It must have a size of at least 24 bytes. * @return an UpdateSketch */ - public static UpdateSketch heapify(final Memory srcMem) { - return heapify(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static UpdateSketch heapify(final MemorySegment srcSeg) { + return heapify(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Instantiates an on-heap UpdateSketch from Memory. - * @param srcMem See Memory + * Instantiates an on-heap UpdateSketch from a MemorySegment. + * @param srcSeg the given MemorySegment. * It must have a size of at least 24 bytes. - * @param expectedSeed the seed used to validate the given Memory image. + * @param expectedSeed the seed used to validate the given MemorySegment image. * See Update Hash Seed. * @return an UpdateSketch */ - public static UpdateSketch heapify(final Memory srcMem, final long expectedSeed) { - Objects.requireNonNull(srcMem, "Source Memory must not be null"); - checkBounds(0, 24, srcMem.getCapacity()); //need min 24 bytes - final Family family = Family.idToFamily(srcMem.getByte(FAMILY_BYTE)); + public static UpdateSketch heapify(final MemorySegment srcSeg, final long expectedSeed) { + Objects.requireNonNull(srcSeg, "Source MemorySegment must not be null"); + checkBounds(0, 24, srcSeg.byteSize()); //need min 24 bytes + final Family family = Family.idToFamily(srcSeg.get(JAVA_BYTE, FAMILY_BYTE)); if (family.equals(Family.ALPHA)) { - return HeapAlphaSketch.heapifyInstance(srcMem, expectedSeed); + return HeapAlphaSketch.heapifyInstance(srcSeg, expectedSeed); } - return HeapQuickSelectSketch.heapifyInstance(srcMem, expectedSeed); + return HeapQuickSelectSketch.heapifyInstance(srcSeg, expectedSeed); } //Sketch interface @Override - public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) { + public CompactSketch compact(final boolean dstOrdered, final MemorySegment dstWSeg) { return componentsToCompact(getThetaLong(), getRetainedEntries(true), getSeedHash(), isEmpty(), - false, false, dstOrdered, dstMem, getCache()); + false, false, dstOrdered, dstWSeg, getCache()); } @Override @@ -160,16 +159,31 @@ int getCurrentDataLongs() { return 1 << getLgArrLongs(); } + @Override + public boolean hasMemorySegment() { + return (this instanceof DirectQuickSelectSketchR && ((DirectQuickSelectSketchR)this).hasMemorySegment()); + } + @Override public boolean isCompact() { return false; } + @Override + public boolean isDirect() { + return (this instanceof DirectQuickSelectSketchR && ((DirectQuickSelectSketchR)this).isDirect()); + } + @Override public boolean isOrdered() { return false; } + @Override + public boolean isSameResource(final MemorySegment that) { + return (this instanceof DirectQuickSelectSketchR && ((DirectQuickSelectSketchR)this).isSameResource(that)); + } + //UpdateSketch interface /** @@ -378,10 +392,10 @@ public UpdateReturnState update(final long[] data) { */ abstract boolean isOutOfSpace(int numEntries); - static void checkUnionQuickSelectFamily(final Memory mem, final int preambleLongs, + static void checkUnionQuickSelectFamily(final MemorySegment seg, final int preambleLongs, final int lgNomLongs) { //Check Family - final int familyID = extractFamilyID(mem); //byte 2 + final int familyID = extractFamilyID(seg); //byte 2 final Family family = Family.idToFamily(familyID); if (family.equals(Family.UNION)) { if (preambleLongs != Family.UNION.getMinPreLongs()) { @@ -402,45 +416,45 @@ else if (family.equals(Family.QUICKSELECT)) { //Check lgNomLongs if (lgNomLongs < ThetaUtil.MIN_LG_NOM_LONGS) { throw new SketchesArgumentException( - "Possible corruption: Current Memory lgNomLongs < min required size: " + "Possible corruption: Current MemorySegment lgNomLongs < min required size: " + lgNomLongs + " < " + ThetaUtil.MIN_LG_NOM_LONGS); } } - static void checkMemIntegrity(final Memory srcMem, final long expectedSeed, final int preambleLongs, + static void checkSegIntegrity(final MemorySegment srcSeg, final long expectedSeed, final int preambleLongs, final int lgNomLongs, final int lgArrLongs) { //Check SerVer - final int serVer = extractSerVer(srcMem); //byte 1 + final int serVer = extractSerVer(srcSeg); //byte 1 if (serVer != SER_VER) { throw new SketchesArgumentException( "Possible corruption: Invalid Serialization Version: " + serVer); } //Check flags - final int flags = extractFlags(srcMem); //byte 5 + final int flags = extractFlags(srcSeg); //byte 5 final int flagsMask = ORDERED_FLAG_MASK | COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK | BIG_ENDIAN_FLAG_MASK; if ((flags & flagsMask) > 0) { throw new SketchesArgumentException( - "Possible corruption: Input srcMem cannot be: big-endian, compact, ordered, or read-only"); + "Possible corruption: Input srcSeg cannot be: big-endian, compact, ordered, nor read-only"); } //Check seed hashes - final short seedHash = checkMemorySeedHash(srcMem, expectedSeed); //byte 6,7 - ThetaUtil.checkSeedHashes(seedHash, ThetaUtil.computeSeedHash(expectedSeed)); + final short seedHash = checkSegmentSeedHash(srcSeg, expectedSeed); //byte 6,7 + Util.checkSeedHashes(seedHash, Util.computeSeedHash(expectedSeed)); - //Check mem capacity, lgArrLongs - final long curCapBytes = srcMem.getCapacity(); - final int minReqBytes = getMemBytes(lgArrLongs, preambleLongs); + //Check seg capacity, lgArrLongs + final long curCapBytes = srcSeg.byteSize(); + final int minReqBytes = getSegBytes(lgArrLongs, preambleLongs); if (curCapBytes < minReqBytes) { throw new SketchesArgumentException( - "Possible corruption: Current Memory size < min required size: " + "Possible corruption: Current MemorySegment size < min required size: " + curCapBytes + " < " + minReqBytes); } //check Theta, p - final float p = extractP(srcMem); //bytes 12-15 - final long thetaLong = extractThetaLong(srcMem); //bytes 16-23 + final float p = extractP(srcSeg); //bytes 12-15 + final long thetaLong = extractThetaLong(srcSeg); //bytes 16-23 final double theta = thetaLong / LONG_MAX_VALUE_AS_DOUBLE; //if (lgArrLongs <= lgNomLongs) the sketch is still resizing, thus theta cannot be < p. if ((lgArrLongs <= lgNomLongs) && (theta < p) ) { @@ -451,19 +465,19 @@ static void checkMemIntegrity(final Memory srcMem, final long expectedSeed, fina } /** - * This checks to see if the memory RF factor was set correctly as early versions may not + * This checks to see if the MemorySegment RF factor was set correctly as early versions may not * have set it. - * @param srcMem the source memory + * @param srcSeg the source MemorySegment * @param lgNomLongs the current lgNomLongs * @param lgArrLongs the current lgArrLongs - * @return true if the the memory RF factor is incorrect and the caller can either + * @return true if the the MemorySegment RF factor is incorrect and the caller can either * correct it or throw an error. */ - static boolean isResizeFactorIncorrect(final Memory srcMem, final int lgNomLongs, + static boolean isResizeFactorIncorrect(final MemorySegment srcSeg, final int lgNomLongs, final int lgArrLongs) { final int lgT = lgNomLongs + 1; final int lgA = lgArrLongs; - final int lgR = extractLgResizeFactor(srcMem); + final int lgR = extractLgResizeFactor(srcSeg); if (lgR == 0) { return lgA != lgT; } return !(((lgT - lgA) % lgR) == 0); } diff --git a/src/main/java/org/apache/datasketches/theta/UpdateSketchBuilder.java b/src/main/java/org/apache/datasketches/theta/UpdateSketchBuilder.java index eec91e881..834778f87 100644 --- a/src/main/java/org/apache/datasketches/theta/UpdateSketchBuilder.java +++ b/src/main/java/org/apache/datasketches/theta/UpdateSketchBuilder.java @@ -23,14 +23,14 @@ import static org.apache.datasketches.common.Util.TAB; import static org.apache.datasketches.common.Util.ceilingPowerOf2; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; import org.apache.datasketches.common.SuppressFBWarnings; -import org.apache.datasketches.memory.DefaultMemoryRequestServer; -import org.apache.datasketches.memory.MemoryRequestServer; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -38,13 +38,12 @@ * * @author Lee Rhodes */ -public class UpdateSketchBuilder { +public final class UpdateSketchBuilder { private int bLgNomLongs; private long bSeed; private ResizeFactor bRF; private Family bFam; private float bP; - private MemoryRequestServer bMemReqSvr; //Fields for concurrent theta sketch private int bNumPoolThreads; @@ -57,30 +56,28 @@ public class UpdateSketchBuilder { * Constructor for building a new UpdateSketch. The default configuration is *
      *
    • Nominal Entries: {@value org.apache.datasketches.thetacommon.ThetaUtil#DEFAULT_NOMINAL_ENTRIES}
      • - *
    • Seed: {@value org.apache.datasketches.thetacommon.ThetaUtil#DEFAULT_UPDATE_SEED}
      • + *
    • Seed: {@value org.apache.datasketches.common.Util#DEFAULT_UPDATE_SEED}
      • *
    • Input Sampling Probability: 1.0
      • *
    • Family: {@link org.apache.datasketches.common.Family#QUICKSELECT}
      • *
    • Resize Factor: The default for sketches on the Java heap is {@link ResizeFactor#X8}. - * For direct sketches, which are targeted for native memory off the Java heap, this value will + * For direct sketches, which are targeted for off-heap, this value will * be fixed at either {@link ResizeFactor#X1} or {@link ResizeFactor#X2}.
      • - *
    • MemoryRequestServer (Direct only): - * {@link org.apache.datasketches.memory.DefaultMemoryRequestServer}.
      • *
    * Parameters unique to the concurrent sketches only: *
      - *
    • Number of local Nominal Entries: 4
      • *
    • Concurrent NumPoolThreads: 3
      • + *
    • Number of local Nominal Entries: 4
      • *
    • Concurrent PropagateOrderedCompact: true
      • *
    • Concurrent MaxConcurrencyError: 0
      • + *
    • Concurrent MaxNumLocalThreads: 1
      • *
    */ public UpdateSketchBuilder() { bLgNomLongs = Integer.numberOfTrailingZeros(ThetaUtil.DEFAULT_NOMINAL_ENTRIES); - bSeed = ThetaUtil.DEFAULT_UPDATE_SEED; + bSeed = Util.DEFAULT_UPDATE_SEED; bP = (float) 1.0; bRF = ResizeFactor.X8; bFam = Family.QUICKSELECT; - bMemReqSvr = new DefaultMemoryRequestServer(); // Default values for concurrent sketch bNumPoolThreads = ConcurrentPropagationService.NUM_POOL_THREADS; bLocalLgNomLongs = 4; //default is smallest legal QS sketch @@ -249,24 +246,6 @@ public Family getFamily() { return bFam; } - /** - * Set the MemoryRequestServer - * @param memReqSvr the given MemoryRequestServer - * @return this UpdateSketchBuilder - */ - public UpdateSketchBuilder setMemoryRequestServer(final MemoryRequestServer memReqSvr) { - bMemReqSvr = memReqSvr; - return this; - } - - /** - * Returns the MemoryRequestServer - * @return the MemoryRequestServer - */ - public MemoryRequestServer getMemoryRequestServer() { - return bMemReqSvr; - } - /** * Sets the number of pool threads used for background propagation in the concurrent sketches. * @param numPoolThreads the given number of pool threads @@ -347,30 +326,30 @@ public UpdateSketch build() { /** * Returns an UpdateSketch with the current configuration of this Builder - * with the specified backing destination Memory store. + * with the specified backing destination MemorySegment store. * Note: this cannot be used with the Alpha Family of sketches. - * @param dstMem The destination Memory. + * @param dstSeg The destination MemorySegment. * @return an UpdateSketch */ - public UpdateSketch build(final WritableMemory dstMem) { + public UpdateSketch build(final MemorySegment dstSeg) { UpdateSketch sketch = null; switch (bFam) { case ALPHA: { - if (dstMem == null) { + if (dstSeg == null) { sketch = HeapAlphaSketch.newHeapInstance(bLgNomLongs, bSeed, bP, bRF); } else { - throw new SketchesArgumentException("AlphaSketch cannot be made Direct to Memory."); + throw new SketchesArgumentException("AlphaSketch cannot be backed by a MemorySegment."); } break; } case QUICKSELECT: { - if (dstMem == null) { + if (dstSeg == null) { sketch = new HeapQuickSelectSketch(bLgNomLongs, bSeed, bP, bRF, false); } else { sketch = new DirectQuickSelectSketch( - bLgNomLongs, bSeed, bP, bRF, bMemReqSvr, dstMem, false); + bLgNomLongs, bSeed, bP, bRF, dstSeg, false); } break; } @@ -404,9 +383,9 @@ public UpdateSketch buildShared() { } /** - * Returns a direct (potentially off-heap) concurrent shared UpdateSketch with the current - * configuration of the Builder and the given destination WritableMemory. If the destination - * WritableMemory is null, this defaults to an on-heap concurrent shared UpdateSketch. + * Returns a concurrent shared UpdateSketch with the current + * configuration of the Builder and the given destination MemorySegment. If the destination + * MemorySegment is null, this defaults to an on-heap concurrent shared UpdateSketch. * *

    The parameters unique to the shared concurrent sketch are: *

      @@ -417,28 +396,28 @@ public UpdateSketch buildShared() { *

      Key parameters that are in common with other Theta sketches: *

        *
      • Nominal Entries or Log Nominal Entries (for the shared concurrent sketch)
        • - *
      • Destination Writable Memory (if not null, returned sketch is Direct. Default is null.)
        • + *
      • Destination MemorySegment (if not null, returned sketch is Direct. Default is null.)
        • *
      * - * @param dstMem the given WritableMemory for Direct, otherwise null. + * @param dstSeg the given MemorySegment for Direct, otherwise null. * @return a concurrent UpdateSketch with the current configuration of the Builder - * and the given destination WritableMemory. + * and the given destination MemorySegment. */ @SuppressFBWarnings(value = "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD", justification = "Harmless in Builder, fix later") - public UpdateSketch buildShared(final WritableMemory dstMem) { + public UpdateSketch buildShared(final MemorySegment dstSeg) { ConcurrentPropagationService.NUM_POOL_THREADS = bNumPoolThreads; - if (dstMem == null) { + if (dstSeg == null) { return new ConcurrentHeapQuickSelectSketch(bLgNomLongs, bSeed, bMaxConcurrencyError); } else { - return new ConcurrentDirectQuickSelectSketch(bLgNomLongs, bSeed, bMaxConcurrencyError, dstMem); + return new ConcurrentDirectQuickSelectSketch(bLgNomLongs, bSeed, bMaxConcurrencyError, dstSeg); } } /** * Returns a direct (potentially off-heap) concurrent shared UpdateSketch with the current * configuration of the Builder, the data from the given sketch, and the given destination - * WritableMemory. If the destination WritableMemory is null, this defaults to an on-heap + * MemorySegment. If the destination MemorySegment is null, this defaults to an on-heap * concurrent shared UpdateSketch. * *

      The parameters unique to the shared concurrent sketch are: @@ -450,23 +429,23 @@ public UpdateSketch buildShared(final WritableMemory dstMem) { *

      Key parameters that are in common with other Theta sketches: *

        *
      • Nominal Entries or Log Nominal Entries (for the shared concurrent sketch)
        • - *
      • Destination Writable Memory (if not null, returned sketch is Direct. Default is null.)
        • + *
      • Destination MemorySegment (if not null, returned sketch is Direct. Default is null.)
        • *
      * * @param sketch a given UpdateSketch from which the data is used to initialize the returned * shared sketch. - * @param dstMem the given WritableMemory for Direct, otherwise null. + * @param dstSeg the given MemorySegment for Direct, otherwise null. * @return a concurrent UpdateSketch with the current configuration of the Builder - * and the given destination WritableMemory. + * and the given destination MemorySegment. */ @SuppressFBWarnings(value = "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD", justification = "Harmless in Builder, fix later") - public UpdateSketch buildSharedFromSketch(final UpdateSketch sketch, final WritableMemory dstMem) { + public UpdateSketch buildSharedFromSketch(final UpdateSketch sketch, final MemorySegment dstSeg) { ConcurrentPropagationService.NUM_POOL_THREADS = bNumPoolThreads; - if (dstMem == null) { + if (dstSeg == null) { return new ConcurrentHeapQuickSelectSketch(sketch, bSeed, bMaxConcurrencyError); } else { - return new ConcurrentDirectQuickSelectSketch(sketch, bSeed, bMaxConcurrencyError, dstMem); + return new ConcurrentDirectQuickSelectSketch(sketch, bSeed, bMaxConcurrencyError, dstSeg); } } @@ -504,8 +483,6 @@ public String toString() { sb.append("p:").append(TAB).append(bP).append(LS); sb.append("ResizeFactor:").append(TAB).append(bRF).append(LS); sb.append("Family:").append(TAB).append(bFam).append(LS); - final String mrsStr = bMemReqSvr.getClass().getSimpleName(); - sb.append("MemoryRequestServer:").append(TAB).append(mrsStr).append(LS); sb.append("Propagate Ordered Compact").append(TAB).append(bPropagateOrderedCompact).append(LS); sb.append("NumPoolThreads").append(TAB).append(bNumPoolThreads).append(LS); sb.append("MaxConcurrencyError").append(TAB).append(bMaxConcurrencyError).append(LS); diff --git a/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java b/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java index 170c98cad..e9a952ab4 100644 --- a/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java +++ b/src/main/java/org/apache/datasketches/theta/WrappedCompactCompressedSketch.java @@ -19,21 +19,21 @@ package org.apache.datasketches.theta; -import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits; import static org.apache.datasketches.theta.PreambleUtil.ENTRY_BITS_BYTE_V4; import static org.apache.datasketches.theta.PreambleUtil.NUM_ENTRIES_BYTES_BYTE_V4; import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE; +import static org.apache.datasketches.theta.PreambleUtil.wholeBytesToHoldBits; import org.apache.datasketches.common.ByteArrayUtil; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; /** * Wrapper around a serialized compact compressed read-only sketch. It is not empty, not a single item. * *

      This sketch can only be associated with a Serialization Version 4 format binary image.

      */ -class WrappedCompactCompressedSketch extends WrappedCompactSketch { - +final class WrappedCompactCompressedSketch extends WrappedCompactSketch { + /** * Construct this sketch with the given bytes. * @param bytes containing serialized compact compressed sketch. @@ -50,7 +50,7 @@ class WrappedCompactCompressedSketch extends WrappedCompactSketch { * @return this sketch */ static WrappedCompactCompressedSketch wrapInstance(final byte[] bytes, final short seedHash) { - ThetaUtil.checkSeedHashes(ByteArrayUtil.getShortLE(bytes, PreambleUtil.SEED_HASH_SHORT), seedHash); + Util.checkSeedHashes(ByteArrayUtil.getShortLE(bytes, PreambleUtil.SEED_HASH_SHORT), seedHash); return new WrappedCompactCompressedSketch(bytes); } @@ -66,7 +66,7 @@ public int getCurrentBytes() { private static final int START_PACKED_DATA_EXACT_MODE = 8; private static final int START_PACKED_DATA_ESTIMATION_MODE = 16; - + @Override public int getRetainedEntries(final boolean valid) { //compact is always valid // number of entries is stored using variable length encoding diff --git a/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java b/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java index 519857d21..a5b67363f 100644 --- a/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/WrappedCompactSketch.java @@ -22,20 +22,19 @@ import static org.apache.datasketches.common.ByteArrayUtil.getIntLE; import static org.apache.datasketches.common.ByteArrayUtil.getLongLE; import static org.apache.datasketches.common.ByteArrayUtil.getShortLE; -import static org.apache.datasketches.theta.CompactOperations.memoryToCompact; +import static org.apache.datasketches.theta.CompactOperations.segmentToCompact; import static org.apache.datasketches.theta.PreambleUtil.EMPTY_FLAG_MASK; -import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE; -import static org.apache.datasketches.theta.PreambleUtil.RETAINED_ENTRIES_INT; +import static org.apache.datasketches.theta.PreambleUtil.ORDERED_FLAG_MASK; import static org.apache.datasketches.theta.PreambleUtil.PREAMBLE_LONGS_BYTE; -import static org.apache.datasketches.theta.PreambleUtil.THETA_LONG; +import static org.apache.datasketches.theta.PreambleUtil.RETAINED_ENTRIES_INT; import static org.apache.datasketches.theta.PreambleUtil.SEED_HASH_SHORT; +import static org.apache.datasketches.theta.PreambleUtil.THETA_LONG; +import java.lang.foreign.MemorySegment; import java.util.Arrays; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; /** * Wrapper around a serialized compact read-only sketch. It is not empty, not a single item. @@ -54,22 +53,22 @@ class WrappedCompactSketch extends CompactSketch { } /** - * Wraps the given Memory, which must be a SerVer 3 CompactSketch image. + * Wraps the given byteArray, which must be a SerVer 3 CompactSketch image. * @param bytes representation of serialized compressed compact sketch. * @param seedHash The update seedHash. * See Seed Hash. * @return this sketch */ static WrappedCompactSketch wrapInstance(final byte[] bytes, final short seedHash) { - ThetaUtil.checkSeedHashes(getShortLE(bytes, SEED_HASH_SHORT), seedHash); + Util.checkSeedHashes(getShortLE(bytes, SEED_HASH_SHORT), seedHash); return new WrappedCompactSketch(bytes); } //Sketch Overrides @Override - public CompactSketch compact(final boolean dstOrdered, final WritableMemory dstMem) { - return memoryToCompact(Memory.wrap(bytes_), dstOrdered, dstMem); + public CompactSketch compact(final boolean dstOrdered, final MemorySegment dstSeg) { + return segmentToCompact(MemorySegment.ofArray(bytes_), dstOrdered, dstSeg); } @Override @@ -91,16 +90,6 @@ public long getThetaLong() { return (preLongs > 2) ? getLongLE(bytes_, THETA_LONG) : Long.MAX_VALUE; } - @Override - public boolean hasMemory() { - return false; - } - - @Override - public boolean isDirect() { - return false; - } - @Override public boolean isEmpty() { return (bytes_[FLAGS_BYTE] & EMPTY_FLAG_MASK) > 0; @@ -111,11 +100,6 @@ public boolean isOrdered() { return (bytes_[FLAGS_BYTE] & ORDERED_FLAG_MASK) > 0; } - @Override - public boolean isSameResource(final Memory that) { - return false; - } - @Override public HashIterator iterator() { return new BytesCompactHashIterator( @@ -153,11 +137,6 @@ int getCurrentPreambleLongs() { return bytes_[PREAMBLE_LONGS_BYTE]; } - @Override - Memory getMemory() { - return null; - } - @Override short getSeedHash() { return getShortLE(bytes_, SEED_HASH_SHORT); diff --git a/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java b/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java index f6b22cb50..7044551f0 100644 --- a/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java +++ b/src/main/java/org/apache/datasketches/thetacommon/HashOperations.java @@ -19,13 +19,14 @@ package org.apache.datasketches.thetacommon; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static java.lang.Math.max; import static org.apache.datasketches.common.Util.ceilingPowerOf2; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; /** * Helper class for the common hash table methods. @@ -183,22 +184,22 @@ public static int hashArrayInsert(final long[] srcArr, final long[] hashTable, return count; } - //With Memory or WritableMemory + //With MemorySegment /** - * This is a classical Knuth-style Open Addressing, Double Hash (OADH) search scheme for Memory. - * Returns the index if found, -1 if not found. + * This is a classical Knuth-style Open Addressing, Double Hash (OADH) search scheme for MemorySegment. + * Returns the index if found, -1 if not found. The input MemorySegment may be read only. * - * @param mem The Memory containing the hash table to search. + * @param seg The MemorySegment containing the hash table to search. * The hash table portion must be a power of 2 in size. * @param lgArrLongs The log_base2(hashTable.length). * See lgArrLongs. * @param hash The hash value to search for. Must not be zero. - * @param memOffsetBytes offset in the memory where the hashTable starts + * @param segOffsetBytes offset in the MemorySegment where the hashTable starts * @return Current probe index if found, -1 if not found. */ - public static int hashSearchMemory(final Memory mem, final int lgArrLongs, final long hash, - final int memOffsetBytes) { + public static int hashSearchMemorySegment(final MemorySegment seg, final int lgArrLongs, final long hash, + final int segOffsetBytes) { if (hash == 0) { throw new SketchesArgumentException("Given hash must not be zero: " + hash); } @@ -207,8 +208,8 @@ public static int hashSearchMemory(final Memory mem, final int lgArrLongs, final int curProbe = (int) (hash & arrayMask); final int loopIndex = curProbe; do { - final int curProbeOffsetBytes = (curProbe << 3) + memOffsetBytes; - final long curArrayHash = mem.getLong(curProbeOffsetBytes); + final int curProbeOffsetBytes = (curProbe << 3) + segOffsetBytes; + final long curArrayHash = seg.get(JAVA_LONG_UNALIGNED, curProbeOffsetBytes); if (curArrayHash == EMPTY) { return -1; } else if (curArrayHash == hash) { return curProbe; } curProbe = (curProbe + stride) & arrayMask; @@ -217,21 +218,21 @@ public static int hashSearchMemory(final Memory mem, final int lgArrLongs, final } /** - * This is a classical Knuth-style Open Addressing, Double Hash (OADH) insert scheme for Memory. + * This is a classical Knuth-style Open Addressing, Double Hash (OADH) insert scheme for MemorySegment. * This method assumes that the input hash is not a duplicate. * Useful for rebuilding tables to avoid unnecessary comparisons. * Returns the index of insertion, which is always positive or zero. * Throws an exception if table has no empty slot. * - * @param wmem The WritableMemory that contains the hashTable to insert into. + * @param wseg The writable MemorySegment that contains the hashTable to insert into. * The size of the hashTable portion must be a power of 2. * @param lgArrLongs The log_base2(hashTable.length. * See lgArrLongs. * @param hash value that must not be zero and will be inserted into the array into an empty slot. - * @param memOffsetBytes offset in the WritableMemory where the hashTable starts + * @param memOffsetBytes offset in the writable MemorySegment where the hashTable starts * @return index of insertion. Always positive or zero. */ - public static int hashInsertOnlyMemory(final WritableMemory wmem, final int lgArrLongs, + public static int hashInsertOnlyMemorySegment(final MemorySegment wseg, final int lgArrLongs, final long hash, final int memOffsetBytes) { final int arrayMask = (1 << lgArrLongs) - 1; // current Size -1 final int stride = getStride(hash, lgArrLongs); @@ -240,9 +241,9 @@ public static int hashInsertOnlyMemory(final WritableMemory wmem, final int lgAr final int loopIndex = curProbe; do { final int curProbeOffsetBytes = (curProbe << 3) + memOffsetBytes; - final long curArrayHash = wmem.getLong(curProbeOffsetBytes); + final long curArrayHash = wseg.get(JAVA_LONG_UNALIGNED, curProbeOffsetBytes); if (curArrayHash == EMPTY) { - wmem.putLong(curProbeOffsetBytes, hash); + wseg.set(JAVA_LONG_UNALIGNED, curProbeOffsetBytes, hash); return curProbe; } curProbe = (curProbe + stride) & arrayMask; @@ -252,19 +253,19 @@ public static int hashInsertOnlyMemory(final WritableMemory wmem, final int lgAr /** * This is a classical Knuth-style Open Addressing, Double Hash insert scheme, but inserts - * values directly into a Memory. + * values directly into a writable MemorySegment. * Returns index ≥ 0 if found (duplicate); < 0 if inserted, inserted at -(index + 1). * Throws an exception if the value is not found and table has no empty slot. * - * @param wmem The WritableMemory that contains the hashTable to insert into. + * @param wseg The writable MemorySegment that contains the hashTable to insert into. * @param lgArrLongs The log_base2(hashTable.length). * See lgArrLongs. * @param hash The hash value to be potentially inserted into an empty slot only if it is not * a duplicate of any other hash value in the table. It must not be zero. - * @param memOffsetBytes offset in the WritableMemory where the hash array starts + * @param memOffsetBytes offset in the writable MemorySegment where the hash array starts * @return index ≥ 0 if found (duplicate); < 0 if inserted, inserted at -(index + 1). */ - public static int hashSearchOrInsertMemory(final WritableMemory wmem, final int lgArrLongs, + public static int hashSearchOrInsertMemorySegment(final MemorySegment wseg, final int lgArrLongs, final long hash, final int memOffsetBytes) { final int arrayMask = (1 << lgArrLongs) - 1; // current Size -1 final int stride = getStride(hash, lgArrLongs); @@ -273,9 +274,9 @@ public static int hashSearchOrInsertMemory(final WritableMemory wmem, final int final int loopIndex = curProbe; do { final int curProbeOffsetBytes = (curProbe << 3) + memOffsetBytes; - final long curArrayHash = wmem.getLong(curProbeOffsetBytes); + final long curArrayHash = wseg.get(JAVA_LONG_UNALIGNED, curProbeOffsetBytes); if (curArrayHash == EMPTY) { - wmem.putLong(curProbeOffsetBytes, hash); + wseg.set(JAVA_LONG_UNALIGNED, curProbeOffsetBytes, hash); return ~curProbe; } else if (curArrayHash == hash) { return curProbe; } // curArrayHash is a duplicate // curArrayHash is not a duplicate and not zero, continue searching diff --git a/src/main/java/org/apache/datasketches/thetacommon/ThetaUtil.java b/src/main/java/org/apache/datasketches/thetacommon/ThetaUtil.java index be209ece1..4012cb412 100644 --- a/src/main/java/org/apache/datasketches/thetacommon/ThetaUtil.java +++ b/src/main/java/org/apache/datasketches/thetacommon/ThetaUtil.java @@ -19,8 +19,6 @@ package org.apache.datasketches.thetacommon; -import static org.apache.datasketches.hash.MurmurHash3.hash; - import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.Util; @@ -55,27 +53,6 @@ public final class ThetaUtil { * See Default Nominal Entries */ public static final int DEFAULT_NOMINAL_ENTRIES = 4096; - /** - * The seed 9001 used in the sketch update methods is a prime number that - * was chosen very early on in experimental testing. Choosing a seed is somewhat arbitrary, and - * the author cannot prove that this particular seed is somehow superior to other seeds. There - * was some early Internet discussion that a seed of 0 did not produce as clean avalanche diagrams - * as non-zero seeds, but this may have been more related to the MurmurHash2 release, which did - * have some issues. As far as the author can determine, MurmurHash3 does not have these problems. - * - *

      In order to perform set operations on two sketches it is critical that the same hash - * function and seed are identical for both sketches, otherwise the assumed 1:1 relationship - * between the original source key value and the hashed bit string would be violated. Once - * you have developed a history of stored sketches you are stuck with it. - * - *

      WARNING: This seed is used internally by library sketches in different - * packages and thus must be declared public. However, this seed value must not be used by library - * users with the MurmurHash3 function. It should be viewed as existing for exclusive, private - * use by the library. - * - *

      See Default Update Seed - */ - public static final long DEFAULT_UPDATE_SEED = 9001L; private ThetaUtil() {} @@ -84,39 +61,6 @@ private ThetaUtil() {} */ public static final int MIN_LG_ARR_LONGS = 5; - /** - * Check if the two seed hashes are equal. If not, throw an SketchesArgumentException. - * @param seedHashA the seedHash A - * @param seedHashB the seedHash B - * @return seedHashA if they are equal - */ - public static short checkSeedHashes(final short seedHashA, final short seedHashB) { - if (seedHashA != seedHashB) { - throw new SketchesArgumentException( - "Incompatible Seed Hashes. " + Integer.toHexString(seedHashA & 0XFFFF) - + ", " + Integer.toHexString(seedHashB & 0XFFFF)); - } - return seedHashA; - } - - /** - * Computes and checks the 16-bit seed hash from the given long seed. - * The seed hash may not be zero in order to maintain compatibility with older serialized - * versions that did not have this concept. - * @param seed See Update Hash Seed - * @return the seed hash. - */ - public static short computeSeedHash(final long seed) { - final long[] seedArr = {seed}; - final short seedHash = (short)(hash(seedArr, 0L)[0] & 0xFFFFL); - if (seedHash == 0) { - throw new SketchesArgumentException( - "The given seed: " + seed + " produced a seedHash of zero. " - + "You must choose a different seed."); - } - return seedHash; - } - /** * Gets the smallest allowed exponent of 2 that it is a sub-multiple of the target by zero, * one or more resize factors. diff --git a/src/main/java/org/apache/datasketches/tuple/AnotB.java b/src/main/java/org/apache/datasketches/tuple/AnotB.java index 908e8f6be..fed710bdd 100644 --- a/src/main/java/org/apache/datasketches/tuple/AnotB.java +++ b/src/main/java/org/apache/datasketches/tuple/AnotB.java @@ -31,9 +31,9 @@ import org.apache.datasketches.common.SketchesStateException; import org.apache.datasketches.common.SuppressFBWarnings; import org.apache.datasketches.thetacommon.SetOperationCornerCases; +import org.apache.datasketches.thetacommon.ThetaUtil; import org.apache.datasketches.thetacommon.SetOperationCornerCases.AnotbAction; import org.apache.datasketches.thetacommon.SetOperationCornerCases.CornerCase; -import org.apache.datasketches.thetacommon.ThetaUtil; /** * Computes a set difference, A-AND-NOT-B, of two generic tuple sketches. diff --git a/src/main/java/org/apache/datasketches/tuple/CompactSketch.java b/src/main/java/org/apache/datasketches/tuple/CompactSketch.java index ec58a5e76..20eac81fb 100644 --- a/src/main/java/org/apache/datasketches/tuple/CompactSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/CompactSketch.java @@ -19,15 +19,18 @@ package org.apache.datasketches.tuple; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.thetacommon.HashOperations.count; +import java.lang.foreign.MemorySegment; import java.lang.reflect.Array; import java.nio.ByteOrder; import org.apache.datasketches.common.ByteArrayUtil; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; /** * CompactSketches are never created directly. They are created as a result of @@ -69,24 +72,24 @@ private enum Flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDER /** * This is to create an instance of a CompactSketch given a serialized form * - * @param mem Memory object with serialized CompactSketch + * @param seg MemorySegment object with serialized CompactSketch * @param deserializer the SummaryDeserializer */ - CompactSketch(final Memory mem, final SummaryDeserializer deserializer) { + CompactSketch(final MemorySegment seg, final SummaryDeserializer deserializer) { super(Long.MAX_VALUE, true, null); int offset = 0; - final byte preambleLongs = mem.getByte(offset++); - final byte version = mem.getByte(offset++); - final byte familyId = mem.getByte(offset++); + final byte preambleLongs = seg.get(JAVA_BYTE, offset++); + final byte version = seg.get(JAVA_BYTE, offset++); + final byte familyId = seg.get(JAVA_BYTE, offset++); SerializerDeserializer.validateFamily(familyId, preambleLongs); if (version > serialVersionUID) { throw new SketchesArgumentException( "Unsupported serial version. Expected: " + serialVersionUID + " or lower, actual: " + version); } SerializerDeserializer - .validateType(mem.getByte(offset++), SerializerDeserializer.SketchType.CompactSketch); + .validateType(seg.get(JAVA_BYTE, offset++), SerializerDeserializer.SketchType.CompactSketch); if (version <= serialVersionUIDLegacy) { // legacy serial format - final byte flags = mem.getByte(offset++); + final byte flags = seg.get(JAVA_BYTE, offset++); final boolean isBigEndian = (flags & 1 << FlagsLegacy.IS_BIG_ENDIAN.ordinal()) > 0; if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { throw new SketchesArgumentException("Byte order mismatch"); @@ -94,7 +97,7 @@ private enum Flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDER empty_ = (flags & 1 << FlagsLegacy.IS_EMPTY.ordinal()) > 0; final boolean isThetaIncluded = (flags & 1 << FlagsLegacy.IS_THETA_INCLUDED.ordinal()) > 0; if (isThetaIncluded) { - thetaLong_ = mem.getLong(offset); + thetaLong_ = seg.get(JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; } else { thetaLong_ = Long.MAX_VALUE; @@ -103,9 +106,9 @@ private enum Flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDER if (hasEntries) { int classNameLength = 0; if (version == serialVersionWithSummaryClassNameUID) { - classNameLength = mem.getByte(offset++); + classNameLength = seg.get(JAVA_BYTE, offset++); } - final int count = mem.getInt(offset); + final int count = seg.get(JAVA_INT_UNALIGNED, offset); offset += Integer.BYTES; if (version == serialVersionWithSummaryClassNameUID) { offset += classNameLength; @@ -113,11 +116,11 @@ private enum Flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDER hashArr_ = new long[count]; for (int i = 0; i < count; i++) { - hashArr_[i] = mem.getLong(offset); + hashArr_[i] = seg.get(JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; } for (int i = 0; i < count; i++) { - offset += readSummary(mem, offset, i, count, deserializer); + offset += readSummary(seg, offset, i, count, deserializer); } } else { hashArr_ = new long[0]; @@ -125,7 +128,7 @@ private enum Flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDER } } else { // current serial format offset++; //skip unused byte - final byte flags = mem.getByte(offset++); + final byte flags = seg.get(JAVA_BYTE, offset++); offset += 2; //skip 2 unused bytes empty_ = (flags & 1 << Flags.IS_EMPTY.ordinal()) > 0; thetaLong_ = Long.MAX_VALUE; @@ -134,11 +137,11 @@ private enum Flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDER if (preambleLongs == 1) { count = 1; } else { - count = mem.getInt(offset); + count = seg.get(JAVA_INT_UNALIGNED, offset); offset += Integer.BYTES; offset += 4; // unused if (preambleLongs > 2) { - thetaLong_ = mem.getLong(offset); + thetaLong_ = seg.get(JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; } } @@ -146,18 +149,18 @@ private enum Flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDER hashArr_ = new long[count]; for (int i = 0; i < count; i++) { - hashArr_[i] = mem.getLong(offset); + hashArr_[i] = seg.get(JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; - offset += readSummary(mem, offset, i, count, deserializer); + offset += readSummary(seg, offset, i, count, deserializer); } } } @SuppressWarnings({"unchecked"}) - private int readSummary(final Memory mem, final int offset, final int i, final int count, + private int readSummary(final MemorySegment seg, final int offset, final int i, final int count, final SummaryDeserializer deserializer) { - final Memory memRegion = mem.region(offset, mem.getCapacity() - offset); - final DeserializeResult result = deserializer.heapifySummary(memRegion); + final MemorySegment segRegion = seg.asSlice(offset, seg.byteSize() - offset); + final DeserializeResult result = deserializer.heapifySummary(segRegion); final S summary = result.getObject(); final Class summaryType = (Class) result.getObject().getClass(); if (summaryArr_ == null) { diff --git a/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java index 4b9afb1a1..708486df1 100644 --- a/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java @@ -19,22 +19,26 @@ package org.apache.datasketches.tuple; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.common.Util.ceilingPowerOf2; import static org.apache.datasketches.common.Util.checkBounds; import static org.apache.datasketches.common.Util.exactLog2OfLong; import static org.apache.datasketches.thetacommon.HashOperations.count; +import java.lang.foreign.MemorySegment; import java.lang.reflect.Array; import java.nio.ByteOrder; import java.util.Objects; import org.apache.datasketches.common.ByteArrayUtil; import org.apache.datasketches.common.Family; +import org.apache.datasketches.common.QuickSelect; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.thetacommon.HashOperations; -import org.apache.datasketches.thetacommon.QuickSelect; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -167,7 +171,7 @@ private QuickSelectSketch( /** * This is to create an instance of a QuickSelectSketch given a serialized form - * @param mem Memory object with serialized QuickSelectSketch + * @param seg MemorySegment object with serialized QuickSelectSketch * @param deserializer the SummaryDeserializer * @param summaryFactory the SummaryFactory * @deprecated As of 3.0.0, heapifying an UpdatableSketch is deprecated. @@ -176,26 +180,26 @@ private QuickSelectSketch( */ @Deprecated QuickSelectSketch( - final Memory mem, + final MemorySegment seg, final SummaryDeserializer deserializer, final SummaryFactory summaryFactory) { - this(new Validate<>(), mem, deserializer, summaryFactory); + this(new Validate<>(), seg, deserializer, summaryFactory); } /* * This private constructor is used to protect against "Finalizer attacks". * The private static inner class Validate performs validation and deserialization - * from the input Memory and may throw exceptions. In order to protect against the attack, we must + * from the input MemorySegment and may throw exceptions. In order to protect against the attack, we must * perform this validation prior to the constructor's super reaches the Object class. * Making QuickSelectSketch final won't work here because UpdatableSketch is a subclass. * Using an empty final finalizer() is not recommended and is deprecated as of Java9. */ private QuickSelectSketch( final Validate val, - final Memory mem, + final MemorySegment seg, final SummaryDeserializer deserializer, final SummaryFactory summaryFactory) { - super(val.validate(mem, deserializer), val.myEmpty, summaryFactory); + super(val.validate(seg, deserializer), val.myEmpty, summaryFactory); nomEntries_ = val.myNomEntries; lgResizeFactor_ = val.myLgResizeFactor; samplingProbability_ = val.mySamplingProbability; @@ -222,43 +226,43 @@ private static final class Validate { @SuppressWarnings("unchecked") long validate( - final Memory mem, + final MemorySegment seg, final SummaryDeserializer deserializer) { - Objects.requireNonNull(mem, "SourceMemory must not be null."); + Objects.requireNonNull(seg, "Source MemorySegment must not be null."); Objects.requireNonNull(deserializer, "Deserializer must not be null."); - checkBounds(0, 8, mem.getCapacity()); + checkBounds(0, 8, seg.byteSize()); int offset = 0; - final byte preambleLongs = mem.getByte(offset++); //byte 0 PreLongs - final byte version = mem.getByte(offset++); //byte 1 SerVer - final byte familyId = mem.getByte(offset++); //byte 2 FamID + final byte preambleLongs = seg.get(JAVA_BYTE, offset++); //byte 0 PreLongs + final byte version = seg.get(JAVA_BYTE, offset++); //byte 1 SerVer + final byte familyId = seg.get(JAVA_BYTE, offset++); //byte 2 FamID SerializerDeserializer.validateFamily(familyId, preambleLongs); if (version > serialVersionUID) { throw new SketchesArgumentException( "Unsupported serial version. Expected: " + serialVersionUID + " or lower, actual: " + version); } - SerializerDeserializer.validateType(mem.getByte(offset++), //byte 3 + SerializerDeserializer.validateType(seg.get(JAVA_BYTE, offset++), //byte 3 SerializerDeserializer.SketchType.QuickSelectSketch); - final byte flags = mem.getByte(offset++); //byte 4 + final byte flags = seg.get(JAVA_BYTE, offset++); //byte 4 final boolean isBigEndian = (flags & 1 << Flags.IS_BIG_ENDIAN.ordinal()) > 0; if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { throw new SketchesArgumentException("Endian byte order mismatch"); } - myNomEntries = 1 << mem.getByte(offset++); //byte 5 - myLgCurrentCapacity = mem.getByte(offset++); //byte 6 - myLgResizeFactor = mem.getByte(offset++); //byte 7 + myNomEntries = 1 << seg.get(JAVA_BYTE, offset++); //byte 5 + myLgCurrentCapacity = seg.get(JAVA_BYTE, offset++); //byte 6 + myLgResizeFactor = seg.get(JAVA_BYTE, offset++); //byte 7 - checkBounds(0, preambleLongs * 8L, mem.getCapacity()); + checkBounds(0, preambleLongs * 8L, seg.byteSize()); final boolean isInSamplingMode = (flags & 1 << Flags.IS_IN_SAMPLING_MODE.ordinal()) > 0; - mySamplingProbability = isInSamplingMode ? mem.getFloat(offset) : 1f; //bytes 8 - 11 + mySamplingProbability = isInSamplingMode ? seg.get(JAVA_FLOAT_UNALIGNED, offset) : 1f; //bytes 8 - 11 if (isInSamplingMode) { offset += Float.BYTES; } final boolean isThetaIncluded = (flags & 1 << Flags.IS_THETA_INCLUDED.ordinal()) > 0; if (isThetaIncluded) { - myThetaLong = mem.getLong(offset); + myThetaLong = seg.get(JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; } else { myThetaLong = (long) (Long.MAX_VALUE * (double) mySamplingProbability); @@ -267,16 +271,16 @@ long validate( int count = 0; final boolean hasEntries = (flags & (1 << Flags.HAS_ENTRIES.ordinal())) > 0; if (hasEntries) { - count = mem.getInt(offset); + count = seg.get(JAVA_INT_UNALIGNED, offset); offset += Integer.BYTES; } final int currentCapacity = 1 << myLgCurrentCapacity; myHashTable = new long[currentCapacity]; for (int i = 0; i < count; i++) { - final long hash = mem.getLong(offset); + final long hash = seg.get(JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; - final Memory memRegion = mem.region(offset, mem.getCapacity() - offset); - final DeserializeResult summaryResult = deserializer.heapifySummary(memRegion); + final MemorySegment segRegion = seg.asSlice(offset, seg.byteSize() - offset); + final DeserializeResult summaryResult = deserializer.heapifySummary(segRegion); final S summary = (S) summaryResult.getObject(); offset += summaryResult.getSize(); //in-place equivalent to insert(hash, summary): diff --git a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java index a30d47edf..2ca7c29df 100644 --- a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java +++ b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java @@ -19,9 +19,12 @@ package org.apache.datasketches.tuple; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; + +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; /** * Multipurpose serializer-deserializer for a collection of sketches defined by the enum. @@ -77,12 +80,12 @@ public static void validateType(final byte sketchTypeByte, final SketchType expe } /** - * Gets the sketch type byte from the given Memory image - * @param mem the given Memory image + * Gets the sketch type byte from the given MemorySegment image + * @param seg the given MemorySegment image * @return the SketchType */ - public static SketchType getSketchType(final Memory mem) { - final byte sketchTypeByte = mem.getByte(TYPE_BYTE_OFFSET); + public static SketchType getSketchType(final MemorySegment seg) { + final byte sketchTypeByte = seg.get(JAVA_BYTE, TYPE_BYTE_OFFSET); return getSketchType(sketchTypeByte); } diff --git a/src/main/java/org/apache/datasketches/tuple/Sketch.java b/src/main/java/org/apache/datasketches/tuple/Sketch.java index f0003bf3c..8fd81f78c 100644 --- a/src/main/java/org/apache/datasketches/tuple/Sketch.java +++ b/src/main/java/org/apache/datasketches/tuple/Sketch.java @@ -24,7 +24,7 @@ import org.apache.datasketches.thetacommon.BinomialBoundsN; /** - * This is an equivalent to org.apache.datasketches.theta.Sketch with + * This is an equivalent to org.apache.datasketches.theta2.Sketch with * addition of a user-defined Summary object associated with every unique entry * in the sketch. * @param Type of Summary diff --git a/src/main/java/org/apache/datasketches/tuple/Sketches.java b/src/main/java/org/apache/datasketches/tuple/Sketches.java index fba8ab604..6eef7f88d 100644 --- a/src/main/java/org/apache/datasketches/tuple/Sketches.java +++ b/src/main/java/org/apache/datasketches/tuple/Sketches.java @@ -19,7 +19,7 @@ package org.apache.datasketches.tuple; -import org.apache.datasketches.memory.Memory; +import java.lang.foreign.MemorySegment; /** * Convenient static methods to instantiate generic tuple sketches. @@ -37,36 +37,36 @@ public static Sketch createEmptySketch() { } /** - * Instantiate a Sketch from a given Memory. + * Instantiate a Sketch from a given MemorySegment. * @param Type of Summary - * @param mem Memory object representing a Sketch + * @param seg MemorySegment object representing a Sketch * @param deserializer instance of SummaryDeserializer - * @return Sketch created from its Memory representation + * @return Sketch created from its MemorySegment representation */ public static Sketch heapifySketch( - final Memory mem, + final MemorySegment seg, final SummaryDeserializer deserializer) { - final SerializerDeserializer.SketchType sketchType = SerializerDeserializer.getSketchType(mem); + final SerializerDeserializer.SketchType sketchType = SerializerDeserializer.getSketchType(seg); if (sketchType == SerializerDeserializer.SketchType.QuickSelectSketch) { - return new QuickSelectSketch<>(mem, deserializer, null); + return new QuickSelectSketch<>(seg, deserializer, null); } - return new CompactSketch<>(mem, deserializer); + return new CompactSketch<>(seg, deserializer); } /** - * Instantiate UpdatableSketch from a given Memory + * Instantiate UpdatableSketch from a given MemorySegment * @param Type of update value * @param Type of Summary - * @param mem Memory object representing a Sketch + * @param seg MemorySegment object representing a Sketch * @param deserializer instance of SummaryDeserializer * @param summaryFactory instance of SummaryFactory - * @return Sketch created from its Memory representation + * @return Sketch created from its MemorySegment representation */ public static > UpdatableSketch heapifyUpdatableSketch( - final Memory mem, + final MemorySegment seg, final SummaryDeserializer deserializer, final SummaryFactory summaryFactory) { - return new UpdatableSketch<>(mem, deserializer, summaryFactory); + return new UpdatableSketch<>(seg, deserializer, summaryFactory); } } diff --git a/src/main/java/org/apache/datasketches/tuple/SummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/SummaryDeserializer.java index 8edbc3318..6393019b1 100644 --- a/src/main/java/org/apache/datasketches/tuple/SummaryDeserializer.java +++ b/src/main/java/org/apache/datasketches/tuple/SummaryDeserializer.java @@ -19,7 +19,7 @@ package org.apache.datasketches.tuple; -import org.apache.datasketches.memory.Memory; +import java.lang.foreign.MemorySegment; /** * Interface for deserializing user-defined Summary @@ -29,14 +29,14 @@ public interface SummaryDeserializer { /** * This is to create an instance of a Summary given a serialized representation. - * The user may assume that the start of the given Memory is the correct place to start + * The user may assume that the start of the given MemorySegment is the correct place to start * deserializing. However, the user must be able to determine the number of bytes required to - * deserialize the summary as the capacity of the given Memory may + * deserialize the summary as the capacity of the given MemorySegment may * include multiple such summaries and may be much larger than required for a single summary. - * @param mem Memory object with serialized representation of a Summary - * @return DeserializedResult object, which contains a Summary object and number of bytes read - * from the Memory + * @param seg MemorySegment object with serialized representation of a Summary + * @return DeserializedResult object, which contains a Summary object and number of bytes read + * from the MemorySegment */ - public DeserializeResult heapifySummary(Memory mem); + public DeserializeResult heapifySummary(MemorySegment seg); } diff --git a/src/main/java/org/apache/datasketches/tuple/Union.java b/src/main/java/org/apache/datasketches/tuple/Union.java index acefa2ab5..f67626d1b 100644 --- a/src/main/java/org/apache/datasketches/tuple/Union.java +++ b/src/main/java/org/apache/datasketches/tuple/Union.java @@ -21,8 +21,8 @@ import static java.lang.Math.min; +import org.apache.datasketches.common.QuickSelect; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.thetacommon.QuickSelect; import org.apache.datasketches.thetacommon.ThetaUtil; /** diff --git a/src/main/java/org/apache/datasketches/tuple/UpdatableSketch.java b/src/main/java/org/apache/datasketches/tuple/UpdatableSketch.java index 36743618b..b706ca11e 100644 --- a/src/main/java/org/apache/datasketches/tuple/UpdatableSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/UpdatableSketch.java @@ -19,11 +19,12 @@ package org.apache.datasketches.tuple; +import static org.apache.datasketches.common.Util.DEFAULT_UPDATE_SEED; + +import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import org.apache.datasketches.hash.MurmurHash3; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.thetacommon.ThetaUtil; /** * An extension of QuickSelectSketch<S>, which can be updated with many types of keys. @@ -63,7 +64,7 @@ public UpdatableSketch(final int nomEntries, final int lgResizeFactor, /** * This is to create an instance of a sketch given a serialized form - * @param srcMem Memory object with data of a serialized UpdatableSketch + * @param srcSeg MemorySegment object with data of a serialized UpdatableSketch * @param deserializer instance of SummaryDeserializer * @param summaryFactory instance of SummaryFactory * @deprecated As of 3.0.0, heapifying an UpdatableSketch is deprecated. @@ -72,10 +73,10 @@ public UpdatableSketch(final int nomEntries, final int lgResizeFactor, */ @Deprecated public UpdatableSketch( - final Memory srcMem, + final MemorySegment srcSeg, final SummaryDeserializer deserializer, final SummaryFactory summaryFactory) { - super(srcMem, deserializer, summaryFactory); + super(srcSeg, deserializer, summaryFactory); } /** @@ -136,7 +137,7 @@ public void update(final String key, final U value) { */ public void update(final byte[] key, final U value) { if ((key == null) || (key.length == 0)) { return; } - insertOrIgnore(MurmurHash3.hash(key, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1, value); + insertOrIgnore(MurmurHash3.hash(key, DEFAULT_UPDATE_SEED)[0] >>> 1, value); } /** @@ -148,7 +149,7 @@ public void update(final byte[] key, final U value) { */ public void update(final ByteBuffer buffer, final U value) { if (buffer == null || buffer.hasRemaining() == false) { return; } - insertOrIgnore(MurmurHash3.hash(buffer, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1, value); + insertOrIgnore(MurmurHash3.hash(buffer, DEFAULT_UPDATE_SEED)[0] >>> 1, value); } /** @@ -160,7 +161,7 @@ public void update(final ByteBuffer buffer, final U value) { */ public void update(final int[] key, final U value) { if ((key == null) || (key.length == 0)) { return; } - insertOrIgnore(MurmurHash3.hash(key, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1, value); + insertOrIgnore(MurmurHash3.hash(key, DEFAULT_UPDATE_SEED)[0] >>> 1, value); } /** @@ -172,7 +173,7 @@ public void update(final int[] key, final U value) { */ public void update(final long[] key, final U value) { if ((key == null) || (key.length == 0)) { return; } - insertOrIgnore(MurmurHash3.hash(key, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1, value); + insertOrIgnore(MurmurHash3.hash(key, DEFAULT_UPDATE_SEED)[0] >>> 1, value); } void insertOrIgnore(final long hash, final U value) { diff --git a/src/main/java/org/apache/datasketches/tuple/Util.java b/src/main/java/org/apache/datasketches/tuple/Util.java index 92193ca56..46f069724 100644 --- a/src/main/java/org/apache/datasketches/tuple/Util.java +++ b/src/main/java/org/apache/datasketches/tuple/Util.java @@ -21,13 +21,11 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.datasketches.common.Util.ceilingPowerOf2; -import static org.apache.datasketches.hash.MurmurHash3.hash; -import static org.apache.datasketches.memory.XxHash.hashCharArr; -import static org.apache.datasketches.memory.XxHash.hashString; +import static org.apache.datasketches.hash.XxHash.hashCharArr; +import static org.apache.datasketches.hash.XxHash.hashString; import java.lang.reflect.Array; -import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -58,36 +56,6 @@ public static final byte[] stringToByteArray(final String value) { return value.getBytes(UTF_8); } - /** - * Computes and checks the 16-bit seed hash from the given long seed. - * The seed hash may not be zero in order to maintain compatibility with older serialized - * versions that did not have this concept. - * @param seed See Update Hash Seed - * @return the seed hash. - */ - public static short computeSeedHash(final long seed) { - final long[] seedArr = {seed}; - final short seedHash = (short)((hash(seedArr, 0L)[0]) & 0xFFFFL); - if (seedHash == 0) { - throw new SketchesArgumentException( - "The given seed: " + seed + " produced a seedHash of zero. " - + "You must choose a different seed."); - } - return seedHash; - } - - /** - * Checks the two given seed hashes. If they are not equal, this method throws an Exception. - * @param seedHashA given seed hash A - * @param seedHashB given seed hash B - */ - public static final void checkSeedHashes(final short seedHashA, final short seedHashB) { - if (seedHashA != seedHashB) { - throw new SketchesArgumentException("Incompatible Seed Hashes. " + seedHashA + ", " - + seedHashB); - } - } - /** * Gets the starting capacity of a new sketch given the Nominal Entries and the log Resize Factor. * @param nomEntries the given Nominal Entries diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java index 1bb9edeca..d51451cf5 100644 --- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSketch.java @@ -19,8 +19,9 @@ package org.apache.datasketches.tuple.adouble; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ResizeFactor; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.tuple.UpdatableSketch; /** @@ -59,17 +60,17 @@ public DoubleSketch(final int lgK, final int lgResizeFactor, final float samplin } /** - * Constructs this sketch from a Memory image, which must be from an DoubleSketch, and + * Constructs this sketch from a MemorySegment image, which must be from an DoubleSketch, and * usually with data. - * @param mem the given Memory + * @param seg the given MemorySegment * @param mode The DoubleSummary mode to be used * @deprecated As of 3.0.0, heapifying an UpdatableSketch is deprecated. * This capability will be removed in a future release. * Heapifying a CompactSketch is not deprecated. */ @Deprecated - public DoubleSketch(final Memory mem, final DoubleSummary.Mode mode) { - super(mem, new DoubleSummaryDeserializer(), new DoubleSummaryFactory(mode)); + public DoubleSketch(final MemorySegment seg, final DoubleSummary.Mode mode) { + super(seg, new DoubleSummaryDeserializer(), new DoubleSummaryFactory(mode)); } @Override diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java index f678fba06..e7268ffef 100644 --- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java +++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java @@ -19,8 +19,12 @@ package org.apache.datasketches.tuple.adouble; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED; + +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ByteArrayUtil; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.tuple.DeserializeResult; import org.apache.datasketches.tuple.UpdatableSummary; @@ -146,13 +150,13 @@ public byte[] toByteArray() { /** * Creates an instance of the DoubleSummary given a serialized representation - * @param mem Memory object with serialized DoubleSummary + * @param seg MemorySegment object with serialized DoubleSummary * @return DeserializedResult object, which contains a DoubleSummary object and number of bytes - * read from the Memory + * read from the MemorySegment */ - public static DeserializeResult fromMemory(final Memory mem) { - return new DeserializeResult<>(new DoubleSummary(mem.getDouble(VALUE_INDEX), - Mode.values()[mem.getByte(MODE_BYTE_INDEX)]), SERIALIZED_SIZE_BYTES); + public static DeserializeResult fromMemorySegment(final MemorySegment seg) { + return new DeserializeResult<>(new DoubleSummary(seg.get(JAVA_DOUBLE_UNALIGNED, VALUE_INDEX), + Mode.values()[seg.get(JAVA_BYTE, MODE_BYTE_INDEX)]), SERIALIZED_SIZE_BYTES); } } diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java index 90e01b72b..b105a6b16 100644 --- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java +++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummaryDeserializer.java @@ -19,7 +19,8 @@ package org.apache.datasketches.tuple.adouble; -import org.apache.datasketches.memory.Memory; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.tuple.DeserializeResult; import org.apache.datasketches.tuple.SummaryDeserializer; @@ -30,8 +31,8 @@ public class DoubleSummaryDeserializer implements SummaryDeserializer { @Override - public DeserializeResult heapifySummary(final Memory mem) { - return DoubleSummary.fromMemory(mem); + public DeserializeResult heapifySummary(final MemorySegment seg) { + return DoubleSummary.fromMemorySegment(seg); } } diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java index 0bad790cc..5fdfe270e 100644 --- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSketch.java @@ -19,8 +19,9 @@ package org.apache.datasketches.tuple.aninteger; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ResizeFactor; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.tuple.UpdatableSketch; /** @@ -59,17 +60,17 @@ public IntegerSketch(final int lgK, final int lgResizeFactor, final float sampli } /** - * Constructs this sketch from a Memory image, which must be from an IntegerSketch, and + * Constructs this sketch from a MemorySegment image, which must be from an IntegerSketch, and * usually with data. - * @param mem the given Memory + * @param seg the given MemorySegment * @param mode The IntegerSummary mode to be used * @deprecated As of 3.0.0, heapifying an UpdatableSketch is deprecated. * This capability will be removed in a future release. * Heapifying a CompactSketch is not deprecated. */ @Deprecated - public IntegerSketch(final Memory mem, final IntegerSummary.Mode mode) { - super(mem, new IntegerSummaryDeserializer(), new IntegerSummaryFactory(mode)); + public IntegerSketch(final MemorySegment seg, final IntegerSummary.Mode mode) { + super(seg, new IntegerSummaryDeserializer(), new IntegerSummaryFactory(mode)); } @Override diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java index a8cfdab56..b8c7c23fb 100644 --- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java +++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java @@ -19,8 +19,12 @@ package org.apache.datasketches.tuple.aninteger; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; + +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ByteArrayUtil; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.tuple.DeserializeResult; import org.apache.datasketches.tuple.UpdatableSummary; @@ -146,13 +150,13 @@ public byte[] toByteArray() { /** * Creates an instance of the IntegerSummary given a serialized representation - * @param mem Memory object with serialized IntegerSummary + * @param seg MemorySegment object with serialized IntegerSummary * @return DeserializedResult object, which contains a IntegerSummary object and number of bytes - * read from the Memory + * read from the MemorySegment */ - public static DeserializeResult fromMemory(final Memory mem) { - return new DeserializeResult<>(new IntegerSummary(mem.getInt(VALUE_INDEX), - Mode.values()[mem.getByte(MODE_BYTE_INDEX)]), SERIALIZED_SIZE_BYTES); + public static DeserializeResult fromMemorySegment(final MemorySegment seg) { + return new DeserializeResult<>(new IntegerSummary(seg.get(JAVA_INT_UNALIGNED, VALUE_INDEX), + Mode.values()[seg.get(JAVA_BYTE, MODE_BYTE_INDEX)]), SERIALIZED_SIZE_BYTES); } } diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java index 68d468785..0941a1088 100644 --- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java +++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummaryDeserializer.java @@ -19,7 +19,8 @@ package org.apache.datasketches.tuple.aninteger; -import org.apache.datasketches.memory.Memory; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.tuple.DeserializeResult; import org.apache.datasketches.tuple.SummaryDeserializer; @@ -30,8 +31,8 @@ public class IntegerSummaryDeserializer implements SummaryDeserializer { @Override - public DeserializeResult heapifySummary(final Memory mem) { - return IntegerSummary.fromMemory(mem); + public DeserializeResult heapifySummary(final MemorySegment seg) { + return IntegerSummary.fromMemorySegment(seg); } } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotB.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotB.java index 5fda000bc..670038c01 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotB.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotB.java @@ -19,7 +19,7 @@ package org.apache.datasketches.tuple.arrayofdoubles; -import org.apache.datasketches.memory.WritableMemory; +import java.lang.foreign.MemorySegment; /** * Computes a set difference of two tuple sketches of type ArrayOfDoubles @@ -48,10 +48,10 @@ public abstract class ArrayOfDoublesAnotB { /** * Gets the result of this operation in the form of a ArrayOfDoublesCompactSketch - * @param mem memory for the result (can be null) - * @return compact sketch representing the result of the operation (off-heap if memory is + * @param seg MemorySegment for the result (can be null) + * @return compact sketch representing the result of the operation (off-heap if MemorySegment is * provided) */ - public abstract ArrayOfDoublesCompactSketch getResult(WritableMemory mem); + public abstract ArrayOfDoublesCompactSketch getResult(MemorySegment seg); } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotBImpl.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotBImpl.java index 44c0ed9b8..8d98b8e3a 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotBImpl.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotBImpl.java @@ -26,16 +26,16 @@ import static org.apache.datasketches.thetacommon.HashOperations.count; import static org.apache.datasketches.thetacommon.HashOperations.hashSearch; +import java.lang.foreign.MemorySegment; import java.util.Arrays; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SuppressFBWarnings; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.SetOperationCornerCases; +import org.apache.datasketches.thetacommon.ThetaUtil; import org.apache.datasketches.thetacommon.SetOperationCornerCases.AnotbAction; import org.apache.datasketches.thetacommon.SetOperationCornerCases.CornerCase; -import org.apache.datasketches.thetacommon.ThetaUtil; -import org.apache.datasketches.tuple.Util; /** * Computes a set difference, A-AND-NOT-B, of two ArrayOfDoublesSketches. @@ -154,8 +154,8 @@ public ArrayOfDoublesCompactSketch getResult() { } @Override - public ArrayOfDoublesCompactSketch getResult(final WritableMemory dstMem) { - return new DirectArrayOfDoublesCompactSketch(keys_, values_, thetaLong_, empty_, numValues_, seedHash_, dstMem); + public ArrayOfDoublesCompactSketch getResult(final MemorySegment dstSeg) { + return new DirectArrayOfDoublesCompactSketch(keys_, values_, thetaLong_, empty_, numValues_, seedHash_, dstSeg); } private static DataArrays getResultArrays( diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesIntersection.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesIntersection.java index 60d6d61d4..7c474ee45 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesIntersection.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesIntersection.java @@ -21,10 +21,11 @@ import static java.lang.Math.min; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.tuple.Util; +import org.apache.datasketches.common.Util; /** * Computes the intersection of two or more tuple sketches of type ArrayOfDoubles. @@ -116,10 +117,10 @@ public ArrayOfDoublesCompactSketch getResult() { /** * Gets the result of stateful intersections so far. - * @param dstMem Memory for the compact sketch (can be null). + * @param dstSeg MemorySegment for the compact sketch (can be null). * @return Result of the intersections so far as a compact sketch. */ - public ArrayOfDoublesCompactSketch getResult(final WritableMemory dstMem) { + public ArrayOfDoublesCompactSketch getResult(final MemorySegment dstSeg) { if (firstCall_) { throw new SketchesStateException( "getResult() with no intervening intersections is not a legal result."); @@ -150,11 +151,11 @@ public ArrayOfDoublesCompactSketch getResult(final WritableMemory dstMem) { } } - return (dstMem == null) + return (dstSeg == null) ? new HeapArrayOfDoublesCompactSketch(hashArrOut, valuesArrOut, thetaLong_, empty_, numValues_, seedHash_) : new DirectArrayOfDoublesCompactSketch(hashArrOut, valuesArrOut, - thetaLong_, empty_, numValues_, seedHash_, dstMem); + thetaLong_, empty_, numValues_, seedHash_, dstSeg); } /** diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesQuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesQuickSelectSketch.java index a6c536816..ddf102451 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesQuickSelectSketch.java @@ -21,9 +21,10 @@ import static org.apache.datasketches.common.Util.ceilingPowerOf2; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.QuickSelect; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.QuickSelect; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -85,7 +86,7 @@ abstract class ArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesUpdatableSk abstract int getSerializedSizeBytes(); - abstract void serializeInto(WritableMemory mem); + abstract void serializeInto(MemorySegment seg); @Override public void trim() { diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSetOperationBuilder.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSetOperationBuilder.java index bb6cc3741..14177b499 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSetOperationBuilder.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSetOperationBuilder.java @@ -19,8 +19,9 @@ package org.apache.datasketches.tuple.arrayofdoubles; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.Util; /** * Builds set operations object for tuple sketches of type ArrayOfDoubles. @@ -47,7 +48,7 @@ public class ArrayOfDoublesSetOperationBuilder { public ArrayOfDoublesSetOperationBuilder() { nomEntries_ = DEFAULT_NOMINAL_ENTRIES; numValues_ = DEFAULT_NUMBER_OF_VALUES; - seed_ = ThetaUtil.DEFAULT_UPDATE_SEED; + seed_ = Util.DEFAULT_UPDATE_SEED; } /** @@ -83,7 +84,7 @@ public ArrayOfDoublesSetOperationBuilder setSeed(final long seed) { /** * Creates an instance of ArrayOfDoublesUnion based on the current configuration of the builder. - * The new instance is allocated on the heap if the memory is not provided. + * The new instance is allocated on the heap. * @return an instance of ArrayOfDoublesUnion */ public ArrayOfDoublesUnion buildUnion() { @@ -92,18 +93,18 @@ public ArrayOfDoublesUnion buildUnion() { /** * Creates an instance of ArrayOfDoublesUnion based on the current configuration of the builder - * and the given destination memory. - * @param dstMem destination memory to be used by the sketch + * and the given destination MemorySegment. + * @param dstSeg destination MemorySegment to be used by the sketch * @return an instance of ArrayOfDoublesUnion */ - public ArrayOfDoublesUnion buildUnion(final WritableMemory dstMem) { - return new DirectArrayOfDoublesUnion(nomEntries_, numValues_, seed_, dstMem); + public ArrayOfDoublesUnion buildUnion(final MemorySegment dstSeg) { + return new DirectArrayOfDoublesUnion(nomEntries_, numValues_, seed_, dstSeg); } /** * Creates an instance of ArrayOfDoublesIntersection based on the current configuration of the * builder. - * The new instance is allocated on the heap if the memory is not provided. + * The new instance is allocated on the heap. * The number of nominal entries is not relevant to this, so it is ignored. * @return an instance of ArrayOfDoublesIntersection */ @@ -112,20 +113,18 @@ public ArrayOfDoublesIntersection buildIntersection() { } /** - * Creates an instance of ArrayOfDoublesIntersection based on the current configuration of the - * builder. - * The new instance is allocated on the heap if the memory is not provided. + * Creates an instance of ArrayOfDoublesIntersection in the given MemorySegment and based on the + * current configuration of the builder. * The number of nominal entries is not relevant to this, so it is ignored. - * @param dstMem destination memory to be used by the sketch + * @param dstSeg destination MemorySegment to be used by the sketch * @return an instance of ArrayOfDoublesIntersection */ - public ArrayOfDoublesIntersection buildIntersection(final WritableMemory dstMem) { - return new DirectArrayOfDoublesIntersection(numValues_, seed_, dstMem); + public ArrayOfDoublesIntersection buildIntersection(final MemorySegment dstSeg) { + return new DirectArrayOfDoublesIntersection(numValues_, seed_, dstSeg); } /** * Creates an instance of ArrayOfDoublesAnotB based on the current configuration of the builder. - * The memory is not relevant to this, so it is ignored if set. * The number of nominal entries is not relevant to this, so it is ignored. * @return an instance of ArrayOfDoublesAnotB */ diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java index 111427b7c..c901f81fd 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketch.java @@ -21,10 +21,10 @@ import static org.apache.datasketches.common.Util.LS; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.BinomialBoundsN; -import org.apache.datasketches.thetacommon.ThetaUtil; import org.apache.datasketches.tuple.SerializerDeserializer; /** @@ -71,49 +71,51 @@ static enum Flags { IS_BIG_ENDIAN, IS_IN_SAMPLING_MODE, IS_EMPTY, HAS_ENTRIES } } /** - * Heapify the given Memory as an ArrayOfDoublesSketch - * @param mem the given Memory + * Heapify the given MemorySegment as an ArrayOfDoublesSketch + * @param seg the given MemorySegment * @return an ArrayOfDoublesSketch */ - public static ArrayOfDoublesSketch heapify(final Memory mem) { - return heapify(mem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesSketch heapify(final MemorySegment seg) { + return heapify(seg, Util.DEFAULT_UPDATE_SEED); } /** - * Heapify the given Memory and seed as a ArrayOfDoublesSketch - * @param mem the given Memory + * Heapify the given MemorySegment and seed as a ArrayOfDoublesSketch + * @param seg the given MemorySegment * @param seed the given seed * @return an ArrayOfDoublesSketch */ - public static ArrayOfDoublesSketch heapify(final Memory mem, final long seed) { - final SerializerDeserializer.SketchType sketchType = SerializerDeserializer.getSketchType(mem); + public static ArrayOfDoublesSketch heapify(final MemorySegment seg, final long seed) { + final SerializerDeserializer.SketchType sketchType = SerializerDeserializer.getSketchType(seg); if (sketchType == SerializerDeserializer.SketchType.ArrayOfDoublesQuickSelectSketch) { - return new HeapArrayOfDoublesQuickSelectSketch(mem, seed); + return new HeapArrayOfDoublesQuickSelectSketch(seg, seed); } - return new HeapArrayOfDoublesCompactSketch(mem, seed); + return new HeapArrayOfDoublesCompactSketch(seg, seed); } /** - * Wrap the given Memory as an ArrayOfDoublesSketch - * @param mem the given Memory + * Wrap the given MemorySegment as an ArrayOfDoublesSketch. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param seg the given MemorySegment * @return an ArrayOfDoublesSketch */ - public static ArrayOfDoublesSketch wrap(final Memory mem) { - return wrap(mem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesSketch wrap(final MemorySegment seg) { + return wrap(seg, Util.DEFAULT_UPDATE_SEED); } /** - * Wrap the given Memory and seed as a ArrayOfDoublesSketch - * @param mem the given Memory + * Wrap the given MemorySegment and seed as a ArrayOfDoublesSketch. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param seg the given MemorySegment * @param seed the given seed * @return an ArrayOfDoublesSketch */ - public static ArrayOfDoublesSketch wrap(final Memory mem, final long seed) { - final SerializerDeserializer.SketchType sketchType = SerializerDeserializer.getSketchType(mem); + public static ArrayOfDoublesSketch wrap(final MemorySegment seg, final long seed) { + final SerializerDeserializer.SketchType sketchType = SerializerDeserializer.getSketchType(seg); if (sketchType == SerializerDeserializer.SketchType.ArrayOfDoublesQuickSelectSketch) { - return new DirectArrayOfDoublesQuickSelectSketchR(mem, seed); + return new DirectArrayOfDoublesQuickSelectSketchR(seg, seed); } - return new DirectArrayOfDoublesCompactSketch(mem, seed); + return new DirectArrayOfDoublesCompactSketch(seg, seed); } /** @@ -152,16 +154,16 @@ public double getLowerBound(final int numStdDev) { } /** - * Returns true if this sketch's data structure is backed by Memory or WritableMemory. - * @return true if this sketch's data structure is backed by Memory or WritableMemory. + * Returns true if this sketch's data structure is backed by MemorySegment. + * @return true if this sketch's data structure is backed by MemorySegment. */ - public abstract boolean hasMemory(); + public abstract boolean hasMemorySegment(); /** - * Returns the Memory object if it exists, otherwise null. - * @return the Memory object if it exists, otherwise null. + * Returns the MemorySegment object if it exists, otherwise null. + * @return the MemorySegment object if it exists, otherwise null. */ - abstract Memory getMemory(); + abstract MemorySegment getMemorySegment(); /** * See Empty @@ -256,10 +258,10 @@ public ArrayOfDoublesCompactSketch compact() { /** * Returns this sketch in compact form, which is immutable. - * @param dstMem the destination WritableMemory + * @param dstSeg the destination MemorySegment * @return this sketch in compact form, which is immutable. */ - public abstract ArrayOfDoublesCompactSketch compact(WritableMemory dstMem); + public abstract ArrayOfDoublesCompactSketch compact(MemorySegment dstSeg); @Override public String toString() { diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketches.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketches.java index 3e9abe6a3..53a145beb 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketches.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketches.java @@ -19,9 +19,9 @@ package org.apache.datasketches.tuple.arrayofdoubles; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.Util; /** * Convenient static methods to instantiate tuple sketches of type ArrayOfDoubles. @@ -29,136 +29,123 @@ public final class ArrayOfDoublesSketches { /** - * Heapify the given Memory as an ArrayOfDoublesSketch - * @param srcMem the given source Memory + * Heapify the given MemorySegment as an ArrayOfDoublesSketch + * @param srcSeg the given source MemorySegment * @return an ArrayOfDoublesSketch */ - public static ArrayOfDoublesSketch heapifySketch(final Memory srcMem) { - return heapifySketch(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesSketch heapifySketch(final MemorySegment srcSeg) { + return heapifySketch(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Heapify the given Memory and seed as a ArrayOfDoublesSketch - * @param srcMem the given source Memory + * Heapify the given MemorySegment and seed as a ArrayOfDoublesSketch + * @param srcSeg the given source MemorySegment * @param seed the given seed * @return an ArrayOfDoublesSketch */ - public static ArrayOfDoublesSketch heapifySketch(final Memory srcMem, final long seed) { - return ArrayOfDoublesSketch.heapify(srcMem, seed); + public static ArrayOfDoublesSketch heapifySketch(final MemorySegment srcSeg, final long seed) { + return ArrayOfDoublesSketch.heapify(srcSeg, seed); } /** - * Heapify the given Memory as an ArrayOfDoublesUpdatableSketch - * @param srcMem the given source Memory + * Heapify the given MemorySegment as an ArrayOfDoublesUpdatableSketch + * @param srcSeg the given source MemorySegment * @return an ArrayOfDoublesUpdatableSketch */ - public static ArrayOfDoublesUpdatableSketch heapifyUpdatableSketch(final Memory srcMem) { - return heapifyUpdatableSketch(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesUpdatableSketch heapifyUpdatableSketch(final MemorySegment srcSeg) { + return heapifyUpdatableSketch(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Heapify the given Memory and seed as a ArrayOfDoublesUpdatableSketch - * @param srcMem the given source Memory + * Heapify the given MemorySegment and seed as a ArrayOfDoublesUpdatableSketch + * @param srcSeg the given source MemorySegment * @param seed the given seed * @return an ArrayOfDoublesUpdatableSketch */ - public static ArrayOfDoublesUpdatableSketch heapifyUpdatableSketch(final Memory srcMem, final long seed) { - return ArrayOfDoublesUpdatableSketch.heapify(srcMem, seed); + public static ArrayOfDoublesUpdatableSketch heapifyUpdatableSketch(final MemorySegment srcSeg, final long seed) { + return ArrayOfDoublesUpdatableSketch.heapify(srcSeg, seed); } /** - * Wrap the given Memory as an ArrayOfDoublesSketch - * @param srcMem the given source Memory + * Wrap the given MemorySegment as an ArrayOfDoublesSketch. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg the given source MemorySegment * @return an ArrayOfDoublesSketch */ - public static ArrayOfDoublesSketch wrapSketch(final Memory srcMem) { - return wrapSketch(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesSketch wrapSketch(final MemorySegment srcSeg) { + return wrapSketch(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Wrap the given Memory and seed as a ArrayOfDoublesSketch - * @param srcMem the given source Memory + * Wrap the given MemorySegment and seed as a ArrayOfDoublesSketch. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg the given source MemorySegment * @param seed the given seed * @return an ArrayOfDoublesSketch */ - public static ArrayOfDoublesSketch wrapSketch(final Memory srcMem, final long seed) { - return ArrayOfDoublesSketch.wrap(srcMem, seed); + public static ArrayOfDoublesSketch wrapSketch(final MemorySegment srcSeg, final long seed) { + return ArrayOfDoublesSketch.wrap(srcSeg, seed); } /** - * Wrap the given WritableMemory as an ArrayOfDoublesUpdatableSketch - * @param srcMem the given source Memory + * Wrap the given MemorySegment as an ArrayOfDoublesUpdatableSketch. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg the given source MemorySegment * @return an ArrayOfDoublesUpdatableSketch */ - public static ArrayOfDoublesUpdatableSketch wrapUpdatableSketch(final WritableMemory srcMem) { - return wrapUpdatableSketch(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesUpdatableSketch wrapUpdatableSketch(final MemorySegment srcSeg) { + return wrapUpdatableSketch(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Wrap the given WritableMemory and seed as a ArrayOfDoublesUpdatableSketch - * @param srcMem the given source Memory + * Wrap the given MemorySegment and seed as a ArrayOfDoublesUpdatableSketch. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg the given source MemorySegment * @param seed the given seed * @return an ArrayOfDoublesUpdatableSketch */ - public static ArrayOfDoublesUpdatableSketch wrapUpdatableSketch(final WritableMemory srcMem, final long seed) { - return ArrayOfDoublesUpdatableSketch.wrap(srcMem, seed); - } - - /** - * Heapify the given Memory as an ArrayOfDoublesUnion - * @param srcMem the given source Memory - * @return an ArrayOfDoublesUnion - */ - public static ArrayOfDoublesUnion heapifyUnion(final Memory srcMem) { - return heapifyUnion(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); - } - - /** - * Heapify the given Memory and seed as an ArrayOfDoublesUnion - * @param srcMem the given source Memory - * @param seed the given seed - * @return an ArrayOfDoublesUnion - */ - public static ArrayOfDoublesUnion heapifyUnion(final Memory srcMem, final long seed) { - return ArrayOfDoublesUnion.heapify(srcMem, seed); + public static ArrayOfDoublesUpdatableSketch wrapUpdatableSketch(final MemorySegment srcSeg, final long seed) { + return ArrayOfDoublesUpdatableSketch.wrap(srcSeg, seed); } /** - * Wrap the given Memory as an ArrayOfDoublesUnion - * @param srcMem the given source Memory + * Heapify the given MemorySegment as an ArrayOfDoublesUnion + * @param srcSeg the given source MemorySegment * @return an ArrayOfDoublesUnion */ - public static ArrayOfDoublesUnion wrapUnion(final Memory srcMem) { - return wrapUnion(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesUnion heapifyUnion(final MemorySegment srcSeg) { + return heapifyUnion(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Wrap the given Memory and seed as an ArrayOfDoublesUnion - * @param srcMem the given source Memory + * Heapify the given MemorySegment and seed as an ArrayOfDoublesUnion + * @param srcSeg the given source MemorySegment * @param seed the given seed * @return an ArrayOfDoublesUnion */ - public static ArrayOfDoublesUnion wrapUnion(final Memory srcMem, final long seed) { - return ArrayOfDoublesUnion.wrap(srcMem, seed); + public static ArrayOfDoublesUnion heapifyUnion(final MemorySegment srcSeg, final long seed) { + return ArrayOfDoublesUnion.heapify(srcSeg, seed); } /** - * Wrap the given Memory as an ArrayOfDoublesUnion - * @param srcMem the given source Memory + * Wrap the given MemorySegment as an ArrayOfDoublesUnion + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg the given source MemorySegment * @return an ArrayOfDoublesUnion */ - public static ArrayOfDoublesUnion wrapUnion(final WritableMemory srcMem) { - return wrapUnion(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesUnion wrapUnion(final MemorySegment srcSeg) { + return wrapUnion(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Wrap the given Memory and seed as an ArrayOfDoublesUnion - * @param srcMem the given source Memory + * Wrap the given MemorySegment and seed as an ArrayOfDoublesUnion + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg the given source MemorySegment * @param seed the given seed * @return an ArrayOfDoublesUnion */ - public static ArrayOfDoublesUnion wrapUnion(final WritableMemory srcMem, final long seed) { - return ArrayOfDoublesUnion.wrap(srcMem, seed); + public static ArrayOfDoublesUnion wrapUnion(final MemorySegment srcSeg, final long seed) { + return ArrayOfDoublesUnion.wrap(srcSeg, seed); } } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java index 9e1db0ada..58a907702 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnion.java @@ -19,15 +19,16 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static java.lang.Math.min; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; import org.apache.datasketches.tuple.SerializerDeserializer; -import org.apache.datasketches.tuple.Util; /** * The base class for unions of tuple sketches of type ArrayOfDoubles. @@ -59,60 +60,43 @@ public abstract class ArrayOfDoublesUnion { } /** - * Heapify the given Memory as an ArrayOfDoublesUnion - * @param srcMem the given source Memory - * @return an ArrayOfDoublesUnion - */ - public static ArrayOfDoublesUnion heapify(final Memory srcMem) { - return heapify(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); - } - - /** - * Heapify the given Memory and seed as an ArrayOfDoublesUnion - * @param srcMem the given source Memory - * @param seed the given seed - * @return an ArrayOfDoublesUnion - */ - public static ArrayOfDoublesUnion heapify(final Memory srcMem, final long seed) { - return HeapArrayOfDoublesUnion.heapifyUnion(srcMem, seed); - } - - /** - * Wrap the given Memory as an ArrayOfDoublesUnion - * @param srcMem the given source Memory + * Heapify the given MemorySegment as an ArrayOfDoublesUnion. + * @param srcSeg the given source MemorySegment * @return an ArrayOfDoublesUnion */ - public static ArrayOfDoublesUnion wrap(final Memory srcMem) { - return wrap(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesUnion heapify(final MemorySegment srcSeg) { + return heapify(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Wrap the given Memory and seed as an ArrayOfDoublesUnion - * @param srcMem the given source Memory + * Heapify the given MemorySegment and seed as an ArrayOfDoublesUnion. + * @param srcSeg the given source MemorySegment * @param seed the given seed * @return an ArrayOfDoublesUnion */ - public static ArrayOfDoublesUnion wrap(final Memory srcMem, final long seed) { - return DirectArrayOfDoublesUnion.wrapUnion((WritableMemory) srcMem, seed, false); + public static ArrayOfDoublesUnion heapify(final MemorySegment srcSeg, final long seed) { + return HeapArrayOfDoublesUnion.heapifyUnion(srcSeg, seed); } /** - * Wrap the given WritableMemory as an ArrayOfDoublesUnion - * @param srcMem the given source Memory + * Wrap the given MemorySegment as an ArrayOfDoublesUnion. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg the given source MemorySegment * @return an ArrayOfDoublesUnion */ - public static ArrayOfDoublesUnion wrap(final WritableMemory srcMem) { - return wrap(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesUnion wrap(final MemorySegment srcSeg) { + return wrap(srcSeg, Util.DEFAULT_UPDATE_SEED); } /** - * Wrap the given WritableMemory and seed as an ArrayOfDoublesUnion - * @param srcMem the given source Memory + * Wrap the given MemorySegment and seed as an ArrayOfDoublesUnion. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param srcSeg the given source MemorySegment * @param seed the given seed * @return an ArrayOfDoublesUnion */ - public static ArrayOfDoublesUnion wrap(final WritableMemory srcMem, final long seed) { - return DirectArrayOfDoublesUnion.wrapUnion(srcMem, seed, true); + public static ArrayOfDoublesUnion wrap(final MemorySegment srcSeg, final long seed) { + return DirectArrayOfDoublesUnion.wrapUnion(srcSeg, seed, !srcSeg.isReadOnly()); } /** @@ -121,7 +105,7 @@ public static ArrayOfDoublesUnion wrap(final WritableMemory srcMem, final long s * *

      Nulls and empty sketches are ignored.

      * - * @param tupleSketch sketch to add to the union + * @param tupleSketch sketch to add to the union. */ public void union(final ArrayOfDoublesSketch tupleSketch) { if (tupleSketch == null) { return; } @@ -151,18 +135,18 @@ public void union(final ArrayOfDoublesSketch tupleSketch) { /** * Returns the resulting union in the form of a compact sketch - * @param dstMem memory for the result (can be null) - * @return compact sketch representing the union (off-heap if memory is provided) + * @param dstSeg MemorySegment for the result (can be null) + * @return compact sketch representing the union (off-heap if MemorySegment is provided) */ - public ArrayOfDoublesCompactSketch getResult(final WritableMemory dstMem) { + public ArrayOfDoublesCompactSketch getResult(final MemorySegment dstSeg) { long unionThetaLong = unionThetaLong_; if (gadget_.getRetainedEntries() > gadget_.getNominalEntries()) { unionThetaLong = Math.min(unionThetaLong, gadget_.getNewThetaLong()); } - if (dstMem == null) { + if (dstSeg == null) { return new HeapArrayOfDoublesCompactSketch(gadget_, unionThetaLong); } - return new DirectArrayOfDoublesCompactSketch(gadget_, unionThetaLong, dstMem); + return new DirectArrayOfDoublesCompactSketch(gadget_, unionThetaLong, dstSeg); } /** @@ -195,14 +179,14 @@ public void reset() { public byte[] toByteArray() { final int sizeBytes = PREAMBLE_SIZE_BYTES + gadget_.getSerializedSizeBytes(); final byte[] byteArray = new byte[sizeBytes]; - final WritableMemory mem = WritableMemory.writableWrap(byteArray); - mem.putByte(PREAMBLE_LONGS_BYTE, (byte) 1); // unused, always 1 - mem.putByte(SERIAL_VERSION_BYTE, serialVersionUID); - mem.putByte(FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); - mem.putByte(SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesUnion.ordinal()); + final MemorySegment seg = MemorySegment.ofArray(byteArray); + seg.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, (byte) 1); // unused, always 1 + seg.set(JAVA_BYTE, SERIAL_VERSION_BYTE, serialVersionUID); + seg.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); + seg.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesUnion.ordinal()); //byte 4-7 automatically zero - mem.putLong(THETA_LONG, unionThetaLong_); - gadget_.serializeInto(mem.writableRegion(PREAMBLE_SIZE_BYTES, mem.getCapacity() - PREAMBLE_SIZE_BYTES)); + seg.set(JAVA_LONG_UNALIGNED, THETA_LONG, unionThetaLong_); + gadget_.serializeInto(seg.asSlice(PREAMBLE_SIZE_BYTES, seg.byteSize() - PREAMBLE_SIZE_BYTES)); return byteArray; } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUpdatableSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUpdatableSketch.java index cd3c1d2e7..dfeffcc4f 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUpdatableSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUpdatableSketch.java @@ -19,13 +19,14 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import static org.apache.datasketches.common.Util.computeSeedHash; +import static org.apache.datasketches.common.Util.DEFAULT_UPDATE_SEED; + +import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.hash.MurmurHash3; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; import org.apache.datasketches.tuple.Util; /** @@ -41,41 +42,43 @@ public abstract class ArrayOfDoublesUpdatableSketch extends ArrayOfDoublesSketch } /** - * Heapify the given Memory as an ArrayOfDoublesUpdatableSketch - * @param mem the given Memory + * Heapify the given MemorySegment as an ArrayOfDoublesUpdatableSketch + * @param seg the given MemorySegment * @return an ArrayOfDoublesUpdatableSketch */ - public static ArrayOfDoublesUpdatableSketch heapify(final Memory mem) { - return heapify(mem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesUpdatableSketch heapify(final MemorySegment seg) { + return heapify(seg, DEFAULT_UPDATE_SEED); } /** - * Heapify the given Memory and seed as a ArrayOfDoublesUpdatableSketch - * @param mem the given Memory + * Heapify the given MemorySegment and seed as a ArrayOfDoublesUpdatableSketch + * @param seg the given MemorySegment * @param seed the given seed * @return an ArrayOfDoublesUpdatableSketch */ - public static ArrayOfDoublesUpdatableSketch heapify(final Memory mem, final long seed) { - return new HeapArrayOfDoublesQuickSelectSketch(mem, seed); + public static ArrayOfDoublesUpdatableSketch heapify(final MemorySegment seg, final long seed) { + return new HeapArrayOfDoublesQuickSelectSketch(seg, seed); } /** - * Wrap the given WritableMemory as an ArrayOfDoublesUpdatableSketch - * @param mem the given Memory + * Wrap the given MemorySegment as an ArrayOfDoublesUpdatableSketch. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param seg the given MemorySegment * @return an ArrayOfDoublesUpdatableSketch */ - public static ArrayOfDoublesUpdatableSketch wrap(final WritableMemory mem) { - return wrap(mem, ThetaUtil.DEFAULT_UPDATE_SEED); + public static ArrayOfDoublesUpdatableSketch wrap(final MemorySegment seg) { + return wrap(seg, DEFAULT_UPDATE_SEED); } /** - * Wrap the given WritableMemory and seed as a ArrayOfDoublesUpdatableSketch - * @param mem the given Memory + * Wrap the given MemorySegment and seed as a ArrayOfDoublesUpdatableSketch. + * If the given source MemorySegment is read-only, the returned Union object will also be read-only. + * @param seg the given MemorySegment * @param seed the given seed * @return an ArrayOfDoublesUpdatableSketch */ - public static ArrayOfDoublesUpdatableSketch wrap(final WritableMemory mem, final long seed) { - return new DirectArrayOfDoublesQuickSelectSketch(mem, seed); + public static ArrayOfDoublesUpdatableSketch wrap(final MemorySegment seg, final long seed) { + return new DirectArrayOfDoublesQuickSelectSketch(seg, seed); } /** @@ -197,16 +200,16 @@ public ArrayOfDoublesCompactSketch compact() { } /** - * Gets an off-heap compact representation of the sketch using the given memory - * @param dstMem memory for the compact sketch (can be null) - * @return compact sketch (off-heap if memory is provided) + * Gets an off-heap compact representation of the sketch using the given MemorySegment + * @param dstSeg MemorySegment for the compact sketch (can be null) + * @return compact sketch (off-heap if MemorySegment is provided) */ @Override - public ArrayOfDoublesCompactSketch compact(final WritableMemory dstMem) { - if (dstMem == null) { + public ArrayOfDoublesCompactSketch compact(final MemorySegment dstSeg) { + if (dstSeg == null) { return new HeapArrayOfDoublesCompactSketch(this); } - return new DirectArrayOfDoublesCompactSketch(this, dstMem); + return new DirectArrayOfDoublesCompactSketch(this, dstSeg); } abstract int getCurrentCapacity(); @@ -217,7 +220,7 @@ long getSeed() { @Override short getSeedHash() { - return Util.computeSeedHash(seed_); + return computeSeedHash(seed_); } /** diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUpdatableSketchBuilder.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUpdatableSketchBuilder.java index e9fd12d25..185186975 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUpdatableSketchBuilder.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUpdatableSketchBuilder.java @@ -19,9 +19,11 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.thetacommon.ThetaUtil; /** @@ -47,7 +49,7 @@ public ArrayOfDoublesUpdatableSketchBuilder() { resizeFactor_ = DEFAULT_RESIZE_FACTOR; numValues_ = DEFAULT_NUMBER_OF_VALUES; samplingProbability_ = DEFAULT_SAMPLING_PROBABILITY; - seed_ = ThetaUtil.DEFAULT_UPDATE_SEED; + seed_ = Util.DEFAULT_UPDATE_SEED; } /** @@ -119,12 +121,12 @@ public ArrayOfDoublesUpdatableSketch build() { /** * Returns an ArrayOfDoublesUpdatableSketch with the current configuration of this Builder. - * @param dstMem instance of Memory to be used by the sketch + * @param dstSeg instance of MemorySegment to be used by the sketch * @return an ArrayOfDoublesUpdatableSketch */ - public ArrayOfDoublesUpdatableSketch build(final WritableMemory dstMem) { + public ArrayOfDoublesUpdatableSketch build(final MemorySegment dstSeg) { return new DirectArrayOfDoublesQuickSelectSketch(nomEntries_, resizeFactor_.lg(), - samplingProbability_, numValues_, seed_, dstMem); + samplingProbability_, numValues_, seed_, dstSeg); } } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java index 52f827149..083b5e06a 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketch.java @@ -19,35 +19,40 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; + +import java.lang.foreign.MemorySegment; import java.nio.ByteOrder; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; import org.apache.datasketches.tuple.SerializerDeserializer; -import org.apache.datasketches.tuple.Util; /** * Direct Compact Sketch of type ArrayOfDoubles. * - *

      This implementation uses data in a given Memory that is owned and managed by the caller. - * This Memory can be off-heap, which if managed properly will greatly reduce the need for + *

      This implementation uses data in a given MemorySegment that is owned and managed by the caller. + * This MemorySegment can be off-heap, which if managed properly will greatly reduce the need for * the JVM to perform garbage collection.

      */ final class DirectArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketch { // this value exists only on heap, never serialized - private Memory mem_; + private MemorySegment seg_; /** * Converts the given UpdatableArrayOfDoublesSketch to this compact form. * @param sketch the given UpdatableArrayOfDoublesSketch - * @param dstMem the given destination Memory. + * @param dstSeg the given destination MemorySegment. */ DirectArrayOfDoublesCompactSketch(final ArrayOfDoublesUpdatableSketch sketch, - final WritableMemory dstMem) { - this(sketch, sketch.getThetaLong(), dstMem); + final MemorySegment dstSeg) { + this(sketch, sketch.getThetaLong(), dstSeg); } /** @@ -55,30 +60,30 @@ final class DirectArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketc * trimming if necessary according to given theta * @param sketch the given UpdatableArrayOfDoublesSketch * @param thetaLong new value of thetaLong - * @param dstMem the given destination Memory. + * @param dstSeg the given destination MemorySegment. */ DirectArrayOfDoublesCompactSketch(final ArrayOfDoublesUpdatableSketch sketch, - final long thetaLong, final WritableMemory dstMem) { + final long thetaLong, final MemorySegment dstSeg) { super(sketch.getNumValues()); - checkIfEnoughMemory(dstMem, sketch.getRetainedEntries(), sketch.getNumValues()); - mem_ = dstMem; - dstMem.putByte(PREAMBLE_LONGS_BYTE, (byte) 1); - dstMem.putByte(SERIAL_VERSION_BYTE, serialVersionUID); - dstMem.putByte(FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); - dstMem.putByte(SKETCH_TYPE_BYTE, (byte) + checkMemorySegmentSize(dstSeg, sketch.getRetainedEntries(), sketch.getNumValues()); + seg_ = dstSeg; + dstSeg.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, (byte) 1); + dstSeg.set(JAVA_BYTE, SERIAL_VERSION_BYTE, serialVersionUID); + dstSeg.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); + dstSeg.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch.ordinal()); final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); isEmpty_ = sketch.isEmpty(); final int count = sketch.getRetainedEntries(); - dstMem.putByte(FLAGS_BYTE, (byte) ( + dstSeg.set(JAVA_BYTE, FLAGS_BYTE, (byte) ( (isBigEndian ? 1 << Flags.IS_BIG_ENDIAN.ordinal() : 0) | (isEmpty_ ? 1 << Flags.IS_EMPTY.ordinal() : 0) | (count > 0 ? 1 << Flags.HAS_ENTRIES.ordinal() : 0) )); - dstMem.putByte(NUM_VALUES_BYTE, (byte) numValues_); - dstMem.putShort(SEED_HASH_SHORT, Util.computeSeedHash(sketch.getSeed())); + dstSeg.set(JAVA_BYTE, NUM_VALUES_BYTE, (byte) numValues_); + dstSeg.set(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT, Util.computeSeedHash(sketch.getSeed())); thetaLong_ = Math.min(sketch.getThetaLong(), thetaLong); - dstMem.putLong(THETA_LONG, thetaLong_); + dstSeg.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong_); if (count > 0) { int keyOffset = ENTRIES_START; int valuesOffset = keyOffset + (SIZE_OF_KEY_BYTES * sketch.getRetainedEntries()); @@ -86,14 +91,14 @@ final class DirectArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketc int actualCount = 0; while (it.next()) { if (it.getKey() < thetaLong_) { - dstMem.putLong(keyOffset, it.getKey()); - dstMem.putDoubleArray(valuesOffset, it.getValues(), 0, numValues_); + dstSeg.set(JAVA_LONG_UNALIGNED, keyOffset, it.getKey()); + MemorySegment.copy(it.getValues(), 0, dstSeg, JAVA_DOUBLE_UNALIGNED, valuesOffset, numValues_); keyOffset += SIZE_OF_KEY_BYTES; valuesOffset += SIZE_OF_VALUE_BYTES * numValues_; actualCount++; } } - dstMem.putInt(RETAINED_ENTRIES_INT, actualCount); + dstSeg.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, actualCount); } } @@ -101,109 +106,108 @@ final class DirectArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketc * Creates an instance from components */ DirectArrayOfDoublesCompactSketch(final long[] keys, final double[] values, final long thetaLong, - final boolean isEmpty, final int numValues, final short seedHash, final WritableMemory dstMem) { + final boolean isEmpty, final int numValues, final short seedHash, final MemorySegment dstSeg) { super(numValues); - checkIfEnoughMemory(dstMem, values.length, numValues); - mem_ = dstMem; - dstMem.putByte(PREAMBLE_LONGS_BYTE, (byte) 1); - dstMem.putByte(SERIAL_VERSION_BYTE, serialVersionUID); - dstMem.putByte(FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); - dstMem.putByte(SKETCH_TYPE_BYTE, (byte) + checkMemorySegmentSize(dstSeg, values.length, numValues); + seg_ = dstSeg; + dstSeg.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, (byte) 1); + dstSeg.set(JAVA_BYTE, SERIAL_VERSION_BYTE, serialVersionUID); + dstSeg.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); + dstSeg.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch.ordinal()); final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); isEmpty_ = isEmpty; final int count = keys.length; - dstMem.putByte(FLAGS_BYTE, (byte) ( + dstSeg.set(JAVA_BYTE, FLAGS_BYTE, (byte) ( (isBigEndian ? 1 << Flags.IS_BIG_ENDIAN.ordinal() : 0) | (isEmpty_ ? 1 << Flags.IS_EMPTY.ordinal() : 0) | (count > 0 ? 1 << Flags.HAS_ENTRIES.ordinal() : 0) )); - dstMem.putByte(NUM_VALUES_BYTE, (byte) numValues_); - dstMem.putShort(SEED_HASH_SHORT, seedHash); + dstSeg.set(JAVA_BYTE, NUM_VALUES_BYTE, (byte) numValues_); + dstSeg.set(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT, seedHash); thetaLong_ = thetaLong; - dstMem.putLong(THETA_LONG, thetaLong_); + dstSeg.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong_); if (count > 0) { - dstMem.putInt(RETAINED_ENTRIES_INT, count); - dstMem.putLongArray(ENTRIES_START, keys, 0, count); - dstMem.putDoubleArray( - ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * count), values, 0, values.length); + dstSeg.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, count); + MemorySegment.copy(keys, 0, dstSeg, JAVA_LONG_UNALIGNED, ENTRIES_START, count); + MemorySegment.copy(values, 0, dstSeg, JAVA_DOUBLE_UNALIGNED, ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * count), values.length); } } /** - * Wraps the given Memory. - * @param mem See Memory + * Wraps the given MemorySegment. + * @param seg the given MemorySegment */ - DirectArrayOfDoublesCompactSketch(final Memory mem) { - super(mem.getByte(NUM_VALUES_BYTE)); - mem_ = mem; - SerializerDeserializer.validateFamily(mem.getByte(FAMILY_ID_BYTE), - mem.getByte(PREAMBLE_LONGS_BYTE)); - SerializerDeserializer.validateType(mem_.getByte(SKETCH_TYPE_BYTE), + DirectArrayOfDoublesCompactSketch(final MemorySegment seg) { + super(seg.get(JAVA_BYTE, NUM_VALUES_BYTE)); + seg_ = seg; + SerializerDeserializer.validateFamily(seg.get(JAVA_BYTE, FAMILY_ID_BYTE), + seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE)); + SerializerDeserializer.validateType(seg_.get(JAVA_BYTE, SKETCH_TYPE_BYTE), SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch); - final byte version = mem_.getByte(SERIAL_VERSION_BYTE); + final byte version = seg_.get(JAVA_BYTE, SERIAL_VERSION_BYTE); if (version != serialVersionUID) { throw new SketchesArgumentException("Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } final boolean isBigEndian = - (mem.getByte(FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; + (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { throw new SketchesArgumentException("Byte order mismatch"); } - isEmpty_ = (mem_.getByte(FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; - thetaLong_ = mem_.getLong(THETA_LONG); + isEmpty_ = (seg_.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; + thetaLong_ = seg_.get(JAVA_LONG_UNALIGNED, THETA_LONG); } /** - * Wraps the given Memory. - * @param mem See Memory + * Wraps the given MemorySegment. + * @param seg the given MemorySegment. * @param seed See seed */ - DirectArrayOfDoublesCompactSketch(final Memory mem, final long seed) { - super(mem.getByte(NUM_VALUES_BYTE)); - mem_ = mem; - SerializerDeserializer.validateFamily(mem.getByte(FAMILY_ID_BYTE), - mem.getByte(PREAMBLE_LONGS_BYTE)); - SerializerDeserializer.validateType(mem_.getByte(SKETCH_TYPE_BYTE), + DirectArrayOfDoublesCompactSketch(final MemorySegment seg, final long seed) { + super(seg.get(JAVA_BYTE, NUM_VALUES_BYTE)); + seg_ = seg; + SerializerDeserializer.validateFamily(seg.get(JAVA_BYTE, FAMILY_ID_BYTE), + seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE)); + SerializerDeserializer.validateType(seg_.get(JAVA_BYTE, SKETCH_TYPE_BYTE), SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch); - final byte version = mem_.getByte(SERIAL_VERSION_BYTE); + final byte version = seg_.get(JAVA_BYTE, SERIAL_VERSION_BYTE); if (version != serialVersionUID) { throw new SketchesArgumentException("Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } final boolean isBigEndian = - (mem.getByte(FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; + (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { throw new SketchesArgumentException("Byte order mismatch"); } - Util.checkSeedHashes(mem.getShort(SEED_HASH_SHORT), Util.computeSeedHash(seed)); - isEmpty_ = (mem_.getByte(FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; - thetaLong_ = mem_.getLong(THETA_LONG); + Util.checkSeedHashes(seg.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT), Util.computeSeedHash(seed)); + isEmpty_ = (seg_.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; + thetaLong_ = seg.get(JAVA_LONG_UNALIGNED, THETA_LONG); } @Override - public ArrayOfDoublesCompactSketch compact(final WritableMemory dstMem) { - if (dstMem == null) { + public ArrayOfDoublesCompactSketch compact(final MemorySegment dstSeg) { + if (dstSeg == null) { return new HeapArrayOfDoublesCompactSketch(getKeys(), getValuesAsOneDimension(), thetaLong_, isEmpty_, numValues_, getSeedHash()); } else { - mem_.copyTo(0, dstMem, 0, mem_.getCapacity()); - return new DirectArrayOfDoublesCompactSketch(dstMem); + MemorySegment.copy(seg_, 0, dstSeg, 0, seg_.byteSize()); + return new DirectArrayOfDoublesCompactSketch(dstSeg); } } @Override public int getRetainedEntries() { final boolean hasEntries = - (mem_.getByte(FLAGS_BYTE) & (1 << Flags.HAS_ENTRIES.ordinal())) != 0; - return (hasEntries ? mem_.getInt(RETAINED_ENTRIES_INT) : 0); + (seg_.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.HAS_ENTRIES.ordinal())) != 0; + return (hasEntries ? seg_.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT) : 0); } @Override - //converts compact Memory array of double[] to compact double[][] + //converts compact MemorySegment array of double[] to compact double[][] public double[][] getValues() { final int count = getRetainedEntries(); final double[][] values = new double[count][]; @@ -211,7 +215,7 @@ public double[][] getValues() { int valuesOffset = ENTRIES_START + (SIZE_OF_KEY_BYTES * count); for (int i = 0; i < count; i++) { final double[] array = new double[numValues_]; - mem_.getDoubleArray(valuesOffset, array, 0, numValues_); + MemorySegment.copy(seg_, JAVA_DOUBLE_UNALIGNED, valuesOffset, array, 0, numValues_); values[i] = array; valuesOffset += SIZE_OF_VALUE_BYTES * numValues_; } @@ -220,26 +224,26 @@ public double[][] getValues() { } @Override - //converts compact Memory array of double[] to compact double[] + //converts compact MemorySegment array of double[] to compact double[] double[] getValuesAsOneDimension() { final int count = getRetainedEntries(); final int numDoubles = count * numValues_; final double[] values = new double[numDoubles]; if (count > 0) { final int valuesOffset = ENTRIES_START + (SIZE_OF_KEY_BYTES * count); - mem_.getDoubleArray(valuesOffset, values, 0, numDoubles); + MemorySegment.copy(seg_, JAVA_DOUBLE_UNALIGNED, valuesOffset, values, 0, numDoubles); } return values; } @Override - //converts compact Memory array of long[] to compact long[] + //converts compact MemorySegment array of long[] to compact long[] long[] getKeys() { final int count = getRetainedEntries(); final long[] keys = new long[count]; if (count > 0) { for (int i = 0; i < count; i++) { - mem_.getLongArray(ENTRIES_START, keys, 0, count); + MemorySegment.copy(seg_, JAVA_LONG_UNALIGNED, ENTRIES_START, keys, 0, count); } } return keys; @@ -249,35 +253,35 @@ long[] getKeys() { public byte[] toByteArray() { final int sizeBytes = getCurrentBytes(); final byte[] byteArray = new byte[sizeBytes]; - final WritableMemory mem = WritableMemory.writableWrap(byteArray); - mem_.copyTo(0, mem, 0, sizeBytes); + final MemorySegment seg = MemorySegment.ofArray(byteArray); + MemorySegment.copy(seg_, 0, seg, 0, sizeBytes); return byteArray; } @Override public ArrayOfDoublesSketchIterator iterator() { return new DirectArrayOfDoublesSketchIterator( - mem_, ENTRIES_START, getRetainedEntries(), numValues_); + seg_, ENTRIES_START, getRetainedEntries(), numValues_); } @Override short getSeedHash() { - return mem_.getShort(SEED_HASH_SHORT); + return seg_.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT); } @Override - public boolean hasMemory() { return true; } + public boolean hasMemorySegment() { return true; } @Override - Memory getMemory() { return mem_; } + MemorySegment getMemorySegment() { return seg_; } - private static void checkIfEnoughMemory(final Memory mem, final int numEntries, + private static void checkMemorySegmentSize(final MemorySegment seg, final int numEntries, final int numValues) { final int sizeNeeded = ENTRIES_START + ((SIZE_OF_KEY_BYTES + (SIZE_OF_VALUE_BYTES * numValues)) * numEntries); - if (sizeNeeded > mem.getCapacity()) { - throw new SketchesArgumentException("Not enough memory: need " + sizeNeeded - + " bytes, got " + mem.getCapacity() + " bytes"); + if (sizeNeeded > seg.byteSize()) { + throw new SketchesArgumentException("Not enough space: need " + sizeNeeded + + " bytes, got " + seg.byteSize() + " bytes"); } } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java index 7c1b1bf07..1aa34dbba 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesIntersection.java @@ -19,34 +19,34 @@ package org.apache.datasketches.tuple.arrayofdoubles; -import org.apache.datasketches.memory.WritableMemory; +import java.lang.foreign.MemorySegment; /** * Direct Intersection operation for tuple sketches of type ArrayOfDoubles. * - *

      This implementation uses data in a given Memory that is owned and managed by the caller. - * This Memory can be off-heap, which if managed properly will greatly reduce the need for + *

      This implementation uses data in a given MemorySegment that is owned and managed by the caller. + * This MemorySegment can be off-heap, which if managed properly will greatly reduce the need for * the JVM to perform garbage collection.

      */ final class DirectArrayOfDoublesIntersection extends ArrayOfDoublesIntersection { - private WritableMemory mem_; + private MemorySegment seg_; /** * Creates an instance of a DirectArrayOfDoublesIntersection with a custom update seed * @param numValues number of double values associated with each key * @param seed See seed - * @param dstMem See Memory + * @param dstSeg the destination MemorySegment */ - DirectArrayOfDoublesIntersection(final int numValues, final long seed, final WritableMemory dstMem) { + DirectArrayOfDoublesIntersection(final int numValues, final long seed, final MemorySegment dstSeg) { super(numValues, seed); - mem_ = dstMem; + seg_ = dstSeg; } @Override protected ArrayOfDoublesQuickSelectSketch createSketch(final int nomEntries, final int numValues, final long seed) { - return new DirectArrayOfDoublesQuickSelectSketch(nomEntries, 0, 1f, numValues, seed, mem_); + return new DirectArrayOfDoublesQuickSelectSketch(nomEntries, 0, 1f, numValues, seed, seg_); } } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java index ae1aa3dc0..0bd6ee65f 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketch.java @@ -19,14 +19,25 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; +import static org.apache.datasketches.common.Util.checkSeedHashes; +import static org.apache.datasketches.common.Util.clear; +import static org.apache.datasketches.common.Util.clearBits; +import static org.apache.datasketches.common.Util.computeSeedHash; +import static org.apache.datasketches.common.Util.setBits; + +import java.lang.foreign.MemorySegment; import java.nio.ByteOrder; import java.util.Arrays; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.HashOperations; import org.apache.datasketches.tuple.SerializerDeserializer; import org.apache.datasketches.tuple.Util; @@ -34,20 +45,20 @@ /** * Direct QuickSelect tuple sketch of type ArrayOfDoubles. * - *

      This implementation uses data in a given Memory that is owned and managed by the caller. - * This Memory can be off-heap, which if managed properly will greatly reduce the need for + *

      This implementation uses data in a given MemorySegment that is owned and managed by the caller. + * This MemorySegment can be off-heap, which if managed properly will greatly reduce the need for * the JVM to perform garbage collection.

      */ class DirectArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelectSketch { // these values exist only on heap, never serialized - private WritableMemory mem_; - // these can be derived from the mem_ contents, but are kept here for performance + private MemorySegment seg_; + // these can be derived from the seg_ contents, but are kept here for performance private int keysOffset_; private int valuesOffset_; /** - * Construct a new sketch using the given Memory as its backing store. + * Construct a new sketch using the given MemorySegment as its backing store. * * @param nomEntries Nominal number of entries. Forced to the nearest power of 2 greater than * given value. @@ -60,7 +71,7 @@ class DirectArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelectSke * See Sampling Probability * @param numValues Number of double values to keep for each key. * @param seed See seed - * @param dstMem See Memory + * @param dstSeg the destination MemorySegment. */ DirectArrayOfDoublesQuickSelectSketch( final int nomEntries, @@ -68,8 +79,8 @@ class DirectArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelectSke final float samplingProbability, final int numValues, final long seed, - final WritableMemory dstMem) { - this(checkMemory(nomEntries, lgResizeFactor, numValues, dstMem), + final MemorySegment dstSeg) { + this(checkMemorySegment(nomEntries, lgResizeFactor, numValues, dstSeg), //SpotBugs CT_CONSTRUCTOR_THROW is false positive. //this construction scheme is compliant with SEI CERT Oracle Coding Standard for Java / OBJ11-J nomEntries, @@ -77,7 +88,7 @@ class DirectArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelectSke samplingProbability, numValues, seed, - dstMem); + dstSeg); } private DirectArrayOfDoublesQuickSelectSketch( @@ -87,89 +98,89 @@ private DirectArrayOfDoublesQuickSelectSketch( final float samplingProbability, final int numValues, final long seed, - final WritableMemory dstMem) { + final MemorySegment dstSeg) { super(numValues, seed); - mem_ = dstMem; + seg_ = dstSeg; final int startingCapacity = Util.getStartingCapacity(nomEntries, lgResizeFactor); - mem_.putByte(PREAMBLE_LONGS_BYTE, (byte) 1); - mem_.putByte(SERIAL_VERSION_BYTE, serialVersionUID); - mem_.putByte(FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); - mem_.putByte(SKETCH_TYPE_BYTE, (byte) + seg_.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, (byte) 1); + seg_.set(JAVA_BYTE, SERIAL_VERSION_BYTE, serialVersionUID); + seg_.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); + seg_.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesQuickSelectSketch.ordinal()); final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); - mem_.putByte(FLAGS_BYTE, (byte) ( + seg_.set(JAVA_BYTE, FLAGS_BYTE, (byte) ( (isBigEndian ? 1 << Flags.IS_BIG_ENDIAN.ordinal() : 0) | (samplingProbability < 1f ? 1 << Flags.IS_IN_SAMPLING_MODE.ordinal() : 0) | (1 << Flags.IS_EMPTY.ordinal()) )); - mem_.putByte(NUM_VALUES_BYTE, (byte) numValues); - mem_.putShort(SEED_HASH_SHORT, Util.computeSeedHash(seed)); + seg_.set(JAVA_BYTE, NUM_VALUES_BYTE, (byte) numValues); + seg_.set(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT, computeSeedHash(seed)); thetaLong_ = (long) (Long.MAX_VALUE * (double) samplingProbability); - mem_.putLong(THETA_LONG, thetaLong_); - mem_.putByte(LG_NOM_ENTRIES_BYTE, (byte) Integer.numberOfTrailingZeros(nomEntries)); - mem_.putByte(LG_CUR_CAPACITY_BYTE, (byte) Integer.numberOfTrailingZeros(startingCapacity)); - mem_.putByte(LG_RESIZE_FACTOR_BYTE, (byte) lgResizeFactor); - mem_.putFloat(SAMPLING_P_FLOAT, samplingProbability); - mem_.putInt(RETAINED_ENTRIES_INT, 0); + seg_.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong_); + seg_.set(JAVA_BYTE, LG_NOM_ENTRIES_BYTE, (byte) Integer.numberOfTrailingZeros(nomEntries)); + seg_.set(JAVA_BYTE, LG_CUR_CAPACITY_BYTE, (byte) Integer.numberOfTrailingZeros(startingCapacity)); + seg_.set(JAVA_BYTE, LG_RESIZE_FACTOR_BYTE, (byte) lgResizeFactor); + seg_.set(JAVA_FLOAT_UNALIGNED, SAMPLING_P_FLOAT, samplingProbability); + seg_.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, 0); keysOffset_ = ENTRIES_START; valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * startingCapacity); - mem_.clear(keysOffset_, (long) SIZE_OF_KEY_BYTES * startingCapacity); // clear keys only + clear(seg_, keysOffset_, (long) SIZE_OF_KEY_BYTES * startingCapacity); //clear keys only lgCurrentCapacity_ = Integer.numberOfTrailingZeros(startingCapacity); setRebuildThreshold(); } - private static final boolean checkMemory( + private static final boolean checkMemorySegment( final int nomEntries, final int lgResizeFactor, final int numValues, - final WritableMemory dstMem) { + final MemorySegment dstSeg) { final int startingCapacity = Util.getStartingCapacity(nomEntries, lgResizeFactor); - checkIfEnoughMemory(dstMem, startingCapacity, numValues); + checkMemorySegmentSize(dstSeg, startingCapacity, numValues); return true; } /** - * Wraps the given Memory. - * @param mem See Memory + * Wraps the given MemorySegment. + * @param seg the given MemorySegment * @param seed update seed */ DirectArrayOfDoublesQuickSelectSketch( - final WritableMemory mem, + final MemorySegment seg, final long seed) { - this(checkSerVer_Endianness(mem), mem, seed); + this(checkSerVer_Endianness(seg), seg, seed); //SpotBugs CT_CONSTRUCTOR_THROW is false positive. //this construction scheme is compliant with SEI CERT Oracle Coding Standard for Java / OBJ11-J } private DirectArrayOfDoublesQuickSelectSketch( final boolean secure, //required part of Finalizer Attack prevention - final WritableMemory mem, + final MemorySegment seg, final long seed) { - super(mem.getByte(NUM_VALUES_BYTE), seed); - mem_ = mem; - SerializerDeserializer.validateFamily(mem.getByte(FAMILY_ID_BYTE), - mem.getByte(PREAMBLE_LONGS_BYTE)); - SerializerDeserializer.validateType(mem_.getByte(SKETCH_TYPE_BYTE), + super(seg.get(JAVA_BYTE, NUM_VALUES_BYTE), seed); + seg_ = seg; + SerializerDeserializer.validateFamily(seg.get(JAVA_BYTE, FAMILY_ID_BYTE), + seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE)); + SerializerDeserializer.validateType(seg_.get(JAVA_BYTE, SKETCH_TYPE_BYTE), SerializerDeserializer.SketchType.ArrayOfDoublesQuickSelectSketch); - Util.checkSeedHashes(mem.getShort(SEED_HASH_SHORT), Util.computeSeedHash(seed)); + checkSeedHashes(seg.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT), computeSeedHash(seed)); keysOffset_ = ENTRIES_START; valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * getCurrentCapacity()); // to do: make parent take care of its own parts lgCurrentCapacity_ = Integer.numberOfTrailingZeros(getCurrentCapacity()); - thetaLong_ = mem_.getLong(THETA_LONG); - isEmpty_ = (mem_.getByte(FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; + thetaLong_ = seg_.get(JAVA_LONG_UNALIGNED, THETA_LONG); + isEmpty_ = (seg_.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; setRebuildThreshold(); } - private static final boolean checkSerVer_Endianness(final Memory mem) { - final byte version = mem.getByte(SERIAL_VERSION_BYTE); + private static final boolean checkSerVer_Endianness(final MemorySegment seg) { + final byte version = seg.get(JAVA_BYTE, SERIAL_VERSION_BYTE); if (version != serialVersionUID) { throw new SketchesArgumentException("Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } final boolean isBigEndian = - (mem.getByte(FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; + (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { throw new SketchesArgumentException("Byte order mismatch"); } @@ -177,7 +188,7 @@ private static final boolean checkSerVer_Endianness(final Memory mem) { } @Override - //converts Memory hashTable of double[] to compacted double[][] + //converts MemorySegment hashTable of double[] to compacted double[][] public double[][] getValues() { final int count = getRetainedEntries(); final double[][] values = new double[count][]; @@ -186,9 +197,9 @@ public double[][] getValues() { long valuesOffset = valuesOffset_; int cnt = 0; for (int j = 0; j < getCurrentCapacity(); j++) { - if (mem_.getLong(keyOffset) != 0) { + if (seg_.get(JAVA_LONG_UNALIGNED, keyOffset) != 0) { final double[] array = new double[numValues_]; - mem_.getDoubleArray(valuesOffset, array, 0, numValues_); + MemorySegment.copy(seg_, JAVA_DOUBLE_UNALIGNED, valuesOffset, array, 0, numValues_); values[cnt++] = array; } keyOffset += SIZE_OF_KEY_BYTES; @@ -209,8 +220,8 @@ public double[][] getValues() { long valuesOffsetBytes = valuesOffset_; int cnt = 0; for (int j = 0; j < cap; j++) { - if (mem_.getLong(keyOffsetBytes) != 0) { - mem_.getDoubleArray(valuesOffsetBytes, values, cnt++ * numValues_, numValues_); + if (seg_.get(JAVA_LONG_UNALIGNED, keyOffsetBytes) != 0) { + MemorySegment.copy(seg_, JAVA_DOUBLE_UNALIGNED, valuesOffsetBytes, values, cnt++ * numValues_, numValues_); } keyOffsetBytes += SIZE_OF_KEY_BYTES; valuesOffsetBytes += (long)SIZE_OF_VALUE_BYTES * numValues_; @@ -231,7 +242,7 @@ long[] getKeys() { int cnt = 0; for (int j = 0; j < cap; j++) { final long key; - if ((key = mem_.getLong(keyOffsetBytes)) != 0) { + if ((key = seg_.get(JAVA_LONG_UNALIGNED, keyOffsetBytes)) != 0) { keys[cnt++] = key; } keyOffsetBytes += SIZE_OF_KEY_BYTES; @@ -243,44 +254,43 @@ long[] getKeys() { @Override public int getRetainedEntries() { - return mem_.getInt(RETAINED_ENTRIES_INT); + return seg_.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT); } @Override public int getNominalEntries() { - return 1 << mem_.getByte(LG_NOM_ENTRIES_BYTE); + return 1 << seg_.get(JAVA_BYTE, LG_NOM_ENTRIES_BYTE); } @Override public ResizeFactor getResizeFactor() { - return ResizeFactor.getRF(mem_.getByte(LG_RESIZE_FACTOR_BYTE)); + return ResizeFactor.getRF(seg_.get(JAVA_BYTE, LG_RESIZE_FACTOR_BYTE)); } @Override public float getSamplingProbability() { - return mem_.getFloat(SAMPLING_P_FLOAT); + return seg_.get(JAVA_FLOAT_UNALIGNED, SAMPLING_P_FLOAT); } @Override public byte[] toByteArray() { final int sizeBytes = getSerializedSizeBytes(); final byte[] byteArray = new byte[sizeBytes]; - final WritableMemory mem = WritableMemory.writableWrap(byteArray); - serializeInto(mem); + final MemorySegment seg = MemorySegment.ofArray(byteArray); + serializeInto(seg); return byteArray; } @Override public ArrayOfDoublesSketchIterator iterator() { - return new DirectArrayOfDoublesSketchIterator(mem_, keysOffset_, getCurrentCapacity(), - numValues_); + return new DirectArrayOfDoublesSketchIterator(seg_, keysOffset_, getCurrentCapacity(), numValues_); } @Override - public boolean hasMemory() { return true; } + public boolean hasMemorySegment() { return true; } @Override - WritableMemory getMemory() { return mem_; } + MemorySegment getMemorySegment() { return seg_; } @Override int getSerializedSizeBytes() { @@ -288,60 +298,60 @@ int getSerializedSizeBytes() { } @Override - void serializeInto(final WritableMemory mem) { - mem_.copyTo(0, mem, 0, mem.getCapacity()); + void serializeInto(final MemorySegment seg) { + MemorySegment.copy(seg_, 0, seg, 0, seg.byteSize()); } @Override public void reset() { if (!isEmpty_) { isEmpty_ = true; - mem_.setBits(FLAGS_BYTE, (byte) (1 << Flags.IS_EMPTY.ordinal())); + setBits(seg_, FLAGS_BYTE, (byte) (1 << Flags.IS_EMPTY.ordinal())); } - final int lgResizeFactor = mem_.getByte(LG_RESIZE_FACTOR_BYTE); - final float samplingProbability = mem_.getFloat(SAMPLING_P_FLOAT); + final int lgResizeFactor = seg_.get(JAVA_BYTE, LG_RESIZE_FACTOR_BYTE); + final float samplingProbability = seg_.get(JAVA_FLOAT_UNALIGNED, SAMPLING_P_FLOAT); final int startingCapacity = Util.getStartingCapacity(getNominalEntries(), lgResizeFactor); thetaLong_ = (long) (Long.MAX_VALUE * (double) samplingProbability); - mem_.putLong(THETA_LONG, thetaLong_); - mem_.putByte(LG_CUR_CAPACITY_BYTE, (byte) Integer.numberOfTrailingZeros(startingCapacity)); - mem_.putInt(RETAINED_ENTRIES_INT, 0); + seg_.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong_); + seg_.set(JAVA_BYTE, LG_CUR_CAPACITY_BYTE, (byte) Integer.numberOfTrailingZeros(startingCapacity)); + seg_.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, 0); keysOffset_ = ENTRIES_START; valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * startingCapacity); - mem_.clear(keysOffset_, (long) SIZE_OF_KEY_BYTES * startingCapacity); // clear keys only + clear(seg_, keysOffset_, (long) SIZE_OF_KEY_BYTES * startingCapacity); //clear keys only lgCurrentCapacity_ = Integer.numberOfTrailingZeros(startingCapacity); setRebuildThreshold(); } @Override protected long getKey(final int index) { - return mem_.getLong(keysOffset_ + ((long) SIZE_OF_KEY_BYTES * index)); + return seg_.get(JAVA_LONG_UNALIGNED, keysOffset_ + ((long) SIZE_OF_KEY_BYTES * index)); } @Override protected void incrementCount() { - final int count = mem_.getInt(RETAINED_ENTRIES_INT); + final int count = seg_.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT); if (count == 0) { - mem_.setBits(FLAGS_BYTE, (byte) (1 << Flags.HAS_ENTRIES.ordinal())); + setBits(seg_, FLAGS_BYTE, (byte) (1 << Flags.HAS_ENTRIES.ordinal())); } - mem_.putInt(RETAINED_ENTRIES_INT, count + 1); + seg_.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, count + 1); } @Override protected final int getCurrentCapacity() { - return 1 << mem_.getByte(LG_CUR_CAPACITY_BYTE); + return 1 << seg_.get(JAVA_BYTE, LG_CUR_CAPACITY_BYTE); } @Override protected void setThetaLong(final long thetaLong) { thetaLong_ = thetaLong; - mem_.putLong(THETA_LONG, thetaLong_); + seg_.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong_); } @Override protected void setValues(final int index, final double[] values) { long offset = valuesOffset_ + ((long) SIZE_OF_VALUE_BYTES * numValues_ * index); for (int i = 0; i < numValues_; i++) { - mem_.putDouble(offset, values[i]); + seg_.set(JAVA_DOUBLE_UNALIGNED, offset, values[i]); offset += SIZE_OF_VALUE_BYTES; } } @@ -350,7 +360,7 @@ protected void setValues(final int index, final double[] values) { protected void updateValues(final int index, final double[] values) { long offset = valuesOffset_ + ((long) SIZE_OF_VALUE_BYTES * numValues_ * index); for (int i = 0; i < numValues_; i++) { - mem_.putDouble(offset, mem_.getDouble(offset) + values[i]); + seg_.set(JAVA_DOUBLE_UNALIGNED, offset, seg_.get(JAVA_DOUBLE_UNALIGNED, offset) + values[i]); offset += SIZE_OF_VALUE_BYTES; } } @@ -359,29 +369,30 @@ protected void updateValues(final int index, final double[] values) { protected void setNotEmpty() { if (isEmpty_) { isEmpty_ = false; - mem_.clearBits(FLAGS_BYTE, (byte) (1 << Flags.IS_EMPTY.ordinal())); + clearBits(seg_, FLAGS_BYTE, (byte) (1 << Flags.IS_EMPTY.ordinal())); + } } @Override protected boolean isInSamplingMode() { - return (mem_.getByte(FLAGS_BYTE) & (1 << Flags.IS_IN_SAMPLING_MODE.ordinal())) != 0; + return (seg_.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_IN_SAMPLING_MODE.ordinal())) != 0; } - // rebuild in the same memory + // rebuild in the same MemorySegment @Override protected void rebuild(final int newCapacity) { final int numValues = getNumValues(); - checkIfEnoughMemory(mem_, newCapacity, numValues); + checkMemorySegmentSize(seg_, newCapacity, numValues); final int currCapacity = getCurrentCapacity(); final long[] keys = new long[currCapacity]; final double[] values = new double[currCapacity * numValues]; - mem_.getLongArray(keysOffset_, keys, 0, currCapacity); - mem_.getDoubleArray(valuesOffset_, values, 0, currCapacity * numValues); - mem_.clear(keysOffset_, - ((long) SIZE_OF_KEY_BYTES * newCapacity) + ((long) SIZE_OF_VALUE_BYTES * newCapacity * numValues)); - mem_.putInt(RETAINED_ENTRIES_INT, 0); - mem_.putByte(LG_CUR_CAPACITY_BYTE, (byte)Integer.numberOfTrailingZeros(newCapacity)); + MemorySegment.copy(seg_, JAVA_LONG_UNALIGNED, keysOffset_, keys, 0, currCapacity); + MemorySegment.copy(seg_, JAVA_DOUBLE_UNALIGNED, valuesOffset_, values, 0, currCapacity * numValues); + + clear(seg_, keysOffset_, ((long) SIZE_OF_KEY_BYTES * newCapacity) + ((long) SIZE_OF_VALUE_BYTES * newCapacity * numValues)); + seg_.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, 0); + seg_.set(JAVA_BYTE, LG_CUR_CAPACITY_BYTE, (byte)Integer.numberOfTrailingZeros(newCapacity)); valuesOffset_ = keysOffset_ + (SIZE_OF_KEY_BYTES * newCapacity); lgCurrentCapacity_ = Integer.numberOfTrailingZeros(newCapacity); for (int i = 0; i < keys.length; i++) { @@ -394,30 +405,30 @@ protected void rebuild(final int newCapacity) { @Override protected int insertKey(final long key) { - return HashOperations.hashInsertOnlyMemory(mem_, lgCurrentCapacity_, key, ENTRIES_START); + return HashOperations.hashInsertOnlyMemorySegment(seg_, lgCurrentCapacity_, key, ENTRIES_START); } @Override protected int findOrInsertKey(final long key) { - return HashOperations.hashSearchOrInsertMemory(mem_, lgCurrentCapacity_, key, ENTRIES_START); + return HashOperations.hashSearchOrInsertMemorySegment(seg_, lgCurrentCapacity_, key, ENTRIES_START); } @Override protected double[] find(final long key) { - final int index = HashOperations.hashSearchMemory(mem_, lgCurrentCapacity_, key, ENTRIES_START); + final int index = HashOperations.hashSearchMemorySegment(seg_, lgCurrentCapacity_, key, ENTRIES_START); if (index == -1) { return null; } final double[] array = new double[numValues_]; - mem_.getDoubleArray(valuesOffset_ + ((long) SIZE_OF_VALUE_BYTES * numValues_ * index), - array, 0, numValues_); + MemorySegment.copy(seg_, JAVA_DOUBLE_UNALIGNED, valuesOffset_ + + ((long) SIZE_OF_VALUE_BYTES * numValues_ * index), array, 0, numValues_); return array; } - private static void checkIfEnoughMemory(final Memory mem, final int numEntries, final int numValues) { + private static void checkMemorySegmentSize(final MemorySegment seg, final int numEntries, final int numValues) { final int sizeNeeded = ENTRIES_START + ((SIZE_OF_KEY_BYTES + (SIZE_OF_VALUE_BYTES * numValues)) * numEntries); - if (sizeNeeded > mem.getCapacity()) { - throw new SketchesArgumentException("Not enough memory: need " - + sizeNeeded + " bytes, got " + mem.getCapacity() + " bytes"); + if (sizeNeeded > seg.byteSize()) { + throw new SketchesArgumentException("Not enough space: need " + + sizeNeeded + " bytes, got " + seg.byteSize() + " bytes"); } } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketchR.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketchR.java index f168c7284..3ff1a260c 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketchR.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketchR.java @@ -19,14 +19,14 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesReadOnlyException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; final class DirectArrayOfDoublesQuickSelectSketchR extends DirectArrayOfDoublesQuickSelectSketch { - DirectArrayOfDoublesQuickSelectSketchR(final Memory mem, final long seed) { - super((WritableMemory) mem, seed); + DirectArrayOfDoublesQuickSelectSketchR(final MemorySegment seg, final long seed) { + super(seg, seed); } @Override diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java index dcdab1313..35f172c3f 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesSketchIterator.java @@ -19,18 +19,21 @@ package org.apache.datasketches.tuple.arrayofdoubles; -import org.apache.datasketches.memory.Memory; +import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; + +import java.lang.foreign.MemorySegment; /** * Iterator over the off-heap, Direct tuple sketch of type ArrayOfDoubles (compact or hash table). * - *

      This implementation uses data in a given Memory that is owned and managed by the caller. - * This Memory can be off-heap, which if managed properly will greatly reduce the need for + *

      This implementation uses data in a given MemorySegment that is owned and managed by the caller. + * This MemorySegment can be off-heap, which if managed properly will greatly reduce the need for * the JVM to perform garbage collection.

      */ final class DirectArrayOfDoublesSketchIterator implements ArrayOfDoublesSketchIterator { - private Memory mem_; + private MemorySegment seg_; private int offset_; private int numEntries_; private int numValues_; @@ -38,9 +41,9 @@ final class DirectArrayOfDoublesSketchIterator implements ArrayOfDoublesSketchIt private static final int SIZE_OF_KEY_BYTES = 8; private static final int SIZE_OF_VALUE_BYTES = 8; - DirectArrayOfDoublesSketchIterator(final Memory mem, final int offset, final int numEntries, + DirectArrayOfDoublesSketchIterator(final MemorySegment seg, final int offset, final int numEntries, final int numValues) { - mem_ = mem; + seg_ = seg; offset_ = offset; numEntries_ = numEntries; numValues_ = numValues; @@ -51,7 +54,8 @@ final class DirectArrayOfDoublesSketchIterator implements ArrayOfDoublesSketchIt public boolean next() { i_++; while (i_ < numEntries_) { - if (mem_.getLong(offset_ + ((long) SIZE_OF_KEY_BYTES * i_)) != 0) { return true; } + final long off = offset_ + ((long) SIZE_OF_KEY_BYTES * i_); + if (seg_.get(JAVA_LONG_UNALIGNED, off) != 0) { return true; } i_++; } return false; @@ -59,19 +63,20 @@ public boolean next() { @Override public long getKey() { - return mem_.getLong(offset_ + ((long) SIZE_OF_KEY_BYTES * i_)); + final long off = offset_ + ((long) SIZE_OF_KEY_BYTES * i_); + return seg_.get(JAVA_LONG_UNALIGNED, off); } @Override public double[] getValues() { + long off; if (numValues_ == 1) { - return new double[] { - mem_.getDouble(offset_ + ((long) SIZE_OF_KEY_BYTES * numEntries_) - + ((long) SIZE_OF_VALUE_BYTES * i_)) }; + off = offset_ + ((long) SIZE_OF_KEY_BYTES * numEntries_) + ((long) SIZE_OF_VALUE_BYTES * i_); + return new double[] { seg_.get(JAVA_DOUBLE_UNALIGNED, off) }; } final double[] array = new double[numValues_]; - mem_.getDoubleArray(offset_ + ((long) SIZE_OF_KEY_BYTES * numEntries_) - + ((long) SIZE_OF_VALUE_BYTES * i_ * numValues_), array, 0, numValues_); + off = offset_ + ((long) SIZE_OF_KEY_BYTES * numEntries_) + ((long) SIZE_OF_VALUE_BYTES * i_ * numValues_); + MemorySegment.copy(seg_, JAVA_DOUBLE_UNALIGNED, off, array, 0, numValues_); return array; } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java index 734019632..0c7242088 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnion.java @@ -19,73 +19,74 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; + +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.tuple.SerializerDeserializer; /** * Direct Union operation for tuple sketches of type ArrayOfDoubles. * - *

      This implementation uses data in a given Memory that is owned and managed by the caller. - * This Memory can be off-heap, which if managed properly will greatly reduce the need for + *

      This implementation uses data in a given MemorySegment that is owned and managed by the caller. + * This MemorySegment can be off-heap, which if managed properly will greatly reduce the need for * the JVM to perform garbage collection.

      */ class DirectArrayOfDoublesUnion extends ArrayOfDoublesUnion { - final WritableMemory mem_; + final MemorySegment seg_; /** * Creates an instance of DirectArrayOfDoublesUnion - * @param nomEntries Nominal number of entries. Forced to the nearest power of 2 greater than - * given value. + * @param nomEntries Nominal number of entries. Forced to the nearest power of 2 greater than given value. * @param numValues Number of double values to keep for each key. * @param seed See seed - * @param dstMem See Memory + * @param dstSeg the destination MemorySegment */ DirectArrayOfDoublesUnion(final int nomEntries, final int numValues, final long seed, - final WritableMemory dstMem) { + final MemorySegment dstSeg) { super(new DirectArrayOfDoublesQuickSelectSketch(nomEntries, 3, 1f, numValues, seed, - dstMem.writableRegion(PREAMBLE_SIZE_BYTES, dstMem.getCapacity() - PREAMBLE_SIZE_BYTES))); - mem_ = dstMem; - mem_.putByte(PREAMBLE_LONGS_BYTE, (byte) 1); // unused, always 1 - mem_.putByte(SERIAL_VERSION_BYTE, serialVersionUID); - mem_.putByte(FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); - mem_.putByte(SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesUnion.ordinal()); - mem_.putLong(THETA_LONG, gadget_.getThetaLong()); + dstSeg.asSlice(PREAMBLE_SIZE_BYTES, dstSeg.byteSize() - PREAMBLE_SIZE_BYTES))); + seg_ = dstSeg; + seg_.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, (byte) 1); // unused, always 1 + seg_.set(JAVA_BYTE, SERIAL_VERSION_BYTE, serialVersionUID); + seg_.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); + seg_.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesUnion.ordinal()); + seg_.set(JAVA_LONG_UNALIGNED, THETA_LONG, gadget_.getThetaLong()); } //Called from wrapUnion below and extended by DirectArrayOfDoublesUnionR - DirectArrayOfDoublesUnion(final ArrayOfDoublesQuickSelectSketch gadget, final WritableMemory mem) { + DirectArrayOfDoublesUnion(final ArrayOfDoublesQuickSelectSketch gadget, final MemorySegment seg) { super(gadget); - mem_ = mem; - unionThetaLong_ = mem.getLong(THETA_LONG); + seg_ = seg; + unionThetaLong_ = seg.get(JAVA_LONG_UNALIGNED, THETA_LONG); } @Override void setUnionThetaLong(final long thetaLong) { super.setUnionThetaLong(thetaLong); - mem_.putLong(THETA_LONG, thetaLong); + seg_.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong); } - static ArrayOfDoublesUnion wrapUnion(final WritableMemory mem, final long seed, final boolean isWritable) { - final byte version = mem.getByte(ArrayOfDoublesUnion.SERIAL_VERSION_BYTE); + static ArrayOfDoublesUnion wrapUnion(final MemorySegment seg, final long seed, final boolean isWritable) { + final byte version = seg.get(JAVA_BYTE, ArrayOfDoublesUnion.SERIAL_VERSION_BYTE); if (version != ArrayOfDoublesUnion.serialVersionUID) { throw new SketchesArgumentException("Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } - SerializerDeserializer.validateFamily(mem.getByte(FAMILY_ID_BYTE), mem.getByte(PREAMBLE_LONGS_BYTE)); - SerializerDeserializer.validateType(mem.getByte(SKETCH_TYPE_BYTE), + SerializerDeserializer.validateFamily(seg.get(JAVA_BYTE, FAMILY_ID_BYTE), seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE)); + SerializerDeserializer.validateType(seg.get(JAVA_BYTE, SKETCH_TYPE_BYTE), SerializerDeserializer.SketchType.ArrayOfDoublesUnion); if (isWritable) { - final WritableMemory sketchMem = mem.writableRegion(PREAMBLE_SIZE_BYTES, - mem.getCapacity() - PREAMBLE_SIZE_BYTES); - return new DirectArrayOfDoublesUnion(new DirectArrayOfDoublesQuickSelectSketch(sketchMem, seed), mem); + final MemorySegment sketchSeg = seg.asSlice(PREAMBLE_SIZE_BYTES, seg.byteSize() - PREAMBLE_SIZE_BYTES); + return new DirectArrayOfDoublesUnion(new DirectArrayOfDoublesQuickSelectSketch(sketchSeg, seed), seg); } - final Memory sketchMem = mem.region(PREAMBLE_SIZE_BYTES, mem.getCapacity() - PREAMBLE_SIZE_BYTES); - return new DirectArrayOfDoublesUnionR(new DirectArrayOfDoublesQuickSelectSketchR(sketchMem, seed), mem); + final MemorySegment sketchSeg = seg.asSlice(PREAMBLE_SIZE_BYTES, seg.byteSize() - PREAMBLE_SIZE_BYTES); + return new DirectArrayOfDoublesUnionR(new DirectArrayOfDoublesQuickSelectSketchR(sketchSeg, seed), seg); } } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnionR.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnionR.java index f865d823b..164526c96 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnionR.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesUnionR.java @@ -19,18 +19,19 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesReadOnlyException; -import org.apache.datasketches.memory.WritableMemory; final class DirectArrayOfDoublesUnionR extends DirectArrayOfDoublesUnion { /** - * Wraps the given Memory. + * Wraps the given MemorySegment. * @param gadget the ArrayOfDoublesQuickSelectSketch - * @param mem See Memory + * @param seg the destination MemorySegment */ - DirectArrayOfDoublesUnionR(final ArrayOfDoublesQuickSelectSketch gadget, final WritableMemory mem) { - super(gadget, mem); + DirectArrayOfDoublesUnionR(final ArrayOfDoublesQuickSelectSketch gadget, final MemorySegment seg) { + super(gadget, seg); } @Override diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketch.java index 3277d4a2c..b8f6fd863 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketch.java @@ -19,16 +19,20 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; + +import java.lang.foreign.MemorySegment; import java.nio.ByteOrder; import java.util.Arrays; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; import org.apache.datasketches.tuple.SerializerDeserializer; -import org.apache.datasketches.tuple.Util; /** * The on-heap implementation of tuple Compact Sketch of type ArrayOfDoubles. @@ -100,57 +104,57 @@ final class HeapArrayOfDoublesCompactSketch extends ArrayOfDoublesCompactSketch /** * This is to create an instance given a serialized form - * @param mem See Memory + * @param seg the destination segment */ - HeapArrayOfDoublesCompactSketch(final Memory mem) { - this(mem, ThetaUtil.DEFAULT_UPDATE_SEED); + HeapArrayOfDoublesCompactSketch(final MemorySegment seg) { + this(seg, Util.DEFAULT_UPDATE_SEED); } /** * This is to create an instance given a serialized form - * @param mem See Memory + * @param seg the source MemorySegment * @param seed See seed */ - HeapArrayOfDoublesCompactSketch(final Memory mem, final long seed) { - super(mem.getByte(NUM_VALUES_BYTE)); - seedHash_ = mem.getShort(SEED_HASH_SHORT); - SerializerDeserializer.validateFamily(mem.getByte(FAMILY_ID_BYTE), - mem.getByte(PREAMBLE_LONGS_BYTE)); - SerializerDeserializer.validateType(mem.getByte(SKETCH_TYPE_BYTE), + HeapArrayOfDoublesCompactSketch(final MemorySegment seg, final long seed) { + super(seg.get(JAVA_BYTE, NUM_VALUES_BYTE)); + seedHash_ = seg.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT); + SerializerDeserializer.validateFamily(seg.get(JAVA_BYTE, FAMILY_ID_BYTE), + seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE)); + SerializerDeserializer.validateType(seg.get(JAVA_BYTE, SKETCH_TYPE_BYTE), SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch); - final byte version = mem.getByte(SERIAL_VERSION_BYTE); + final byte version = seg.get(JAVA_BYTE, SERIAL_VERSION_BYTE); if (version != serialVersionUID) { throw new SketchesArgumentException( "Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } final boolean isBigEndian = - (mem.getByte(FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; + (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_BIG_ENDIAN.ordinal())) != 0; if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { throw new SketchesArgumentException("Byte order mismatch"); } Util.checkSeedHashes(seedHash_, Util.computeSeedHash(seed)); - isEmpty_ = (mem.getByte(FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; - thetaLong_ = mem.getLong(THETA_LONG); + isEmpty_ = (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.IS_EMPTY.ordinal())) != 0; + thetaLong_ = seg.get(JAVA_LONG_UNALIGNED, THETA_LONG); final boolean hasEntries = - (mem.getByte(FLAGS_BYTE) & (1 << Flags.HAS_ENTRIES.ordinal())) != 0; + (seg.get(JAVA_BYTE, FLAGS_BYTE) & (1 << Flags.HAS_ENTRIES.ordinal())) != 0; if (hasEntries) { - final int count = mem.getInt(RETAINED_ENTRIES_INT); + final int count = seg.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT); keys_ = new long[count]; values_ = new double[count * numValues_]; - mem.getLongArray(ENTRIES_START, keys_, 0, count); - mem.getDoubleArray(ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * count), values_, 0, values_.length); + MemorySegment.copy(seg, JAVA_LONG_UNALIGNED, ENTRIES_START, keys_, 0, count); + MemorySegment.copy(seg, JAVA_DOUBLE_UNALIGNED, ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * count), values_, 0, values_.length); } } @Override - public ArrayOfDoublesCompactSketch compact(final WritableMemory dstMem) { - if (dstMem == null) { + public ArrayOfDoublesCompactSketch compact(final MemorySegment dstSeg) { + if (dstSeg == null) { return new HeapArrayOfDoublesCompactSketch(keys_.clone(), values_.clone(), thetaLong_, isEmpty_, numValues_, seedHash_); } else { final byte[] byteArr = this.toByteArray(); - dstMem.putByteArray(0, byteArr, 0, byteArr.length); - return new DirectArrayOfDoublesCompactSketch(dstMem); + MemorySegment.copy(byteArr, 0, dstSeg, JAVA_BYTE, 0, byteArr.length); + return new DirectArrayOfDoublesCompactSketch(dstSeg); } } @@ -164,25 +168,24 @@ public byte[] toByteArray() { final int count = getRetainedEntries(); final int sizeBytes = getCurrentBytes(); final byte[] bytes = new byte[sizeBytes]; - final WritableMemory mem = WritableMemory.writableWrap(bytes); - mem.putByte(PREAMBLE_LONGS_BYTE, (byte) 1); - mem.putByte(SERIAL_VERSION_BYTE, serialVersionUID); - mem.putByte(FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); - mem.putByte(SKETCH_TYPE_BYTE, - (byte) SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch.ordinal()); + final MemorySegment seg = MemorySegment.ofArray(bytes); + seg.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, (byte) 1); + seg.set(JAVA_BYTE, SERIAL_VERSION_BYTE, serialVersionUID); + seg.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); + seg.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesCompactSketch.ordinal()); final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); - mem.putByte(FLAGS_BYTE, (byte) ( + seg.set(JAVA_BYTE, FLAGS_BYTE, (byte) ( ((isBigEndian ? 1 : 0) << Flags.IS_BIG_ENDIAN.ordinal()) | ((isEmpty() ? 1 : 0) << Flags.IS_EMPTY.ordinal()) | ((count > 0 ? 1 : 0) << Flags.HAS_ENTRIES.ordinal()) )); - mem.putByte(NUM_VALUES_BYTE, (byte) numValues_); - mem.putShort(SEED_HASH_SHORT, seedHash_); - mem.putLong(THETA_LONG, thetaLong_); + seg.set(JAVA_BYTE, NUM_VALUES_BYTE, (byte) numValues_); + seg.set(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT, seedHash_); + seg.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong_); if (count > 0) { - mem.putInt(RETAINED_ENTRIES_INT, count); - mem.putLongArray(ENTRIES_START, keys_, 0, count); - mem.putDoubleArray(ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * count), values_, 0, values_.length); + seg.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, count); + MemorySegment.copy(keys_, 0, seg, JAVA_LONG_UNALIGNED, ENTRIES_START, count); + MemorySegment.copy(values_, 0, seg, JAVA_DOUBLE_UNALIGNED, ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * count), values_.length); } return bytes; } @@ -222,8 +225,8 @@ short getSeedHash() { } @Override - public boolean hasMemory() { return false; } + public boolean hasMemorySegment() { return false; } @Override - Memory getMemory() { return null; } + MemorySegment getMemorySegment() { return null; } } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesIntersection.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesIntersection.java index 39b8fb340..a87b349bb 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesIntersection.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesIntersection.java @@ -35,8 +35,7 @@ final class HeapArrayOfDoublesIntersection extends ArrayOfDoublesIntersection { } @Override - protected ArrayOfDoublesQuickSelectSketch createSketch(final int nomEntries, final int numValues, - final long seed) { + protected ArrayOfDoublesQuickSelectSketch createSketch(final int nomEntries, final int numValues, final long seed) { return new HeapArrayOfDoublesQuickSelectSketch(nomEntries, 0, 1f, numValues, seed); } diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketch.java index 9409a5828..beab988d3 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketch.java @@ -19,17 +19,24 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_FLOAT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; import static org.apache.datasketches.common.Util.ceilingPowerOf2; +import static org.apache.datasketches.common.Util.checkSeedHashes; +import static org.apache.datasketches.common.Util.computeSeedHash; import static org.apache.datasketches.common.Util.exactLog2OfLong; +import java.lang.foreign.MemorySegment; import java.nio.ByteOrder; import java.util.Arrays; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.HashOperations; import org.apache.datasketches.tuple.SerializerDeserializer; import org.apache.datasketches.tuple.Util; @@ -79,40 +86,41 @@ final class HeapArrayOfDoublesQuickSelectSketch extends ArrayOfDoublesQuickSelec /** * This is to create an instance given a serialized form - * @param mem See Memory + * @param seg the source MemorySegment * @param seed See seed */ - HeapArrayOfDoublesQuickSelectSketch(final Memory mem, final long seed) { - super(mem.getByte(NUM_VALUES_BYTE), seed); - SerializerDeserializer.validateFamily(mem.getByte(FAMILY_ID_BYTE), - mem.getByte(PREAMBLE_LONGS_BYTE)); - SerializerDeserializer.validateType(mem.getByte(SKETCH_TYPE_BYTE), + HeapArrayOfDoublesQuickSelectSketch(final MemorySegment seg, final long seed) { + super(seg.get(JAVA_BYTE, NUM_VALUES_BYTE), seed); + SerializerDeserializer.validateFamily(seg.get(JAVA_BYTE, FAMILY_ID_BYTE), + seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE)); + SerializerDeserializer.validateType(seg.get(JAVA_BYTE, SKETCH_TYPE_BYTE), SerializerDeserializer.SketchType.ArrayOfDoublesQuickSelectSketch); - final byte version = mem.getByte(SERIAL_VERSION_BYTE); + final byte version = seg.get(JAVA_BYTE, SERIAL_VERSION_BYTE); if (version != serialVersionUID) { throw new SketchesArgumentException("Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } - final byte flags = mem.getByte(FLAGS_BYTE); + final byte flags = seg.get(JAVA_BYTE, FLAGS_BYTE); final boolean isBigEndian = (flags & (1 << Flags.IS_BIG_ENDIAN.ordinal())) > 0; if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { throw new SketchesArgumentException("Byte order mismatch"); } - Util.checkSeedHashes(mem.getShort(SEED_HASH_SHORT), Util.computeSeedHash(seed)); + checkSeedHashes(seg.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT), computeSeedHash(seed)); isEmpty_ = (flags & (1 << Flags.IS_EMPTY.ordinal())) > 0; - lgNomEntries_ = mem.getByte(LG_NOM_ENTRIES_BYTE); - thetaLong_ = mem.getLong(THETA_LONG); - final int currentCapacity = 1 << mem.getByte(LG_CUR_CAPACITY_BYTE); - lgResizeFactor_ = mem.getByte(LG_RESIZE_FACTOR_BYTE); - samplingProbability_ = mem.getFloat(SAMPLING_P_FLOAT); + lgNomEntries_ = seg.get(JAVA_BYTE, LG_NOM_ENTRIES_BYTE); + thetaLong_ = seg.get(JAVA_LONG_UNALIGNED, THETA_LONG); + final int currentCapacity = 1 << seg.get(JAVA_BYTE, LG_CUR_CAPACITY_BYTE); + lgResizeFactor_ = seg.get(JAVA_BYTE, LG_RESIZE_FACTOR_BYTE); + samplingProbability_ = seg.get(JAVA_FLOAT_UNALIGNED, SAMPLING_P_FLOAT); keys_ = new long[currentCapacity]; values_ = new double[currentCapacity * numValues_]; final boolean hasEntries = (flags & (1 << Flags.HAS_ENTRIES.ordinal())) > 0; - count_ = hasEntries ? mem.getInt(RETAINED_ENTRIES_INT) : 0; + count_ = hasEntries ? seg.get(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT) : 0; if (count_ > 0) { - mem.getLongArray(ENTRIES_START, keys_, 0, currentCapacity); - mem.getDoubleArray(ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * currentCapacity), values_, 0, - currentCapacity * numValues_); + MemorySegment.copy(seg, JAVA_LONG_UNALIGNED, ENTRIES_START, keys_, 0, currentCapacity); + final long off = ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * currentCapacity); + MemorySegment.copy(seg, JAVA_DOUBLE_UNALIGNED, off, values_, 0, currentCapacity * numValues_); + } setRebuildThreshold(); lgCurrentCapacity_ = Integer.numberOfTrailingZeros(currentCapacity); @@ -191,8 +199,8 @@ public ResizeFactor getResizeFactor() { @Override public byte[] toByteArray() { final byte[] byteArray = new byte[getSerializedSizeBytes()]; - final WritableMemory mem = WritableMemory.writableWrap(byteArray); - serializeInto(mem); + final MemorySegment seg = MemorySegment.ofArray(byteArray); + serializeInto(seg); return byteArray; } @@ -224,38 +232,39 @@ int getSerializedSizeBytes() { // || Values Array doubles * values[] Length | @Override - void serializeInto(final WritableMemory mem) { - mem.putByte(PREAMBLE_LONGS_BYTE, (byte) 1); - mem.putByte(SERIAL_VERSION_BYTE, serialVersionUID); - mem.putByte(FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); - mem.putByte(SKETCH_TYPE_BYTE, + void serializeInto(final MemorySegment seg) { + seg.set(JAVA_BYTE, PREAMBLE_LONGS_BYTE, (byte) 1); + seg.set(JAVA_BYTE, SERIAL_VERSION_BYTE, serialVersionUID); + seg.set(JAVA_BYTE, FAMILY_ID_BYTE, (byte) Family.TUPLE.getID()); + seg.set(JAVA_BYTE, SKETCH_TYPE_BYTE, (byte) SerializerDeserializer.SketchType.ArrayOfDoublesQuickSelectSketch.ordinal()); final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); - mem.putByte(FLAGS_BYTE, (byte)( + seg.set(JAVA_BYTE, FLAGS_BYTE, (byte)( (isBigEndian ? 1 << Flags.IS_BIG_ENDIAN.ordinal() : 0) | (isInSamplingMode() ? 1 << Flags.IS_IN_SAMPLING_MODE.ordinal() : 0) | (isEmpty_ ? 1 << Flags.IS_EMPTY.ordinal() : 0) | (count_ > 0 ? 1 << Flags.HAS_ENTRIES.ordinal() : 0) )); - mem.putByte(NUM_VALUES_BYTE, (byte) numValues_); - mem.putShort(SEED_HASH_SHORT, Util.computeSeedHash(seed_)); - mem.putLong(THETA_LONG, thetaLong_); - mem.putByte(LG_NOM_ENTRIES_BYTE, (byte) lgNomEntries_); - mem.putByte(LG_CUR_CAPACITY_BYTE, (byte) Integer.numberOfTrailingZeros(keys_.length)); - mem.putByte(LG_RESIZE_FACTOR_BYTE, (byte) lgResizeFactor_); - mem.putFloat(SAMPLING_P_FLOAT, samplingProbability_); - mem.putInt(RETAINED_ENTRIES_INT, count_); + seg.set(JAVA_BYTE, NUM_VALUES_BYTE, (byte) numValues_); + seg.set(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT, computeSeedHash(seed_)); + seg.set(JAVA_LONG_UNALIGNED, THETA_LONG, thetaLong_); + seg.set(JAVA_BYTE, LG_NOM_ENTRIES_BYTE, (byte) lgNomEntries_); + seg.set(JAVA_BYTE, LG_CUR_CAPACITY_BYTE, (byte) Integer.numberOfTrailingZeros(keys_.length)); + seg.set(JAVA_BYTE, LG_RESIZE_FACTOR_BYTE, (byte) lgResizeFactor_); + seg.set(JAVA_FLOAT_UNALIGNED, SAMPLING_P_FLOAT, samplingProbability_); + seg.set(JAVA_INT_UNALIGNED, RETAINED_ENTRIES_INT, count_); if (count_ > 0) { - mem.putLongArray(ENTRIES_START, keys_, 0, keys_.length); - mem.putDoubleArray(ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * keys_.length), values_, 0, values_.length); + MemorySegment.copy(keys_, 0, seg, JAVA_LONG_UNALIGNED, ENTRIES_START, keys_.length); + final long off = ENTRIES_START + ((long) SIZE_OF_KEY_BYTES * keys_.length); + MemorySegment.copy(values_, 0, seg, JAVA_DOUBLE_UNALIGNED, off, values_.length); } } @Override - public boolean hasMemory() { return false; } + public boolean hasMemorySegment() { return false; } @Override - Memory getMemory() { return null; } + MemorySegment getMemorySegment() { return null; } @Override public void reset() { diff --git a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesUnion.java b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesUnion.java index f112f1823..7a9b7fd75 100644 --- a/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesUnion.java +++ b/src/main/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesUnion.java @@ -19,8 +19,12 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; + +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.tuple.SerializerDeserializer; /** @@ -47,23 +51,23 @@ final class HeapArrayOfDoublesUnion extends ArrayOfDoublesUnion { /** * This is to create an instance given a serialized form and a custom seed - * @param mem See Memory + * @param seg the source MemorySegment * @param seed See seed * @return a ArrayOfDoublesUnion on the Java heap */ - static ArrayOfDoublesUnion heapifyUnion(final Memory mem, final long seed) { - final byte version = mem.getByte(SERIAL_VERSION_BYTE); + static ArrayOfDoublesUnion heapifyUnion(final MemorySegment seg, final long seed) { + final byte version = seg.get(JAVA_BYTE, SERIAL_VERSION_BYTE); if (version != serialVersionUID) { throw new SketchesArgumentException("Serial version mismatch. Expected: " + serialVersionUID + ", actual: " + version); } - SerializerDeserializer.validateFamily(mem.getByte(FAMILY_ID_BYTE), mem.getByte(PREAMBLE_LONGS_BYTE)); - SerializerDeserializer.validateType(mem.getByte(SKETCH_TYPE_BYTE), + SerializerDeserializer.validateFamily(seg.get(JAVA_BYTE, FAMILY_ID_BYTE), seg.get(JAVA_BYTE, PREAMBLE_LONGS_BYTE)); + SerializerDeserializer.validateType(seg.get(JAVA_BYTE, SKETCH_TYPE_BYTE), SerializerDeserializer.SketchType.ArrayOfDoublesUnion); - final Memory sketchMem = mem.region(PREAMBLE_SIZE_BYTES, mem.getCapacity() - PREAMBLE_SIZE_BYTES); - final ArrayOfDoublesQuickSelectSketch sketch = new HeapArrayOfDoublesQuickSelectSketch(sketchMem, seed); - return new HeapArrayOfDoublesUnion(sketch, mem.getLong(THETA_LONG)); + final MemorySegment sketchSeg = seg.asSlice(PREAMBLE_SIZE_BYTES, seg.byteSize() - PREAMBLE_SIZE_BYTES); + final ArrayOfDoublesQuickSelectSketch sketch = new HeapArrayOfDoublesQuickSelectSketch(sketchSeg, seed); + return new HeapArrayOfDoublesUnion(sketch, seg.get(JAVA_LONG_UNALIGNED, THETA_LONG)); } } diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java index 99ddeee16..66ba4f96e 100644 --- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketch.java @@ -21,8 +21,9 @@ import static org.apache.datasketches.tuple.Util.stringArrHash; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ResizeFactor; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.tuple.UpdatableSketch; /** @@ -62,16 +63,16 @@ public ArrayOfStringsSketch(final int lgK, final ResizeFactor rf, final float p) } /** - * Constructs this sketch from a Memory image, which must be from an ArrayOfStringsSketch, and + * Constructs this sketch from a MemorySegment image, which must be from an ArrayOfStringsSketch, and * usually with data. - * @param mem the given Memory + * @param seg the given MemorySegment * @deprecated As of 3.0.0, heapifying an UpdatableSketch is deprecated. * This capability will be removed in a future release. * Heapifying a CompactSketch is not deprecated. */ @Deprecated - public ArrayOfStringsSketch(final Memory mem) { - super(mem, new ArrayOfStringsSummaryDeserializer(), new ArrayOfStringsSummaryFactory()); + public ArrayOfStringsSketch(final MemorySegment seg) { + super(seg, new ArrayOfStringsSummaryDeserializer(), new ArrayOfStringsSummaryFactory()); } /** diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java index 48b2d6d6a..12d587398 100644 --- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java +++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummary.java @@ -19,15 +19,15 @@ package org.apache.datasketches.tuple.strings; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.datasketches.tuple.Util.stringArrHash; import static org.apache.datasketches.tuple.Util.stringConcat; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Buffer; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableBuffer; -import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.tuple.UpdatableSummary; /** @@ -36,55 +36,76 @@ */ public final class ArrayOfStringsSummary implements UpdatableSummary { - private String[] nodesArr = null; + private String[] stringArr = null; ArrayOfStringsSummary() { //required for ArrayOfStringsSummaryFactory - nodesArr = null; + stringArr = null; } //Used by copy() and in test - ArrayOfStringsSummary(final String[] nodesArr) { - this.nodesArr = nodesArr.clone(); - checkNumNodes(nodesArr.length); + ArrayOfStringsSummary(final String[] stringArr) { + this.stringArr = stringArr.clone(); + checkNumNodes(stringArr.length); } - //used by fromMemory and in test - ArrayOfStringsSummary(final Memory mem) { - final Buffer buf = mem.asBuffer(); - final int totBytes = buf.getInt(); - checkInBytes(mem, totBytes); - final int nodes = buf.getByte(); + //used by fromMemorySegment and in test + /** + * This reads a MemorySegment that has a layout similar to the C struct: + * {@snippet : + * typedef struct { + * int totBytes; + * byte nodes; //number of Nodes. + * Node[nodes] = { Node[0], Node[1], ... } + * } + * } + * Where a Node has a layout similar to the C struct: + * {@snippet : + * typedef struct { + * int numBytes; + * byte[] byteArray; //UTF-8 byte array. Not null terminated. + * } + * } + * @param seg the MemorySegment containing the Summary data + */ + ArrayOfStringsSummary(final MemorySegment seg) { + int pos = 0; + final int totBytes = seg.get(JAVA_INT_UNALIGNED, pos); pos += Integer.BYTES; + checkInBytes(seg, totBytes); + final int nodes = seg.get(JAVA_BYTE, pos); pos += Byte.BYTES; checkNumNodes(nodes); - final String[] nodesArr = new String[nodes]; + final String[] stringArr = new String[nodes]; for (int i = 0; i < nodes; i++) { - final int len = buf.getInt(); + final int len = seg.get(JAVA_INT_UNALIGNED, pos); pos += Integer.BYTES; final byte[] byteArr = new byte[len]; - buf.getByteArray(byteArr, 0, len); - nodesArr[i] = new String(byteArr, UTF_8); + MemorySegment.copy(seg, JAVA_BYTE, pos, byteArr, 0, len); pos += len; + stringArr[i] = new String(byteArr, UTF_8); } - this.nodesArr = nodesArr; + assert pos == totBytes; + this.stringArr = stringArr; } @Override public ArrayOfStringsSummary copy() { - final ArrayOfStringsSummary nodes = new ArrayOfStringsSummary(nodesArr); + final ArrayOfStringsSummary nodes = new ArrayOfStringsSummary(stringArr); return nodes; } @Override public byte[] toByteArray() { - final ComputeBytes cb = new ComputeBytes(nodesArr); + final ComputeBytes cb = new ComputeBytes(stringArr); final int totBytes = cb.totBytes_; final byte[] out = new byte[totBytes]; - final WritableMemory wmem = WritableMemory.writableWrap(out); - final WritableBuffer wbuf = wmem.asWritableBuffer(); - wbuf.putInt(totBytes); - wbuf.putByte(cb.numNodes_); - for (int i = 0; i < cb.numNodes_; i++) { - wbuf.putInt(cb.nodeLengthsArr_[i]); - wbuf.putByteArray(cb.nodeBytesArr_[i], 0, cb.nodeLengthsArr_[i]); + final MemorySegment wseg = MemorySegment.ofArray(out); + int pos = 0; + wseg.set(JAVA_INT_UNALIGNED, pos, totBytes); pos += Integer.BYTES; + final int numNodes = cb.numNodes_; + wseg.set(JAVA_BYTE, pos, (byte)numNodes); pos += Byte.BYTES; + for (int i = 0; i < numNodes; i++) { + final int nodeLen = cb.nodeLengthsArr_[i]; + wseg.set(JAVA_INT_UNALIGNED, pos, nodeLen); pos += Integer.BYTES; + MemorySegment.copy(cb.nodeBytesArr_[i], 0, wseg, JAVA_BYTE, pos, nodeLen); pos += nodeLen; } - assert wbuf.getPosition() == totBytes; + assert pos == totBytes; return out; } @@ -92,8 +113,8 @@ public byte[] toByteArray() { @Override public ArrayOfStringsSummary update(final String[] value) { - if (nodesArr == null) { - nodesArr = value.clone(); + if (stringArr == null) { + stringArr = value.clone(); } return this; } @@ -102,7 +123,7 @@ public ArrayOfStringsSummary update(final String[] value) { @Override public int hashCode() { - return (int) stringArrHash(nodesArr); + return (int) stringArrHash(stringArr); } @Override @@ -110,8 +131,8 @@ public boolean equals(final Object summary) { if (summary == null || !(summary instanceof ArrayOfStringsSummary)) { return false; } - final String thatStr = stringConcat(((ArrayOfStringsSummary) summary).nodesArr); - final String thisStr = stringConcat(nodesArr); + final String thatStr = stringConcat(((ArrayOfStringsSummary) summary).stringArr); + final String thisStr = stringConcat(stringArr); return thisStr.equals(thatStr); } @@ -120,37 +141,40 @@ public boolean equals(final Object summary) { * @return the nodes array for this summary. */ public String[] getValue() { - return nodesArr.clone(); + return stringArr.clone(); } //also used in test static void checkNumNodes(final int numNodes) { - if (numNodes > 127) { - throw new SketchesArgumentException("Number of nodes cannot exceed 127."); + if (numNodes > 127 || numNodes < 0) { + throw new SketchesArgumentException("Number of nodes cannot exceed 127 or be negative."); } } //also used in test - static void checkInBytes(final Memory mem, final int totBytes) { - if (mem.getCapacity() < totBytes) { - throw new SketchesArgumentException("Incoming Memory has insufficient capacity."); + static void checkInBytes(final MemorySegment seg, final int totBytes) { + if (seg.byteSize() < totBytes) { + throw new SketchesArgumentException("Incoming MemorySegment has insufficient capacity."); } } + /** + * Computes total bytes and number of nodes from the given string array. + */ private static class ComputeBytes { final byte numNodes_; final int[] nodeLengthsArr_; final byte[][] nodeBytesArr_; final int totBytes_; - ComputeBytes(final String[] nodesArr) { - numNodes_ = (byte) nodesArr.length; + ComputeBytes(final String[] stringArr) { + numNodes_ = (byte) stringArr.length; checkNumNodes(numNodes_); nodeLengthsArr_ = new int[numNodes_]; nodeBytesArr_ = new byte[numNodes_][]; int sumNodeBytes = 0; for (int i = 0; i < numNodes_; i++) { - nodeBytesArr_[i] = nodesArr[i].getBytes(UTF_8); + nodeBytesArr_[i] = stringArr[i].getBytes(UTF_8); nodeLengthsArr_[i] = nodeBytesArr_[i].length; sumNodeBytes += nodeLengthsArr_[i]; } diff --git a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java index 16467f338..5363605d4 100644 --- a/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java +++ b/src/main/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryDeserializer.java @@ -19,7 +19,10 @@ package org.apache.datasketches.tuple.strings; -import org.apache.datasketches.memory.Memory; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; + +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.tuple.DeserializeResult; import org.apache.datasketches.tuple.SummaryDeserializer; @@ -30,18 +33,18 @@ public class ArrayOfStringsSummaryDeserializer implements SummaryDeserializer { @Override - public DeserializeResult heapifySummary(final Memory mem) { - return ArrayOfStringsSummaryDeserializer.fromMemory(mem); + public DeserializeResult heapifySummary(final MemorySegment seg) { + return ArrayOfStringsSummaryDeserializer.fromMemorySegment(seg); } /** * Also used in test. - * @param mem the given memory + * @param seg the given MemorySegment * @return the DeserializeResult */ - static DeserializeResult fromMemory(final Memory mem) { - final ArrayOfStringsSummary nsum = new ArrayOfStringsSummary(mem); - final int totBytes = mem.getInt(0); + static DeserializeResult fromMemorySegment(final MemorySegment seg) { + final ArrayOfStringsSummary nsum = new ArrayOfStringsSummary(seg); + final int totBytes = seg.get(JAVA_INT_UNALIGNED, 0); return new DeserializeResult<>(nsum, totBytes); } diff --git a/src/test/java/org/apache/datasketches/common/UtilTest.java b/src/test/java/org/apache/datasketches/common/UtilTest.java index ca83dadd6..fac2dea1c 100644 --- a/src/test/java/org/apache/datasketches/common/UtilTest.java +++ b/src/test/java/org/apache/datasketches/common/UtilTest.java @@ -416,15 +416,15 @@ static void checkConvertToLongArray() { byte[] arr = {1,2,3,4,5,6,7,8,9,10,11,12}; long[] out = convertToLongArray(arr, false); - String s = org.apache.datasketches.common.Util.zeroPad(Long.toHexString(out[0]), 16); + String s = zeroPad(Long.toHexString(out[0]), 16); assertEquals(s, "0807060504030201"); - s = org.apache.datasketches.common.Util.zeroPad(Long.toHexString(out[1]), 16); + s = zeroPad(Long.toHexString(out[1]), 16); assertEquals(s, "000000000c0b0a09"); out = convertToLongArray(arr, true); - s = org.apache.datasketches.common.Util.zeroPad(Long.toHexString(out[0]), 16); + s = zeroPad(Long.toHexString(out[0]), 16); assertEquals(s, "0102030405060708"); - s = org.apache.datasketches.common.Util.zeroPad(Long.toHexString(out[1]), 16); + s = zeroPad(Long.toHexString(out[1]), 16); assertEquals(s, "00000000090a0b0c"); } diff --git a/src/test/java/org/apache/datasketches/cpc/CpcSketchTest.java b/src/test/java/org/apache/datasketches/cpc/CpcSketchTest.java index 1c03fb4bb..3782bdc34 100644 --- a/src/test/java/org/apache/datasketches/cpc/CpcSketchTest.java +++ b/src/test/java/org/apache/datasketches/cpc/CpcSketchTest.java @@ -29,8 +29,8 @@ import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.common.Util; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.thetacommon.ThetaUtil; import org.testng.annotations.Test; /** @@ -189,10 +189,10 @@ public void checkIconHipUBLBLg15() { @Test public void checkHeapify() { int lgK = 10; - CpcSketch sk = new CpcSketch(lgK, ThetaUtil.DEFAULT_UPDATE_SEED); + CpcSketch sk = new CpcSketch(lgK, Util.DEFAULT_UPDATE_SEED); assertTrue(sk.isEmpty()); byte[] byteArray = sk.toByteArray(); - CpcSketch sk2 = CpcSketch.heapify(byteArray, ThetaUtil.DEFAULT_UPDATE_SEED); + CpcSketch sk2 = CpcSketch.heapify(byteArray, Util.DEFAULT_UPDATE_SEED); assertTrue(specialEquals(sk2, sk, false, false)); } @@ -210,7 +210,7 @@ public void checkHeapify2() { @Test public void checkRowColUpdate() { int lgK = 10; - CpcSketch sk = new CpcSketch(lgK, ThetaUtil.DEFAULT_UPDATE_SEED); + CpcSketch sk = new CpcSketch(lgK, Util.DEFAULT_UPDATE_SEED); sk.rowColUpdate(0); assertEquals(sk.getFlavor(), Flavor.SPARSE); } diff --git a/src/test/java/org/apache/datasketches/cpc/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/cpc/PreambleUtilTest.java index d09653196..8577b8a3d 100644 --- a/src/test/java/org/apache/datasketches/cpc/PreambleUtilTest.java +++ b/src/test/java/org/apache/datasketches/cpc/PreambleUtilTest.java @@ -54,17 +54,17 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; +import org.apache.datasketches.common.Util; import org.apache.datasketches.cpc.PreambleUtil.HiField; /** * @author Lee Rhodes */ public class PreambleUtilTest { - static final short defaultSeedHash = ThetaUtil.computeSeedHash(ThetaUtil.DEFAULT_UPDATE_SEED) ; + static final short defaultSeedHash = Util.computeSeedHash(Util.DEFAULT_UPDATE_SEED) ; private static void checkFirst8(WritableMemory wmem, Format format, int lgK, int fiCol) { assertEquals(getFormat(wmem), format); diff --git a/src/test/java/org/apache/datasketches/cpc/TestAllTest.java b/src/test/java/org/apache/datasketches/cpc/TestAllTest.java index ce847fc36..4f94fce47 100644 --- a/src/test/java/org/apache/datasketches/cpc/TestAllTest.java +++ b/src/test/java/org/apache/datasketches/cpc/TestAllTest.java @@ -25,7 +25,7 @@ import java.io.PrintWriter; import java.util.Arrays; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; import org.testng.annotations.Test; /** @@ -89,7 +89,7 @@ public void singleRowColCheck() { CompressedState state = CompressedState.compress(srcSketch); ps.println(CompressedState.toString(state, true)); - CpcSketch uncSketch = CpcSketch.uncompress(state, ThetaUtil.DEFAULT_UPDATE_SEED); + CpcSketch uncSketch = CpcSketch.uncompress(state, Util.DEFAULT_UPDATE_SEED); ps.println(uncSketch.toString(true)); } diff --git a/src/test/java/org/apache/datasketches/fdt/FdtSketchTest.java b/src/test/java/org/apache/datasketches/fdt/FdtSketchTest.java index 14945bac0..4c1527b7a 100644 --- a/src/test/java/org/apache/datasketches/fdt/FdtSketchTest.java +++ b/src/test/java/org/apache/datasketches/fdt/FdtSketchTest.java @@ -23,10 +23,13 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; import java.util.List; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.fdt.FdtSketch; +import org.apache.datasketches.fdt.Group; +import org.apache.datasketches.fdt.PostProcessor; import org.apache.datasketches.tuple.TupleSketchIterator; import org.apache.datasketches.tuple.strings.ArrayOfStringsSummary; import org.testng.annotations.Test; @@ -59,8 +62,8 @@ public void checkFdtSketch() { //serialize final byte[] byteArr = sketch.toByteArray(); //deserialize - Memory mem = Memory.wrap(byteArr); - FdtSketch sketch2 = new FdtSketch(mem); + MemorySegment seg = MemorySegment.ofArray(byteArr); + FdtSketch sketch2 = new FdtSketch(seg); //check output final TupleSketchIterator it2 = sketch2.iterator(); diff --git a/src/test/java/org/apache/datasketches/fdt/GroupTest.java b/src/test/java/org/apache/datasketches/fdt/GroupTest.java index 21427fb0b..2d84a0d1d 100644 --- a/src/test/java/org/apache/datasketches/fdt/GroupTest.java +++ b/src/test/java/org/apache/datasketches/fdt/GroupTest.java @@ -21,6 +21,7 @@ import static org.testng.Assert.assertEquals; +import org.apache.datasketches.fdt.Group; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3Test.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3Test.java index 9caf42e1d..21ae20783 100644 --- a/src/test/java/org/apache/datasketches/hash/MurmurHash3Test.java +++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3Test.java @@ -20,6 +20,7 @@ package org.apache.datasketches.hash; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.datasketches.common.Util.longToHexBytes; import static org.apache.datasketches.hash.MurmurHash3.hash; import java.nio.ByteBuffer; @@ -265,37 +266,37 @@ public void checkCrossTypeHashConsistency() { println("Bytes"); byte[] bArr = {1,2,3,4,5,6,7,8, 9,10,11,12,13,14,15,16, 17,18,19,20,21,22,23,24}; long[] out1 = hash(bArr, 0L); - println(org.apache.datasketches.common.Util.longToHexBytes(out1[0])); - println(org.apache.datasketches.common.Util.longToHexBytes(out1[1])); + println(longToHexBytes(out1[0])); + println(longToHexBytes(out1[1])); println("ByteBuffer"); ByteBuffer bBuf = ByteBuffer.wrap(bArr); out = hash(bBuf, 0L); Assert.assertEquals(out, out1); - println(org.apache.datasketches.common.Util.longToHexBytes(out1[0])); - println(org.apache.datasketches.common.Util.longToHexBytes(out1[1])); + println(longToHexBytes(out1[0])); + println(longToHexBytes(out1[1])); println("Chars"); char[] cArr = {0X0201, 0X0403, 0X0605, 0X0807, 0X0a09, 0X0c0b, 0X0e0d, 0X100f, 0X1211, 0X1413, 0X1615, 0X1817}; out = hash(cArr, 0L); Assert.assertEquals(out, out1); - println(org.apache.datasketches.common.Util.longToHexBytes(out[0])); - println(org.apache.datasketches.common.Util.longToHexBytes(out[1])); + println(longToHexBytes(out[0])); + println(longToHexBytes(out[1])); println("Ints"); int[] iArr = {0X04030201, 0X08070605, 0X0c0b0a09, 0X100f0e0d, 0X14131211, 0X18171615}; out = hash(iArr, 0L); Assert.assertEquals(out, out1); - println(org.apache.datasketches.common.Util.longToHexBytes(out[0])); - println(org.apache.datasketches.common.Util.longToHexBytes(out[1])); + println(longToHexBytes(out[0])); + println(longToHexBytes(out[1])); println("Longs"); long[] lArr = {0X0807060504030201L, 0X100f0e0d0c0b0a09L, 0X1817161514131211L}; out = hash(lArr, 0L); Assert.assertEquals(out, out1); - println(org.apache.datasketches.common.Util.longToHexBytes(out[0])); - println(org.apache.datasketches.common.Util.longToHexBytes(out[1])); + println(longToHexBytes(out[0])); + println(longToHexBytes(out[1])); } diff --git a/src/test/java/org/apache/datasketches/hash/XxHash64LoopingTest.java b/src/test/java/org/apache/datasketches/hash/XxHash64LoopingTest.java new file mode 100644 index 000000000..4ae2b4956 --- /dev/null +++ b/src/test/java/org/apache/datasketches/hash/XxHash64LoopingTest.java @@ -0,0 +1,1082 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.hash; + +import static org.testng.Assert.assertEquals; + +import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; + +/** + * @author Lee Rhodes + */ +public class XxHash64LoopingTest { + + /* + * This test is adapted from + * + * OpenHFT/Zero-Allocation-Hashing to test hash compatibility with that implementation. + * See LICENSE. + */ + @Test + public void testWithSeed() { + long seed = 42L; + for (int i = 0; i < 1025; i++) { + byte[] byteArr = new byte[i]; + for (int j = 0; j < byteArr.length; j++) { byteArr[j] = (byte) j; } + WritableMemory wmem = WritableMemory.writableWrap(byteArr); + long hash = wmem.xxHash64(0, byteArr.length, seed); + assertEquals(hash, HASHES_OF_LOOPING_BYTES_WITH_SEED_42[i]); + } + } + + /*This data is from + * + * OpenHFT/Zero-Allocation-Hashing to test hash compatibility with that implementation. + * See LICENSE. + */ + private static final long[] HASHES_OF_LOOPING_BYTES_WITH_SEED_42 = { + -7444071767201028348L, + -8959994473701255385L, + 7116559933691734543L, + 6019482000716350659L, + -6625277557348586272L, + -5507563483608914162L, + 1540412690865189709L, + 4522324563441226749L, + -7143238906056518746L, + -7989831429045113014L, + -7103973673268129917L, + -2319060423616348937L, + -7576144055863289344L, + -8903544572546912743L, + 6376815151655939880L, + 5913754614426879871L, + 6466567997237536608L, + -869838547529805462L, + -2416009472486582019L, + -3059673981515537339L, + 4211239092494362041L, + 1414635639471257331L, + 166863084165354636L, + -3761330575439628223L, + 3524931906845391329L, + 6070229753198168844L, + -3740381894759773016L, + -1268276809699008557L, + 1518581707938531581L, + 7988048690914090770L, + -4510281763783422346L, + -8988936099728967847L, + -8644129751861931918L, + 2046936095001747419L, + 339737284852751748L, + -8493525091666023417L, + -3962890767051635164L, + -5799948707353228709L, + -6503577434416464161L, + 7718729912902936653L, + 191197390694726650L, + -2677870679247057207L, + 20411540801847004L, + 2738354376741059902L, + -3754251900675510347L, + -3208495075154651980L, + 5505877218642938179L, + 6710910171520780908L, + -9060809096139575515L, + 6936438027860748388L, + -6675099569841255629L, + -5358120966884144380L, + -4970515091611332076L, + -1810965683604454696L, + -516197887510505242L, + 1240864593087756274L, + 6033499571835033332L, + 7223146028771530185L, + 909128106589125206L, + 1567720774747329341L, + -1867353301780159863L, + 4655107429511759333L, + 5356891185236995950L, + 182631115370802890L, + -3582744155969569138L, + 595148673029792797L, + 495183136068540256L, + 5536689004903505647L, + -8472683670935785889L, + -4335021702965928166L, + 7306662983232020244L, + 4285260837125010956L, + 8288813008819191181L, + -3442351913745287612L, + 4883297703151707194L, + 9135546183059994964L, + 123663780425483012L, + 509606241253238381L, + 5940344208569311369L, + -2650142344608291176L, + 3232776678942440459L, + -922581627593772181L, + 7617977317085633049L, + 7154902266379028518L, + -5806388675416795571L, + 4368003766009575737L, + -2922716024457242064L, + 4771160713173250118L, + 3275897444752647349L, + -297220751499763878L, + 5095659287766176401L, + 1181843887132908826L, + 9058283605301070357L, + 3984713963471276643L, + 6050484112980480005L, + 1551535065359244224L, + 565337293533335618L, + 7412521035272884309L, + -4735469481351389369L, + 6998597101178745656L, + -9107075101236275961L, + 5879828914430779796L, + 6034964979406620806L, + 5666406915264701514L, + -4666218379625258428L, + 2749972203764815656L, + -782986256139071446L, + 6830581400521008570L, + 2588852022632995043L, + -5484725487363818922L, + -3319556935687817112L, + 6481961252981840893L, + 2204492445852963006L, + -5301091763401031066L, + -2615065677047206256L, + -6769817545131782460L, + -8421640685322953142L, + -3669062629317949176L, + -9167016978640750490L, + 2783671191687959562L, + -7599469568522039782L, + -7589134103255480011L, + -5932706841188717592L, + -8689756354284562694L, + -3934347391198581249L, + -1344748563236040701L, + 2172701592984478834L, + -5322052340624064417L, + -8493945390573620511L, + 3349021988137788403L, + -1806262525300459538L, + -8091524448239736618L, + 4022306289903960690L, + -8346915997379834224L, + -2106001381993805461L, + -5784123934724688161L, + 6775158099649720388L, + -3869682756870293568L, + 4356490186652082006L, + 8469371446702290916L, + -2972961082318458602L, + -7188106622222784561L, + -4961006366631572412L, + 3199991182014172900L, + 2917435868590434179L, + 8385845305547872127L, + 7706824402560674655L, + -1587379863634865277L, + -4212156212298809650L, + -1305209322000720233L, + -7866728337506665880L, + 8195089740529247049L, + -4876930125798534239L, + 798222697981617129L, + -2441020897729372845L, + -3926158482651178666L, + -1254795122048514130L, + 5192463866522217407L, + -5426289318796042964L, + -3267454004443530826L, + 471043133625225785L, + -660956397365869974L, + -6149209189144999161L, + -2630977660039166559L, + 8512219789663151219L, + -3309844068134074620L, + -6211275327487847132L, + -2130171729366885995L, + 6569302074205462321L, + 4855778342281619706L, + 3867211421508653033L, + -3002480002418725542L, + -8297543107467502696L, + 8049642289208775831L, + -5439825716055425635L, + 7251760070798756432L, + -4774526021749797528L, + -3892389575184442548L, + 5162451061244344424L, + 6000530226398686578L, + -5713092252241819676L, + 8740913206879606081L, + -8693282419677309723L, + 1576205127972543824L, + 5760354502610401246L, + 3173225529903529385L, + 1785166236732849743L, + -1024443476832068882L, + -7389053248306187459L, + 1171021620017782166L, + 1471572212217428724L, + 7720766400407679932L, + -8844781213239282804L, + -7030159830170200877L, + 2195066352895261150L, + 1343620937208608634L, + 9178233160016731645L, + -757883447602665223L, + 3303032934975960867L, + -3685775162104101116L, + -4454903657585596656L, + -5721532367620482629L, + 8453227136542829644L, + 5397498317904798888L, + 7820279586106842836L, + -2369852356421022546L, + 3910437403657116169L, + 6072677490463894877L, + -2651044781586183960L, + 5173762670440434510L, + -2970017317595590978L, + -1024698859439768763L, + -3098335260967738522L, + -1983156467650050768L, + -8132353894276010246L, + -1088647368768943835L, + -3942884234250555927L, + 7169967005748210436L, + 2870913702735953746L, + -2207022373847083021L, + 1104181306093040609L, + 5026420573696578749L, + -5874879996794598513L, + -4777071762424874671L, + -7506667858329720470L, + -2926679936584725232L, + -5530649174168373609L, + 5282408526788020384L, + 3589529249264153135L, + -6220724706210580398L, + -7141769650716479812L, + 5142537361821482047L, + -7029808662366864423L, + -6593520217660744466L, + 1454581737122410695L, + -139542971769349865L, + 1727752089112067235L, + -775001449688420017L, + -5011311035350652032L, + -8671171179275033159L, + -2850915129917664667L, + -5258897903906998781L, + -6954153088230718761L, + -4070351752166223959L, + -6902592976462171099L, + -7850366369290661391L, + -4562443925864904705L, + 3186922928616271015L, + 2208521081203400591L, + -2727824999830592777L, + -3817861137262331295L, + 2236720618756809066L, + -4888946967413746075L, + -446884183491477687L, + -43021963625359034L, + -5857689226703189898L, + -2156533592262354883L, + -2027655907961967077L, + 7151844076490292500L, + -5029149124756905464L, + 526404452686156976L, + 8741076980297445408L, + 7962851518384256467L, + -105985852299572102L, + -2614605270539434398L, + -8265006689379110448L, + 8158561071761524496L, + -6923530157382047308L, + 5551949335037580397L, + 565709346370307061L, + -4780869469938333359L, + 6931895917517004830L, + 565234767538051407L, + -8663136372880869656L, + 1427340323685448983L, + 6492705666640232290L, + 1481585578088475369L, + -1712711110946325531L, + 3281685342714380741L, + 6441384790483098576L, + -1073539554682358394L, + 5704050067194788964L, + -5495724689443043319L, + -5425043165837577535L, + 8349736730194941321L, + -4123620508872850061L, + 4687874980541143573L, + -468891940172550975L, + -3212254545038049829L, + -6830802881920725628L, + 9033050533972480988L, + 4204031879107709260L, + -677513987701096310L, + -3286978557209370155L, + 1644111582609113135L, + 2040089403280131741L, + 3323690950628902653L, + -7686964480987925756L, + -4664519769497402737L, + 3358384147145476542L, + -4699919744264452277L, + -4795197464927839170L, + 5051607253379734527L, + -8987703459734976898L, + 8993686795574431834L, + -2688919474688811047L, + 375938183536293311L, + 1049459889197081920L, + -1213022037395838295L, + 4932989235110984138L, + -6647247877090282452L, + -7698817539128166242L, + -3264029336002462659L, + 6487828018122309795L, + -2660821091484592878L, + 7104391069028909121L, + -1765840012354703384L, + 85428166783788931L, + -6732726318028261938L, + 7566202549055682933L, + 229664898114413280L, + -1474237851782211353L, + -1571058880058007603L, + -7926453582850712144L, + 2487148368914275243L, + 8740031015380673473L, + 1908345726881363169L, + -2510061320536523178L, + 7854780026906019630L, + -6023415596650016493L, + -6264841978089051107L, + 4024998278016087488L, + -4266288992025826072L, + -3222176619422665563L, + -1999258726038299316L, + 1715270077442385636L, + 6764658837948099754L, + -8646962299105812577L, + -51484064212171546L, + -1482515279051057493L, + -8663965522608868414L, + -256555202123523670L, + 1973279596140303801L, + -7280796173024508575L, + -5691760367231354704L, + -5915786562256300861L, + -3697715074906156565L, + 3710290115318541949L, + 6796151623958134374L, + -935299482515386356L, + -7078378973978660385L, + 5379481350768846927L, + -9011221735308556302L, + 5936568631579608418L, + -6060732654964511813L, + -4243141607840017809L, + 3198488845875349355L, + -7809288876010447646L, + 4371587872421472389L, + -1304197371105522943L, + 7389861473143460103L, + -1892352887992004024L, + 2214828764044713398L, + 6347546952883613388L, + 1275694314105480954L, + -5262663163358903733L, + 1524757505892047607L, + 1474285098416162746L, + -7976447341881911786L, + 4014100291977623265L, + 8994982266451461043L, + -7737118961020539453L, + -2303955536994331092L, + 1383016539349937136L, + 1771516393548245271L, + -5441914919967503849L, + 5449813464890411403L, + -3321280356474552496L, + 4084073849712624363L, + 4290039323210935932L, + 2449523715173349652L, + 7494827882138362156L, + 9035007221503623051L, + 5722056230130603177L, + -5443061851556843748L, + -7554957764207092109L, + 447883090204372074L, + 533916651576859197L, + -3104765246501904165L, + -4002281505194601516L, + -8402008431255610992L, + -408273018037005304L, + 214196458752109430L, + 6458513309998070914L, + 2665048360156607904L, + 96698248584467992L, + -3238403026096269033L, + 6759639479763272920L, + -4231971627796170796L, + -2149574977639731179L, + -1437035755788460036L, + -6000005629185669767L, + 145244292800946348L, + -3056352941404947199L, + 3748284277779018970L, + 7328354565489106580L, + -2176895260373660284L, + 3077983936372755601L, + 1215485830019410079L, + 683050801367331140L, + -3173237622987755212L, + -1951990779107873701L, + -4714366021269652421L, + 4934690664256059008L, + 1674823104333774474L, + -3974408282362828040L, + 2001478896492417760L, + -4115105568354384199L, + -2039694725495941666L, + -587763432329933431L, + -391276713546911316L, + -5543400904809469053L, + 1882564440421402418L, + -4991793588968693036L, + 3454088185914578321L, + 2290855447126188424L, + 3027910585026909453L, + 2136873580213167431L, + -6243562989966916730L, + 5887939953208193029L, + -3491821629467655741L, + -3138303216306660662L, + 8572629205737718669L, + 4154439973110146459L, + 5542921963475106759L, + -2025215496720103521L, + -4047933760493641640L, + -169455456138383823L, + -1164572689128024473L, + -8551078127234162906L, + -7247713218016599028L, + 8725299775220778242L, + 6263466461599623132L, + 7931568057263751768L, + 7365493014712655238L, + -7343740914722477108L, + 8294118602089088477L, + 7677867223984211483L, + -7052188421655969232L, + -3739992520633991431L, + 772835781531324307L, + 881441588914692737L, + 6321450879891466401L, + 5682516032668315027L, + 8493068269270840662L, + -3895212467022280567L, + -3241911302335746277L, + -7199586338775635848L, + -4606922569968527974L, + -806850906331637768L, + 2433670352784844513L, + -5787982146811444512L, + 7852193425348711165L, + 8669396209073850051L, + -6898875695148963118L, + 6523939610287206782L, + -8084962379210153174L, + 8159432443823995836L, + -2631068535470883494L, + -338649779993793113L, + 6514650029997052016L, + 3926259678521802094L, + 5443275905907218528L, + 7312187582713433551L, + -2993773587362997676L, + -1068335949405953411L, + 4499730398606216151L, + 8538015793827433712L, + -4057209365270423575L, + -1504284818438273559L, + -6460688570035010846L, + 1765077117408991117L, + 8278320303525164177L, + 8510128922449361533L, + 1305722765578569816L, + 7250861238779078656L, + -576624504295396147L, + -4363714566147521011L, + -5932111494795524073L, + 1837387625936544674L, + -4186755953373944712L, + -7657073597826358867L, + 140408487263951108L, + 5578463635002659628L, + 3400326044813475885L, + -6092804808386714986L, + -2410324417287268694L, + 3222007930183458970L, + 4932471983280850419L, + 3554114546976144528L, + -7216067928362857082L, + -6115289896923351748L, + -6769646077108881947L, + 4263895947722578066L, + 2939136721007694271L, + 1426030606447416658L, + -1316192446807442076L, + 5366182640480055129L, + 6527003877470258527L, + 5849680119000207603L, + 5263993237214222328L, + -6936533648789185663L, + -9063642143790846605L, + 3795892210758087672L, + 4987213125282940176L, + 2505500970421590750L, + -1014022559552365387L, + -3574736245968367770L, + 1180676507127340259L, + -2261908445207512503L, + -8416682633172243509L, + 1114990703652673283L, + 7753746660364401380L, + 1874908722469707905L, + 2033421444403047677L, + 21412168602505589L, + 385957952615286205L, + 2053171460074727107L, + 1915131899400103774L, + 6680879515029368390L, + 568807208929724162L, + -6211541450459087674L, + -5026690733412145448L, + 1384781941404886235L, + -98027820852587266L, + 1806580495924249669L, + 6322077317403503963L, + 9078162931419569939L, + -2809061215428363978L, + 7697867577577415733L, + -5270063855897737274L, + 5649864555290587388L, + -6970990547695444247L, + 579684606137331754L, + 3871931565451195154L, + 2030008578322050218L, + -5012357307111799829L, + -2271365921756144065L, + 4551962665158074190L, + -3385474923040271312L, + -7647625164191633577L, + 6634635380316963029L, + -5201190933687061585L, + 8864818738548593973L, + 2855828214210882907L, + 9154512990734024165L, + -6945306719789457786L, + 1200243352799481087L, + 875998327415853787L, + 1275313054449881011L, + -6105772045375948736L, + -2926927684328291437L, + 9200050852144954779L, + 5188726645765880663L, + 5197037323312705176L, + 3434926231010121611L, + -5054013669361906544L, + 2582959199749224670L, + -6053757512723474059L, + -5016308176846054473L, + -2509827316698626133L, + 7700343644503853204L, + -1997627249894596731L, + 3993168688325352290L, + -8181743677541277704L, + 3719056119682565597L, + -7264411659282947790L, + 7177028972346484464L, + -5460831176884283278L, + 1799904662416293978L, + -6549616005092764514L, + 5472403994001122052L, + 8683463751708388502L, + -7873363037838316398L, + 689134758256487260L, + -1287443614028696450L, + 4452712919702709507L, + 762909374167538893L, + 6594302592326281411L, + 1183786629674781984L, + 5021847859620133476L, + -2490098069181538915L, + 5105145136026716679L, + 4437836948098585718L, + 1987270426215858862L, + 6170312798826946249L, + 634297557126003407L, + -1672811625495999581L, + 6282971595586218191L, + 4549149305727581687L, + -5652165370435317782L, + 1064501550023753890L, + -5334885527127139723L, + -6904378001629481237L, + -1807576691784201230L, + -205688432992053911L, + 7621619053293393289L, + 6258649161313982470L, + -1111634238359342096L, + -8044260779481691987L, + 400270655839010807L, + -7806833581382890725L, + -2970563349459508036L, + -7392591524816802798L, + 2918924613160219805L, + -6444161627929149002L, + 6096497501321778876L, + -1477975665655830038L, + 1690651307597306138L, + -2364076888826085362L, + -6521987420014905821L, + -4419193480146960582L, + 3538587780233092477L, + 8374665961716940404L, + 7492412312405424500L, + 6311662249091276767L, + -1240235198282023566L, + 5478559631401166447L, + 3476714419313462133L, + 377427285984503784L, + 2570472638778991109L, + -2741381313777447835L, + -7123472905503039596L, + 2493658686946955193L, + 1024677789035847585L, + -2916713904339582981L, + -4532003852004642304L, + -2202143560366234111L, + 5832267856442755135L, + -261740607772957384L, + 239435959690278014L, + 5755548341947719409L, + 6138795458221887696L, + -7709506987360146385L, + -6657487758065140444L, + -7006376793203657499L, + 6544409861846502033L, + 3171929352014159247L, + 1051041925048792869L, + 2617300158375649749L, + 952652799620095175L, + -576661730162168147L, + -1634191369221345988L, + 4833656816115993519L, + 647566759700005786L, + 2473810683785291822L, + 3005977181064745326L, + -3321881966853149523L, + 7595337666427588699L, + 6004093624251057224L, + -563917505657690279L, + 6117428527147449302L, + -6287297509522976113L, + -4527219334756214406L, + 742626429298092489L, + 3057351806086972041L, + 645967551210272605L, + -4428701157828864227L, + 3236379103879435414L, + -8477089892132066300L, + -6127365537275859058L, + -4052490484706946358L, + -8004854976625046469L, + -3679456917426613424L, + -8212793762082595299L, + -818288739465424130L, + 1358812099481667095L, + 7835987612195254310L, + -3663247409614323059L, + -2931105150130396604L, + 7296136776835614792L, + -2014557408985889628L, + 7267662411237959788L, + 3699280615819277743L, + -212010675469091396L, + -6518374332458360120L, + 145026010541628849L, + 1879297324213501001L, + -7146296067751816833L, + -5002958800391379931L, + 6060682439924517608L, + -432234782921170964L, + -6669688947353256956L, + 7728943532792041267L, + 830911367341171721L, + 3396934884314289432L, + -779464156662780749L, + 2330041851883352285L, + -4783350380736276693L, + -5758476056890049254L, + -7551552301614791791L, + 1253334187723911710L, + -2685018208308798978L, + 5379636036360946454L, + 6154668487114681217L, + -8641287462255458898L, + 4676087643800649558L, + -2405142641398691475L, + 1088685126864246881L, + 6431149082338374041L, + -607357695335069155L, + -720970692129524140L, + 2648766932394044468L, + 8408344790179354573L, + -6193808387735667350L, + 7722524628524697419L, + -6975433852560238120L, + -2925851029234475295L, + -4274458387165211028L, + -8355836377702147319L, + 5278146397877332061L, + 8502098812383680707L, + 2292836642336580326L, + -6127608082651070062L, + 2222301962240611208L, + -1930887695854799378L, + 7640503480494894592L, + 1162652186586436094L, + -1918002592943761683L, + 7648998601717261840L, + -8472603250832757057L, + -988877663117552456L, + 2368458128168026494L, + -6480813811998475245L, + -5896967824416018967L, + -2593783161701820446L, + 6950098417530252598L, + 6362589545555771236L, + 7981389665448567125L, + 3954017080198558850L, + 1626078615050230622L, + 6650159066527969109L, + 697345338922935394L, + -1226816215461768626L, + 8740408765973837440L, + -4194155864629568323L, + 7016680023232424746L, + 6043281358142429469L, + -4201005667174376809L, + 1216727117859013155L, + 6367202436544203935L, + 35414869396444636L, + 3715622794033998412L, + 488654435687670554L, + -2503747297224687460L, + 3147101919441470388L, + -8248611218693190922L, + 970697264481229955L, + 3411465763826851418L, + 9117405004661599969L, + -5204346498331519734L, + -19637460819385174L, + -5039124225167977219L, + 2990108874601696668L, + -2623857460235459202L, + 4256291692861397446L, + 6724147860870760443L, + 3558616688507246537L, + 6487680097936412800L, + -6470792832935928161L, + 4314814550912237614L, + -1292878983006062345L, + 6791915152630414174L, + 5971652079925815310L, + 2557529546662864312L, + 466175054322801580L, + -585216717310746872L, + -2486640422147349036L, + 7212029603994220134L, + 3958995069888972500L, + 4950471855791412790L, + -3721948842035712763L, + -6184503487488243051L, + 4079570444585775332L, + -3952156172546996872L, + 4543894231118208322L, + -1739995588466209963L, + 9155948355455935530L, + 5821980345462207860L, + -2431287667309520417L, + -3890108130519441316L, + -558124689277030490L, + 6079823537335801717L, + 5409742395192364262L, + -2329885777717160453L, + -7332804342513677651L, + 1466490574975950555L, + -420549419907427929L, + -5249909814389692516L, + -5145692168206210661L, + 5934113980649113921L, + 3241618428555359661L, + -6622110266160980250L, + 5048250878669516223L, + 5747219637359976174L, + 2975906212588223728L, + 5730216838646273215L, + -176713127129024690L, + 6734624279336671146L, + 5127866734316017180L, + 7111761230887705595L, + 3457811808274317235L, + 3362961434604932375L, + -1877869936854991246L, + 7171428594877765665L, + -8252167178400462374L, + -6306888185035821047L, + -6684702191247683887L, + -7754928454824190529L, + -1902605599135704386L, + -4037319846689421239L, + 8493746058123583457L, + -8156648963857047193L, + 2051510355149839497L, + -1256416624177218909L, + -3344927996254072010L, + -1838853051925943568L, + 316927471680974556L, + -1502257066700798003L, + -5836095610125837606L, + -1594125583615895424L, + 1442211486559637962L, + -144295071206619569L, + 5159850900959273410L, + 4589139881166423678L, + -7038726987463097509L, + 2886082400772974595L, + 2780759114707171916L, + 5694649587906297495L, + 1260349041268169667L, + 4921517488271434890L, + 644696475796073018L, + 6262811963753436289L, + -6128198676595868773L, + -3625352083004760261L, + -8751453332943236675L, + 8749249479868749221L, + -2450808199545048250L, + -6517435817046180917L, + -3433321727429234998L, + -2591586258908763451L, + 3847750870868804507L, + 6603614438546398643L, + -7598682191291031287L, + 8710261565627204971L, + 4753389483755344355L, + -4645333069458786881L, + -6742695046613492214L, + 643070478568866643L, + -7543096104151965610L, + 7171495384655926161L, + 595063872610714431L, + 3292310150781130424L, + 4326847806055440904L, + -4580020566072794152L, + 3142286571820373678L, + 5530356537440155930L, + 546372639737516181L, + 7401214477400367500L, + 7406531960402873109L, + 3287639667219172570L, + 4977301681213633671L, + 5253257820925174498L, + 2906216636104297878L, + 6142955758238347523L, + -3498651268741727235L, + -5875053958265588593L, + 3896719087169993883L, + -910904726885775073L, + 380107493197368177L, + -4993591912695447004L, + 2970487257212582761L, + 2551762717569548774L, + 953061649962736812L, + 8949739538606589463L, + -2962839167079475801L, + -1375673191272573835L, + 3761793818361866390L, + -389577789190726878L, + 5661262051502180269L, + -6558556411143987683L, + -702798336372315031L, + -336662820551371779L, + 998576401126580155L, + -5945021269112582755L, + 6108533925730179871L, + 2207095297001999618L, + -9042779159998880435L, + -6177868444342118372L, + 6775965402605895077L, + -3788428885163306576L, + 7790055010527190387L, + 3581587652196995358L, + -6176354155561607694L, + -5859381340906321207L, + 395898765763528395L, + 8132967590863909348L, + -3329092504090544483L, + -6785855381158040247L, + 1497218517051796750L, + -5352392845588925911L, + -6271364901230559194L, + 2314830370653350118L, + -7617588269001325450L, + 1423166885758213795L, + 8538612578307869519L, + -61918791718295474L, + -8177103503192338593L, + -4740086042584326695L, + 3677931948215558698L, + 6558856291580149558L, + 2674975452453336335L, + 5133796555646930522L, + 5139252693299337100L, + 7949476871295347205L, + 4407815324662880678L, + -3758305875280581215L, + 6066309507576587415L, + -7368508486398350973L, + -3181640264332856492L, + 6905100869343314145L, + 3677177673848733417L, + 8862933624870506941L, + -8575223195813810568L, + 9178470351355678144L, + 4677809017145408358L, + -1194833416287894989L, + 3436364743255571183L, + -5204770725795363579L, + 560599448536335263L, + -3192077522964776200L, + -751575299648803575L, + 6334581746534596579L, + -8358187891202563300L, + -1462480609823525055L, + 5605961062646987941L, + 4968399805931440889L, + 7968693270782626653L, + -5868205923557518188L, + 1830234928743560617L, + -8435261076693154407L, + 2138416970728681332L, + 8088740745199685138L, + 806532400344230520L, + 1800590379902909333L, + -8909128842071238901L, + -7357495566969170860L, + 3679766664126940553L, + 2060050474865839094L, + 2363972840121763414L, + 525695004292982714L, + -1224842191746529593L, + 7011317848855545003L, + -6337167558180299938L, + -5184688833363785939L, + -8426673387248359061L, + -5035438815930785229L, + 3521810320608058994L, + 4803742557254962242L, + 6623527039545786598L, + -1221475882122634738L, + -3344794405518401087L, + 6510298498414053658L, + 2844753907937720338L, + 90502309714994895L, + -750403235344282494L, + -4825474181021465833L, + -3405519947983849510L, + 3503875590944089793L, + 7286294700691822468L, + 7828126881500292486L, + 8437899353709338096L, + 136052254470293480L, + 1113259077339995086L, + -8244887265606191121L, + 8089569503800461649L, + -1429698194850157567L, + 1575595674002364989L, + 3576095286627428675L, + -7653655285807569222L, + -6053506977362539111L, + -3923855345805787169L, + -8001149080454232377L, + -4382867706931832271L, + 4212860258835896297L, + 4207674254247034014L, + 5519424058779519159L, + -754483042161434654L, + 1434113479814210082L, + -6416645032698336896L, + 5624329676066514819L, + -8229557208322175959L, + 3922640911653270376L, + 7826932478782081910L, + -4862787164488635842L, + 1449234668827944573L, + -1781657689570106327L, + 5442827552725289699L, + 3589862161007644641L, + 4787115581650652778L, + -3512152721942525726L, + -6750103117958685206L, + 5012970446659949261L, + 6797752795961689017L, + 5086454597639943700L, + -7616068364979994076L, + 1492846825433110217L, + 2967476304433704510L, + -8413824338284112078L, + -1319049442043273974L, + -1756090916806844109L, + -9061091728950139525L, + -6864767830358160810L, + 4879532090226251157L, + 5528644708740739488L + }; +} diff --git a/src/test/java/org/apache/datasketches/hash/XxHash64Test.java b/src/test/java/org/apache/datasketches/hash/XxHash64Test.java new file mode 100644 index 000000000..4b9c31ab9 --- /dev/null +++ b/src/test/java/org/apache/datasketches/hash/XxHash64Test.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.hash; + +import static org.apache.datasketches.hash.XxHash.hashByteArr; +import static org.apache.datasketches.hash.XxHash.hashCharArr; +import static org.apache.datasketches.hash.XxHash.hashDoubleArr; +import static org.apache.datasketches.hash.XxHash.hashFloatArr; +import static org.apache.datasketches.hash.XxHash.hashIntArr; +import static org.apache.datasketches.hash.XxHash.hashLong; +import static org.apache.datasketches.hash.XxHash.hashLongArr; +import static org.apache.datasketches.hash.XxHash.hashShortArr; +import static org.apache.datasketches.hash.XxHash.hashString; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; + +/** + * @author Lee Rhodes + */ +public class XxHash64Test { + + @Test + public void offsetChecks() { + long seed = 12345; + int blocks = 6; + int cap = blocks * 16; + + long hash; + + WritableMemory wmem = WritableMemory.allocate(cap); + for (int i = 0; i < cap; i++) { wmem.putByte(i, (byte)(-128 + i)); } + + for (int offset = 0; offset < 16; offset++) { + int arrLen = cap - offset; + hash = wmem.xxHash64(offset, arrLen, seed); + assertTrue(hash != 0); + } + } + + @Test + public void byteArrChecks() { + long seed = 0; + int offset = 0; + int bytes = 16; + + for (int j = 1; j < bytes; j++) { + byte[] in = new byte[bytes]; + + WritableMemory wmem = WritableMemory.writableWrap(in); + for (int i = 0; i < j; i++) { wmem.putByte(i, (byte) (-128 + i)); } + + long hash =wmem.xxHash64(offset, bytes, seed); + assertTrue(hash != 0); + } + } + + /* + * This test is adapted from + * + * OpenHFT/Zero-Allocation-Hashing to test hash compatibility with that implementation. + * It is licensed under Apache License, version 2.0. See LICENSE. + */ + @Test + public void collisionTest() { + WritableMemory wmem = WritableMemory.allocate(128); + wmem.putLong(0, 1); + wmem.putLong(16, 42); + wmem.putLong(32, 2); + long h1 = wmem.xxHash64(0, wmem.getCapacity(), 0); + + wmem.putLong(0, 1L + 0xBA79078168D4BAFL); + wmem.putLong(32, 2L + 0x9C90005B80000000L); + long h2 = wmem.xxHash64(0, wmem.getCapacity(), 0); + assertEquals(h1, h2); + + wmem.putLong(0, 1L + (0xBA79078168D4BAFL * 2)); + wmem.putLong(32, 2L + (0x392000b700000000L)); //= (0x9C90005B80000000L * 2) fix overflow false pos + + long h3 = wmem.xxHash64(0, wmem.getCapacity(), 0); + assertEquals(h2, h3); + } + +// This test had to be disabled because the net.openhft.hashing.LongHashFunction is obsolete and depends on sun.misc.unsafe. +// /** +// * This simple test compares the output of {@link Resource#xxHash64(long, long, long)} with the +// * output of {@link net.openhft.hashing.LongHashFunction}, that itself is tested against the +// * reference implementation in C. This increases confidence that the xxHash function implemented +// * in this package is in fact the same xxHash function implemented in C. +// * +// * @author Roman Leventov +// * @author Lee Rhodes +// */ +// @Test +// public void testXxHash() { +// Random random = ThreadLocalRandom.current(); +// for (int len = 0; len < 100; len++) { +// byte[] bytes = new byte[len]; +// for (int i = 0; i < 10; i++) { +// long zahXxHash = LongHashFunction.xx().hashBytes(bytes); +// long memoryXxHash = Memory.wrap(bytes).xxHash64(0, len, 0); +// assertEquals(memoryXxHash, zahXxHash); +// random.nextBytes(bytes); +// } +// } +// } + + private static final byte[] barr = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + + @Test + public void testArrHashes() { + WritableMemory wmem = WritableMemory.writableWrap(barr); + long hash0 = wmem.xxHash64(8, 8, 0); + long hash1 = hashByteArr(barr, 8, 8, 0); + assertEquals(hash1, hash0); + + char[] carr = new char[8]; + wmem.getCharArray(0, carr, 0, 8); + hash1 = hashCharArr(carr, 4, 4, 0); + assertEquals(hash1, hash0); + + short[] sarr = new short[8]; + wmem.getShortArray(0, sarr, 0, 8); + hash1 = hashShortArr(sarr, 4, 4, 0); + assertEquals(hash1, hash0); + + int[] iarr = new int[4]; + wmem.getIntArray(0, iarr, 0, 4); + hash1 = hashIntArr(iarr, 2, 2, 0); + assertEquals(hash1, hash0); + + float[] farr = new float[4]; + wmem.getFloatArray(0, farr, 0, 4); + hash1 = hashFloatArr(farr, 2, 2, 0); + assertEquals(hash1, hash0); + + long[] larr = new long[2]; + wmem.getLongArray(0, larr, 0, 2); + hash1 = hashLongArr(larr, 1, 1, 0); + long in = wmem.getLong(8); + long hash2 = hashLong(in, 00); //tests the single long hash + assertEquals(hash1, hash0); + assertEquals(hash2, hash0); + + double[] darr = new double[2]; + wmem.getDoubleArray(0, darr, 0, 2); + hash1 = hashDoubleArr(darr, 1, 1, 0); + assertEquals(hash1, hash0); + } + + @Test + public void testString() { + String s = "Now is the time for all good men to come to the aid of their country."; + char[] arr = s.toCharArray(); + long hash0 = hashString(s, 0, s.length(), 0); + long hash1 = hashCharArr(arr, 0, arr.length, 0); + assertEquals(hash1, hash0); + } + +} diff --git a/src/test/java/org/apache/datasketches/hash/XxHashTest.java b/src/test/java/org/apache/datasketches/hash/XxHashTest.java deleted file mode 100644 index 4ad433b2d..000000000 --- a/src/test/java/org/apache/datasketches/hash/XxHashTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.hash; - -import static org.testng.Assert.assertEquals; - -import org.testng.annotations.Test; - -import org.apache.datasketches.memory.Memory; - -/** - * @author Lee Rhodes - */ -public class XxHashTest { - - @Test - public void longCheck() { - long seed = 0; - long hash1 = XxHash.hash(123L, seed); - long[] arr = new long[1]; - arr[0] = 123L; - Memory mem = Memory.wrap(arr); - long hash2 = XxHash.hash(mem, 0, 8, 0); - assertEquals(hash2, hash1); - } - -} diff --git a/src/test/java/org/apache/datasketches/theta/AnotBimplTest.java b/src/test/java/org/apache/datasketches/theta/AnotBimplTest.java index 722a96e83..e400dd1fa 100644 --- a/src/test/java/org/apache/datasketches/theta/AnotBimplTest.java +++ b/src/test/java/org/apache/datasketches/theta/AnotBimplTest.java @@ -24,10 +24,17 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; +import org.apache.datasketches.theta.AnotB; +import org.apache.datasketches.theta.AnotBimpl; +import org.apache.datasketches.theta.CompactSketch; +import org.apache.datasketches.theta.SetOperation; +import org.apache.datasketches.theta.Sketch; +import org.apache.datasketches.theta.Sketches; +import org.apache.datasketches.theta.UpdateSketch; import org.testng.annotations.Test; /** @@ -53,7 +60,7 @@ public void checkExactAnotB_AvalidNoOverlap() { assertTrue(aNb.isEmpty()); //only applies to stateful assertTrue(aNb.getCache().length == 0); //only applies to stateful assertEquals(aNb.getThetaLong(), Long.MAX_VALUE); //only applies to stateful - assertEquals(aNb.getSeedHash(), ThetaUtil.computeSeedHash(ThetaUtil.DEFAULT_UPDATE_SEED)); + assertEquals(aNb.getSeedHash(), Util.computeSeedHash(Util.DEFAULT_UPDATE_SEED)); aNb.setA(usk1); aNb.notB(usk2); @@ -70,16 +77,16 @@ public void checkExactAnotB_AvalidNoOverlap() { assertEquals(rsk1.getEstimate(), k/2.0); final int bytes = rsk1.getCurrentBytes(); - final WritableMemory wmem = WritableMemory.allocate(bytes); + final MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); aNb.setA(usk1); aNb.notB(usk2); - rsk1 = aNb.getResult(false, wmem, true); //unordered, reset + rsk1 = aNb.getResult(false, wseg, true); //unordered, reset assertEquals(rsk1.getEstimate(), k/2.0); aNb.setA(usk1); aNb.notB(usk2); - rsk1 = aNb.getResult(true, wmem, true); //ordered, reset + rsk1 = aNb.getResult(true, wseg, true); //ordered, reset assertEquals(rsk1.getEstimate(), k/2.0); } @@ -214,12 +221,12 @@ public void checkAnotBnotC() { final UpdateSketch cU = UpdateSketch.builder().setNominalEntries(k).build(); for (int i=k/2; i<3*k/4; i++) { cU.update(i); } //third 256 - final int memBytes = Sketch.getMaxUpdateSketchBytes(k); + final int segBytes = Sketch.getMaxUpdateSketchBytes(k); CompactSketch result1, result2, result3; - final WritableMemory wmem1 = WritableMemory.allocate(memBytes); - final WritableMemory wmem2 = WritableMemory.allocate(memBytes); - final WritableMemory wmem3 = WritableMemory.allocate(memBytes); + final MemorySegment wseg1 = MemorySegment.ofArray(new byte[segBytes]); + final MemorySegment wseg2 = MemorySegment.ofArray(new byte[segBytes]); + final MemorySegment wseg3 = MemorySegment.ofArray(new byte[segBytes]); final AnotB aNb = SetOperation.builder().buildANotB(); @@ -227,11 +234,11 @@ public void checkAnotBnotC() { aNb.setA(aU); //stateful - result1 = aNb.aNotB(aU, bU, ordered, wmem1); //stateless + result1 = aNb.aNotB(aU, bU, ordered, wseg1); //stateless aNb.notB(bU); //stateful - result2 = aNb.aNotB(result1, cU, ordered, wmem2); //stateless + result2 = aNb.aNotB(result1, cU, ordered, wseg2); //stateless aNb.notB(cU); //stateful @@ -239,13 +246,13 @@ public void checkAnotBnotC() { println("est: "+est2); assertEquals(est2, k/4.0, 0.0); - result3 = aNb.getResult(ordered, wmem3, true); //stateful result, then reset + result3 = aNb.getResult(ordered, wseg3, true); //stateful result, then reset final double est3 = result3.getEstimate(); assertEquals(est3, k/4.0, 0.0); } @Test - public void checkAnotBnotC_sameMemory() { + public void checkAnotBnotC_sameMemorySegment() { final int k = 1024; final boolean ordered = true; @@ -258,8 +265,8 @@ public void checkAnotBnotC_sameMemory() { final UpdateSketch c = UpdateSketch.builder().setNominalEntries(k).build(); for (int i=k/2; i<3*k/4; i++) { c.update(i); } //third 256 - final int memBytes = Sketch.getMaxCompactSketchBytes(a.getRetainedEntries(true)); - final WritableMemory mem = WritableMemory.allocate(memBytes); + final int segBytes = Sketch.getMaxCompactSketchBytes(a.getRetainedEntries(true)); + final MemorySegment seg = MemorySegment.ofArray(new byte[segBytes]); CompactSketch result1, result2; final AnotB aNb = SetOperation.builder().buildANotB(); @@ -268,15 +275,15 @@ public void checkAnotBnotC_sameMemory() { aNb.setA(a); //stateful - result1 = aNb.aNotB(a, b, ordered, mem); //stateless + result1 = aNb.aNotB(a, b, ordered, seg); //stateless aNb.notB(b); //stateful - result1 = aNb.aNotB(result1, c, ordered, mem); //stateless + result1 = aNb.aNotB(result1, c, ordered, seg); //stateless aNb.notB(c); //stateful - result2 = aNb.getResult(ordered, mem, true); //stateful result, then reset + result2 = aNb.getResult(ordered, seg, true); //stateful result, then reset final double est1 = result1.getEstimate(); //check stateless result println("est: "+est1); @@ -308,7 +315,7 @@ public void checkGetResult() { @Test public void checkGetFamily() { //cheap trick - final AnotBimpl anotb = new AnotBimpl(ThetaUtil.DEFAULT_UPDATE_SEED); + final AnotBimpl anotb = new AnotBimpl(Util.DEFAULT_UPDATE_SEED); assertEquals(anotb.getFamily(), Family.A_NOT_B); } diff --git a/src/test/java/org/apache/datasketches/theta/BackwardConversions.java b/src/test/java/org/apache/datasketches/theta/BackwardConversions.java index 18cd6e865..a0688cbba 100644 --- a/src/test/java/org/apache/datasketches/theta/BackwardConversions.java +++ b/src/test/java/org/apache/datasketches/theta/BackwardConversions.java @@ -19,10 +19,17 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; +import static java.lang.foreign.ValueLayout.JAVA_SHORT_UNALIGNED; + +import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; +import org.apache.datasketches.theta.CompactSketch; +import org.apache.datasketches.theta.EmptyCompactSketch; +import org.apache.datasketches.theta.SingleItemSketch; /** * This class converts current compact sketches into prior SerVer 1 and SerVer 2 format for testing. @@ -32,7 +39,7 @@ public class BackwardConversions { /** - * Converts a SerVer3 ordered, heap CompactSketch to a SerVer1 ordered, SetSketch in Memory. + * Converts a SerVer3 ordered, heap CompactSketch to a SerVer1 ordered, SetSketch in MemorySegment. * This is exclusively for testing purposes. * *

      V1 dates from roughly Aug 2014 to about May 2015. @@ -75,36 +82,37 @@ public class BackwardConversions { *

    * * @param skV3 a SerVer3, ordered CompactSketch - * @return a SerVer1 SetSketch as Memory object. + * @return a SerVer1 SetSketch as MemorySegment object. */ - public static Memory convertSerVer3toSerVer1(final CompactSketch skV3) { + public static MemorySegment convertSerVer3toSerVer1(final CompactSketch skV3) { //Check input sketch - final boolean validIn = skV3.isCompact() && skV3.isOrdered() && !skV3.hasMemory(); + final boolean validIn = skV3.isCompact() && skV3.isOrdered() && !skV3.hasMemorySegment(); if (!validIn) { throw new SketchesArgumentException("Invalid input sketch."); } - //Build V1 SetSketch in memory + //Build V1 SetSketch in MemorySegment final int curCount = skV3.getRetainedEntries(true); - final WritableMemory wmem = WritableMemory.allocate((3 + curCount) << 3); + final int bytes = (3 + curCount) << 3; + final MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]);//Util.newHeapSegment(bytes); //Pre0 - wmem.putByte(0, (byte) 3); //preLongs - wmem.putByte(1, (byte) 1); //SerVer - wmem.putByte(2, (byte) 3); //Compact (SetSketch) - wmem.putByte(6, (byte) 2); //Flags ReadOnly, LittleEndian + wseg.set(JAVA_BYTE, 0, (byte) 3); //preLongs + wseg.set(JAVA_BYTE, 1, (byte) 1); //SerVer + wseg.set(JAVA_BYTE, 2, (byte) 3); //Compact (SetSketch) + wseg.set(JAVA_BYTE, 6, (byte) 2); //Flags ReadOnly, LittleEndian //Pre1 - wmem.putInt(8, curCount); + wseg.set(JAVA_INT_UNALIGNED, 8, curCount); //Pre2 - wmem.putLong(16, skV3.getThetaLong()); + wseg.set(JAVA_LONG_UNALIGNED, 16, skV3.getThetaLong()); //Data if (curCount > 0) { - wmem.putLongArray(24, skV3.getCache(), 0, curCount); + MemorySegment.copy(skV3.getCache(), 0, wseg, JAVA_LONG_UNALIGNED, 24, curCount); } - return wmem; + return wseg; } /** - * Converts a SerVer3 ordered, heap CompactSketch to a SerVer2 ordered, SetSketch in Memory. + * Converts a SerVer3 ordered, heap CompactSketch to a SerVer2 ordered, SetSketch in MemorySegment. * This is exclusively for testing purposes. * *

    V2 is short-lived and dates from roughly Mid May 2015 to about June 1st, 2015. @@ -179,54 +187,54 @@ public static Memory convertSerVer3toSerVer1(final CompactSketch skV3) { * * @param skV3 a SerVer3, ordered CompactSketch * @param seed used for checking the seed hash (if one exists). - * @return a SerVer2 SetSketch as Memory object. + * @return a SerVer2 SetSketch as MemorySegment object. */ - public static Memory convertSerVer3toSerVer2(final CompactSketch skV3, final long seed) { - final short seedHash = ThetaUtil.computeSeedHash(seed); - WritableMemory wmem = null; + public static MemorySegment convertSerVer3toSerVer2(final CompactSketch skV3, final long seed) { + final short seedHash = Util.computeSeedHash(seed); + MemorySegment wseg = null; if (skV3 instanceof EmptyCompactSketch) { - wmem = WritableMemory.allocate(8); - wmem.putByte(0, (byte) 1); //preLongs - wmem.putByte(1, (byte) 2); //SerVer - wmem.putByte(2, (byte) 3); //SetSketch + wseg = MemorySegment.ofArray(new long[1]); + wseg.set(JAVA_BYTE, 0, (byte) 1); //preLongs + wseg.set(JAVA_BYTE, 1, (byte) 2); //SerVer + wseg.set(JAVA_BYTE, 2, (byte) 3); //SetSketch final byte flags = (byte) 0xE; //NoRebuild, Empty, ReadOnly, LE - wmem.putByte(5, flags); - wmem.putShort(6, seedHash); - return wmem; + wseg.set(JAVA_BYTE, 5, flags); + wseg.set(JAVA_SHORT_UNALIGNED, 6, seedHash); + return wseg; } if (skV3 instanceof SingleItemSketch) { final SingleItemSketch sis = (SingleItemSketch) skV3; - wmem = WritableMemory.allocate(24); - wmem.putByte(0, (byte) 2); //preLongs - wmem.putByte(1, (byte) 2); //SerVer - wmem.putByte(2, (byte) 3); //SetSketch + wseg = MemorySegment.ofArray(new long[3]); + wseg.set(JAVA_BYTE, 0, (byte) 2); //preLongs + wseg.set(JAVA_BYTE, 1, (byte) 2); //SerVer + wseg.set(JAVA_BYTE, 2, (byte) 3); //SetSketch final byte flags = (byte) 0xA; //NoRebuild, notEmpty, ReadOnly, LE - wmem.putByte(5, flags); - wmem.putShort(6, seedHash); - wmem.putInt(8, 1); + wseg.set(JAVA_BYTE, 5, flags); + wseg.set(JAVA_SHORT_UNALIGNED, 6, seedHash); + wseg.set(JAVA_INT_UNALIGNED, 8, 1); final long[] arr = sis.getCache(); - wmem.putLong(16, arr[0]); - return wmem; + wseg.set(JAVA_LONG_UNALIGNED, 16, arr[0]); + return wseg; } //General CompactSketch final int preLongs = skV3.getCompactPreambleLongs(); final int entries = skV3.getRetainedEntries(true); final boolean unordered = !(skV3.isOrdered()); final byte flags = (byte) (0xA | (unordered ? 16 : 0)); //Unordered, NoRebuild, notEmpty, ReadOnly, LE - wmem = WritableMemory.allocate((preLongs + entries) << 3); - wmem.putByte(0, (byte) preLongs); //preLongs - wmem.putByte(1, (byte) 2); //SerVer - wmem.putByte(2, (byte) 3); //SetSketch + wseg = MemorySegment.ofArray(new byte[(preLongs + entries) << 3]); + wseg.set(JAVA_BYTE, 0, (byte) preLongs); //preLongs + wseg.set(JAVA_BYTE, 1, (byte) 2); //SerVer + wseg.set(JAVA_BYTE, 2, (byte) 3); //SetSketch - wmem.putByte(5, flags); - wmem.putShort(6, seedHash); - wmem.putInt(8, entries); + wseg.set(JAVA_BYTE, 5, flags); + wseg.set(JAVA_SHORT_UNALIGNED, 6, seedHash); + wseg.set(JAVA_INT_UNALIGNED, 8, entries); if (preLongs == 3) { - wmem.putLong(16, skV3.getThetaLong()); + wseg.set(JAVA_LONG_UNALIGNED, 16, skV3.getThetaLong()); } final long[] arr = skV3.getCache(); - wmem.putLongArray(preLongs * 8L, arr, 0, entries); - return wmem; + MemorySegment.copy(arr, 0, wseg, JAVA_LONG_UNALIGNED, preLongs << 3, entries); + return wseg; } } diff --git a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java index c155bef77..35de3e342 100644 --- a/src/test/java/org/apache/datasketches/theta/BitPackingTest.java +++ b/src/test/java/org/apache/datasketches/theta/BitPackingTest.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertEquals; import org.apache.datasketches.common.Util; +import org.apache.datasketches.theta.BitPacking; import org.testng.annotations.Test; public class BitPackingTest { diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java index 906c1914b..8541ed6bc 100644 --- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java @@ -26,10 +26,21 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.theta.CompactSketch; +import org.apache.datasketches.theta.DirectCompactSketch; +import org.apache.datasketches.theta.EmptyCompactSketch; +import org.apache.datasketches.theta.HashIterator; +import org.apache.datasketches.theta.HeapCompactSketch; +import org.apache.datasketches.theta.Intersection; +import org.apache.datasketches.theta.SingleItemSketch; +import org.apache.datasketches.theta.Sketch; +import org.apache.datasketches.theta.Sketches; +import org.apache.datasketches.theta.UpdateSketch; +import org.apache.datasketches.theta.WrappedCompactCompressedSketch; +import org.apache.datasketches.theta.WrappedCompactSketch; import org.testng.annotations.Test; import java.lang.foreign.Arena; @@ -60,36 +71,36 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) { usk.update(i); } - /****ON HEAP MEMORY -- HEAPIFY****/ + /****ON HEAP MemorySegment -- HEAPIFY****/ CompactSketch refSk = usk.compact(ordered, null); byte[] barr = refSk.toByteArray(); - Memory srcMem = Memory.wrap(barr); - CompactSketch testSk = (CompactSketch) Sketch.heapify(srcMem); + MemorySegment srcSeg = MemorySegment.ofArray(barr); + CompactSketch testSk = (CompactSketch) Sketch.heapify(srcSeg); checkByRange(refSk, testSk, u, ordered); /**Via byte[]**/ byte[] byteArray = refSk.toByteArray(); - Memory heapROMem = Memory.wrap(byteArray); - testSk = (CompactSketch)Sketch.heapify(heapROMem); + MemorySegment heapROSeg = MemorySegment.ofArray(byteArray).asReadOnly(); + testSk = (CompactSketch)Sketch.heapify(heapROSeg); checkByRange(refSk, testSk, u, ordered); - /****OFF HEAP MEMORY -- WRAP****/ - //Prepare Memory for direct + /****OFF HEAP MemorySegment -- WRAP****/ + //Prepare MemorySegment for direct int bytes = usk.getCompactBytes(); //for Compact try (Arena arena = Arena.ofConfined()) { - WritableMemory directMem = WritableMemory.allocateDirect(bytes, arena); + MemorySegment directSeg = arena.allocate(bytes); /**Via CompactSketch.compact**/ - refSk = usk.compact(ordered, directMem); - testSk = (CompactSketch)Sketch.wrap(directMem); + refSk = usk.compact(ordered, directSeg); + testSk = (CompactSketch)Sketch.wrap(directSeg); checkByRange(refSk, testSk, u, ordered); /**Via CompactSketch.compact**/ - testSk = (CompactSketch)Sketch.wrap(directMem); + testSk = (CompactSketch)Sketch.wrap(directSeg); checkByRange(refSk, testSk, u, ordered); } catch (final Exception e) { throw new RuntimeException(e); @@ -111,9 +122,9 @@ private static void checkEmptySketch(Sketch testSk) { assertTrue(testSk instanceof EmptyCompactSketch); assertTrue(testSk.isEmpty()); assertTrue(testSk.isOrdered()); - assertNull(testSk.getMemory()); + assertNull(testSk.getMemorySegment()); assertFalse(testSk.isDirect()); - assertFalse(testSk.hasMemory()); + assertFalse(testSk.hasMemorySegment()); assertEquals(testSk.getSeedHash(), 0); assertEquals(testSk.getRetainedEntries(true), 0); assertEquals(testSk.getEstimate(), 0.0, 0.0); @@ -129,9 +140,9 @@ private static void checkSingleItemSketch(Sketch testSk, Sketch refSk) { assertTrue(testSk instanceof SingleItemSketch); assertFalse(testSk.isEmpty()); assertTrue(testSk.isOrdered()); - assertNull(testSk.getMemory()); + assertNull(testSk.getMemorySegment()); assertFalse(testSk.isDirect()); - assertFalse(testSk.hasMemory()); + assertFalse(testSk.hasMemorySegment()); assertEquals(testSk.getSeedHash(), refSk.getSeedHash()); assertEquals(testSk.getRetainedEntries(true), 1); assertEquals(testSk.getEstimate(), 1.0, 0.0); @@ -147,9 +158,9 @@ private static void checkOtherCompactSketch(Sketch testSk, Sketch refSk, boolean assertFalse(testSk.isEmpty()); assertNotNull(testSk.iterator()); assertEquals(testSk.isOrdered(), ordered); - if (refSk.hasMemory()) { - assertTrue(testSk.hasMemory()); - assertNotNull(testSk.getMemory()); + if (refSk.hasMemorySegment()) { + assertTrue(testSk.hasMemorySegment()); + assertNotNull(testSk.getMemorySegment()); if (ordered) { assertTrue(testSk.isOrdered()); } else { @@ -161,7 +172,7 @@ private static void checkOtherCompactSketch(Sketch testSk, Sketch refSk, boolean assertFalse(testSk.isDirect()); } } else { - assertFalse(testSk.hasMemory()); + assertFalse(testSk.hasMemorySegment()); assertTrue(testSk instanceof HeapCompactSketch); } assertEquals(testSk.getSeedHash(), refSk.getSeedHash()); @@ -178,14 +189,14 @@ public void checkDirectSingleItemSketch() { UpdateSketch sk = Sketches.updateSketchBuilder().build(); sk.update(1); int bytes = sk.getCompactBytes(); - WritableMemory wmem = WritableMemory.allocate(bytes); - sk.compact(true, wmem); - Sketch csk2 = Sketch.heapify(wmem); + MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + sk.compact(true, wseg); + Sketch csk2 = Sketch.heapify(wseg); assertTrue(csk2 instanceof SingleItemSketch); } @Test(expectedExceptions = SketchesArgumentException.class) - public void checkMemTooSmall() { + public void checkSegTooSmall() { int k = 512; int u = k; boolean ordered = false; @@ -196,12 +207,12 @@ public void checkMemTooSmall() { int bytes = usk.getCompactBytes(); byte[] byteArray = new byte[bytes -8]; //too small - WritableMemory mem = WritableMemory.writableWrap(byteArray); - usk.compact(ordered, mem); + MemorySegment seg = MemorySegment.ofArray(byteArray); + usk.compact(ordered, seg); } @Test(expectedExceptions = SketchesArgumentException.class) - public void checkMemTooSmallOrdered() { + public void checkSegTooSmallOrdered() { int k = 512; int u = k; boolean ordered = true; @@ -212,8 +223,8 @@ public void checkMemTooSmallOrdered() { int bytes = usk.getCompactBytes(); byte[] byteArray = new byte[bytes -8]; //too small - WritableMemory mem = WritableMemory.writableWrap(byteArray); - usk.compact(ordered, mem); + MemorySegment seg = MemorySegment.ofArray(byteArray); + usk.compact(ordered, seg); } @Test @@ -230,38 +241,38 @@ public void checkCompactCachePart() { private static final boolean COMPACT = true; private static final boolean EMPTY = true; private static final boolean DIRECT = true; - private static final boolean MEMORY = true; + private static final boolean SEGMENT = true; private static final boolean ORDERED = true; private static final boolean ESTIMATION = true; @Test /** - * Empty, memory-based Compact sketches are always ordered + * Empty, segment-based Compact sketches are always ordered */ - public void checkEmptyMemoryCompactSketch() { + public void checkEmptyMemorySegmentCompactSketch() { UpdateSketch sk = Sketches.updateSketchBuilder().build(); - WritableMemory wmem1 = WritableMemory.allocate(16); - CompactSketch csk1 = sk.compact(false, wmem1); //the first parameter is ignored when empty - State state1 = new State("DirectCompactSketch", 0, 8, COMPACT, EMPTY, !DIRECT, MEMORY, ORDERED, !ESTIMATION); + MemorySegment wseg1 = MemorySegment.ofArray(new byte[16]); + CompactSketch csk1 = sk.compact(false, wseg1); //the first parameter is ignored when empty + State state1 = new State("DirectCompactSketch", 0, 8, COMPACT, EMPTY, !DIRECT, SEGMENT, ORDERED, !ESTIMATION); state1.check(csk1); - WritableMemory wmem2 = WritableMemory.allocate(16); - CompactSketch csk2 = sk.compact(false, wmem2); + MemorySegment wseg2 = MemorySegment.ofArray(new byte[16]); + CompactSketch csk2 = sk.compact(false, wseg2); state1.check(csk2); - assertNotEquals(csk1, csk2); //different object because memory is valid + assertNotEquals(csk1, csk2); //different object because MemorySegment is valid assertFalse(csk1 == csk2); - WritableMemory wmem3 = WritableMemory.allocate(16); - CompactSketch csk3 = csk1.compact(false, wmem3); + MemorySegment wseg3 = MemorySegment.ofArray(new byte[16]); + CompactSketch csk3 = csk1.compact(false, wseg3); state1.check(csk3); - assertNotEquals(csk1, csk3); //different object because memory is valid + assertNotEquals(csk1, csk3); //different object because MemorySegment is valid assertFalse(csk1 == csk3); CompactSketch csk4 = csk1.compact(false, null); - State state4 = new State("EmptyCompactSketch", 0, 8, COMPACT, EMPTY, !DIRECT, !MEMORY, ORDERED, !ESTIMATION); + State state4 = new State("EmptyCompactSketch", 0, 8, COMPACT, EMPTY, !DIRECT, !SEGMENT, ORDERED, !ESTIMATION); state4.check(csk4); assertNotEquals(csk1, csk4); //different object because on heap @@ -276,29 +287,29 @@ public void checkEmptyMemoryCompactSketch() { @Test /** - * Single-Item, memory-based Compact sketches are always ordered: + * Single-Item, segment-based Compact sketches are always ordered: */ - public void checkSingleItemMemoryCompactSketch() { + public void checkSingleItemMemorySegmentCompactSketch() { UpdateSketch sk = Sketches.updateSketchBuilder().build(); sk.update(1); - WritableMemory wmem1 = WritableMemory.allocate(16); - CompactSketch csk1 = sk.compact(false, wmem1); //the first parameter is ignored when single item - State state1 = new State("DirectCompactSketch", 1, 16, COMPACT, !EMPTY, !DIRECT, MEMORY, ORDERED, !ESTIMATION); + MemorySegment wseg1 = MemorySegment.ofArray(new byte[16]); + CompactSketch csk1 = sk.compact(false, wseg1); //the first parameter is ignored when single item + State state1 = new State("DirectCompactSketch", 1, 16, COMPACT, !EMPTY, !DIRECT, SEGMENT, ORDERED, !ESTIMATION); state1.check(csk1); - WritableMemory wmem2 = WritableMemory.allocate(16); - CompactSketch csk2 = sk.compact(false, wmem2); //the first parameter is ignored when single item + MemorySegment wseg2 = MemorySegment.ofArray(new byte[16]); + CompactSketch csk2 = sk.compact(false, wseg2); //the first parameter is ignored when single item state1.check(csk2); - assertNotEquals(csk1, csk2); //different object because memory is valid + assertNotEquals(csk1, csk2); //different object because segment is valid assertFalse(csk1 == csk2); - WritableMemory wmem3 = WritableMemory.allocate(16); - CompactSketch csk3 = csk1.compact(false, wmem3); + MemorySegment wseg3 = MemorySegment.ofArray(new byte[16]); + CompactSketch csk3 = csk1.compact(false, wseg3); state1.check(csk3); - assertNotEquals(csk1, csk3); //different object because memory is valid + assertNotEquals(csk1, csk3); //different object because segment is valid assertFalse(csk1 == csk3); CompactSketch cskc = csk1.compact(); @@ -309,31 +320,31 @@ public void checkSingleItemMemoryCompactSketch() { } @Test - public void checkMultipleItemMemoryCompactSketch() { + public void checkMultipleItemMemorySegmentCompactSketch() { UpdateSketch sk = Sketches.updateSketchBuilder().build(); //This sequence is naturally out-of-order by the hash values. sk.update(1); sk.update(2); sk.update(3); - WritableMemory wmem1 = WritableMemory.allocate(50); - CompactSketch csk1 = sk.compact(true, wmem1); - State state1 = new State("DirectCompactSketch", 3, 40, COMPACT, !EMPTY, !DIRECT, MEMORY, ORDERED, !ESTIMATION); + MemorySegment wseg1 = MemorySegment.ofArray(new byte[50]); + CompactSketch csk1 = sk.compact(true, wseg1); + State state1 = new State("DirectCompactSketch", 3, 40, COMPACT, !EMPTY, !DIRECT, SEGMENT, ORDERED, !ESTIMATION); state1.check(csk1); - WritableMemory wmem2 = WritableMemory.allocate(50); - CompactSketch csk2 = sk.compact(false, wmem2); - State state2 = new State("DirectCompactSketch", 3, 40, COMPACT, !EMPTY, !DIRECT, MEMORY, !ORDERED, !ESTIMATION); + MemorySegment wseg2 = MemorySegment.ofArray(new byte[50]); + CompactSketch csk2 = sk.compact(false, wseg2); + State state2 = new State("DirectCompactSketch", 3, 40, COMPACT, !EMPTY, !DIRECT, SEGMENT, !ORDERED, !ESTIMATION); state2.check(csk2); - assertNotEquals(csk1, csk2); //different object because memory is valid + assertNotEquals(csk1, csk2); //different object because segment is valid assertFalse(csk1 == csk2); - WritableMemory wmem3 = WritableMemory.allocate(50); - CompactSketch csk3 = csk1.compact(false, wmem3); + MemorySegment wseg3 = MemorySegment.ofArray(new byte[50]); + CompactSketch csk3 = csk1.compact(false, wseg3); state2.check(csk3); - assertNotEquals(csk1, csk3); //different object because memory is valid + assertNotEquals(csk1, csk3); //different object because segment is valid assertFalse(csk1 == csk3); CompactSketch cskc = csk1.compact(); @@ -352,7 +363,7 @@ public void checkEmptyHeapCompactSketch() { UpdateSketch sk = Sketches.updateSketchBuilder().build(); CompactSketch csk1 = sk.compact(false, null); //the first parameter is ignored when empty - State state1 = new State("EmptyCompactSketch", 0, 8, COMPACT, EMPTY, !DIRECT, !MEMORY, ORDERED, !ESTIMATION); + State state1 = new State("EmptyCompactSketch", 0, 8, COMPACT, EMPTY, !DIRECT, !SEGMENT, ORDERED, !ESTIMATION); state1.check(csk1); CompactSketch csk2 = sk.compact(false, null); //the first parameter is ignored when empty @@ -383,7 +394,7 @@ public void checkSingleItemHeapCompactSketch() { sk.update(1); CompactSketch csk1 = sk.compact(false, null); //the first parameter is ignored when single item - State state1 = new State("SingleItemSketch", 1, 16, COMPACT, !EMPTY, !DIRECT, !MEMORY, ORDERED, !ESTIMATION); + State state1 = new State("SingleItemSketch", 1, 16, COMPACT, !EMPTY, !DIRECT, !SEGMENT, ORDERED, !ESTIMATION); state1.check(csk1); CompactSketch csk2 = sk.compact(false, null); //the first parameter is ignored when single item @@ -414,11 +425,11 @@ public void checkMultipleItemHeapCompactSketch() { sk.update(3); CompactSketch csk1 = sk.compact(true, null); //creates a new object - State state1 = new State("HeapCompactSketch", 3, 40, COMPACT, !EMPTY, !DIRECT, !MEMORY, ORDERED, !ESTIMATION); + State state1 = new State("HeapCompactSketch", 3, 40, COMPACT, !EMPTY, !DIRECT, !SEGMENT, ORDERED, !ESTIMATION); state1.check(csk1); CompactSketch csk2 = sk.compact(false, null); //creates a new object, unordered - State state2 = new State("HeapCompactSketch", 3, 40, COMPACT, !EMPTY, !DIRECT, !MEMORY, !ORDERED, !ESTIMATION); + State state2 = new State("HeapCompactSketch", 3, 40, COMPACT, !EMPTY, !DIRECT, !SEGMENT, !ORDERED, !ESTIMATION); state2.check(csk2); assertNotEquals(csk1, csk2); //order is different and different objects @@ -427,10 +438,10 @@ public void checkMultipleItemHeapCompactSketch() { CompactSketch csk3 = csk1.compact(true, null); state1.check(csk3); - assertEquals(csk1, csk3); //the same object because wmem = null and csk1.ordered = dstOrdered + assertEquals(csk1, csk3); //the same object because wseg = null and csk1.ordered = dstOrdered assertTrue(csk1 == csk3); - assertNotEquals(csk2, csk3); //different object because wmem = null and csk2.ordered = false && dstOrdered = true + assertNotEquals(csk2, csk3); //different object because wseg = null and csk2.ordered = false && dstOrdered = true assertFalse(csk2 == csk3); CompactSketch cskc = csk1.compact(); @@ -445,19 +456,19 @@ public void checkHeapifySingleItemSketch() { UpdateSketch sk = Sketches.updateSketchBuilder().build(); sk.update(1); int bytes = Sketches.getMaxCompactSketchBytes(2); //1 more than needed - WritableMemory wmem = WritableMemory.allocate(bytes); - sk.compact(false, wmem); - Sketch csk = Sketch.heapify(wmem); + MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + sk.compact(false, wseg); + Sketch csk = Sketch.heapify(wseg); assertTrue(csk instanceof SingleItemSketch); } @Test public void checkHeapifyEmptySketch() { UpdateSketch sk = Sketches.updateSketchBuilder().build(); - WritableMemory wmem = WritableMemory.allocate(16); //empty, but extra bytes - CompactSketch csk = sk.compact(false, wmem); //ignores order because it is empty + MemorySegment wseg = MemorySegment.ofArray(new byte[16]); //empty, but extra bytes + CompactSketch csk = sk.compact(false, wseg); //ignores order because it is empty assertTrue(csk instanceof DirectCompactSketch); - Sketch csk2 = Sketch.heapify(wmem); + Sketch csk2 = Sketch.heapify(wseg); assertTrue(csk2 instanceof EmptyCompactSketch); } @@ -466,7 +477,7 @@ public void checkGetCache() { UpdateSketch sk = Sketches.updateSketchBuilder().setP((float).5).build(); sk.update(7); int bytes = sk.getCompactBytes(); - CompactSketch csk = sk.compact(true, WritableMemory.allocate(bytes)); + CompactSketch csk = sk.compact(true, MemorySegment.ofArray(new byte[bytes])); long[] cache = csk.getCache(); assertTrue(cache.length == 0); } @@ -484,12 +495,12 @@ public void checkHeapCompactSketchCompact() { /** * This is checking the empty, single, exact and estimating cases of an off-heap * sketch to make sure they are being stored properly and to check the new capability - * of calling compact(boolean, Memory) on an already compact sketch. This allows the + * of calling compact(boolean, MemorySegment) on an already compact sketch. This allows the * user to be able to change the order and heap status of an already compact sketch. */ @Test public void checkDirectCompactSketchCompact() { - WritableMemory wmem1, wmem2; + MemorySegment wseg1, wseg2; CompactSketch csk1, csk2; int bytes; int lgK = 6; @@ -497,12 +508,12 @@ public void checkDirectCompactSketchCompact() { //empty UpdateSketch sk = Sketches.updateSketchBuilder().setLogNominalEntries(lgK).build(); bytes = sk.getCompactBytes(); //empty, 8 bytes - wmem1 = WritableMemory.allocate(bytes); - wmem2 = WritableMemory.allocate(bytes); - csk1 = sk.compact(false, wmem1); //place into memory as unordered + wseg1 = MemorySegment.ofArray(new byte[bytes]); + wseg2 = MemorySegment.ofArray(new byte[bytes]); + csk1 = sk.compact(false, wseg1); //place into MemorySegment as unordered assertTrue(csk1 instanceof DirectCompactSketch); assertTrue(csk1.isOrdered()); //empty is always ordered - csk2 = csk1.compact(false, wmem2); //set to unordered again + csk2 = csk1.compact(false, wseg2); //set to unordered again assertTrue(csk2 instanceof DirectCompactSketch); assertTrue(csk2.isOrdered()); //empty is always ordered assertTrue(csk2.getSeedHash() == 0); //empty has no seed hash @@ -511,12 +522,12 @@ public void checkDirectCompactSketchCompact() { //single sk.update(1); bytes = sk.getCompactBytes(); //single, 16 bytes - wmem1 = WritableMemory.allocate(bytes); - wmem2 = WritableMemory.allocate(bytes); - csk1 = sk.compact(false, wmem1); //place into memory as unordered + wseg1 = MemorySegment.ofArray(new byte[bytes]); + wseg2 = MemorySegment.ofArray(new byte[bytes]); + csk1 = sk.compact(false, wseg1); //place into MemorySegment as unordered assertTrue(csk1 instanceof DirectCompactSketch); assertTrue(csk1.isOrdered()); //single is always ordered - csk2 = csk1.compact(false, wmem2); //set to unordered again + csk2 = csk1.compact(false, wseg2); //set to unordered again assertTrue(csk2 instanceof DirectCompactSketch); assertTrue(csk2.isOrdered()); //single is always ordered assertTrue(csk2.getSeedHash() != 0); //has a seed hash @@ -525,12 +536,12 @@ public void checkDirectCompactSketchCompact() { //exact sk.update(2); bytes = sk.getCompactBytes(); //exact, 16 bytes preamble, 16 bytes data - wmem1 = WritableMemory.allocate(bytes); - wmem2 = WritableMemory.allocate(bytes); - csk1 = sk.compact(false, wmem1); //place into memory as unordered + wseg1 = MemorySegment.ofArray(new byte[bytes]); + wseg2 = MemorySegment.ofArray(new byte[bytes]); + csk1 = sk.compact(false, wseg1); //place into MemorySegment as unordered assertTrue(csk1 instanceof DirectCompactSketch); assertFalse(csk1.isOrdered()); //should be unordered - csk2 = csk1.compact(true, wmem2); //set to ordered + csk2 = csk1.compact(true, wseg2); //set to ordered assertTrue(csk2 instanceof DirectCompactSketch); assertTrue(csk2.isOrdered()); //should be ordered assertTrue(csk2.getSeedHash() != 0); //has a seed hash @@ -540,12 +551,12 @@ public void checkDirectCompactSketchCompact() { int n = 1 << (lgK + 1); for (int i = 2; i < n; i++) { sk.update(i); } bytes = sk.getCompactBytes(); //24 bytes preamble + curCount * 8, - wmem1 = WritableMemory.allocate(bytes); - wmem2 = WritableMemory.allocate(bytes); - csk1 = sk.compact(false, wmem1); //place into memory as unordered + wseg1 = MemorySegment.ofArray(new byte[bytes]); + wseg2 = MemorySegment.ofArray(new byte[bytes]); + csk1 = sk.compact(false, wseg1); //place into MemorySegment as unordered assertTrue(csk1 instanceof DirectCompactSketch); assertFalse(csk1.isOrdered()); //should be unordered - csk2 = csk1.compact(true, wmem2); //set to ordered + csk2 = csk1.compact(true, wseg2); //set to ordered assertTrue(csk2 instanceof DirectCompactSketch); assertTrue(csk2.isOrdered()); //should be ordered assertTrue(csk2.getSeedHash() != 0); //has a seed hash @@ -561,7 +572,7 @@ public void serializeDeserializeHeapV4() { } CompactSketch cs1 = sk.compact(); byte[] bytes = cs1.toByteArrayCompressed(); - CompactSketch cs2 = CompactSketch.heapify(Memory.wrap(bytes)); + CompactSketch cs2 = CompactSketch.heapify(MemorySegment.ofArray(bytes)); assertEquals(cs1.getRetainedEntries(), cs2.getRetainedEntries()); HashIterator it1 = cs1.iterator(); HashIterator it2 = cs2.iterator(); @@ -576,9 +587,9 @@ public void serializeDeserializeDirectV4() { for (int i = 0; i < 10000; i++) { sk.update(i); } - CompactSketch cs1 = sk.compact(true, WritableMemory.allocate(sk.getCompactBytes())); + CompactSketch cs1 = sk.compact(true, MemorySegment.ofArray(new byte[sk.getCompactBytes()])); byte[] bytes = cs1.toByteArrayCompressed(); - CompactSketch cs2 = CompactSketch.wrap(Memory.wrap(bytes)); + CompactSketch cs2 = CompactSketch.wrap(MemorySegment.ofArray(bytes)); assertEquals(cs1.getRetainedEntries(), cs2.getRetainedEntries()); HashIterator it1 = cs1.iterator(); HashIterator it2 = cs2.iterator(); @@ -630,19 +641,19 @@ private static class State { boolean compact = false; boolean empty = false; boolean direct = false; - boolean memory = false; + boolean hasSeg = false; boolean ordered = false; boolean estimation = false; State(String classType, int count, int bytes, boolean compact, boolean empty, boolean direct, - boolean memory, boolean ordered, boolean estimation) { + boolean hasSeg, boolean ordered, boolean estimation) { this.classType = classType; this.count = count; this.bytes = bytes; this.compact = compact; this.empty = empty; this.direct = direct; - this.memory = memory; + this.hasSeg = hasSeg; this.ordered = ordered; this.estimation = estimation; } @@ -654,7 +665,7 @@ void check(CompactSketch csk) { assertEquals(csk.isCompact(), compact, "Compact"); assertEquals(csk.isEmpty(), empty, "Empty"); assertEquals(csk.isDirect(), direct, "Direct"); - assertEquals(csk.hasMemory(), memory, "Memory"); + assertEquals(csk.hasMemorySegment(), hasSeg, "MemorySegment"); assertEquals(csk.isOrdered(), ordered, "Ordered"); assertEquals(csk.isEstimationMode(), estimation, "Estimation"); } diff --git a/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java index 6d6af7047..ac08fd9e6 100644 --- a/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketchTest.java @@ -19,6 +19,7 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static org.apache.datasketches.theta.ConcurrentHeapQuickSelectSketchTest.waitForBgPropagationToComplete; import static org.apache.datasketches.theta.PreambleUtil.FAMILY_BYTE; import static org.apache.datasketches.theta.PreambleUtil.LG_NOM_LONGS_BYTE; @@ -27,37 +28,46 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; +import org.apache.datasketches.theta.CompactSketch; +import org.apache.datasketches.theta.ConcurrentDirectQuickSelectSketch; +import org.apache.datasketches.theta.ConcurrentHeapThetaBuffer; +import org.apache.datasketches.theta.ConcurrentSharedThetaSketch; +import org.apache.datasketches.theta.DirectQuickSelectSketch; +import org.apache.datasketches.theta.Sketch; +import org.apache.datasketches.theta.Sketches; +import org.apache.datasketches.theta.UpdateSketch; +import org.apache.datasketches.theta.UpdateSketchBuilder; import org.apache.datasketches.theta.ConcurrentHeapQuickSelectSketchTest.SharedLocal; import org.apache.datasketches.thetacommon.HashOperations; -import org.apache.datasketches.thetacommon.ThetaUtil; import org.testng.annotations.Test; /** * @author eshcar */ public class ConcurrentDirectQuickSelectSketchTest { - private static final long SEED = ThetaUtil.DEFAULT_UPDATE_SEED; + private static final long SEED = Util.DEFAULT_UPDATE_SEED; @Test public void checkDirectCompactConversion() { int lgK = 9; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); assertTrue(sl.shared instanceof ConcurrentDirectQuickSelectSketch); assertTrue(sl.shared.compact().isCompact()); } @Test - public void checkHeapifyMemoryEstimating() { + public void checkHeapifyMemorySegmentEstimating() { int lgK = 9; int k = 1 << lgK; int u = 2*k; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); UpdateSketch shared = sl.shared; //off-heap UpdateSketch local = sl.local; @@ -69,10 +79,10 @@ public void checkHeapifyMemoryEstimating() { assertEquals(local.getClass().getSimpleName(), "ConcurrentHeapThetaBuffer"); //This sharedHeap is not linked to the concurrent local buffer - UpdateSketch sharedHeap = Sketches.heapifyUpdateSketch(sl.wmem); + UpdateSketch sharedHeap = Sketches.heapifyUpdateSketch(sl.wseg); assertEquals(sharedHeap.getClass().getSimpleName(), "HeapQuickSelectSketch"); - checkMemoryDirectProxyMethods(local, shared); + checkMemorySegmentDirectProxyMethods(local, shared); checkOtherProxyMethods(local, shared); checkOtherProxyMethods(local, sharedHeap); @@ -90,8 +100,8 @@ public void checkHeapifyMemoryEstimating() { public void checkHeapifyByteArrayExact() { int lgK = 9; int k = 1 << lgK; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); UpdateSketch shared = sl.shared; UpdateSketch local = sl.local; @@ -99,13 +109,13 @@ public void checkHeapifyByteArrayExact() { waitForBgPropagationToComplete(shared); byte[] serArr = shared.toByteArray(); - Memory srcMem = Memory.wrap(serArr); - Sketch recoveredShared = Sketch.heapify(srcMem); + MemorySegment srcSeg = MemorySegment.ofArray(serArr).asReadOnly(); + Sketch recoveredShared = Sketch.heapify(srcSeg); //reconstruct to Native/Direct final int bytes = Sketch.getMaxUpdateSketchBytes(k); - final WritableMemory wmem = WritableMemory.allocate(bytes); - shared = sl.bldr.buildSharedFromSketch((UpdateSketch)recoveredShared, wmem); + final MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + shared = sl.bldr.buildSharedFromSketch((UpdateSketch)recoveredShared, wseg); UpdateSketch local2 = sl.bldr.buildLocal(shared); assertEquals(local2.getEstimate(), k, 0.0); @@ -127,8 +137,8 @@ public void checkHeapifyByteArrayEstimating() { int k = 1 << lgK; int u = 2*k; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); UpdateSketch shared = sl.shared; UpdateSketch local = sl.local; @@ -141,13 +151,13 @@ public void checkHeapifyByteArrayEstimating() { assertEquals(local.isEstimationMode(), true); byte[] serArr = shared.toByteArray(); - Memory srcMem = Memory.wrap(serArr); - Sketch recoveredShared = Sketch.heapify(srcMem); + MemorySegment srcSeg = MemorySegment.ofArray(serArr).asReadOnly(); + Sketch recoveredShared = Sketch.heapify(srcSeg); //reconstruct to Native/Direct final int bytes = Sketch.getMaxUpdateSketchBytes(k); - final WritableMemory wmem = WritableMemory.allocate(bytes); - shared = sl.bldr.buildSharedFromSketch((UpdateSketch)recoveredShared, wmem); + final MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + shared = sl.bldr.buildSharedFromSketch((UpdateSketch)recoveredShared, wseg); UpdateSketch local2 = sl.bldr.buildLocal(shared); assertEquals(local2.getEstimate(), uskEst); @@ -159,14 +169,14 @@ public void checkHeapifyByteArrayEstimating() { } @Test - public void checkWrapMemoryEst() { + public void checkWrapMemorySegmentEst() { int lgK = 9; int k = 1 << lgK; int u = 2*k; //boolean estimating = (u > k); - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); UpdateSketch shared = sl.shared; UpdateSketch local = sl.local; @@ -178,7 +188,7 @@ public void checkWrapMemoryEst() { double sk1ub = local.getUpperBound(2); assertTrue(local.isEstimationMode()); - Sketch local2 = Sketch.wrap(sl.wmem); + Sketch local2 = Sketch.wrap(sl.wseg); assertEquals(local2.getEstimate(), sk1est); assertEquals(local2.getLowerBound(2), sk1lb); @@ -193,14 +203,14 @@ public void checkDQStoCompactForms() { int k = 1 << lgK; int u = 4*k; //boolean estimating = (u > k); - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); UpdateSketch shared = sl.shared; UpdateSketch local = sl.local; assertEquals(local.getClass().getSimpleName(), "ConcurrentHeapThetaBuffer"); assertFalse(local.isDirect()); - assertTrue(local.hasMemory()); + assertTrue(local.hasMemorySegment()); for (int i=0; i lgNomLongs +1. @Test - public void checkResizeInBigMem() { + public void checkResizeInBigSeg() { int lgK = 14; int u = 1 << 20; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, SEED, useMem, true, 8); //mem is 8X larger than needed + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, SEED, useSeg, true, 8); //seg is 8X larger than needed UpdateSketch local = sl.local; for (int i = 0; i < u; i++) { local.update(i); } @@ -536,37 +546,37 @@ public void checkResizeInBigMem() { @Test(expectedExceptions = SketchesArgumentException.class) public void checkConstructorKtooSmall() { int lgK = 3; - boolean useMem = true; - new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + new SharedLocal(lgK, lgK, useSeg); } @Test(expectedExceptions = SketchesArgumentException.class) - public void checkConstructorMemTooSmall() { + public void checkConstructorSegTooSmall() { int lgK = 4; int k = 1 << lgK; - WritableMemory wmem = WritableMemory.allocate(k/2); + MemorySegment wseg = MemorySegment.ofArray(new byte[k/2]); UpdateSketchBuilder bldr = new UpdateSketchBuilder(); bldr.setLogNominalEntries(lgK); - bldr.buildShared(wmem); + bldr.buildShared(wseg); } @Test(expectedExceptions = SketchesArgumentException.class) public void checkHeapifyIllegalFamilyID_heapify() { int lgK = 9; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); - sl.wmem.putByte(FAMILY_BYTE, (byte) 0); //corrupt the Family ID byte - //try to heapify the corrupted mem - Sketch.heapify(sl.wmem); //catch in Sketch.constructHeapSketch + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); + sl.wseg.set(JAVA_BYTE, FAMILY_BYTE, (byte) 0); //corrupt the Family ID byte + //try to heapify the corrupted seg + Sketch.heapify(sl.wseg); //catch in Sketch.constructHeapSketch } @Test(expectedExceptions = SketchesArgumentException.class) public void checkBadLgNomLongs() { int lgK = 4; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); - sl.wmem.putByte(LG_NOM_LONGS_BYTE, (byte) 3); //Corrupt LgNomLongs byte - DirectQuickSelectSketch.writableWrap(sl.wmem, ThetaUtil.DEFAULT_UPDATE_SEED); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); + sl.wseg.set(JAVA_BYTE, LG_NOM_LONGS_BYTE, (byte) 3); //Corrupt LgNomLongs byte + DirectQuickSelectSketch.writableWrap(sl.wseg, Util.DEFAULT_UPDATE_SEED); } @Test @@ -574,8 +584,8 @@ public void checkBackgroundPropagation() { int lgK = 4; int k = 1 << lgK; int u = 10*k; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); UpdateSketch shared = sl.shared; UpdateSketch local = sl.local; assertTrue(local.isEmpty()); @@ -612,8 +622,8 @@ public void checkBackgroundPropagation() { public void checkBadSerVer() { int lgK = 9; int k = 1 << lgK; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); UpdateSketch shared = sl.shared; UpdateSketch local = sl.local; assertTrue(local.isEmpty()); @@ -625,60 +635,60 @@ public void checkBadSerVer() { assertEquals(local.getEstimate(), k, 0.0); assertEquals(shared.getRetainedEntries(false), k); - sl.wmem.putByte(SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte - Sketch.wrap(sl.wmem); + sl.wseg.set(JAVA_BYTE, SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte + Sketch.wrap(sl.wseg); } @Test(expectedExceptions = SketchesArgumentException.class) public void checkWrapIllegalFamilyID_wrap() { int lgK = 9; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); - sl.wmem.putByte(FAMILY_BYTE, (byte) 0); //corrupt the Sketch ID byte - //try to wrap the corrupted mem - Sketch.wrap(sl.wmem); //catch in Sketch.constructDirectSketch + sl.wseg.set(JAVA_BYTE, FAMILY_BYTE, (byte) 0); //corrupt the Sketch ID byte + //try to wrap the corrupted seg + Sketch.wrap(sl.wseg); //catch in Sketch.constructDirectSketch } @Test(expectedExceptions = SketchesArgumentException.class) public void checkWrapIllegalFamilyID_direct() { int lgK = 9; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); - sl.wmem.putByte(FAMILY_BYTE, (byte) 0); //corrupt the Sketch ID byte - //try to wrap the corrupted mem - DirectQuickSelectSketch.writableWrap(sl.wmem, ThetaUtil.DEFAULT_UPDATE_SEED); + sl.wseg.set(JAVA_BYTE, FAMILY_BYTE, (byte) 0); //corrupt the Sketch ID byte + //try to wrap the corrupted seg + DirectQuickSelectSketch.writableWrap(sl.wseg, Util.DEFAULT_UPDATE_SEED); } @Test(expectedExceptions = SketchesArgumentException.class) public void checkHeapifySeedConflict() { int lgK = 9; long seed1 = 1021; - long seed2 = ThetaUtil.DEFAULT_UPDATE_SEED; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, seed1, useMem, true, 1); + long seed2 = Util.DEFAULT_UPDATE_SEED; + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, seed1, useSeg, true, 1); UpdateSketch shared = sl.shared; - Memory srcMem = Memory.wrap(shared.toByteArray()); - Sketch.heapify(srcMem, seed2); + MemorySegment srcSeg = MemorySegment.ofArray(shared.toByteArray()).asReadOnly(); + Sketch.heapify(srcSeg, seed2); } @Test(expectedExceptions = SketchesArgumentException.class) public void checkCorruptLgNomLongs() { int lgK = 4; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); - sl.wmem.putByte(LG_NOM_LONGS_BYTE, (byte)2); //corrupt - Sketch.heapify(sl.wmem, ThetaUtil.DEFAULT_UPDATE_SEED); + sl.wseg.set(JAVA_BYTE, LG_NOM_LONGS_BYTE, (byte)2); //corrupt + Sketch.heapify(sl.wseg, Util.DEFAULT_UPDATE_SEED); } @Test(expectedExceptions = UnsupportedOperationException.class) public void checkIllegalHashUpdate() { int lgK = 4; - boolean useMem = true; - SharedLocal sl = new SharedLocal(lgK, lgK, useMem); + boolean useSeg = true; + SharedLocal sl = new SharedLocal(lgK, lgK, useSeg); UpdateSketch shared = sl.shared; shared.hashUpdate(1); } @@ -695,12 +705,14 @@ static void println(String s) { //System.out.println(s); //disable here } - private static void checkMemoryDirectProxyMethods(Sketch local, Sketch shared) { - assertEquals(local.hasMemory(), shared.hasMemory()); + private static void checkMemorySegmentDirectProxyMethods(Sketch local, Sketch shared) { + assertEquals( + local.hasMemorySegment(), + shared.hasMemorySegment()); assertEquals(local.isDirect(), shared.isDirect()); } - //Does not check hasMemory(), isDirect() + //Does not check hasMemorySegment(), isDirect() private static void checkOtherProxyMethods(Sketch local, Sketch shared) { assertEquals(local.getCompactBytes(), shared.getCompactBytes()); assertEquals(local.getCurrentBytes(), shared.getCurrentBytes()); diff --git a/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java index 84ddcb80e..e8c517afd 100644 --- a/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketchTest.java @@ -19,6 +19,7 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static org.apache.datasketches.theta.PreambleUtil.FAMILY_BYTE; import static org.apache.datasketches.theta.PreambleUtil.LG_NOM_LONGS_BYTE; import static org.apache.datasketches.theta.PreambleUtil.SER_VER_BYTE; @@ -27,13 +28,23 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; import java.util.Arrays; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; +import org.apache.datasketches.theta.CompactSketch; +import org.apache.datasketches.theta.ConcurrentHeapQuickSelectSketch; +import org.apache.datasketches.theta.ConcurrentHeapThetaBuffer; +import org.apache.datasketches.theta.ConcurrentPropagationService; +import org.apache.datasketches.theta.ConcurrentSharedThetaSketch; +import org.apache.datasketches.theta.HeapQuickSelectSketch; +import org.apache.datasketches.theta.PreambleUtil; +import org.apache.datasketches.theta.Sketch; +import org.apache.datasketches.theta.Sketches; +import org.apache.datasketches.theta.UpdateSketch; +import org.apache.datasketches.theta.UpdateSketchBuilder; import org.testng.annotations.Test; /** @@ -63,12 +74,12 @@ public void checkBadSerVer() { assertEquals(shared.getRetainedEntries(false), u); byte[] serArr = shared.toByteArray(); - WritableMemory mem = WritableMemory.writableWrap(serArr); - Sketch sk = Sketch.heapify(mem, sl.seed); + MemorySegment seg = MemorySegment.ofArray(serArr); + Sketch sk = Sketch.heapify(seg, sl.seed); assertTrue(sk instanceof HeapQuickSelectSketch); //Intentional promotion to Parent - mem.putByte(SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte - Sketch.heapify(mem, sl.seed); + seg.set(JAVA_BYTE, SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte + Sketch.heapify(seg, sl.seed); } @Test @@ -111,22 +122,22 @@ public void checkIllegalSketchID_UpdateSketch() { assertEquals(local.getEstimate(), u, 0.0); assertEquals(shared.getRetainedEntries(false), u); byte[] byteArray = shared.toByteArray(); - WritableMemory mem = WritableMemory.writableWrap(byteArray); - mem.putByte(FAMILY_BYTE, (byte) 0); //corrupt the Sketch ID byte + MemorySegment seg = MemorySegment.ofArray(byteArray); + seg.set(JAVA_BYTE, FAMILY_BYTE, (byte) 0); //corrupt the Sketch ID byte - //try to heapify the corrupted mem - Sketch.heapify(mem, sl.seed); + //try to heapify the corrupted seg + Sketch.heapify(seg, sl.seed); } @Test(expectedExceptions = SketchesArgumentException.class) public void checkHeapifySeedConflict() { int lgK = 9; long seed = 1021; - long seed2 = ThetaUtil.DEFAULT_UPDATE_SEED; + long seed2 = Util.DEFAULT_UPDATE_SEED; SharedLocal sl = new SharedLocal(lgK, lgK, seed); byte[] byteArray = sl.shared.toByteArray(); - Memory srcMem = Memory.wrap(byteArray); - Sketch.heapify(srcMem, seed2); + MemorySegment srcSeg = MemorySegment.ofArray(byteArray); + Sketch.heapify(srcSeg, seed2); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -134,9 +145,9 @@ public void checkHeapifyCorruptLgNomLongs() { int lgK = 4; SharedLocal sl = new SharedLocal(lgK); byte[] serArr = sl.shared.toByteArray(); - WritableMemory srcMem = WritableMemory.writableWrap(serArr); - srcMem.putByte(LG_NOM_LONGS_BYTE, (byte)2); //corrupt - Sketch.heapify(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + MemorySegment srcSeg = MemorySegment.ofArray(serArr); + srcSeg.set(JAVA_BYTE, LG_NOM_LONGS_BYTE, (byte)2); //corrupt + Sketch.heapify(srcSeg, Util.DEFAULT_UPDATE_SEED); } @Test(expectedExceptions = UnsupportedOperationException.class) @@ -161,13 +172,13 @@ public void checkHeapifyByteArrayExact() { waitForBgPropagationToComplete(shared); byte[] serArr = shared.toByteArray(); - Memory srcMem = Memory.wrap(serArr); - Sketch recoveredShared = Sketches.heapifyUpdateSketch(srcMem); + MemorySegment srcSeg = MemorySegment.ofArray(serArr).asReadOnly(); + Sketch recoveredShared = Sketches.heapifyUpdateSketch(srcSeg); //reconstruct to Native/Direct final int bytes = Sketch.getMaxUpdateSketchBytes(k); - final WritableMemory wmem = WritableMemory.allocate(bytes); - shared = sl.bldr.buildSharedFromSketch((UpdateSketch)recoveredShared, wmem); + final MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + shared = sl.bldr.buildSharedFromSketch((UpdateSketch)recoveredShared, wseg); UpdateSketch local2 = sl.bldr.buildLocal(shared); assertEquals(local2.getEstimate(), u, 0.0); @@ -200,12 +211,12 @@ public void checkHeapifyByteArrayEstimating() { assertTrue(local.isEstimationMode()); byte[] serArr = shared.toByteArray(); - Memory srcMem = Memory.wrap(serArr); - UpdateSketch recoveredShared = UpdateSketch.heapify(srcMem, sl.seed); + MemorySegment srcSeg = MemorySegment.ofArray(serArr).asReadOnly(); + UpdateSketch recoveredShared = UpdateSketch.heapify(srcSeg, sl.seed); final int bytes = Sketch.getMaxUpdateSketchBytes(k); - final WritableMemory wmem = WritableMemory.allocate(bytes); - shared = sl.bldr.buildSharedFromSketch(recoveredShared, wmem); + final MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + shared = sl.bldr.buildSharedFromSketch(recoveredShared, wseg); UpdateSketch local2 = sl.bldr.buildLocal(shared); assertEquals(local2.getEstimate(), localEst); assertEquals(local2.getLowerBound(2), localLB); @@ -216,7 +227,7 @@ public void checkHeapifyByteArrayEstimating() { } @Test - public void checkHeapifyMemoryEstimating() { + public void checkHeapifyMemorySegmentEstimating() { int lgK = 9; int k = 1 << lgK; int u = 2*k; //thus estimating @@ -235,16 +246,16 @@ public void checkHeapifyMemoryEstimating() { double localUB = local.getUpperBound(2); assertTrue(local.isEstimationMode()); assertFalse(local.isDirect()); - assertFalse(local.hasMemory()); + assertFalse(local.hasMemorySegment()); byte[] serArr = shared.toByteArray(); - Memory srcMem = Memory.wrap(serArr); - UpdateSketch recoveredShared = UpdateSketch.heapify(srcMem, ThetaUtil.DEFAULT_UPDATE_SEED); + MemorySegment srcSeg = MemorySegment.ofArray(serArr).asReadOnly(); + UpdateSketch recoveredShared = UpdateSketch.heapify(srcSeg, Util.DEFAULT_UPDATE_SEED); final int bytes = Sketch.getMaxUpdateSketchBytes(k); - final WritableMemory wmem = WritableMemory.allocate(bytes); - shared = sl.bldr.buildSharedFromSketch(recoveredShared, wmem); + final MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + shared = sl.bldr.buildSharedFromSketch(recoveredShared, wseg); UpdateSketch local2 = sl.bldr.buildLocal(shared); assertEquals(local2.getEstimate(), localEst); @@ -268,7 +279,7 @@ public void checkHQStoCompactForms() { assertEquals(local.getClass().getSimpleName(), "ConcurrentHeapThetaBuffer"); assertFalse(local.isDirect()); - assertFalse(local.hasMemory()); + assertFalse(local.hasMemorySegment()); for (int i=0; i k); println("Est: "+est); - final byte[] memArr3 = inter2.toByteArray(); - final WritableMemory srcMem2 = WritableMemory.writableWrap(memArr3); - inter3 = Sketches.wrapIntersection(srcMem2); + final byte[] segArr3 = inter2.toByteArray(); + final MemorySegment srcSeg2 = MemorySegment.ofArray(segArr3); + inter3 = Sketches.wrapIntersection(srcSeg2); resultComp2 = inter3.getResult(false, null); est2 = resultComp2.getEstimate(); println("Est2: "+est2); @@ -660,15 +669,15 @@ public void checkWrap() { @Test public void checkDefaultMinSize() { final int k = 32; - final WritableMemory mem = WritableMemory.writableWrap(new byte[k*8 + PREBYTES]); - IntersectionImpl.initNewDirectInstance(ThetaUtil.DEFAULT_UPDATE_SEED, mem); + final MemorySegment seg = MemorySegment.ofArray(new byte[k*8 + PREBYTES]); + IntersectionImpl.initNewDirectInstance(Util.DEFAULT_UPDATE_SEED, seg); } @Test(expectedExceptions = SketchesArgumentException.class) public void checkExceptionMinSize() { final int k = 16; - final WritableMemory mem = WritableMemory.writableWrap(new byte[k*8 + PREBYTES]); - IntersectionImpl.initNewDirectInstance(ThetaUtil.DEFAULT_UPDATE_SEED, mem); + final MemorySegment seg = MemorySegment.ofArray(new byte[k*8 + PREBYTES]); + IntersectionImpl.initNewDirectInstance(Util.DEFAULT_UPDATE_SEED, seg); } @Test @@ -676,11 +685,11 @@ public void checkGetResult() { final int k = 1024; final UpdateSketch sk = Sketches.updateSketchBuilder().build(); - final int memBytes = getMaxIntersectionBytes(k); - final byte[] memArr = new byte[memBytes]; - final WritableMemory iMem = WritableMemory.writableWrap(memArr); + final int segBytes = getMaxIntersectionBytes(k); + final byte[] segArr = new byte[segBytes]; + final MemorySegment iSeg = MemorySegment.ofArray(segArr); - final Intersection inter = Sketches.setOperationBuilder().buildIntersection(iMem); + final Intersection inter = Sketches.setOperationBuilder().buildIntersection(iSeg); inter.intersect(sk); final CompactSketch csk = inter.getResult(); assertEquals(csk.getCompactBytes(), 8); @@ -690,31 +699,31 @@ public void checkGetResult() { public void checkFamily() { //cheap trick final int k = 16; - final WritableMemory mem = WritableMemory.writableWrap(new byte[k*16 + PREBYTES]); - final IntersectionImpl impl = IntersectionImpl.initNewDirectInstance(ThetaUtil.DEFAULT_UPDATE_SEED, mem); + final MemorySegment seg = MemorySegment.ofArray(new byte[k*16 + PREBYTES]); + final IntersectionImpl impl = IntersectionImpl.initNewDirectInstance(Util.DEFAULT_UPDATE_SEED, seg); assertEquals(impl.getFamily(), Family.INTERSECTION); } @Test(expectedExceptions = SketchesArgumentException.class) public void checkExceptions1() { final int k = 16; - final WritableMemory mem = WritableMemory.writableWrap(new byte[k*16 + PREBYTES]); - IntersectionImpl.initNewDirectInstance(ThetaUtil.DEFAULT_UPDATE_SEED, mem); + final MemorySegment seg = MemorySegment.ofArray(new byte[k*16 + PREBYTES]); + IntersectionImpl.initNewDirectInstance(Util.DEFAULT_UPDATE_SEED, seg); //corrupt SerVer - mem.putByte(PreambleUtil.SER_VER_BYTE, (byte) 2); - IntersectionImpl.wrapInstance(mem, ThetaUtil.DEFAULT_UPDATE_SEED, false); + seg.set(JAVA_BYTE, PreambleUtil.SER_VER_BYTE, (byte) 2); + IntersectionImpl.wrapInstance(seg, Util.DEFAULT_UPDATE_SEED, false); } @Test(expectedExceptions = SketchesArgumentException.class) public void checkExceptions2() { final int k = 16; - final WritableMemory mem = WritableMemory.writableWrap(new byte[k*16 + PREBYTES]); - IntersectionImpl.initNewDirectInstance(ThetaUtil.DEFAULT_UPDATE_SEED, mem); - //mem now has non-empty intersection + final MemorySegment seg = MemorySegment.ofArray(new byte[k*16 + PREBYTES]); + IntersectionImpl.initNewDirectInstance(Util.DEFAULT_UPDATE_SEED, seg); + //seg now has non-empty intersection //corrupt empty and CurCount - mem.setBits(PreambleUtil.FLAGS_BYTE, (byte) PreambleUtil.EMPTY_FLAG_MASK); - mem.putInt(PreambleUtil.RETAINED_ENTRIES_INT, 2); - IntersectionImpl.wrapInstance(mem, ThetaUtil.DEFAULT_UPDATE_SEED, false); + Util.setBits(seg, PreambleUtil.FLAGS_BYTE, (byte) PreambleUtil.EMPTY_FLAG_MASK); + seg.set(JAVA_INT_UNALIGNED, PreambleUtil.RETAINED_ENTRIES_INT, 2); + IntersectionImpl.wrapInstance(seg, Util.DEFAULT_UPDATE_SEED, false); } //Check Alex's bug intersecting 2 direct full sketches with only overlap of 2 @@ -722,26 +731,26 @@ public void checkExceptions2() { @Test public void checkOverlappedDirect() { final int k = 1 << 4; - final int memBytes = 2*k*16 +PREBYTES; //plenty of room + final int segBytes = 2*k*16 +PREBYTES; //plenty of room final UpdateSketch sk1 = Sketches.updateSketchBuilder().setNominalEntries(k).build(); final UpdateSketch sk2 = Sketches.updateSketchBuilder().setNominalEntries(k).build(); for (int i=0; i>> 1); //corrupt theta and - mem1.putByte(LG_ARR_LONGS_BYTE, (byte) 10); //corrupt lgArrLongs + seg1.set(JAVA_LONG_UNALIGNED, THETA_LONG, Long.MAX_VALUE >>> 1); //corrupt theta and + seg1.set(JAVA_BYTE, LG_ARR_LONGS_BYTE, (byte) 10); //corrupt lgArrLongs try { - usk2 = DirectQuickSelectSketch.writableWrap(mem1, ThetaUtil.DEFAULT_UPDATE_SEED); + usk2 = DirectQuickSelectSketch.writableWrap(seg1, Util.DEFAULT_UPDATE_SEED); fail("Expected SketchesArgumentException"); } catch (SketchesArgumentException e) { //pass } - mem1.putLong(THETA_LONG, Long.MAX_VALUE); //fix theta and - mem1.putByte(LG_ARR_LONGS_BYTE, (byte) 11); //fix lgArrLongs + seg1.set(JAVA_LONG_UNALIGNED, THETA_LONG, Long.MAX_VALUE); //fix theta and + seg1.set(JAVA_BYTE, LG_ARR_LONGS_BYTE, (byte) 11); //fix lgArrLongs byte badFlags = (byte) (BIG_ENDIAN_FLAG_MASK | COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK | ORDERED_FLAG_MASK); - mem1.putByte(FLAGS_BYTE, badFlags); + seg1.set(JAVA_BYTE, FLAGS_BYTE, badFlags); try { - usk2 = DirectQuickSelectSketch.writableWrap(mem1, ThetaUtil.DEFAULT_UPDATE_SEED); + usk2 = DirectQuickSelectSketch.writableWrap(seg1, Util.DEFAULT_UPDATE_SEED); fail("Expected SketchesArgumentException"); } catch (SketchesArgumentException e) { //pass } byte[] arr2 = Arrays.copyOfRange(arr1, 0, bytes-1); //corrupt length - WritableMemory mem2 = WritableMemory.writableWrap(arr2); + MemorySegment seg2 = MemorySegment.ofArray(arr2); try { - usk2 = DirectQuickSelectSketch.writableWrap(mem2, ThetaUtil.DEFAULT_UPDATE_SEED); + usk2 = DirectQuickSelectSketch.writableWrap(seg2, Util.DEFAULT_UPDATE_SEED); fail("Expected SketchesArgumentException"); } catch (SketchesArgumentException e) { //pass @@ -826,42 +832,42 @@ public void checkCorruptRFWithInsufficientArray() { int bytes = Sketches.getMaxUpdateSketchBytes(k); byte[] arr = new byte[bytes]; - WritableMemory mem = WritableMemory.writableWrap(arr); + MemorySegment seg = MemorySegment.ofArray(arr); ResizeFactor rf = ResizeFactor.X8; // 3 - UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).setResizeFactor(rf).build(mem); + UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).setResizeFactor(rf).build(seg); usk.update(0); - insertLgResizeFactor(mem, 0); // corrupt RF: X1 - UpdateSketch dqss = DirectQuickSelectSketch.writableWrap(mem, ThetaUtil.DEFAULT_UPDATE_SEED); + insertLgResizeFactor(seg, 0); // corrupt RF: X1 + UpdateSketch dqss = DirectQuickSelectSketch.writableWrap(seg, Util.DEFAULT_UPDATE_SEED); assertEquals(dqss.getResizeFactor(), ResizeFactor.X2); // force-promote to X2 } @Test public void checkFamilyAndRF() { int k = 16; - WritableMemory mem = WritableMemory.writableWrap(new byte[(k*16) +24]); - UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(mem); + MemorySegment seg = MemorySegment.ofArray(new byte[(k*16) + 24]); + UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(seg); assertEquals(sketch.getFamily(), Family.QUICKSELECT); assertEquals(sketch.getResizeFactor(), ResizeFactor.X8); } //checks Alex's bug where lgArrLongs > lgNomLongs +1. @Test - public void checkResizeInBigMem() { + public void checkResizeInBigSeg() { int k = 1 << 14; int u = 1 << 20; - WritableMemory mem = WritableMemory.writableWrap(new byte[(8*k*16) +24]); - UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(mem); + MemorySegment seg = MemorySegment.ofArray(new byte[(8*k*16) +24]); + UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(seg); for (int i=0; i k); println(quick1.toString()); - println(PreambleUtil.preambleToString(mem)); + println(PreambleUtil.preambleToString(seg)); - final WritableMemory uMem = WritableMemory.writableWrap(new byte[getMaxUnionBytes(k)]); - final Union union = SetOperation.builder().setNominalEntries(k).buildUnion(uMem); + final MemorySegment uSeg = MemorySegment.ofArray(new byte[getMaxUnionBytes(k)]); + final Union union = SetOperation.builder().setNominalEntries(k).buildUnion(uSeg); union.union(quick1); - println(PreambleUtil.preambleToString(uMem)); + println(PreambleUtil.preambleToString(uSeg)); } @Test @@ -120,7 +126,7 @@ public void checkPreambleToStringExceptions() { byteArr = new byte[8]; byteArr[0] = (byte) 2; //needs min capacity of 16 try { //check preLongs == 2 fails - Sketch.toString(Memory.wrap(byteArr)); + Sketch.toString(MemorySegment.ofArray(byteArr).asReadOnly()); fail("Did not throw SketchesArgumentException."); } catch (final SketchesArgumentException e) { //expected @@ -130,7 +136,7 @@ public void checkPreambleToStringExceptions() { @Test(expectedExceptions = SketchesArgumentException.class) public void checkBadSeedHashFromSeed() { //In the first 64K values 50541 produces a seedHash of 0, - ThetaUtil.computeSeedHash(50541); + Util.computeSeedHash(50541); } @Test @@ -150,70 +156,70 @@ public void checkPreLongs() { } comp = sketch.compact(false, null); byteArr = comp.toByteArray(); - println(Sketch.toString(Memory.wrap(byteArr))); //PreLongs = 3 + println(Sketch.toString(MemorySegment.ofArray(byteArr).asReadOnly())); //PreLongs = 3 } @Test public void checkInsertsAndExtracts() { final byte[] arr = new byte[32]; - final WritableMemory wmem = WritableMemory.writableWrap(arr); + final MemorySegment wseg = MemorySegment.ofArray(arr); int v = 0; - insertPreLongs(wmem, ++v); - assertEquals(extractPreLongs(wmem), v); - insertPreLongs(wmem, 0); + insertPreLongs(wseg, ++v); + assertEquals(extractPreLongs(wseg), v); + insertPreLongs(wseg, 0); - insertLgResizeFactor(wmem, 3); //limited to 2 bits - assertEquals(extractLgResizeFactor(wmem), 3); - insertLgResizeFactor(wmem, 0); + insertLgResizeFactor(wseg, 3); //limited to 2 bits + assertEquals(extractLgResizeFactor(wseg), 3); + insertLgResizeFactor(wseg, 0); - insertSerVer(wmem, ++v); - assertEquals(extractSerVer(wmem), v); - insertSerVer(wmem, 0); + insertSerVer(wseg, ++v); + assertEquals(extractSerVer(wseg), v); + insertSerVer(wseg, 0); - insertFamilyID(wmem, ++v); - assertEquals(extractFamilyID(wmem), v); - insertFamilyID(wmem, 0); + insertFamilyID(wseg, ++v); + assertEquals(extractFamilyID(wseg), v); + insertFamilyID(wseg, 0); - insertLgNomLongs(wmem, ++v); - assertEquals(extractLgNomLongs(wmem), v); - insertLgNomLongs(wmem, 0); + insertLgNomLongs(wseg, ++v); + assertEquals(extractLgNomLongs(wseg), v); + insertLgNomLongs(wseg, 0); - insertLgArrLongs(wmem, ++v); - assertEquals(extractLgArrLongs(wmem), v); - insertLgArrLongs(wmem, 0); + insertLgArrLongs(wseg, ++v); + assertEquals(extractLgArrLongs(wseg), v); + insertLgArrLongs(wseg, 0); - insertFlags(wmem, 3); - assertEquals(extractFlags(wmem), 3); - assertEquals(extractLgResizeRatioV1(wmem), 3); //also at byte 5, limited to 2 bits - insertFlags(wmem, 0); + insertFlags(wseg, 3); + assertEquals(extractFlags(wseg), 3); + assertEquals(extractLgResizeRatioV1(wseg), 3); //also at byte 5, limited to 2 bits + insertFlags(wseg, 0); - insertSeedHash(wmem, ++v); - assertEquals(extractSeedHash(wmem), v); - assertEquals(extractFlagsV1(wmem), v); //also at byte 6 - insertSeedHash(wmem, 0); + insertSeedHash(wseg, ++v); + assertEquals(extractSeedHash(wseg), v); + assertEquals(extractFlagsV1(wseg), v); //also at byte 6 + insertSeedHash(wseg, 0); - insertCurCount(wmem, ++v); - assertEquals(extractCurCount(wmem), v); - insertCurCount(wmem, 0); + insertCurCount(wseg, ++v); + assertEquals(extractCurCount(wseg), v); + insertCurCount(wseg, 0); - insertP(wmem, (float) 1.0); - assertEquals(extractP(wmem), (float) 1.0); - insertP(wmem, (float) 0.0); + insertP(wseg, (float) 1.0); + assertEquals(extractP(wseg), (float) 1.0); + insertP(wseg, (float) 0.0); - insertThetaLong(wmem, ++v); - assertEquals(extractThetaLong(wmem), v); - insertThetaLong(wmem, 0L); + insertThetaLong(wseg, ++v); + assertEquals(extractThetaLong(wseg), v); + insertThetaLong(wseg, 0L); - insertUnionThetaLong(wmem, ++v); - assertEquals(extractUnionThetaLong(wmem), v); - insertUnionThetaLong(wmem, 0L); + insertUnionThetaLong(wseg, ++v); + assertEquals(extractUnionThetaLong(wseg), v); + insertUnionThetaLong(wseg, 0L); - setEmpty(wmem); - assertTrue(isEmptyFlag(wmem)); + setEmpty(wseg); + assertTrue(isEmptyFlag(wseg)); - clearEmpty(wmem); - assertFalse(isEmptyFlag(wmem)); + clearEmpty(wseg); + assertFalse(isEmptyFlag(wseg)); } @Test diff --git a/src/test/java/org/apache/datasketches/theta/ReadOnlyMemoryTest.java b/src/test/java/org/apache/datasketches/theta/ReadOnlyMemorySegmentTest.java similarity index 61% rename from src/test/java/org/apache/datasketches/theta/ReadOnlyMemoryTest.java rename to src/test/java/org/apache/datasketches/theta/ReadOnlyMemorySegmentTest.java index b77f3d660..cc9672a4e 100644 --- a/src/test/java/org/apache/datasketches/theta/ReadOnlyMemoryTest.java +++ b/src/test/java/org/apache/datasketches/theta/ReadOnlyMemorySegmentTest.java @@ -20,26 +20,34 @@ package org.apache.datasketches.theta; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.apache.datasketches.common.SketchesReadOnlyException; -import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.theta.Intersection; +import org.apache.datasketches.theta.SetOperation; +import org.apache.datasketches.theta.Sketch; +import org.apache.datasketches.theta.Sketches; +import org.apache.datasketches.theta.Union; +import org.apache.datasketches.theta.UpdateSketch; import org.testng.Assert; import org.testng.annotations.Test; -public class ReadOnlyMemoryTest { +public class ReadOnlyMemorySegmentTest { @Test public void wrapAndTryUpdatingUpdateSketch() { UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - Memory mem = Memory.wrap(ByteBuffer.wrap(updateSketch.toByteArray()) - .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - UpdateSketch sketch = (UpdateSketch) Sketch.wrap(mem); + MemorySegment seg = MemorySegment.ofBuffer( + ByteBuffer.wrap(updateSketch.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); + UpdateSketch sketch = (UpdateSketch) Sketch.wrap(seg); assertEquals(sketch.getEstimate(), 1.0); + assertTrue(seg.isReadOnly()); boolean thrown = false; try { @@ -54,76 +62,83 @@ public void wrapAndTryUpdatingUpdateSketch() { public void wrapCompactUnorderedSketch() { UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - Memory mem = Memory.wrap(ByteBuffer.wrap(updateSketch.compact(false, null).toByteArray()) - .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Sketch sketch = Sketch.wrap(mem); + MemorySegment seg = MemorySegment.ofBuffer( + ByteBuffer.wrap(updateSketch.compact(false, null).toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); + Sketch sketch = Sketch.wrap(seg); assertEquals(sketch.getEstimate(), 1.0); + assertTrue(seg.isReadOnly()); } @Test public void wrapCompactOrderedSketch() { UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - Memory mem = Memory.wrap(ByteBuffer.wrap(updateSketch.compact().toByteArray()) + MemorySegment seg = MemorySegment.ofBuffer(ByteBuffer.wrap(updateSketch.compact().toByteArray()) .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Sketch sketch = Sketch.wrap(mem); + Sketch sketch = Sketch.wrap(seg); assertEquals(sketch.getEstimate(), 1.0); + assertTrue(seg.isReadOnly()); } @Test public void heapifyUpdateSketch() { UpdateSketch us1 = UpdateSketch.builder().build(); us1.update(1); - Memory mem = Memory.wrap(ByteBuffer.wrap(us1.toByteArray()) - .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); + MemorySegment seg = MemorySegment.ofBuffer( + ByteBuffer.wrap(us1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); // downcasting is not recommended, for testing only - UpdateSketch us2 = (UpdateSketch) Sketch.heapify(mem); + UpdateSketch us2 = (UpdateSketch) Sketch.heapify(seg); us2.update(2); assertEquals(us2.getEstimate(), 2.0); + assertTrue(seg.isReadOnly()); } @Test public void heapifyCompactUnorderedSketch() { UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - Memory mem = Memory.wrap(ByteBuffer.wrap(updateSketch.compact(false, null).toByteArray()) - .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Sketch sketch = Sketch.heapify(mem); + MemorySegment seg = MemorySegment.ofBuffer( + ByteBuffer.wrap(updateSketch.compact(false, null).toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); + Sketch sketch = Sketch.heapify(seg); assertEquals(sketch.getEstimate(), 1.0); + assertTrue(seg.isReadOnly()); } @Test public void heapifyCompactOrderedSketch() { UpdateSketch updateSketch = UpdateSketch.builder().build(); updateSketch.update(1); - Memory mem = Memory.wrap(ByteBuffer.wrap(updateSketch.compact().toByteArray()) - .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Sketch sketch = Sketch.heapify(mem); + MemorySegment seg = MemorySegment.ofBuffer( + ByteBuffer.wrap(updateSketch.compact().toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); + Sketch sketch = Sketch.heapify(seg); assertEquals(sketch.getEstimate(), 1.0); + assertTrue(seg.isReadOnly()); } @Test public void heapifyUnion() { Union u1 = SetOperation.builder().buildUnion(); u1.update(1); - Memory mem = Memory.wrap(ByteBuffer.wrap(u1.toByteArray()) - .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Union u2 = (Union) SetOperation.heapify(mem); + MemorySegment seg = MemorySegment.ofBuffer( + ByteBuffer.wrap(u1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); + Union u2 = (Union) SetOperation.heapify(seg); u2.update(2); Assert.assertEquals(u2.getResult().getEstimate(), 2.0); + assertTrue(seg.isReadOnly()); } @Test public void wrapAndTryUpdatingUnion() { Union u1 = SetOperation.builder().buildUnion(); u1.update(1); - Memory mem = Memory.wrap(ByteBuffer.wrap(u1.toByteArray()) - .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); + MemorySegment seg = MemorySegment.ofBuffer( + ByteBuffer.wrap(u1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Union u2 = (Union) Sketches.wrapSetOperation(mem); - Union u3 = Sketches.wrapUnion(mem); + Union u2 = (Union) Sketches.wrapSetOperation(seg); + Union u3 = Sketches.wrapUnion(seg); Assert.assertEquals(u2.getResult().getEstimate(), 1.0); Assert.assertEquals(u3.getResult().getEstimate(), 1.0); + assertTrue(seg.isReadOnly()); try { u2.update(2); @@ -152,11 +167,12 @@ public void heapifyIntersection() { Intersection i1 = SetOperation.builder().buildIntersection(); i1.intersect(us1); i1.intersect(us2); - Memory mem = Memory.wrap(ByteBuffer.wrap(i1.toByteArray()) - .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Intersection i2 = (Intersection) SetOperation.heapify(mem); + MemorySegment seg = MemorySegment.ofBuffer( + ByteBuffer.wrap(i1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); + Intersection i2 = (Intersection) SetOperation.heapify(seg); i2.intersect(us1); Assert.assertEquals(i2.getResult().getEstimate(), 1.0); + assertTrue(seg.isReadOnly()); } @Test @@ -171,9 +187,9 @@ public void wrapIntersection() { Intersection i1 = SetOperation.builder().buildIntersection(); i1.intersect(us1); i1.intersect(us2); - Memory mem = Memory.wrap(ByteBuffer.wrap(i1.toByteArray()) - .asReadOnlyBuffer().order(ByteOrder.nativeOrder())); - Intersection i2 = (Intersection) SetOperation.wrap(mem); + MemorySegment seg = MemorySegment.ofBuffer( + ByteBuffer.wrap(i1.toByteArray()).asReadOnlyBuffer().order(ByteOrder.nativeOrder())); + Intersection i2 = (Intersection) SetOperation.wrap(seg); Assert.assertEquals(i2.getResult().getEstimate(), 1.0); boolean thrown = false; @@ -183,6 +199,7 @@ public void wrapIntersection() { thrown = true; } Assert.assertTrue(thrown); + assertTrue(seg.isReadOnly()); } @Test diff --git a/src/test/java/org/apache/datasketches/theta/SetOperationTest.java b/src/test/java/org/apache/datasketches/theta/SetOperationTest.java index ee4350744..7ba06bb73 100644 --- a/src/test/java/org/apache/datasketches/theta/SetOperationTest.java +++ b/src/test/java/org/apache/datasketches/theta/SetOperationTest.java @@ -23,20 +23,27 @@ import static org.apache.datasketches.theta.Sketch.getMaxUpdateSketchBytes; import static org.apache.datasketches.thetacommon.HashOperations.minLgHashTableSize; import static org.testng.Assert.assertEquals; -//import static org.testng.Assert.assertTrue; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.DefaultMemoryRequestServer; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.MemoryRequestServer; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; +import org.apache.datasketches.theta.AnotB; +import org.apache.datasketches.theta.CompactSketch; +import org.apache.datasketches.theta.Intersection; +import org.apache.datasketches.theta.PreambleUtil; +import org.apache.datasketches.theta.SetOperation; +import org.apache.datasketches.theta.SetOperationBuilder; +import org.apache.datasketches.theta.Sketch; +import org.apache.datasketches.theta.Sketches; +import org.apache.datasketches.theta.Union; +import org.apache.datasketches.theta.UpdateSketch; import org.apache.datasketches.thetacommon.ThetaUtil; import org.testng.annotations.Test; @@ -95,10 +102,6 @@ public void checkBuilder2() { bldr.setNominalEntries(k); assertEquals(bldr.getLgNominalEntries(), lgK); - final MemoryRequestServer mrs = new DefaultMemoryRequestServer(); - bldr.setMemoryRequestServer(mrs); - assertEquals(bldr.getMemoryRequestServer(), mrs); - println(bldr.toString()); } @@ -131,9 +134,9 @@ public void checkBuilderValidP() { } @Test(expectedExceptions = SketchesArgumentException.class) - public void checkBuilderAnotB_noMem() { - final WritableMemory mem = WritableMemory.writableWrap(new byte[64]); - SetOperation.builder().build(Family.A_NOT_B, mem); + public void checkBuilderAnotB_noSeg() { + final MemorySegment seg = MemorySegment.ofArray(new byte[64]); + SetOperation.builder().build(Family.A_NOT_B, seg); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -174,8 +177,8 @@ public void checkIllegalSetOpHeapify() { usk1.update(i); //64 } final byte[] byteArray = usk1.toByteArray(); - final Memory mem = Memory.wrap(byteArray); - SetOperation.heapify(mem); + final MemorySegment seg = MemorySegment.ofArray(byteArray).asReadOnly(); + SetOperation.heapify(seg); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -186,8 +189,8 @@ public void checkIllegalSetOpWrap() { usk1.update(i); //64 } final byte[] byteArray = usk1.toByteArray(); - final Memory mem = Memory.wrap(byteArray); - Sketches.wrapIntersection(mem); + final MemorySegment seg = MemorySegment.ofArray(byteArray).asReadOnly(); + Sketches.wrapIntersection(seg); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -197,10 +200,10 @@ public void checkIllegalSetOpWrap2() { for (int i=0; i k, true); assertEquals(csk.getThetaLong() < Long.MAX_VALUE, true); assertEquals(csk.isDirect(), false); - assertEquals(csk.hasMemory(), false); + assertEquals(csk.hasMemorySegment(), false); assertEquals(csk.isOrdered(), true); csk = generate(State.THLT1_CNT0_FALSE, k); @@ -411,7 +419,7 @@ public void checkGenerator() { assertEquals(csk.getRetainedEntries(true), 0); assertEquals(csk.getThetaLong() < Long.MAX_VALUE, true); assertEquals(csk.isDirect(), false); - assertEquals(csk.hasMemory(), false); + assertEquals(csk.hasMemorySegment(), false); assertEquals(csk.isOrdered(), true); csk = generate(State.THEQ1_CNT0_TRUE, k); @@ -420,20 +428,20 @@ public void checkGenerator() { assertEquals(csk.getRetainedEntries(true), 0); assertEquals(csk.getThetaLong() < Long.MAX_VALUE, false); assertEquals(csk.isDirect(), false); - assertEquals(csk.hasMemory(), false); + assertEquals(csk.hasMemorySegment(), false); assertEquals(csk.isOrdered(), true); - csk = generate(State.EST_MEMORY_UNORDERED, k); + csk = generate(State.EST_SEGMENT_UNORDERED, k); assertEquals(csk.isEmpty(), false); assertEquals(csk.isEstimationMode(), true); assertEquals(csk.getRetainedEntries(true) > k, true); assertEquals(csk.getThetaLong() < Long.MAX_VALUE, true); assertEquals(csk.isDirect(), false); - assertEquals(csk.hasMemory(), true); + assertEquals(csk.hasMemorySegment(), true); assertEquals(csk.isOrdered(), false); } - enum State {NULL, EMPTY, SINGLE, EXACT, EST_HEAP, THLT1_CNT0_FALSE, THEQ1_CNT0_TRUE, EST_MEMORY_UNORDERED} + enum State {NULL, EMPTY, SINGLE, EXACT, EST_HEAP, THLT1_CNT0_FALSE, THEQ1_CNT0_TRUE, EST_SEGMENT_UNORDERED} private static CompactSketch generate(State state, int k) { UpdateSketch sk = null; @@ -483,15 +491,15 @@ private static CompactSketch generate(State state, int k) { csk = sk.compact(true, null); //compact as {Th < 1.0, 0, T} break; } - case EST_MEMORY_UNORDERED : { + case EST_SEGMENT_UNORDERED : { sk = Sketches.updateSketchBuilder().setNominalEntries(k).build(); for (int i = 0; i < (4 * k); i++) { sk.update(i); } int bytes = Sketch.getMaxCompactSketchBytes(sk.getRetainedEntries(true)); byte[] byteArr = new byte[bytes]; - WritableMemory mem = WritableMemory.writableWrap(byteArr); - csk = sk.compact(false, mem); + MemorySegment wseg = MemorySegment.ofArray(byteArr); + csk = sk.compact(false, wseg); break; } } diff --git a/src/test/java/org/apache/datasketches/theta/SingleItemSketchTest.java b/src/test/java/org/apache/datasketches/theta/SingleItemSketchTest.java index 567fb1a56..2dd4df315 100644 --- a/src/test/java/org/apache/datasketches/theta/SingleItemSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/SingleItemSketchTest.java @@ -19,6 +19,8 @@ package org.apache.datasketches.theta; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED; import static org.apache.datasketches.hash.MurmurHash3.hash; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; @@ -26,17 +28,26 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; +import org.apache.datasketches.theta.AnotB; +import org.apache.datasketches.theta.CompactSketch; +import org.apache.datasketches.theta.DirectCompactSketch; +import org.apache.datasketches.theta.Intersection; +import org.apache.datasketches.theta.SingleItemSketch; +import org.apache.datasketches.theta.Sketch; +import org.apache.datasketches.theta.Sketches; +import org.apache.datasketches.theta.Union; +import org.apache.datasketches.theta.UpdateSketch; +import org.apache.datasketches.theta.UpdateSketchBuilder; import org.testng.annotations.Test; /** * @author Lee Rhodes */ public class SingleItemSketchTest { - final static short DEFAULT_SEED_HASH = (short) (ThetaUtil.computeSeedHash(ThetaUtil.DEFAULT_UPDATE_SEED) & 0XFFFFL); + final static short DEFAULT_SEED_HASH = (short) (Util.computeSeedHash(Util.DEFAULT_UPDATE_SEED) & 0XFFFFL); @Test public void check1() { @@ -79,7 +90,7 @@ public void check1() { @Test public void check2() { - long seed = ThetaUtil.DEFAULT_UPDATE_SEED; + long seed = Util.DEFAULT_UPDATE_SEED; Union union = Sketches.setOperationBuilder().buildUnion(); union.union(SingleItemSketch.create(1, seed)); union.union(SingleItemSketch.create(1.0, seed)); @@ -126,7 +137,7 @@ public void checkSketchInterface() { assertEquals(sis.getRetainedEntries(true), 1); assertEquals(sis.getUpperBound(1), 1.0); assertFalse(sis.isDirect()); - assertFalse(sis.hasMemory()); + assertFalse(sis.hasMemorySegment()); assertFalse(sis.isEmpty()); assertTrue(sis.isOrdered()); } @@ -135,7 +146,7 @@ public void checkSketchInterface() { public void checkLessThanThetaLong() { for (int i = 0; i < 10; i++) { long[] data = { i }; - long h = hash(data, ThetaUtil.DEFAULT_UPDATE_SEED)[0] >>> 1; + long h = hash(data, Util.DEFAULT_UPDATE_SEED)[0] >>> 1; SingleItemSketch sis = SingleItemSketch.create(i); long halfMax = Long.MAX_VALUE >> 1; int count = sis.getCountLessThanThetaLong(halfMax); @@ -147,12 +158,12 @@ public void checkLessThanThetaLong() { public void checkSerDe() { SingleItemSketch sis = SingleItemSketch.create(1); byte[] byteArr = sis.toByteArray(); - Memory mem = Memory.wrap(byteArr); - final short defaultSeedHash = ThetaUtil.computeSeedHash(ThetaUtil.DEFAULT_UPDATE_SEED); - SingleItemSketch sis2 = SingleItemSketch.heapify(mem, defaultSeedHash); + MemorySegment seg = MemorySegment.ofArray(byteArr); + final short defaultSeedHash = Util.computeSeedHash(Util.DEFAULT_UPDATE_SEED); + SingleItemSketch sis2 = SingleItemSketch.heapify(seg, defaultSeedHash); assertEquals(sis2.getEstimate(), 1.0); - SingleItemSketch sis3 = SingleItemSketch.heapify(mem, defaultSeedHash); + SingleItemSketch sis3 = SingleItemSketch.heapify(seg , defaultSeedHash); assertEquals(sis3.getEstimate(), 1.0); Union union = Sketches.setOperationBuilder().buildUnion(); @@ -167,7 +178,7 @@ public void checkSerDe() { @Test public void checkRestricted() { SingleItemSketch sis = SingleItemSketch.create(1); - assertNull(sis.getMemory()); + assertNull(sis.getMemorySegment()); assertEquals(sis.getCompactPreambleLongs(), 1); } @@ -175,8 +186,8 @@ public void checkRestricted() { public void unionWrapped() { Sketch sketch = SingleItemSketch.create(1); Union union = Sketches.setOperationBuilder().buildUnion(); - Memory mem = Memory.wrap(sketch.toByteArray()); - union.union(mem); + MemorySegment seg = MemorySegment.ofArray(sketch.toByteArray()); + union.union(seg ); assertEquals(union.getResult().getEstimate(), 1, 0); } @@ -195,8 +206,8 @@ public void buildAndCompact() { //Off-heap bytes = Sketches.getMaxUpdateSketchBytes(32); - WritableMemory wmem = WritableMemory.writableWrap(new byte[bytes]); - sk1= Sketches.updateSketchBuilder().setNominalEntries(32).build(wmem); + MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + sk1= Sketches.updateSketchBuilder().setNominalEntries(32).build(wseg ); sk1.update(1); csk = sk1.compact(true, null); assertTrue(csk instanceof SingleItemSketch); @@ -204,10 +215,10 @@ public void buildAndCompact() { assertTrue(csk instanceof SingleItemSketch); bytes = Sketches.getMaxCompactSketchBytes(1); - wmem = WritableMemory.writableWrap(new byte[bytes]); - csk = sk1.compact(true, wmem); + wseg = MemorySegment.ofArray(new byte[bytes]); + csk = sk1.compact(true, wseg ); assertTrue(csk.isOrdered()); - csk = sk1.compact(false, wmem); + csk = sk1.compact(false, wseg ); assertTrue(csk.isOrdered()); } @@ -230,8 +241,8 @@ public void intersection() { //Intersection off-heap bytes = Sketches.getMaxIntersectionBytes(32); - WritableMemory wmem = WritableMemory.writableWrap(new byte[bytes]); - inter = Sketches.setOperationBuilder().buildIntersection(wmem); + MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + inter = Sketches.setOperationBuilder().buildIntersection(wseg ); inter.intersect(sk1); inter.intersect(sk2); csk = inter.getResult(true, null); @@ -258,8 +269,8 @@ public void union() { //Union off-heap bytes = Sketches.getMaxUnionBytes(32); - WritableMemory wmem = WritableMemory.writableWrap(new byte[bytes]); - union = Sketches.setOperationBuilder().buildUnion(wmem); + MemorySegment wseg = MemorySegment.ofArray(new byte[bytes]); + union = Sketches.setOperationBuilder().buildUnion(wseg ); union.union(sk1); union.union(sk2); csk = union.getResult(true, null); @@ -294,24 +305,24 @@ public void checkHeapifyInstance() { Intersection inter = Sketches.setOperationBuilder().buildIntersection(); inter.intersect(sk1); inter.intersect(sk2); - WritableMemory wmem = WritableMemory.writableWrap(new byte[16]); - CompactSketch csk = inter.getResult(false, wmem); + MemorySegment wseg = MemorySegment.ofArray(new byte[16]); + CompactSketch csk = inter.getResult(false, wseg ); assertTrue(csk.isOrdered()); - Sketch csk2 = Sketches.heapifySketch(wmem); + Sketch csk2 = Sketches.heapifySketch(wseg ); assertTrue(csk2 instanceof SingleItemSketch); println(csk2.toString(true, true, 1, true)); } @Test public void checkSingleItemBadFlags() { - final short defaultSeedHash = ThetaUtil.computeSeedHash(ThetaUtil.DEFAULT_UPDATE_SEED); + final short defaultSeedHash = Util.computeSeedHash(Util.DEFAULT_UPDATE_SEED); UpdateSketch sk1 = new UpdateSketchBuilder().build(); sk1.update(1); - WritableMemory wmem = WritableMemory.allocate(16); - sk1.compact(true, wmem); - wmem.putByte(5, (byte) 0); //corrupt flags to zero + MemorySegment wseg = MemorySegment.ofArray(new byte[16]); + sk1.compact(true, wseg ); + wseg .set(JAVA_BYTE, 5, (byte) 0); //corrupt flags to zero try { - SingleItemSketch.heapify(wmem, defaultSeedHash); //fails due to corrupted flags bytes + SingleItemSketch.heapify(wseg , defaultSeedHash); //fails due to corrupted flags bytes fail(); } catch (SketchesArgumentException e) { } } @@ -334,7 +345,7 @@ public void checkSingleItemCompact() { assertTrue(csk instanceof SingleItemSketch); CompactSketch csk2 = csk.compact(); assertEquals(csk, csk2); - CompactSketch csk3 = csk.compact(true, WritableMemory.allocate(16)); + CompactSketch csk3 = csk.compact(true, MemorySegment.ofArray(new byte[16])); assertTrue(csk3 instanceof DirectCompactSketch); assertEquals(csk2.getCurrentPreambleLongs(), 1); assertEquals(csk3.getCurrentPreambleLongs(), 1); @@ -345,20 +356,20 @@ public void checkSingleItemCompact() { static final long SiSkPre0WoutSiFlag = 0x93cc1a0000030301L; static final long Hash = 0x05a186bdcb7df915L; - static Memory siSkWithSiFlag24Bytes() { + static MemorySegment siSkWithSiFlag24Bytes() { int cap = 24; //8 extra bytes - WritableMemory wmem = WritableMemory.allocate(cap); - wmem.putLong(0, SiSkPre0WithSiFlag); - wmem.putLong(8, Hash); - return wmem; + MemorySegment wseg = MemorySegment.ofArray(new byte[cap]); + wseg .set(JAVA_LONG_UNALIGNED, 0, SiSkPre0WithSiFlag); + wseg .set(JAVA_LONG_UNALIGNED, 8, Hash); + return wseg ; } - static Memory siSkWoutSiFlag24Bytes() { + static MemorySegment siSkWoutSiFlag24Bytes() { int cap = 24; //8 extra bytes - WritableMemory wmem = WritableMemory.allocate(cap); - wmem.putLong(0, SiSkPre0WoutSiFlag); - wmem.putLong(8, Hash); - return wmem; + MemorySegment wseg = MemorySegment.ofArray(new byte[cap]); + wseg .set(JAVA_LONG_UNALIGNED, 0, SiSkPre0WoutSiFlag); + wseg .set(JAVA_LONG_UNALIGNED, 8, Hash); + return wseg; } @Test diff --git a/src/test/java/org/apache/datasketches/theta/SketchTest.java b/src/test/java/org/apache/datasketches/theta/SketchTest.java index 341d8eb20..592453749 100644 --- a/src/test/java/org/apache/datasketches/theta/SketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/SketchTest.java @@ -26,7 +26,7 @@ import static org.apache.datasketches.common.ResizeFactor.X2; import static org.apache.datasketches.common.ResizeFactor.X4; import static org.apache.datasketches.common.ResizeFactor.X8; -import static org.apache.datasketches.common.Util.*; +import static org.apache.datasketches.common.Util.isSameResource; import static org.apache.datasketches.theta.BackwardConversions.convertSerVer3toSerVer1; import static org.apache.datasketches.theta.BackwardConversions.convertSerVer3toSerVer2; import static org.apache.datasketches.theta.CompactOperations.computeCompactPreLongs; @@ -34,16 +34,25 @@ import static org.apache.datasketches.theta.PreambleUtil.FLAGS_BYTE; import static org.apache.datasketches.theta.PreambleUtil.READ_ONLY_FLAG_MASK; import static org.apache.datasketches.theta.Sketch.getMaxCompactSketchBytes; +import static org.apache.datasketches.common.Util.LONG_MAX_VALUE_AS_DOUBLE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.common.Util; +import org.apache.datasketches.theta.CompactSketch; +import org.apache.datasketches.theta.DirectCompactSketch; +import org.apache.datasketches.theta.PreambleUtil; +import org.apache.datasketches.theta.SetOperation; +import org.apache.datasketches.theta.Sketch; +import org.apache.datasketches.theta.Sketches; +import org.apache.datasketches.theta.Union; +import org.apache.datasketches.theta.UpdateSketch; import org.apache.datasketches.thetacommon.ThetaUtil; import org.testng.annotations.Test; @@ -138,7 +147,7 @@ public void checkBuilder() { nameS1 = sk1.getClass().getSimpleName(); assertEquals(nameS1, "HeapQuickSelectSketch"); assertEquals(sk1.getLgNomLongs(), Integer.numberOfTrailingZeros(ThetaUtil.DEFAULT_NOMINAL_ENTRIES)); - assertEquals(sk1.getSeed(), ThetaUtil.DEFAULT_UPDATE_SEED); + assertEquals(sk1.getSeed(), Util.DEFAULT_UPDATE_SEED); assertEquals(sk1.getP(), (float)1.0); assertEquals(sk1.getResizeFactor(), ResizeFactor.X8); } @@ -181,8 +190,8 @@ public void checkBuilderResizeFactor() { public void checkWrapBadFamily() { UpdateSketch sketch = UpdateSketch.builder().setFamily(Family.ALPHA).setNominalEntries(1024).build(); byte[] byteArr = sketch.toByteArray(); - Memory srcMem = Memory.wrap(byteArr); - Sketch.wrap(srcMem); + MemorySegment srcSeg = MemorySegment.ofArray(byteArr); + Sketch.wrap(srcSeg); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -195,12 +204,12 @@ public void checkBadFamily() { public void checkSerVer() { UpdateSketch sketch = UpdateSketch.builder().setNominalEntries(1024).build(); byte[] sketchArray = sketch.toByteArray(); - Memory mem = Memory.wrap(sketchArray); - int serVer = Sketch.getSerializationVersion(mem); + MemorySegment seg = MemorySegment.ofArray(sketchArray); + int serVer = Sketch.getSerializationVersion(seg); assertEquals(serVer, 3); - WritableMemory wmem = WritableMemory.writableWrap(sketchArray); - UpdateSketch sk2 = UpdateSketch.wrap(wmem); - serVer = sk2.getSerializationVersion(wmem); + MemorySegment wseg = MemorySegment.ofArray(sketchArray); + UpdateSketch sk2 = UpdateSketch.wrap(wseg); + serVer = sk2.getSerializationVersion(wseg); assertEquals(serVer, 3); } @@ -209,10 +218,10 @@ public void checkHeapifyAlphaCompactExcep() { int k = 512; Sketch sketch1 = UpdateSketch.builder().setFamily(ALPHA).setNominalEntries(k).build(); byte[] byteArray = sketch1.toByteArray(); - WritableMemory mem = WritableMemory.writableWrap(byteArray); + MemorySegment seg = MemorySegment.ofArray(byteArray); //corrupt: - mem.setBits(FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); - Sketch.heapify(mem); + Util.setBits(seg, FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); + Sketch.heapify(seg); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -220,10 +229,10 @@ public void checkHeapifyQSCompactExcep() { int k = 512; Sketch sketch1 = UpdateSketch.builder().setFamily(QUICKSELECT).setNominalEntries(k).build(); byte[] byteArray = sketch1.toByteArray(); - WritableMemory mem = WritableMemory.writableWrap(byteArray); + MemorySegment seg = MemorySegment.ofArray(byteArray); //corrupt: - mem.setBits(FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); - Sketch.heapify(mem); + Util.setBits(seg, FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); + Sketch.heapify(seg); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -232,11 +241,11 @@ public void checkHeapifyNotCompactExcep() { UpdateSketch sketch1 = UpdateSketch.builder().setFamily(QUICKSELECT).setNominalEntries(k).build(); int bytes = Sketch.getMaxCompactSketchBytes(0); byte[] byteArray = new byte[bytes]; - WritableMemory mem = WritableMemory.writableWrap(byteArray); - sketch1.compact(false, mem); + MemorySegment seg = MemorySegment.ofArray(byteArray); + sketch1.compact(false, seg); //corrupt: - mem.clearBits(FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); - Sketch.heapify(mem); + Util.clearBits(seg, FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); + Sketch.heapify(seg); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -244,9 +253,9 @@ public void checkHeapifyFamilyExcep() { int k = 512; Union union = SetOperation.builder().setNominalEntries(k).buildUnion(); byte[] byteArray = union.toByteArray(); - Memory mem = Memory.wrap(byteArray); + MemorySegment seg = MemorySegment.ofArray(byteArray); //Improper use - Sketch.heapify(mem); + Sketch.heapify(seg); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -254,10 +263,10 @@ public void checkWrapAlphaCompactExcep() { int k = 512; Sketch sketch1 = UpdateSketch.builder().setFamily(ALPHA).setNominalEntries(k).build(); byte[] byteArray = sketch1.toByteArray(); - WritableMemory mem = WritableMemory.writableWrap(byteArray); + MemorySegment seg = MemorySegment.ofArray(byteArray); //corrupt: - mem.setBits(FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); - Sketch.wrap(mem); + Util.setBits(seg, FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); + Sketch.wrap(seg); } @@ -266,10 +275,10 @@ public void checkWrapQSCompactExcep() { int k = 512; Sketch sketch1 = UpdateSketch.builder().setFamily(QUICKSELECT).setNominalEntries(k).build(); byte[] byteArray = sketch1.toByteArray(); - WritableMemory mem = WritableMemory.writableWrap(byteArray); + MemorySegment seg = MemorySegment.ofArray(byteArray); //corrupt: - mem.setBits(FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); - Sketch.wrap(mem); + Util.setBits(seg, FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); + Sketch.wrap(seg); } @Test(expectedExceptions = SketchesArgumentException.class) @@ -278,11 +287,11 @@ public void checkWrapNotCompactExcep() { UpdateSketch sketch1 = UpdateSketch.builder().setFamily(QUICKSELECT).setNominalEntries(k).build(); int bytes = Sketch.getMaxCompactSketchBytes(0); byte[] byteArray = new byte[bytes]; - WritableMemory mem = WritableMemory.writableWrap(byteArray); - sketch1.compact(false, mem); + MemorySegment seg = MemorySegment.ofArray(byteArray); + sketch1.compact(false, seg); //corrupt: - mem.clearBits(FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); - Sketch.wrap(mem); + Util.clearBits(seg, FLAGS_BYTE, (byte) COMPACT_FLAG_MASK); + Sketch.wrap(seg); } @Test @@ -297,45 +306,46 @@ public void checkValidSketchID() { public void checkWrapToHeapifyConversion1() { int k = 512; UpdateSketch sketch1 = UpdateSketch.builder().setNominalEntries(k).build(); - for (int i=0; i 0) && !(((lgT - lgA) % lgR) == 0); boolean rf0 = (lgR == 0) && (lgA != lgT); assertTrue((lgRbad == rf0) || (lgRbad == rf123)); @@ -206,8 +208,8 @@ public void checkIsResizeFactorIncorrect() { @SuppressWarnings("unused") @Test - public void checkCompactOpsMemoryToCompact() { - WritableMemory skwmem, cskwmem1, cskwmem2, cskwmem3; + public void checkCompactOpsMemorySegmentToCompact() { + MemorySegment skwseg, cskwseg1, cskwseg2, cskwseg3; CompactSketch csk1, csk2, csk3; int lgK = 6; UpdateSketch sk = Sketches.updateSketchBuilder().setLogNominalEntries(lgK).build(); @@ -215,15 +217,15 @@ public void checkCompactOpsMemoryToCompact() { for (int i = 2; i < n; i++) { sk.update(i); } int cbytes = sk.getCompactBytes(); byte[] byteArr = sk.toByteArray(); - skwmem = WritableMemory.writableWrap(byteArr); - cskwmem1 = WritableMemory.allocate(cbytes); - cskwmem2 = WritableMemory.allocate(cbytes); - cskwmem3 = WritableMemory.allocate(cbytes); - csk1 = sk.compact(true, cskwmem1); - csk2 = CompactOperations.memoryToCompact(skwmem, true, cskwmem2); - csk3 = CompactOperations.memoryToCompact(cskwmem1, true, cskwmem3); - assertTrue(cskwmem1.equalTo(cskwmem2)); - assertTrue(cskwmem1.equalTo(cskwmem3)); + skwseg = MemorySegment.ofArray(byteArr); + cskwseg1 = MemorySegment.ofArray(new byte[cbytes]); + cskwseg2 = MemorySegment.ofArray(new byte[cbytes]); + cskwseg3 = MemorySegment.ofArray(new byte[cbytes]); + csk1 = sk.compact(true, cskwseg1); + csk2 = CompactOperations.segmentToCompact(skwseg, true, cskwseg2); + csk3 = CompactOperations.segmentToCompact(cskwseg1, true, cskwseg3); + assertTrue(equalContents(cskwseg1,cskwseg2)); + assertTrue(equalContents(cskwseg1, cskwseg3)); } @Test diff --git a/src/test/java/org/apache/datasketches/thetacommon/BinomialBoundsNTest.java b/src/test/java/org/apache/datasketches/thetacommon/BinomialBoundsNTest.java index 823b4104f..c991432a9 100644 --- a/src/test/java/org/apache/datasketches/thetacommon/BinomialBoundsNTest.java +++ b/src/test/java/org/apache/datasketches/thetacommon/BinomialBoundsNTest.java @@ -27,6 +27,7 @@ import static org.testng.Assert.fail; import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.thetacommon.BinomialBoundsN; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInThetaSketchedSetsTest.java b/src/test/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInThetaSketchedSetsTest.java index bf9a3d753..ed8833f7a 100644 --- a/src/test/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInThetaSketchedSetsTest.java +++ b/src/test/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInThetaSketchedSetsTest.java @@ -27,6 +27,7 @@ import org.apache.datasketches.theta.Intersection; import org.apache.datasketches.theta.Sketches; import org.apache.datasketches.theta.UpdateSketch; +import org.apache.datasketches.thetacommon.BoundsOnRatiosInThetaSketchedSets; import org.testng.annotations.Test; public class BoundsOnRatiosInThetaSketchedSetsTest { diff --git a/src/test/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInTupleSketchedSetsTest.java b/src/test/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInTupleSketchedSetsTest.java index a0d9cf086..4a083b8e8 100644 --- a/src/test/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInTupleSketchedSetsTest.java +++ b/src/test/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInTupleSketchedSetsTest.java @@ -22,13 +22,14 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.theta.UpdateSketch; import org.apache.datasketches.theta.UpdateSketchBuilder; +import org.apache.datasketches.thetacommon.BoundsOnRatiosInTupleSketchedSets; +import org.apache.datasketches.tuple.Intersection; import org.apache.datasketches.tuple.Sketch; import org.apache.datasketches.tuple.UpdatableSketch; import org.apache.datasketches.tuple.UpdatableSketchBuilder; import org.apache.datasketches.tuple.adouble.DoubleSummary; import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; import org.apache.datasketches.tuple.adouble.DoubleSummarySetOperations; -import org.apache.datasketches.tuple.Intersection; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; diff --git a/src/test/java/org/apache/datasketches/thetacommon/HashOperationsTest.java b/src/test/java/org/apache/datasketches/thetacommon/HashOperationsTest.java index 0d35b8fd5..e13a0b498 100644 --- a/src/test/java/org/apache/datasketches/thetacommon/HashOperationsTest.java +++ b/src/test/java/org/apache/datasketches/thetacommon/HashOperationsTest.java @@ -25,19 +25,20 @@ import static org.apache.datasketches.thetacommon.HashOperations.continueCondition; import static org.apache.datasketches.thetacommon.HashOperations.hashArrayInsert; import static org.apache.datasketches.thetacommon.HashOperations.hashInsertOnly; -import static org.apache.datasketches.thetacommon.HashOperations.hashInsertOnlyMemory; +import static org.apache.datasketches.thetacommon.HashOperations.hashInsertOnlyMemorySegment; import static org.apache.datasketches.thetacommon.HashOperations.hashSearch; -import static org.apache.datasketches.thetacommon.HashOperations.hashSearchMemory; +import static org.apache.datasketches.thetacommon.HashOperations.hashSearchMemorySegment; import static org.apache.datasketches.thetacommon.HashOperations.hashSearchOrInsert; -import static org.apache.datasketches.thetacommon.HashOperations.hashSearchOrInsertMemory; +import static org.apache.datasketches.thetacommon.HashOperations.hashSearchOrInsertMemorySegment; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; public class HashOperationsTest { @@ -105,20 +106,20 @@ public void testHashInsertOnlyWithStride() { } @Test - public void testHashInsertOnlyMemoryNoStride() { + public void testHashInsertOnlyMemorySegmentNoStride() { final long[] table = new long[32]; - final WritableMemory mem = WritableMemory.writableWrap(table); - final int index = hashInsertOnlyMemory(mem, 5, 1, 0); + final MemorySegment seg = MemorySegment.ofArray(table); + final int index = hashInsertOnlyMemorySegment(seg, 5, 1, 0); assertEquals(index, 1); assertEquals(table[1], 1L); } @Test - public void testHashInsertOnlyMemoryWithStride() { + public void testHashInsertOnlyMemorySegmentWithStride() { final long[] table = new long[32]; table[1] = 1; - final WritableMemory mem = WritableMemory.writableWrap(table); - final int index = hashInsertOnlyMemory(mem, 5, 1, 0); + final MemorySegment seg = MemorySegment.ofArray(table); + final int index = hashInsertOnlyMemorySegment(seg, 5, 1, 0); assertEquals(index, 2); assertEquals(table[2], 1L); } @@ -152,24 +153,24 @@ public void checkFullHeapTableCatchesInfiniteLoop() { @Test public void checkFullDirectTableCatchesInfiniteLoop() { final long[] table = new long[32]; - final WritableMemory mem = WritableMemory.writableWrap(table); + final MemorySegment seg = MemorySegment.ofArray(table); for (int i = 1; i <= 32; ++i) { - hashInsertOnlyMemory(mem, 5, i, 0); + hashInsertOnlyMemorySegment(seg, 5, i, 0); } // table full; search returns not found, others throw exception - final int retVal = hashSearchMemory(mem, 5, 33, 0); + final int retVal = hashSearchMemorySegment(seg, 5, 33, 0); assertEquals(retVal, -1); try { - hashInsertOnlyMemory(mem, 5, 33, 0); + hashInsertOnlyMemorySegment(seg, 5, 33, 0); fail(); } catch (final SketchesArgumentException e) { // expected } try { - hashSearchOrInsertMemory(mem, 5, 33, 0); + hashSearchOrInsertMemorySegment(seg, 5, 33, 0); fail(); } catch (final SketchesArgumentException e) { // expected @@ -179,22 +180,22 @@ public void checkFullDirectTableCatchesInfiniteLoop() { @Test public void checkFullFastDirectTableCatchesInfiniteLoop() { final long[] table = new long[32]; - final WritableMemory wmem = WritableMemory.writableWrap(table); + final MemorySegment wseg = MemorySegment.ofArray(table); for (int i = 1; i <= 32; ++i) { - hashInsertOnlyMemory(wmem, 5, i, 0); + hashInsertOnlyMemorySegment(wseg, 5, i, 0); } // table full; throws exception try { - hashInsertOnlyMemory(wmem, 5, 33, 0); + hashInsertOnlyMemorySegment(wseg, 5, 33, 0); fail(); } catch (final SketchesArgumentException e) { // expected } try { - hashSearchOrInsertMemory(wmem, 5, 33, 0); + hashSearchOrInsertMemorySegment(wseg, 5, 33, 0); fail(); } catch (final SketchesArgumentException e) { // expected diff --git a/src/test/java/org/apache/datasketches/thetacommon/QuickSelectTest.java b/src/test/java/org/apache/datasketches/thetacommon/QuickSelectTest.java index 2d7e3c77e..f812a7509 100644 --- a/src/test/java/org/apache/datasketches/thetacommon/QuickSelectTest.java +++ b/src/test/java/org/apache/datasketches/thetacommon/QuickSelectTest.java @@ -20,9 +20,9 @@ package org.apache.datasketches.thetacommon; import static java.lang.String.format; -import static org.apache.datasketches.thetacommon.QuickSelect.select; -import static org.apache.datasketches.thetacommon.QuickSelect.selectExcludingZeros; -import static org.apache.datasketches.thetacommon.QuickSelect.selectIncludingZeros; +import static org.apache.datasketches.common.QuickSelect.select; +import static org.apache.datasketches.common.QuickSelect.selectExcludingZeros; +import static org.apache.datasketches.common.QuickSelect.selectIncludingZeros; import java.util.Random; diff --git a/src/test/java/org/apache/datasketches/thetacommon/ThetaUtilTest.java b/src/test/java/org/apache/datasketches/thetacommon/ThetaUtilTest.java index 3a4ed9574..2a7628470 100644 --- a/src/test/java/org/apache/datasketches/thetacommon/ThetaUtilTest.java +++ b/src/test/java/org/apache/datasketches/thetacommon/ThetaUtilTest.java @@ -20,6 +20,7 @@ package org.apache.datasketches.thetacommon; import org.apache.datasketches.quantilescommon.QuantilesUtil; +import org.apache.datasketches.thetacommon.ThetaUtil; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/src/test/java/org/apache/datasketches/tuple/CompactSketchWithDoubleSummaryTest.java b/src/test/java/org/apache/datasketches/tuple/CompactSketchWithDoubleSummaryTest.java index 045777c19..a093d4feb 100644 --- a/src/test/java/org/apache/datasketches/tuple/CompactSketchWithDoubleSummaryTest.java +++ b/src/test/java/org/apache/datasketches/tuple/CompactSketchWithDoubleSummaryTest.java @@ -19,12 +19,19 @@ package org.apache.datasketches.tuple; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.tuple.CompactSketch; +import org.apache.datasketches.tuple.Sketch; +import org.apache.datasketches.tuple.Sketches; +import org.apache.datasketches.tuple.TupleSketchIterator; +import org.apache.datasketches.tuple.UpdatableSketch; +import org.apache.datasketches.tuple.UpdatableSketchBuilder; import org.apache.datasketches.tuple.adouble.DoubleSummary; -import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.apache.datasketches.tuple.adouble.DoubleSummaryDeserializer; import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; +import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.testng.Assert; import org.testng.annotations.Test; @@ -124,7 +131,7 @@ public void serializeDeserializeSmallExact() { us.update("c", 1.0); CompactSketch sketch1 = us.compact(); Sketch sketch2 = - Sketches.heapifySketch(Memory.wrap(sketch1.toByteArray()), + Sketches.heapifySketch(MemorySegment.ofArray(sketch1.toByteArray()), new DoubleSummaryDeserializer()); Assert.assertFalse(sketch2.isEmpty()); Assert.assertFalse(sketch2.isEstimationMode()); @@ -158,7 +165,7 @@ public void serializeDeserializeEstimation() throws Exception { //TestUtil.writeBytesToFile(bytes, "CompactSketchWithDoubleSummary4K.sk"); Sketch sketch2 = - Sketches.heapifySketch(Memory.wrap(bytes), new DoubleSummaryDeserializer()); + Sketches.heapifySketch(MemorySegment.ofArray(bytes), new DoubleSummaryDeserializer()); Assert.assertFalse(sketch2.isEmpty()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), sketch1.getEstimate()); @@ -180,7 +187,7 @@ public void deserializeWrongType() { us.update(i, 1.0); } CompactSketch sketch1 = us.compact(); - Sketches.heapifyUpdatableSketch(Memory.wrap(sketch1.toByteArray()), + Sketches.heapifyUpdatableSketch(MemorySegment.ofArray(sketch1.toByteArray()), new DoubleSummaryDeserializer(), new DoubleSummaryFactory(mode)); } diff --git a/src/test/java/org/apache/datasketches/tuple/IntegerSummary.java b/src/test/java/org/apache/datasketches/tuple/IntegerSummary.java index e476573d3..68c74b0a5 100644 --- a/src/test/java/org/apache/datasketches/tuple/IntegerSummary.java +++ b/src/test/java/org/apache/datasketches/tuple/IntegerSummary.java @@ -19,8 +19,13 @@ package org.apache.datasketches.tuple; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; + +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ByteArrayUtil; -import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.tuple.DeserializeResult; +import org.apache.datasketches.tuple.UpdatableSummary; /** * Summary for generic tuple sketches of type Integer. @@ -67,12 +72,12 @@ public byte[] toByteArray() { /** * Creates an instance of the IntegerSummary given a serialized representation - * @param mem Memory object with serialized IntegerSummary + * @param seg MemorySegment object with serialized IntegerSummary * @return DeserializedResult object, which contains a IntegerSummary object and number of bytes - * read from the Memory + * read from the MemorySegment */ - public static DeserializeResult fromMemory(final Memory mem) { - return new DeserializeResult<>(new IntegerSummary(mem.getInt(VALUE_INDEX)), SERIALIZED_SIZE_BYTES); + public static DeserializeResult fromMemorySegment(final MemorySegment seg) { + return new DeserializeResult<>(new IntegerSummary(seg.get(JAVA_INT_UNALIGNED, VALUE_INDEX)), SERIALIZED_SIZE_BYTES); } } diff --git a/src/test/java/org/apache/datasketches/tuple/IntegerSummaryDeserializer.java b/src/test/java/org/apache/datasketches/tuple/IntegerSummaryDeserializer.java index 33c0e34f8..56a222758 100644 --- a/src/test/java/org/apache/datasketches/tuple/IntegerSummaryDeserializer.java +++ b/src/test/java/org/apache/datasketches/tuple/IntegerSummaryDeserializer.java @@ -19,13 +19,16 @@ package org.apache.datasketches.tuple; -import org.apache.datasketches.memory.Memory; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.tuple.DeserializeResult; +import org.apache.datasketches.tuple.SummaryDeserializer; public class IntegerSummaryDeserializer implements SummaryDeserializer { @Override - public DeserializeResult heapifySummary(final Memory mem) { - return IntegerSummary.fromMemory(mem); + public DeserializeResult heapifySummary(final MemorySegment seg) { + return IntegerSummary.fromMemorySegment(seg); } } diff --git a/src/test/java/org/apache/datasketches/tuple/IntegerSummaryFactory.java b/src/test/java/org/apache/datasketches/tuple/IntegerSummaryFactory.java index cc21ffffd..3a61a4aee 100644 --- a/src/test/java/org/apache/datasketches/tuple/IntegerSummaryFactory.java +++ b/src/test/java/org/apache/datasketches/tuple/IntegerSummaryFactory.java @@ -19,6 +19,8 @@ package org.apache.datasketches.tuple; +import org.apache.datasketches.tuple.SummaryFactory; + /** * Factory for IntegerSummary. */ diff --git a/src/test/java/org/apache/datasketches/tuple/JaccardSimilarityTest.java b/src/test/java/org/apache/datasketches/tuple/JaccardSimilarityTest.java index d3c9a1bd0..1553596c9 100644 --- a/src/test/java/org/apache/datasketches/tuple/JaccardSimilarityTest.java +++ b/src/test/java/org/apache/datasketches/tuple/JaccardSimilarityTest.java @@ -19,13 +19,15 @@ package org.apache.datasketches.tuple; +import org.apache.datasketches.theta.UpdateSketch; +import org.apache.datasketches.theta.UpdateSketchBuilder; +import org.apache.datasketches.tuple.UpdatableSketch; +import org.apache.datasketches.tuple.UpdatableSketchBuilder; import org.apache.datasketches.tuple.adouble.DoubleSummary; import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; import org.apache.datasketches.tuple.adouble.DoubleSummarySetOperations; import org.testng.annotations.Test; -import org.apache.datasketches.theta.UpdateSketch; -import org.apache.datasketches.theta.UpdateSketchBuilder; import static org.apache.datasketches.tuple.JaccardSimilarity.dissimilarityTest; import static org.apache.datasketches.tuple.JaccardSimilarity.exactlyEqual; import static org.apache.datasketches.tuple.JaccardSimilarity.jaccard; @@ -450,8 +452,8 @@ public void printlnTest() { /** * @param s value to print */ - static void println(String s) { - //System.out.println(s); //disable here + static void println(Object o) { + //System.out.println(o.toString()); //disable here } } diff --git a/src/test/java/org/apache/datasketches/tuple/MiscTest.java b/src/test/java/org/apache/datasketches/tuple/MiscTest.java index 23ff08b2d..e4eb97064 100644 --- a/src/test/java/org/apache/datasketches/tuple/MiscTest.java +++ b/src/test/java/org/apache/datasketches/tuple/MiscTest.java @@ -23,9 +23,12 @@ import static org.testng.Assert.assertTrue; import org.apache.datasketches.thetacommon.SetOperationCornerCases.CornerCase; +import org.apache.datasketches.tuple.UpdatableSketch; +import org.apache.datasketches.tuple.UpdatableSketchBuilder; +import org.apache.datasketches.tuple.Util; import org.apache.datasketches.tuple.adouble.DoubleSummary; -import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; +import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/tuple/ReadOnlyMemoryTest.java b/src/test/java/org/apache/datasketches/tuple/ReadOnlyMemorySegmentTest.java similarity index 92% rename from src/test/java/org/apache/datasketches/tuple/ReadOnlyMemoryTest.java rename to src/test/java/org/apache/datasketches/tuple/ReadOnlyMemorySegmentTest.java index 3da256d6d..9e6d4b231 100644 --- a/src/test/java/org/apache/datasketches/tuple/ReadOnlyMemoryTest.java +++ b/src/test/java/org/apache/datasketches/tuple/ReadOnlyMemorySegmentTest.java @@ -19,8 +19,9 @@ package org.apache.datasketches.tuple; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesReadOnlyException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSetOperationBuilder; import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch; import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches; @@ -30,14 +31,14 @@ import org.testng.Assert; import org.testng.annotations.Test; -public class ReadOnlyMemoryTest { +public class ReadOnlyMemorySegmentTest { @Test public void wrapAndTryUpdatingSketch() { final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); sketch1.update(1, new double[] {1}); final ArrayOfDoublesUpdatableSketch sketch2 = (ArrayOfDoublesUpdatableSketch) - ArrayOfDoublesSketches.wrapSketch(Memory.wrap(sketch1.toByteArray())); + ArrayOfDoublesSketches.wrapSketch(MemorySegment.ofArray(sketch1.toByteArray())); Assert.assertEquals(sketch2.getEstimate(), 1.0); sketch2.toByteArray(); boolean thrown = false; @@ -60,7 +61,7 @@ public void heapifyAndUpdateSketch() { sketch1.update(1, new double[] {1}); // downcasting is not recommended, for testing only final ArrayOfDoublesUpdatableSketch sketch2 = (ArrayOfDoublesUpdatableSketch) - ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray())); + ArrayOfDoublesSketches.heapifySketch(MemorySegment.ofArray(sketch1.toByteArray())); sketch2.update(2, new double[] {1}); Assert.assertEquals(sketch2.getEstimate(), 2.0); } @@ -75,7 +76,7 @@ public void wrapAndTryUpdatingUnionEstimationMode() { } final ArrayOfDoublesUnion union1 = new ArrayOfDoublesSetOperationBuilder().buildUnion(); union1.union(sketch1); - final ArrayOfDoublesUnion union2 = ArrayOfDoublesSketches.wrapUnion(Memory.wrap(union1.toByteArray())); + final ArrayOfDoublesUnion union2 = ArrayOfDoublesSketches.wrapUnion(MemorySegment.ofArray(union1.toByteArray()).asReadOnly()); final ArrayOfDoublesSketch resultSketch = union2.getResult(); Assert.assertTrue(resultSketch.isEstimationMode()); Assert.assertEquals(resultSketch.getEstimate(), numUniques, numUniques * 0.04); @@ -105,7 +106,7 @@ public void heapifyAndUpdateUnion() { } final ArrayOfDoublesUnion union1 = new ArrayOfDoublesSetOperationBuilder().buildUnion(); union1.union(sketch1); - final ArrayOfDoublesUnion union2 = ArrayOfDoublesSketches.heapifyUnion(Memory.wrap(union1.toByteArray())); + final ArrayOfDoublesUnion union2 = ArrayOfDoublesSketches.heapifyUnion(MemorySegment.ofArray(union1.toByteArray())); final ArrayOfDoublesSketch resultSketch = union2.getResult(); Assert.assertTrue(resultSketch.isEstimationMode()); Assert.assertEquals(resultSketch.getEstimate(), numUniques, numUniques * 0.04); diff --git a/src/test/java/org/apache/datasketches/tuple/SerializerDeserializerTest.java b/src/test/java/org/apache/datasketches/tuple/SerializerDeserializerTest.java index 2a905694a..92a8599b7 100644 --- a/src/test/java/org/apache/datasketches/tuple/SerializerDeserializerTest.java +++ b/src/test/java/org/apache/datasketches/tuple/SerializerDeserializerTest.java @@ -19,9 +19,13 @@ package org.apache.datasketches.tuple; +import static org.apache.datasketches.common.Util.computeSeedHash; + +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.tuple.SerializerDeserializer; import org.testng.Assert; import org.testng.annotations.Test; @@ -31,22 +35,16 @@ public class SerializerDeserializerTest { public void validSketchType() { byte[] bytes = new byte[4]; bytes[SerializerDeserializer.TYPE_BYTE_OFFSET] = (byte) SerializerDeserializer.SketchType.CompactSketch.ordinal(); - Assert.assertEquals(SerializerDeserializer.getSketchType(Memory.wrap(bytes)), SerializerDeserializer.SketchType.CompactSketch); + Assert.assertEquals(SerializerDeserializer.getSketchType(MemorySegment.ofArray(bytes)), SerializerDeserializer.SketchType.CompactSketch); } @Test(expectedExceptions = SketchesArgumentException.class) public void invalidSketchType() { byte[] bytes = new byte[4]; bytes[SerializerDeserializer.TYPE_BYTE_OFFSET] = 33; - SerializerDeserializer.getSketchType(Memory.wrap(bytes)); + SerializerDeserializer.getSketchType(MemorySegment.ofArray(bytes)); } -// @Test(expectedExceptions = SketchesArgumentException.class) -// public void deserializeFromMemoryUsupportedClass() { -// Memory mem = null; -// SerializerDeserializer.deserializeFromMemory(mem, 0, "bogus"); -// } - @Test(expectedExceptions = SketchesArgumentException.class) public void validateFamilyNotTuple() { SerializerDeserializer.validateFamily((byte) 1, (byte) 0); @@ -59,6 +57,6 @@ public void validateFamilyWrongPreambleLength() { @Test(expectedExceptions = SketchesArgumentException.class) public void checkBadSeedHash() { - org.apache.datasketches.tuple.Util.computeSeedHash(50541); + computeSeedHash(50541); } } diff --git a/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java index e6ea1e79f..f20cb99dc 100644 --- a/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java @@ -27,13 +27,17 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; import java.io.IOException; import java.nio.file.Files; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.TestUtil; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.tuple.Sketch; +import org.apache.datasketches.tuple.Sketches; +import org.apache.datasketches.tuple.TupleSketchIterator; +import org.apache.datasketches.tuple.UpdatableSketch; +import org.apache.datasketches.tuple.UpdatableSketchBuilder; import org.apache.datasketches.tuple.adouble.DoubleSummary; import org.apache.datasketches.tuple.adouble.DoubleSummaryDeserializer; import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUnion; @@ -45,7 +49,7 @@ public class TupleCrossLanguageTest { @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) public void serialVersion1Compatibility() { final byte[] byteArr = TestUtil.getResourceBytes("CompactSketchWithDoubleSummary4K_serialVersion1.sk"); - Sketch sketch = Sketches.heapifySketch(Memory.wrap(byteArr), new DoubleSummaryDeserializer()); + Sketch sketch = Sketches.heapifySketch(MemorySegment.ofArray(byteArr), new DoubleSummaryDeserializer()); Assert.assertTrue(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 8192, 8192 * 0.99); Assert.assertEquals(sketch.getRetainedEntries(), 4096); @@ -61,7 +65,7 @@ public void serialVersion1Compatibility() { @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) public void version2Compatibility() { final byte[] byteArr = TestUtil.getResourceBytes("TupleWithTestIntegerSummary4kTrimmedSerVer2.sk"); - Sketch sketch1 = Sketches.heapifySketch(Memory.wrap(byteArr), new IntegerSummaryDeserializer()); + Sketch sketch1 = Sketches.heapifySketch(MemorySegment.ofArray(byteArr), new IntegerSummaryDeserializer()); // construct the same way final int lgK = 12; @@ -87,7 +91,7 @@ public void deserializeFromCppIntegerSummary() throws IOException { for (int n: nArr) { final byte[] bytes = Files.readAllBytes(cppPath.resolve("tuple_int_n" + n + "_cpp.sk")); final Sketch sketch = - Sketches.heapifySketch(Memory.wrap(bytes), new IntegerSummaryDeserializer()); + Sketches.heapifySketch(MemorySegment.ofArray(bytes), new IntegerSummaryDeserializer()); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertTrue(n > 1000 ? sketch.isEstimationMode() : !sketch.isEstimationMode()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -105,7 +109,9 @@ public void generateForCppIntegerSummary() throws IOException { for (int n: nArr) { final UpdatableSketch sk = new UpdatableSketchBuilder<>(new IntegerSummaryFactory()).build(); - for (int i = 0; i < n; i++) sk.update(i, i); + for (int i = 0; i < n; i++) { + sk.update(i, i); + } Files.newOutputStream(javaPath.resolve("tuple_int_n" + n + "_java.sk")).write(sk.compact().toByteArray()); } } @@ -113,13 +119,13 @@ public void generateForCppIntegerSummary() throws IOException { @Test(expectedExceptions = SketchesArgumentException.class, groups = {CHECK_CPP_HISTORICAL_FILES}) public void noSupportHeapifyV0_9_1() throws Exception { final byte[] byteArr = TestUtil.getResourceBytes("ArrayOfDoublesUnion_v0.9.1.sk"); - ArrayOfDoublesUnion.heapify(Memory.wrap(byteArr)); + ArrayOfDoublesUnion.heapify(MemorySegment.ofArray(byteArr)); } @Test(expectedExceptions = SketchesArgumentException.class, groups = {CHECK_CPP_HISTORICAL_FILES}) public void noSupportWrapV0_9_1() throws Exception { final byte[] byteArr = TestUtil.getResourceBytes("ArrayOfDoublesUnion_v0.9.1.sk"); - ArrayOfDoublesUnion.wrap(WritableMemory.writableWrap(byteArr)); + ArrayOfDoublesUnion.wrap(MemorySegment.ofArray(byteArr)); } } diff --git a/src/test/java/org/apache/datasketches/tuple/TupleExamples2Test.java b/src/test/java/org/apache/datasketches/tuple/TupleExamples2Test.java index 802fe1404..c26f6f394 100644 --- a/src/test/java/org/apache/datasketches/tuple/TupleExamples2Test.java +++ b/src/test/java/org/apache/datasketches/tuple/TupleExamples2Test.java @@ -23,10 +23,16 @@ import org.apache.datasketches.theta.UpdateSketch; import org.apache.datasketches.theta.UpdateSketchBuilder; +import org.apache.datasketches.tuple.CompactSketch; +import org.apache.datasketches.tuple.Intersection; +import org.apache.datasketches.tuple.TupleSketchIterator; +import org.apache.datasketches.tuple.Union; +import org.apache.datasketches.tuple.UpdatableSketch; +import org.apache.datasketches.tuple.UpdatableSketchBuilder; import org.apache.datasketches.tuple.adouble.DoubleSummary; -import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; import org.apache.datasketches.tuple.adouble.DoubleSummarySetOperations; +import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/tuple/TupleExamplesTest.java b/src/test/java/org/apache/datasketches/tuple/TupleExamplesTest.java index ace6c5ff4..f6941e334 100644 --- a/src/test/java/org/apache/datasketches/tuple/TupleExamplesTest.java +++ b/src/test/java/org/apache/datasketches/tuple/TupleExamplesTest.java @@ -23,10 +23,16 @@ import org.apache.datasketches.theta.UpdateSketch; import org.apache.datasketches.theta.UpdateSketchBuilder; +import org.apache.datasketches.tuple.CompactSketch; +import org.apache.datasketches.tuple.Intersection; +import org.apache.datasketches.tuple.TupleSketchIterator; +import org.apache.datasketches.tuple.Union; +import org.apache.datasketches.tuple.UpdatableSketch; +import org.apache.datasketches.tuple.UpdatableSketchBuilder; import org.apache.datasketches.tuple.aninteger.IntegerSummary; -import org.apache.datasketches.tuple.aninteger.IntegerSummary.Mode; import org.apache.datasketches.tuple.aninteger.IntegerSummaryFactory; import org.apache.datasketches.tuple.aninteger.IntegerSummarySetOperations; +import org.apache.datasketches.tuple.aninteger.IntegerSummary.Mode; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleAnotBTest.java b/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleAnotBTest.java index 7f20a3f90..a878e50dc 100644 --- a/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleAnotBTest.java +++ b/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleAnotBTest.java @@ -32,6 +32,8 @@ import org.apache.datasketches.tuple.TupleSketchIterator; import org.apache.datasketches.tuple.UpdatableSketch; import org.apache.datasketches.tuple.UpdatableSketchBuilder; +import org.apache.datasketches.tuple.adouble.DoubleSummary; +import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleIntersectionTest.java b/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleIntersectionTest.java index 4950eb332..b7ee014d6 100644 --- a/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleIntersectionTest.java +++ b/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleIntersectionTest.java @@ -28,10 +28,13 @@ import org.apache.datasketches.tuple.CompactSketch; import org.apache.datasketches.tuple.Intersection; import org.apache.datasketches.tuple.Sketch; -import org.apache.datasketches.tuple.TupleSketchIterator; import org.apache.datasketches.tuple.Sketches; +import org.apache.datasketches.tuple.TupleSketchIterator; import org.apache.datasketches.tuple.UpdatableSketch; import org.apache.datasketches.tuple.UpdatableSketchBuilder; +import org.apache.datasketches.tuple.adouble.DoubleSummary; +import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; +import org.apache.datasketches.tuple.adouble.DoubleSummarySetOperations; import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleTest.java b/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleTest.java index a1cfc1a7d..d66f2a044 100644 --- a/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleTest.java +++ b/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleTest.java @@ -21,16 +21,20 @@ import static org.testng.Assert.assertEquals; +import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.tuple.Sketch; import org.apache.datasketches.tuple.Sketches; import org.apache.datasketches.tuple.TupleSketchIterator; import org.apache.datasketches.tuple.UpdatableSketch; import org.apache.datasketches.tuple.UpdatableSketchBuilder; +import org.apache.datasketches.tuple.adouble.DoubleSketch; +import org.apache.datasketches.tuple.adouble.DoubleSummary; +import org.apache.datasketches.tuple.adouble.DoubleSummaryDeserializer; +import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.testng.Assert; import org.testng.annotations.Test; @@ -75,8 +79,8 @@ public void serDeTest() { a1Sk.update(key, 1.0); } final double est1 = a1Sk.getEstimate(); - final Memory mem = Memory.wrap(a1Sk.toByteArray()); - final DoubleSketch a1Sk2 = new DoubleSketch(mem, Mode.AlwaysOne); + final MemorySegment seg = MemorySegment.ofArray(a1Sk.toByteArray()); + final DoubleSketch a1Sk2 = new DoubleSketch(seg, Mode.AlwaysOne); final double est2 = a1Sk2.getEstimate(); assertEquals(est1, est2); } @@ -331,7 +335,7 @@ public void serializeDeserializeExact() throws Exception { sketch1.update(1, 1.0); final UpdatableSketch sketch2 = Sketches.heapifyUpdatableSketch( - Memory.wrap(sketch1.toByteArray()), + MemorySegment.ofArray(sketch1.toByteArray()), new DoubleSummaryDeserializer(), new DoubleSummaryFactory(mode)); Assert.assertEquals(sketch2.getEstimate(), 1.0); @@ -366,7 +370,7 @@ public void serializeDeserializeEstimationNoResizing() throws Exception { //TestUtil.writeBytesToFile(bytes, "UpdatableSketchWithDoubleSummary4K.sk"); final Sketch sketch2 = - Sketches.heapifySketch(Memory.wrap(bytes), new DoubleSummaryDeserializer()); + Sketches.heapifySketch(MemorySegment.ofArray(bytes), new DoubleSummaryDeserializer()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 8192, 8192 * 0.99); Assert.assertEquals(sketch1.getTheta(), sketch2.getTheta()); @@ -391,7 +395,7 @@ public void serializeDeserializeSampling() throws Exception { sketch1.update(i, 1.0); } final Sketch sketch2 = Sketches.heapifySketch( - Memory.wrap(sketch1.toByteArray()), new DoubleSummaryDeserializer()); + MemorySegment.ofArray(sketch1.toByteArray()), new DoubleSummaryDeserializer()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate() / numberOfUniques, 1.0, 0.01); Assert.assertEquals(sketch2.getRetainedEntries() / (double) numberOfUniques, 0.5, 0.01); diff --git a/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleUnionTest.java b/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleUnionTest.java index dcacf8eed..c9a3d5887 100644 --- a/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleUnionTest.java +++ b/src/test/java/org/apache/datasketches/tuple/adouble/AdoubleUnionTest.java @@ -29,6 +29,9 @@ import org.apache.datasketches.tuple.Union; import org.apache.datasketches.tuple.UpdatableSketch; import org.apache.datasketches.tuple.UpdatableSketchBuilder; +import org.apache.datasketches.tuple.adouble.DoubleSummary; +import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; +import org.apache.datasketches.tuple.adouble.DoubleSummarySetOperations; import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/src/test/java/org/apache/datasketches/tuple/adouble/FilterTest.java b/src/test/java/org/apache/datasketches/tuple/adouble/FilterTest.java index be7aa264b..5e4f24d8e 100644 --- a/src/test/java/org/apache/datasketches/tuple/adouble/FilterTest.java +++ b/src/test/java/org/apache/datasketches/tuple/adouble/FilterTest.java @@ -26,6 +26,8 @@ import org.apache.datasketches.tuple.Sketches; import org.apache.datasketches.tuple.UpdatableSketch; import org.apache.datasketches.tuple.UpdatableSketchBuilder; +import org.apache.datasketches.tuple.adouble.DoubleSummary; +import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; import org.apache.datasketches.tuple.adouble.DoubleSummary.Mode; import org.testng.Assert; import org.testng.annotations.Test; diff --git a/src/test/java/org/apache/datasketches/tuple/aninteger/CornerCaseTupleSetOperationsTest.java b/src/test/java/org/apache/datasketches/tuple/aninteger/CornerCaseTupleSetOperationsTest.java index f9818bb38..0df275001 100644 --- a/src/test/java/org/apache/datasketches/tuple/aninteger/CornerCaseTupleSetOperationsTest.java +++ b/src/test/java/org/apache/datasketches/tuple/aninteger/CornerCaseTupleSetOperationsTest.java @@ -23,13 +23,16 @@ import static org.testng.Assert.assertTrue; import org.apache.datasketches.common.ResizeFactor; +import org.apache.datasketches.common.Util; import org.apache.datasketches.theta.UpdateSketch; import org.apache.datasketches.theta.UpdateSketchBuilder; -import org.apache.datasketches.thetacommon.ThetaUtil; import org.apache.datasketches.tuple.AnotB; import org.apache.datasketches.tuple.CompactSketch; import org.apache.datasketches.tuple.Intersection; import org.apache.datasketches.tuple.Union; +import org.apache.datasketches.tuple.aninteger.IntegerSketch; +import org.apache.datasketches.tuple.aninteger.IntegerSummary; +import org.apache.datasketches.tuple.aninteger.IntegerSummarySetOperations; import org.testng.annotations.Test; public class CornerCaseTupleSetOperationsTest { @@ -625,6 +628,6 @@ private static void checkInvalidUpdate(float p, long updateKey) { } static long getLongHash(long v) { - return (hash(v, ThetaUtil.DEFAULT_UPDATE_SEED)[0]) >>> 1; + return (hash(v, Util.DEFAULT_UPDATE_SEED)[0]) >>> 1; } } diff --git a/src/test/java/org/apache/datasketches/tuple/aninteger/EngagementTest.java b/src/test/java/org/apache/datasketches/tuple/aninteger/EngagementTest.java index 871822e5b..c467fb7af 100644 --- a/src/test/java/org/apache/datasketches/tuple/aninteger/EngagementTest.java +++ b/src/test/java/org/apache/datasketches/tuple/aninteger/EngagementTest.java @@ -28,6 +28,9 @@ import org.apache.datasketches.tuple.CompactSketch; import org.apache.datasketches.tuple.TupleSketchIterator; import org.apache.datasketches.tuple.Union; +import org.apache.datasketches.tuple.aninteger.IntegerSketch; +import org.apache.datasketches.tuple.aninteger.IntegerSummary; +import org.apache.datasketches.tuple.aninteger.IntegerSummarySetOperations; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/tuple/aninteger/IntegerSketchTest.java b/src/test/java/org/apache/datasketches/tuple/aninteger/IntegerSketchTest.java index 1d3a5dc7d..951129425 100644 --- a/src/test/java/org/apache/datasketches/tuple/aninteger/IntegerSketchTest.java +++ b/src/test/java/org/apache/datasketches/tuple/aninteger/IntegerSketchTest.java @@ -21,10 +21,14 @@ import static org.testng.Assert.assertEquals; -import org.apache.datasketches.memory.Memory; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.tuple.AnotB; import org.apache.datasketches.tuple.CompactSketch; import org.apache.datasketches.tuple.Intersection; +import org.apache.datasketches.tuple.aninteger.IntegerSketch; +import org.apache.datasketches.tuple.aninteger.IntegerSummary; +import org.apache.datasketches.tuple.aninteger.IntegerSummarySetOperations; import org.testng.annotations.Test; /** @@ -44,8 +48,8 @@ public void serDeTest() { a1Sk.update(i, 1); } final double est1 = a1Sk.getEstimate(); - final Memory mem = Memory.wrap(a1Sk.toByteArray()); - final IntegerSketch a1Sk2 = new IntegerSketch(mem, a1Mode); + final MemorySegment seg = MemorySegment.ofArray(a1Sk.toByteArray()); + final IntegerSketch a1Sk2 = new IntegerSketch(seg, a1Mode); final double est2 = a1Sk2.getEstimate(); assertEquals(est1, est2); } diff --git a/src/test/java/org/apache/datasketches/tuple/aninteger/MikhailsBugTupleTest.java b/src/test/java/org/apache/datasketches/tuple/aninteger/MikhailsBugTupleTest.java index 1e5aad3d2..0809e972d 100644 --- a/src/test/java/org/apache/datasketches/tuple/aninteger/MikhailsBugTupleTest.java +++ b/src/test/java/org/apache/datasketches/tuple/aninteger/MikhailsBugTupleTest.java @@ -22,6 +22,9 @@ import org.apache.datasketches.tuple.AnotB; import org.apache.datasketches.tuple.CompactSketch; import org.apache.datasketches.tuple.Intersection; +import org.apache.datasketches.tuple.aninteger.IntegerSketch; +import org.apache.datasketches.tuple.aninteger.IntegerSummary; +import org.apache.datasketches.tuple.aninteger.IntegerSummarySetOperations; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/tuple/aninteger/ParameterLeakageTest.java b/src/test/java/org/apache/datasketches/tuple/aninteger/ParameterLeakageTest.java index 3411cea4f..51eee988b 100644 --- a/src/test/java/org/apache/datasketches/tuple/aninteger/ParameterLeakageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/aninteger/ParameterLeakageTest.java @@ -25,10 +25,12 @@ import org.apache.datasketches.tuple.AnotB; import org.apache.datasketches.tuple.CompactSketch; import org.apache.datasketches.tuple.Intersection; -//import org.apache.datasketches.tuple.UpdatableSketch; import org.apache.datasketches.tuple.Sketch; import org.apache.datasketches.tuple.TupleSketchIterator; import org.apache.datasketches.tuple.Union; +import org.apache.datasketches.tuple.aninteger.IntegerSketch; +import org.apache.datasketches.tuple.aninteger.IntegerSummary; +import org.apache.datasketches.tuple.aninteger.IntegerSummarySetOperations; import org.testng.annotations.Test; /** diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java index e3b0db819..faca658d8 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java @@ -27,10 +27,14 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; import java.io.IOException; import java.nio.file.Files; -import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; import org.testng.annotations.Test; /** @@ -44,7 +48,9 @@ public void generateBinariesForCompatibilityTestingOneValue() throws IOException final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n: nArr) { final ArrayOfDoublesUpdatableSketch sk = new ArrayOfDoublesUpdatableSketchBuilder().build(); - for (int i = 0; i < n; i++) sk.update(i, new double[] {i}); + for (int i = 0; i < n; i++) { + sk.update(i, new double[] {i}); + } Files.newOutputStream(javaPath.resolve("aod_1_n" + n + "_java.sk")).write(sk.compact().toByteArray()); } } @@ -54,7 +60,9 @@ public void generateBinariesForCompatibilityTestingThreeValues() throws IOExcept final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n: nArr) { final ArrayOfDoublesUpdatableSketch sk = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(3).build(); - for (int i = 0; i < n; i++) sk.update(i, new double[] {i, i, i}); + for (int i = 0; i < n; i++) { + sk.update(i, new double[] {i, i, i}); + } Files.newOutputStream(javaPath.resolve("aod_3_n" + n + "_java.sk")).write(sk.compact().toByteArray()); } } @@ -74,7 +82,7 @@ public void deserializeFromCppOneValue() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (int n: nArr) { final byte[] bytes = Files.readAllBytes(cppPath.resolve("aod_1_n" + n + "_cpp.sk")); - final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(Memory.wrap(bytes)); + final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); assertEquals(sketch.getNumValues(), 1); @@ -90,7 +98,7 @@ public void deserializeFromCppThreeValues() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (int n: nArr) { final byte[] bytes = Files.readAllBytes(cppPath.resolve("aod_3_n" + n + "_cpp.sk")); - final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(Memory.wrap(bytes)); + final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); assertEquals(sketch.getNumValues(), 3); @@ -106,7 +114,7 @@ public void deserializeFromCppThreeValues() throws IOException { @Test(groups = {CHECK_CPP_FILES}) public void deserializeFromCppOneValueNonEmptyNoEntries() throws IOException { final byte[] bytes = Files.readAllBytes(cppPath.resolve("aod_1_non_empty_no_entries_cpp.sk")); - final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(Memory.wrap(bytes)); + final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertFalse(sketch.isEmpty()); assertEquals(sketch.getRetainedEntries(), 0); } diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotBTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotBTest.java index 571558c92..48f60cdcc 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotBTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesAnotBTest.java @@ -21,9 +21,17 @@ import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesAnotB; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesCompactSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSetOperationBuilder; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; import org.testng.Assert; import org.testng.annotations.Test; @@ -238,7 +246,7 @@ public void estimationMode() { // same operation, but compact sketches and off-heap result aNotB.update(sketchA.compact(), sketchB.compact()); - result = aNotB.getResult(WritableMemory.writableWrap(new byte[1000000])); + result = aNotB.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 4096.0, 4096 * 0.03); // crude estimate of RSE(95%) = 2 / sqrt(result.getRetainedEntries()) Assert.assertTrue(result.getLowerBound(1) <= result.getEstimate()); @@ -278,7 +286,7 @@ public void estimationModeLargeB() { // same operation, but compact sketches and off-heap result aNotB.update(sketchA.compact(), sketchB.compact()); - result = aNotB.getResult(WritableMemory.writableWrap(new byte[1000000])); + result = aNotB.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getEstimate(), expected, expected * 0.1); // crude estimate of RSE(95%) = 2 / sqrt(result.getRetainedEntries()) Assert.assertTrue(result.getLowerBound(1) <= result.getEstimate()); diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesCompactSketchTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesCompactSketchTest.java index 43e281c2c..7bd2ee14b 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesCompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesCompactSketchTest.java @@ -21,9 +21,17 @@ import static org.testng.Assert.assertEquals; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.Util; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesCompactSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; +import org.apache.datasketches.tuple.arrayofdoubles.DirectArrayOfDoublesCompactSketch; +import org.apache.datasketches.tuple.arrayofdoubles.DirectArrayOfDoublesQuickSelectSketchR; +import org.apache.datasketches.tuple.arrayofdoubles.HeapArrayOfDoublesCompactSketch; +import org.apache.datasketches.tuple.arrayofdoubles.HeapArrayOfDoublesQuickSelectSketch; import org.testng.Assert; import org.testng.annotations.Test; @@ -41,8 +49,8 @@ public void heapToDirectExactTwoDoubles() { sketch1.update("c", new double[] {1, 2}); sketch1.update("d", new double[] {1, 2}); ArrayOfDoublesCompactSketch csk = sketch1.compact(); - Memory mem = Memory.wrap(csk.toByteArray()); - ArrayOfDoublesSketch sketch2 = new DirectArrayOfDoublesCompactSketch(mem); + MemorySegment seg = MemorySegment.ofArray(csk.toByteArray()); + ArrayOfDoublesSketch sketch2 = new DirectArrayOfDoublesCompactSketch(seg); Assert.assertFalse(sketch2.isEmpty()); Assert.assertFalse(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 4.0); @@ -62,7 +70,7 @@ public void heapToDirectExactTwoDoubles() { @Test public void directToHeapExactTwoDoubles() { ArrayOfDoublesUpdatableSketch sketch1 = - new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(2).build(MemorySegment.ofArray(new byte[1000000])); sketch1.update("a", new double[] {1, 2}); sketch1.update("b", new double[] {1, 2}); sketch1.update("c", new double[] {1, 2}); @@ -73,7 +81,7 @@ public void directToHeapExactTwoDoubles() { sketch1.update("d", new double[] {1, 2}); ArrayOfDoublesSketch sketch2 = new HeapArrayOfDoublesCompactSketch( - Memory.wrap(sketch1.compact(WritableMemory.writableWrap(new byte[1000000])).toByteArray())); + MemorySegment.ofArray(sketch1.compact(MemorySegment.ofArray(new byte[1000000])).toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertFalse(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 4.0); @@ -113,10 +121,10 @@ public void checkGetValuesAndKeysMethods() { assertEquals(values1d2, values1d); assertEquals(keys2, keys); - Memory hqsskMem = Memory.wrap(hqssk.toByteArray()); + MemorySegment hqsskSeg = MemorySegment.ofArray(hqssk.toByteArray()); DirectArrayOfDoublesQuickSelectSketchR dqssk = - (DirectArrayOfDoublesQuickSelectSketchR)ArrayOfDoublesSketch.wrap(hqsskMem, ThetaUtil.DEFAULT_UPDATE_SEED); + (DirectArrayOfDoublesQuickSelectSketchR)ArrayOfDoublesSketch.wrap(hqsskSeg, Util.DEFAULT_UPDATE_SEED); final double[][] values3 = dqssk.getValues(); final double[] values1d3 = dqssk.getValuesAsOneDimension(); final long[] keys3 = dqssk.getKeys(); @@ -124,10 +132,10 @@ public void checkGetValuesAndKeysMethods() { assertEquals(values1d3, values1d); assertEquals(keys3, keys); - Memory hcskMem = Memory.wrap(hcsk.toByteArray()); + MemorySegment hcskSeg = MemorySegment.ofArray(hcsk.toByteArray()); DirectArrayOfDoublesCompactSketch dcsk2 = - (DirectArrayOfDoublesCompactSketch)ArrayOfDoublesSketch.wrap(hcskMem, ThetaUtil.DEFAULT_UPDATE_SEED); + (DirectArrayOfDoublesCompactSketch)ArrayOfDoublesSketch.wrap(hcskSeg, Util.DEFAULT_UPDATE_SEED); final double[][] values4 = dqssk.getValues(); final double[] values1d4 = dqssk.getValuesAsOneDimension(); final long[] keys4 = dqssk.getKeys(); diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesIntersectionTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesIntersectionTest.java index 164557b98..6a54adbdd 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesIntersectionTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesIntersectionTest.java @@ -21,8 +21,15 @@ import static org.testng.Assert.fail; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesCombiner; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesCompactSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesIntersection; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSetOperationBuilder; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; import org.testng.Assert; import org.testng.annotations.Test; @@ -105,19 +112,19 @@ public void heapExactWithEmpty() { @Test public void directExactWithEmpty() { final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder() - .build(WritableMemory.writableWrap(new byte[1000000])); + .build(MemorySegment.ofArray(new byte[1000000])); sketch1.update(1, new double[] {1}); sketch1.update(2, new double[] {1}); sketch1.update(3, new double[] {1}); final ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder() - .build(WritableMemory.writableWrap(new byte[1000000])); + .build(MemorySegment.ofArray(new byte[1000000])); final ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder(). - buildIntersection(WritableMemory.writableWrap(new byte[1000000])); + buildIntersection(MemorySegment.ofArray(new byte[1000000])); intersection.intersect(sketch1, null); intersection.intersect(sketch2, null); - final ArrayOfDoublesCompactSketch result = intersection.getResult(WritableMemory.writableWrap(new byte[1000000])); + final ArrayOfDoublesCompactSketch result = intersection.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertTrue(result.isEmpty()); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertEquals(result.getEstimate(), 0.0); @@ -190,22 +197,22 @@ public void heapDisjointEstimationMode() { public void directDisjointEstimationMode() { int key = 0; final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder(). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } final ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder(). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } final ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder(). - buildIntersection(WritableMemory.writableWrap(new byte[1000000])); + buildIntersection(MemorySegment.ofArray(new byte[1000000])); intersection.intersect(sketch1, combiner); intersection.intersect(sketch2, combiner); - final ArrayOfDoublesCompactSketch result = intersection.getResult(WritableMemory.writableWrap(new byte[1000000])); + final ArrayOfDoublesCompactSketch result = intersection.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getRetainedEntries(), 0); Assert.assertEquals(result.getEstimate(), 0.0); @@ -245,21 +252,21 @@ public void heapEstimationMode() { @Test public void directEstimationMode() { int key = 0; - final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 4096; // overlap half of the entries - final ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + final ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } - final ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder().buildIntersection(WritableMemory.writableWrap(new byte[1000000])); + final ArrayOfDoublesIntersection intersection = new ArrayOfDoublesSetOperationBuilder().buildIntersection(MemorySegment.ofArray(new byte[1000000])); intersection.intersect(sketch1, combiner); intersection.intersect(sketch2, combiner); - final ArrayOfDoublesCompactSketch result = intersection.getResult(WritableMemory.writableWrap(new byte[1000000])); + final ArrayOfDoublesCompactSketch result = intersection.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 4096.0, 4096 * 0.03); // crude estimate of RSE(95%) = 2 / sqrt(result.getRetainedEntries()) Assert.assertTrue(result.getLowerBound(1) <= result.getEstimate()); diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesQuickSelectSketchTest.java index 46b1a0631..a54c12d20 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesQuickSelectSketchTest.java @@ -19,9 +19,12 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesQuickSelectSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; import org.testng.Assert; import org.testng.annotations.Test; @@ -43,7 +46,7 @@ public void heapToDirectExactTwoDoubles() { sketch1.update("d", valuesArr); sketch1.update("a", valuesArr); noopUpdates(sketch1, valuesArr); - ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.wrap(WritableMemory.writableWrap(sketch1.toByteArray())); + ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.wrap(MemorySegment.ofArray(sketch1.toByteArray())); sketch2.update("b", valuesArr); sketch2.update("c", valuesArr); sketch2.update("d", valuesArr); @@ -74,7 +77,7 @@ public void heapToDirectWithSeed() { sketch1.update("b", values); sketch1.update("c", values); - ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.wrap(WritableMemory.writableWrap(sketch1.toByteArray()), seed); + ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.wrap(MemorySegment.ofArray(sketch1.toByteArray()), seed); sketch2.update("b", values); sketch2.update("c", values); sketch2.update("d", values); @@ -94,14 +97,14 @@ public void directToHeapExactTwoDoubles() { double[] valuesArr = {1.0, 2.0}; ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder(). - setNumberOfValues(2).build(WritableMemory.writableWrap(new byte[1000000])); + setNumberOfValues(2).build(MemorySegment.ofArray(new byte[1000000])); sketch1.update("a", valuesArr); sketch1.update("b", valuesArr); sketch1.update("c", valuesArr); sketch1.update("d", valuesArr); sketch1.update("a", valuesArr); noopUpdates(sketch1, valuesArr); - ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.heapify(Memory.wrap(sketch1.toByteArray())); + ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.heapify(MemorySegment.ofArray(sketch1.toByteArray())); sketch2.update("b", valuesArr); sketch2.update("c", valuesArr); sketch2.update("d", valuesArr); @@ -128,12 +131,12 @@ public void directToHeapWithSeed() { ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed).build( - WritableMemory.writableWrap(new byte[1000000])); + MemorySegment.ofArray(new byte[1000000])); sketch1.update("a", values); sketch1.update("b", values); sketch1.update("c", values); - ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.heapify(Memory.wrap(sketch1.toByteArray()), seed); + ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.heapify(MemorySegment.ofArray(sketch1.toByteArray()), seed); sketch2.update("b", values); sketch2.update("c", values); sketch2.update("d", values); diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnionTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnionTest.java index 82e3936d4..ea507b89c 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnionTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesUnionTest.java @@ -19,11 +19,18 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import java.lang.foreign.MemorySegment; import java.util.Arrays; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesCompactSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSetOperationBuilder; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUnion; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; import org.testng.Assert; import org.testng.annotations.Test; @@ -58,8 +65,8 @@ public void heapExactMode() { Assert.assertEquals(values[1][0], 3.0); Assert.assertEquals(values[2][0], 3.0); - final WritableMemory wmem = WritableMemory.writableWrap(union.toByteArray()); - final ArrayOfDoublesUnion wrappedUnion = ArrayOfDoublesSketches.wrapUnion(wmem); + final MemorySegment wseg = MemorySegment.ofArray(union.toByteArray()); + final ArrayOfDoublesUnion wrappedUnion = ArrayOfDoublesSketches.wrapUnion(wseg); result = wrappedUnion.getResult(); Assert.assertEquals(result.getEstimate(), 3.0); values = result.getValues(); @@ -181,7 +188,7 @@ public void heapSerializeDeserialize() { union1.union(sketch1); union1.union(sketch2); - final ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.heapify(Memory.wrap(union1.toByteArray())); + final ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.heapify(MemorySegment.ofArray(union1.toByteArray())); ArrayOfDoublesCompactSketch result = union2.getResult(); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); @@ -218,7 +225,7 @@ public void heapSerializeDeserializeWithSeed() { union1.union(sketch1); union1.union(sketch2); - final ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.heapify(Memory.wrap(union1.toByteArray()), seed); + final ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.heapify(MemorySegment.ofArray(union1.toByteArray()), seed); final ArrayOfDoublesCompactSketch result = union2.getResult(); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); } @@ -227,25 +234,25 @@ public void heapSerializeDeserializeWithSeed() { public void directSerializeDeserialize() { int key = 0; final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build( - WritableMemory.writableWrap(new byte[1000000])); + MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 4096; // overlap half of the entries final ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build( - WritableMemory.writableWrap(new byte[1000000])); + MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } final ArrayOfDoublesUnion union1 = new ArrayOfDoublesSetOperationBuilder().buildUnion( - WritableMemory.writableWrap(new byte[1000000])); + MemorySegment.ofArray(new byte[1000000])); union1.union(sketch1); union1.union(sketch2); - final ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.wrap(WritableMemory.writableWrap(union1.toByteArray())); - ArrayOfDoublesCompactSketch result = union2.getResult(WritableMemory.writableWrap(new byte[1000000])); + final ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.wrap(MemorySegment.ofArray(union1.toByteArray())); + ArrayOfDoublesCompactSketch result = union2.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); union2.reset(); @@ -267,39 +274,39 @@ public void directSerializeDeserializeWithSeed() { final long seed = 1; int key = 0; final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed) - .build(WritableMemory.writableWrap(new byte[1000000])); + .build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 4096; // overlap half of the entries final ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().setSeed(seed) - .build(WritableMemory.writableWrap(new byte[1000000])); + .build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } final ArrayOfDoublesUnion union1 = new ArrayOfDoublesSetOperationBuilder().setSeed(seed) - .buildUnion(WritableMemory.writableWrap(new byte[1000000])); + .buildUnion(MemorySegment.ofArray(new byte[1000000])); union1.union(sketch1); union1.union(sketch2); - final ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.wrap(WritableMemory.writableWrap(union1.toByteArray()), seed); - final ArrayOfDoublesCompactSketch result = union2.getResult(WritableMemory.writableWrap(new byte[1000000])); + final ArrayOfDoublesUnion union2 = ArrayOfDoublesUnion.wrap(MemorySegment.ofArray(union1.toByteArray()), seed); + final ArrayOfDoublesCompactSketch result = union2.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); } @Test public void directExactMode() { final ArrayOfDoublesUpdatableSketch sketch1 = - new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); sketch1.update(1, new double[] {1.0}); sketch1.update(1, new double[] {1.0}); sketch1.update(1, new double[] {1.0}); sketch1.update(2, new double[] {1.0}); final ArrayOfDoublesUpdatableSketch sketch2 = - new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); sketch2.update(2, new double[] {1.0}); sketch2.update(2, new double[] {1.0}); sketch2.update(3, new double[] {1.0}); @@ -307,10 +314,10 @@ public void directExactMode() { sketch2.update(3, new double[] {1.0}); final ArrayOfDoublesUnion union = - new ArrayOfDoublesSetOperationBuilder().buildUnion(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesSetOperationBuilder().buildUnion(MemorySegment.ofArray(new byte[1000000])); union.union(sketch1); union.union(sketch2); - ArrayOfDoublesCompactSketch result = union.getResult(WritableMemory.writableWrap(new byte[1000000])); + ArrayOfDoublesCompactSketch result = union.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertEquals(result.getEstimate(), 3.0); final double[][] values = result.getValues(); Assert.assertEquals(values[0][0], 3.0); @@ -331,23 +338,23 @@ public void directExactMode() { public void directEstimationMode() { int key = 0; final ArrayOfDoublesUpdatableSketch sketch1 = - new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch1.update(key++, new double[] {1.0}); } key -= 4096; // overlap half of the entries final ArrayOfDoublesUpdatableSketch sketch2 = - new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { sketch2.update(key++, new double[] {1.0}); } final ArrayOfDoublesUnion union = - new ArrayOfDoublesSetOperationBuilder().buildUnion(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesSetOperationBuilder().buildUnion(MemorySegment.ofArray(new byte[1000000])); union.union(sketch1); union.union(sketch2); - ArrayOfDoublesCompactSketch result = union.getResult(WritableMemory.writableWrap(new byte[1000000])); + ArrayOfDoublesCompactSketch result = union.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertEquals(result.getEstimate(), 12288.0, 12288 * 0.01); union.reset(); @@ -379,10 +386,10 @@ public void heapToDirect() { heapUnion.union(sketch1); final ArrayOfDoublesUnion directUnion = - ArrayOfDoublesUnion.wrap(WritableMemory.writableWrap(heapUnion.toByteArray())); - directUnion.union(sketch2); + ArrayOfDoublesUnion.wrap(MemorySegment.ofArray(heapUnion.toByteArray())); + directUnion.union(sketch2); //throws - final ArrayOfDoublesCompactSketch result = directUnion.getResult(WritableMemory.writableWrap(new byte[1000000])); + final ArrayOfDoublesCompactSketch result = directUnion.getResult(MemorySegment.ofArray(new byte[1000000])); Assert.assertFalse(result.isEmpty()); Assert.assertEquals(result.getEstimate(), 3.0); final double[][] values = result.getValues(); @@ -408,10 +415,10 @@ public void directToHeap() { sketch2.update(3, new double[] {1.0}); final ArrayOfDoublesUnion directUnion = - new ArrayOfDoublesSetOperationBuilder().buildUnion(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesSetOperationBuilder().buildUnion(MemorySegment.ofArray(new byte[1000000])); directUnion.union(sketch1); - final ArrayOfDoublesUnion heapUnion = ArrayOfDoublesUnion.heapify(Memory.wrap(directUnion.toByteArray())); + final ArrayOfDoublesUnion heapUnion = ArrayOfDoublesUnion.heapify(MemorySegment.ofArray(directUnion.toByteArray())); heapUnion.union(sketch2); final ArrayOfDoublesCompactSketch result = heapUnion.getResult(); @@ -448,8 +455,8 @@ public void incompatibleInputSketchMoreValues() { @Test public void directDruidUsageOneSketch() { - final WritableMemory mem = WritableMemory.writableWrap(new byte[1_000_000]); - new ArrayOfDoublesSetOperationBuilder().buildUnion(mem); // just set up memory to wrap later + final MemorySegment seg = MemorySegment.ofArray(new byte[1_000_000]); + new ArrayOfDoublesSetOperationBuilder().buildUnion(seg); // just set up MemorySegment to wrap later final int n = 100_000; // estimation mode final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().build(); @@ -458,22 +465,22 @@ public void directDruidUsageOneSketch() { } sketch.trim(); // pretend this is a result from a union - // as Druid wraps memory - WritableMemory mem2 = WritableMemory.writableWrap(new byte[1_000_000]); - ArrayOfDoublesCompactSketch dcsk = sketch.compact(mem2); - ArrayOfDoublesUnion union = ArrayOfDoublesSketches.wrapUnion(mem); //empty union + // as Druid wraps MemorySegment + MemorySegment seg2 = MemorySegment.ofArray(new byte[1_000_000]); + ArrayOfDoublesCompactSketch dcsk = sketch.compact(seg2); + ArrayOfDoublesUnion union = ArrayOfDoublesSketches.wrapUnion(seg); //empty union union.union(dcsk); - //ArrayOfDoublesSketches.wrapUnion(mem).union(sketch.compact(WritableMemory.writableWrap(new byte[1_000_000]))); + //ArrayOfDoublesSketches.wrapUnion(seg).union(sketch.compact(MemorySegment.ofArray(new byte[1_000_000]))); - final ArrayOfDoublesSketch result = ArrayOfDoublesUnion.wrap(mem).getResult(); + final ArrayOfDoublesSketch result = ArrayOfDoublesUnion.wrap(seg).getResult(); Assert.assertEquals(result.getEstimate(), sketch.getEstimate());//expected [98045.91060164096] but found [4096.0] Assert.assertEquals(result.isEstimationMode(), sketch.isEstimationMode()); } @Test public void directDruidUsageTwoSketches() { - final WritableMemory mem = WritableMemory.writableWrap(new byte[1000000]); - new ArrayOfDoublesSetOperationBuilder().buildUnion(mem); // just set up memory to wrap later + final MemorySegment seg = MemorySegment.ofArray(new byte[1000000]); + new ArrayOfDoublesSetOperationBuilder().buildUnion(seg); // just set up MemorySegment to wrap later int key = 0; @@ -482,16 +489,16 @@ public void directDruidUsageTwoSketches() { for (int i = 0; i < n1; i++) { sketch1.update(key++, new double[] {1.0}); } - // as Druid wraps memory - ArrayOfDoublesSketches.wrapUnion(mem).union(sketch1.compact(WritableMemory.writableWrap(new byte[1000000]))); + // as Druid wraps MemorySegment + ArrayOfDoublesSketches.wrapUnion(seg).union(sketch1.compact(MemorySegment.ofArray(new byte[1000000]))); final int n2 = 1000000; // estimation mode final ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build(); for (int i = 0; i < n2; i++) { sketch2.update(key++, new double[] {1.0}); } - // as Druid wraps memory - ArrayOfDoublesSketches.wrapUnion(mem).union(sketch2.compact(WritableMemory.writableWrap(new byte[1000000]))); + // as Druid wraps MemorySegment + ArrayOfDoublesSketches.wrapUnion(seg).union(sketch2.compact(MemorySegment.ofArray(new byte[1000000]))); // build one sketch that must be the same as union key = 0; // reset to have the same keys @@ -502,7 +509,7 @@ public void directDruidUsageTwoSketches() { } expected.trim(); // union result is trimmed, so we need to trim this sketch for valid comparison - final ArrayOfDoublesSketch result = ArrayOfDoublesUnion.wrap(mem).getResult(); + final ArrayOfDoublesSketch result = ArrayOfDoublesUnion.wrap(seg).getResult(); Assert.assertEquals(result.getEstimate(), expected.getEstimate()); Assert.assertEquals(result.isEstimationMode(), expected.isEstimationMode()); Assert.assertEquals(result.getUpperBound(1), expected.getUpperBound(1)); diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/CornerCaseArrayOfDoublesSetOperationsTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/CornerCaseArrayOfDoublesSetOperationsTest.java index 635a41c64..f1ede92e6 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/CornerCaseArrayOfDoublesSetOperationsTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/CornerCaseArrayOfDoublesSetOperationsTest.java @@ -23,7 +23,15 @@ import static org.apache.datasketches.hash.MurmurHash3.hash; import static org.testng.Assert.assertTrue; -import org.apache.datasketches.thetacommon.ThetaUtil; +import org.apache.datasketches.common.Util; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesAnotB; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesCombiner; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesCompactSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesIntersection; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSetOperationBuilder; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUnion; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; import org.testng.annotations.Test; public class CornerCaseArrayOfDoublesSetOperationsTest { @@ -551,13 +559,13 @@ public void printTable() { println(LS + "Longs"); for (long v = 1L; v < 10; v++) { - long hash = (hash(v, ThetaUtil.DEFAULT_UPDATE_SEED)[0]) >>> 1; + long hash = (hash(v, Util.DEFAULT_UPDATE_SEED)[0]) >>> 1; printLong(v, hash); } } static long getLongHash(long v) { - return (hash(v, ThetaUtil.DEFAULT_UPDATE_SEED)[0]) >>> 1; + return (hash(v, Util.DEFAULT_UPDATE_SEED)[0]) >>> 1; } static void printLong(long v, long hash) { diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketchTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketchTest.java index b86a6839f..4cf9693ca 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesCompactSketchTest.java @@ -19,10 +19,16 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableMemory; -import org.apache.datasketches.thetacommon.ThetaUtil; -import org.apache.datasketches.tuple.Util; +import org.apache.datasketches.common.Util; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesCompactSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; import org.testng.Assert; import org.testng.annotations.Test; @@ -31,8 +37,8 @@ public class DirectArrayOfDoublesCompactSketchTest { @Test public void emptyFromQuickSelectSketch() { ArrayOfDoublesUpdatableSketch us = - new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); - ArrayOfDoublesCompactSketch sketch = us.compact(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); + ArrayOfDoublesCompactSketch sketch = us.compact(MemorySegment.ofArray(new byte[1000000])); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); @@ -52,14 +58,14 @@ public void emptyFromQuickSelectSketch() { @Test public void exactModeFromQuickSelectSketch() { ArrayOfDoublesUpdatableSketch us = - new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); us.update(1, new double[] {1.0}); us.update(2, new double[] {1.0}); us.update(3, new double[] {1.0}); us.update(1, new double[] {1.0}); us.update(2, new double[] {1.0}); us.update(3, new double[] {1.0}); - ArrayOfDoublesCompactSketch sketch = us.compact(WritableMemory.writableWrap(new byte[1000000])); + ArrayOfDoublesCompactSketch sketch = us.compact(MemorySegment.ofArray(new byte[1000000])); Assert.assertFalse(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 3.0); @@ -68,7 +74,7 @@ public void exactModeFromQuickSelectSketch() { Assert.assertEquals(sketch.getRetainedEntries(), 3); Assert.assertEquals(sketch.getThetaLong(), Long.MAX_VALUE); Assert.assertEquals(sketch.getTheta(), 1.0); - Assert.assertEquals(sketch.getSeedHash(), Util.computeSeedHash(ThetaUtil.DEFAULT_UPDATE_SEED)); + Assert.assertEquals(sketch.getSeedHash(), Util.computeSeedHash(Util.DEFAULT_UPDATE_SEED)); double[][] values = sketch.getValues(); Assert.assertEquals(values.length, 3); for (double[] array: values) { @@ -78,12 +84,12 @@ public void exactModeFromQuickSelectSketch() { @Test public void serializeDeserializeSmallExact() { - ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); us.update("a", new double[] {1.0}); us.update("b", new double[] {1.0}); us.update("c", new double[] {1.0}); - ArrayOfDoublesCompactSketch sketch1 = us.compact(WritableMemory.writableWrap(new byte[1000000])); - ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.wrapSketch(WritableMemory.writableWrap(sketch1.toByteArray())); + ArrayOfDoublesCompactSketch sketch1 = us.compact(MemorySegment.ofArray(new byte[1000000])); + ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.wrapSketch(MemorySegment.ofArray(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertFalse(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 3.0); @@ -101,12 +107,12 @@ public void serializeDeserializeSmallExact() { @Test public void serializeDeserializeEstimation() { - ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } - ArrayOfDoublesCompactSketch sketch1 = us.compact(WritableMemory.writableWrap(new byte[1000000])); - ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.wrapSketch(WritableMemory.writableWrap(sketch1.toByteArray())); + ArrayOfDoublesCompactSketch sketch1 = us.compact(MemorySegment.ofArray(new byte[1000000])); + ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketches.wrapSketch(MemorySegment.ofArray(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), sketch1.getEstimate()); @@ -115,19 +121,19 @@ public void serializeDeserializeEstimation() { @Test(expectedExceptions = SketchesArgumentException.class) public void deserializeWithWrongSeed() { - ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } - ArrayOfDoublesCompactSketch sketch1 = us.compact(WritableMemory.writableWrap(new byte[1000000])); - ArrayOfDoublesSketches.wrapSketch(WritableMemory.writableWrap(sketch1.toByteArray()), 123); + ArrayOfDoublesCompactSketch sketch1 = us.compact(MemorySegment.ofArray(new byte[1000000])); + ArrayOfDoublesSketches.wrapSketch(MemorySegment.ofArray(sketch1.toByteArray()), 123); } @Test(expectedExceptions = SketchesArgumentException.class) - public void fromQuickSelectSketchNotEnoughMemory() { - ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(WritableMemory.writableWrap(new byte[1000000])); + public void fromQuickSelectSketchNotEnoughMemorySegment() { + ArrayOfDoublesUpdatableSketch us = new ArrayOfDoublesUpdatableSketchBuilder().build(MemorySegment.ofArray(new byte[1000000])); us.update(1, new double[] {1.0}); - us.compact(WritableMemory.writableWrap(new byte[39])); + us.compact(MemorySegment.ofArray(new byte[39])); } } diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketchTest.java index a8e066f4d..781f3893b 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/DirectArrayOfDoublesQuickSelectSketchTest.java @@ -19,11 +19,16 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; +import org.apache.datasketches.tuple.arrayofdoubles.DirectArrayOfDoublesQuickSelectSketch; import org.testng.Assert; import org.testng.annotations.Test; @@ -32,7 +37,7 @@ public class DirectArrayOfDoublesQuickSelectSketchTest { public void isEmpty() { final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder(). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); Assert.assertTrue(sketch.isEmpty()); Assert.assertFalse(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 0.0); @@ -52,7 +57,7 @@ public void isEmptyWithSampling() { final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder(). setSamplingProbability(samplingProbability). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); Assert.assertTrue(sketch.isEmpty()); Assert.assertTrue(((DirectArrayOfDoublesQuickSelectSketch)sketch).isInSamplingMode()); Assert.assertFalse(sketch.isEstimationMode()); @@ -72,7 +77,7 @@ public void sampling() { final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder(). setSamplingProbability(samplingProbability). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); sketch.update("a", new double[] {1.0}); Assert.assertFalse(sketch.isEmpty()); Assert.assertTrue(sketch.isEstimationMode()); @@ -88,7 +93,7 @@ public void sampling() { public void exactMode() { final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder(). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); Assert.assertTrue(sketch.isEmpty()); Assert.assertEquals(sketch.getEstimate(), 0.0); for (int i = 0; i < 4096; i++) { @@ -136,7 +141,7 @@ public void exactMode() { public void estimationMode() { final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder(). - build(WritableMemory.writableWrap(new byte[4096 * 2 * 16 + 32])); + build(MemorySegment.ofArray(new byte[4096 * 2 * 16 + 32])); Assert.assertEquals(sketch.getEstimate(), 0.0); for (int i = 1; i <= 8192; i++) { sketch.update(i, new double[] {1.0}); @@ -177,7 +182,7 @@ public void estimationMode() { public void updatesOfAllKeyTypes() { final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder(). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); sketch.update(1L, new double[] {1.0}); sketch.update(2.0, new double[] {1.0}); final byte[] bytes = new byte[] {3, 4}; @@ -194,7 +199,7 @@ public void updatesOfAllKeyTypes() { public void doubleSum() { final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder(). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); sketch.update(1, new double[] {1.0}); Assert.assertEquals(sketch.getRetainedEntries(), 1); Assert.assertEquals(sketch.getValues()[0][0], 1.0); @@ -210,10 +215,10 @@ public void doubleSum() { public void serializeDeserializeExact() throws Exception { final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder(). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); sketch1.update(1, new double[] {1.0}); - final ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.wrap(WritableMemory.writableWrap(sketch1.toByteArray())); + final ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.wrap(MemorySegment.ofArray(sketch1.toByteArray())); Assert.assertEquals(sketch2.getEstimate(), 1.0); final double[][] values = sketch2.getValues(); @@ -232,7 +237,7 @@ public void serializeDeserializeExact() throws Exception { public void serializeDeserializeEstimationNoResize() throws Exception { final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().setResizeFactor(ResizeFactor.X1). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); for (int j = 0; j < 10; j++) { for (int i = 0; i < 8192; i++) { sketch1.update(i, new double[] {1.0}); @@ -243,7 +248,7 @@ public void serializeDeserializeEstimationNoResize() throws Exception { //for visual testing //TestUtil.writeBytesToFile(byteArray, "ArrayOfDoublesQuickSelectSketch4K.data"); - final ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketch.wrap(WritableMemory.writableWrap(byteArray)); + final ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(byteArray)); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 8192, 8192 * 0.99); Assert.assertEquals(sketch1.getTheta(), sketch2.getTheta()); @@ -261,12 +266,12 @@ public void serializeDeserializeSampling() { final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder(). setNominalEntries(sketchSize).setSamplingProbability(0.5f). - build(WritableMemory.writableWrap(new byte[1000000])); + build(MemorySegment.ofArray(new byte[1000000])); for (int i = 0; i < numberOfUniques; i++) { sketch1.update(i, new double[] {1.0}); } final ArrayOfDoublesSketch sketch2 = - ArrayOfDoublesSketch.wrap(WritableMemory.writableWrap(sketch1.toByteArray())); + ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(sketch1.toByteArray())); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate() / numberOfUniques, 1.0, 0.01); Assert.assertEquals(sketch2.getRetainedEntries() / (double) numberOfUniques, 0.5, 0.01); @@ -274,8 +279,8 @@ public void serializeDeserializeSampling() { } @Test(expectedExceptions = SketchesArgumentException.class) - public void notEnoughMemory() { + public void memorySegmentNotLargeEnough() { new ArrayOfDoublesUpdatableSketchBuilder(). - setNominalEntries(32).build(WritableMemory.writableWrap(new byte[1055])); + setNominalEntries(32).build(MemorySegment.ofArray(new byte[1055])); } } diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketchTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketchTest.java index 055914d3a..0583c139f 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesCompactSketchTest.java @@ -19,9 +19,15 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesCompactSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; import org.testng.Assert; import org.testng.annotations.Test; @@ -80,7 +86,7 @@ public void serializeDeserializeSmallExact() { us.update("c", new double[] {1.0}); ArrayOfDoublesCompactSketch sketch1 = us.compact(); ArrayOfDoublesSketch sketch2 = - ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray())); + ArrayOfDoublesSketches.heapifySketch(MemorySegment.ofArray(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertFalse(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 3.0); @@ -102,14 +108,14 @@ public void serializeDeserializeEstimation() { for (int i = 0; i < 8192; i++) { us.update(i, new double[] {1.0}); } - WritableMemory wmem = WritableMemory.writableWrap(us.toByteArray()); - ArrayOfDoublesUpdatableSketch wrappedUS = ArrayOfDoublesSketches.wrapUpdatableSketch(wmem); + MemorySegment wseg = MemorySegment.ofArray(us.toByteArray()); + ArrayOfDoublesUpdatableSketch wrappedUS = ArrayOfDoublesSketches.wrapUpdatableSketch(wseg); Assert.assertFalse(wrappedUS.isEmpty()); Assert.assertTrue(wrappedUS.isEstimationMode()); Assert.assertEquals(wrappedUS.getEstimate(), us.getEstimate()); Assert.assertEquals(wrappedUS.getThetaLong(), us.getThetaLong()); - ArrayOfDoublesUpdatableSketch heapUS = ArrayOfDoublesSketches.heapifyUpdatableSketch(wmem); + ArrayOfDoublesUpdatableSketch heapUS = ArrayOfDoublesSketches.heapifyUpdatableSketch(wseg); Assert.assertFalse(heapUS.isEmpty()); Assert.assertTrue(heapUS.isEstimationMode()); Assert.assertEquals(heapUS.getEstimate(), us.getEstimate()); @@ -117,7 +123,7 @@ public void serializeDeserializeEstimation() { ArrayOfDoublesCompactSketch sketch1 = us.compact(); ArrayOfDoublesSketch sketch2 = - ArrayOfDoublesSketches.heapifySketch(Memory.wrap(sketch1.toByteArray())); + ArrayOfDoublesSketches.heapifySketch(MemorySegment.ofArray(sketch1.toByteArray())); Assert.assertFalse(sketch2.isEmpty()); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), sketch1.getEstimate()); @@ -131,7 +137,7 @@ public void deserializeWithWrongSeed() { us.update(i, new double[] {1.0}); } ArrayOfDoublesCompactSketch sketch1 = us.compact(); - Memory mem = Memory.wrap(sketch1.toByteArray()); - ArrayOfDoublesSketches.heapifySketch(mem, 123); + MemorySegment seg = MemorySegment.ofArray(sketch1.toByteArray()); + ArrayOfDoublesSketches.heapifySketch(seg, 123); } } diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketchTest.java index 15de291ee..9fe991b36 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/HeapArrayOfDoublesQuickSelectSketchTest.java @@ -19,9 +19,13 @@ package org.apache.datasketches.tuple.arrayofdoubles; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.common.ResizeFactor; -import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableMemory; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch; +import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder; import org.testng.Assert; import org.testng.annotations.Test; @@ -186,7 +190,7 @@ public void serializeDeserializeExact() { final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build(); sketch1.update(1, new double[] {1.0}); - final ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.heapify(WritableMemory.writableWrap(sketch1.toByteArray())); + final ArrayOfDoublesUpdatableSketch sketch2 = ArrayOfDoublesUpdatableSketch.heapify(MemorySegment.ofArray(sketch1.toByteArray())); Assert.assertEquals(sketch2.getEstimate(), 1.0); final double[][] values = sketch2.getValues(); @@ -215,7 +219,7 @@ public void serializeDeserializeEstimationNoResize() throws Exception { //for visual testing //TestUtil.writeBytesToFile(byteArray, "ArrayOfDoublesQuickSelectSketch4K.data"); - final ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketch.heapify(Memory.wrap(byteArray)); + final ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketch.heapify(MemorySegment.ofArray(byteArray)); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate(), 8192, 8192 * 0.99); Assert.assertEquals(sketch1.getTheta(), sketch2.getTheta()); @@ -234,7 +238,7 @@ public void serializeDeserializeSampling() { for (int i = 0; i < numberOfUniques; i++) { sketch1.update(i, new double[] {1.0}); } - final ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketch.heapify(Memory.wrap(sketch1.toByteArray())); + final ArrayOfDoublesSketch sketch2 = ArrayOfDoublesSketch.heapify(MemorySegment.ofArray(sketch1.toByteArray())); Assert.assertTrue(sketch2.isEstimationMode()); Assert.assertEquals(sketch2.getEstimate() / numberOfUniques, 1.0, 0.01); Assert.assertEquals(sketch2.getRetainedEntries() / (double) numberOfUniques, 0.5, 0.01); diff --git a/src/test/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketchTest.java b/src/test/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketchTest.java index 803c2f591..7287c3429 100644 --- a/src/test/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketchTest.java +++ b/src/test/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSketchTest.java @@ -22,12 +22,16 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import org.apache.datasketches.memory.WritableMemory; +import java.lang.foreign.MemorySegment; + import org.apache.datasketches.tuple.AnotB; import org.apache.datasketches.tuple.CompactSketch; import org.apache.datasketches.tuple.Intersection; import org.apache.datasketches.tuple.TupleSketchIterator; import org.apache.datasketches.tuple.Union; +import org.apache.datasketches.tuple.strings.ArrayOfStringsSketch; +import org.apache.datasketches.tuple.strings.ArrayOfStringsSummary; +import org.apache.datasketches.tuple.strings.ArrayOfStringsSummarySetOperations; import org.testng.annotations.Test; /** @@ -48,8 +52,8 @@ public void checkSketch() { sketch1.update(strArrArr[0], strArrArr[0]); //insert duplicate printSummaries(sketch1.iterator()); byte[] array = sketch1.toByteArray(); - WritableMemory wmem = WritableMemory.writableWrap(array); - ArrayOfStringsSketch sketch2 = new ArrayOfStringsSketch(wmem); + MemorySegment wseg = MemorySegment.ofArray(array); + ArrayOfStringsSketch sketch2 = new ArrayOfStringsSketch(wseg); printSummaries(sketch2.iterator()); checkSummaries(sketch2, sketch2); diff --git a/src/test/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryTest.java b/src/test/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryTest.java index 99cc2bc9d..1b1d94bed 100644 --- a/src/test/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryTest.java +++ b/src/test/java/org/apache/datasketches/tuple/strings/ArrayOfStringsSummaryTest.java @@ -23,10 +23,13 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import java.lang.foreign.MemorySegment; + import org.testng.annotations.Test; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.memory.Memory; import org.apache.datasketches.tuple.DeserializeResult; +import org.apache.datasketches.tuple.strings.ArrayOfStringsSummary; +import org.apache.datasketches.tuple.strings.ArrayOfStringsSummaryDeserializer; /** * @author Lee Rhodes @@ -41,15 +44,15 @@ public void checkToByteArray() { assertTrue(copy.equals(nsum)); byte[] out = nsum.toByteArray(); - Memory mem = Memory.wrap(out); - ArrayOfStringsSummary nsum2 = new ArrayOfStringsSummary(mem); + MemorySegment seg = MemorySegment.ofArray(out); + ArrayOfStringsSummary nsum2 = new ArrayOfStringsSummary(seg); String[] nodesArr = nsum2.getValue(); for (String s : nodesArr) { println(s); } - println("\nfromMemory(mem)"); - DeserializeResult dres = ArrayOfStringsSummaryDeserializer.fromMemory(mem); + println("\nfromMemorySegment(seg)"); + DeserializeResult dres = ArrayOfStringsSummaryDeserializer.fromMemorySegment(seg); ArrayOfStringsSummary nsum3 = dres.getObject(); nodesArr = nsum3.getValue(); for (String s : nodesArr) { @@ -64,8 +67,8 @@ public void checkNumNodes() { @Test(expectedExceptions = SketchesArgumentException.class) public void checkInBytes() { - Memory mem = Memory.wrap(new byte[100]); - ArrayOfStringsSummary.checkInBytes(mem, 200); + MemorySegment seg = MemorySegment.ofArray(new byte[100]); + ArrayOfStringsSummary.checkInBytes(seg, 200); } @SuppressWarnings("unlikely-arg-type")