From c787aed1e9f1cbcccb6ee9d6f481065c95b47937 Mon Sep 17 00:00:00 2001 From: "Jin,Zhonghui" Date: Thu, 2 Nov 2023 23:39:13 +0800 Subject: [PATCH 1/6] [Backport] 8264759: x86_32 Minimal VM build failure after JDK-8262355 Summary: 8264759: x86_32 Minimal VM build failure after JDK-8262355 Test Plan: ci jtreg Reviewed-by: JoshuaZhuwj Issue: https://github.com/dragonwell-project/dragonwell11/issues/600 --- src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp index 93f5fad63b2..8242d8e3cc2 100644 --- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp +++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp @@ -217,6 +217,7 @@ OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_ } } +#ifdef COMPILER2 if (save_vectors) { __ subptr(rsp, ymm_bytes); // Save upper half of YMM registers @@ -236,6 +237,10 @@ OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_ } } } +#else + assert(!save_vectors, "vectors are generated only by C2"); +#endif + __ vzeroupper(); // Set an oopmap for the call site. This oopmap will map all From 4f7bcb3f9437c588885f8b4d6a2d09d1d3d1a922 Mon Sep 17 00:00:00 2001 From: "Jin,Zhonghui" Date: Fri, 3 Nov 2023 00:00:39 +0800 Subject: [PATCH 2/6] [Backport] 8264104: Eliminate unnecessary vector mask conversion during VectorUnbox for floating point VectorMask Summary: 8264104: Eliminate unnecessary vector mask conversion during VectorUnbox for floating point VectorMask without new files in src/cpu/aarch64/ Test Plan: ci jtreg Reviewed-by: JoshuaZhuwj Issue: https://github.com/dragonwell-project/dragonwell11/issues/602 --- src/hotspot/share/adlc/formssel.cpp | 3 ++- src/hotspot/share/opto/classes.hpp | 1 + src/hotspot/share/opto/vectornode.cpp | 7 +++++++ src/hotspot/share/opto/vectornode.hpp | 11 +++++++++++ src/hotspot/share/runtime/vmStructs.cpp | 1 + 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp index 6363448dec8..64277008d8b 100644 --- a/src/hotspot/share/adlc/formssel.cpp +++ b/src/hotspot/share/adlc/formssel.cpp @@ -4201,7 +4201,8 @@ bool MatchRule::is_vector() const { "FmaVD", "FmaVF","PopCountVI", // Next are not supported currently. "PackB","PackS","PackI","PackL","PackF","PackD","Pack2L","Pack2D", - "ExtractB","ExtractUB","ExtractC","ExtractS","ExtractI","ExtractL","ExtractF","ExtractD" + "ExtractB","ExtractUB","ExtractC","ExtractS","ExtractI","ExtractL","ExtractF","ExtractD", + "VectorMaskCast" }; int cnt = sizeof(vector_list)/sizeof(char*); if (_rChild) { diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 37aecbf0899..47182f66d70 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -430,6 +430,7 @@ macro(VectorBoxAllocate) macro(VectorUnbox) macro(VectorMaskWrapper) macro(VectorMaskCmp) +macro(VectorMaskCast) macro(VectorTest) macro(VectorBlend) macro(VectorRearrange) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index c9fbe40401c..9975f2ab6e0 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -1095,6 +1095,13 @@ Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) { bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); if (is_vector_mask) { + if (in_vt->length_in_bytes() == out_vt->length_in_bytes() && + Matcher::match_rule_supported_vector(Op_VectorMaskCast, out_vt->length(), out_vt->element_basic_type())) { + // Apply "VectorUnbox (VectorBox vmask) ==> VectorMaskCast (vmask)" + // directly. This could avoid the transformation ordering issue from + // "VectorStoreMask (VectorLoadMask vmask) => vmask". + return new VectorMaskCastNode(value, out_vt); + } // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask) value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length())); return new VectorLoadMaskNode(value, out_vt); diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index d2fd269616d..4362bf7f4e5 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1215,6 +1215,17 @@ class VectorStoreMaskNode : public VectorNode { static VectorStoreMaskNode* make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem); }; +class VectorMaskCastNode : public VectorNode { + public: + VectorMaskCastNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) { + const TypeVect* in_vt = in->bottom_type()->is_vect(); + assert(in_vt->length() == vt->length(), "vector length must match"); + assert(type2aelembytes(in_vt->element_basic_type()) == type2aelembytes(vt->element_basic_type()), "element size must match"); + } + + virtual int Opcode() const; +}; + // This is intended for use as a simple reinterpret node that has no cast. class VectorReinterpretNode : public VectorNode { private: diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp index 480fc9be3c3..4b38536869f 100644 --- a/src/hotspot/share/runtime/vmStructs.cpp +++ b/src/hotspot/share/runtime/vmStructs.cpp @@ -1924,6 +1924,7 @@ typedef PaddedEnd PaddedObjectMonitor; declare_c2_type(VectorInsertNode, VectorNode) \ declare_c2_type(VectorUnboxNode, VectorNode) \ declare_c2_type(VectorReinterpretNode, VectorNode) \ + declare_c2_type(VectorMaskCastNode, VectorNode) \ declare_c2_type(VectorBoxNode, Node) \ declare_c2_type(VectorBoxAllocateNode, CallStaticJavaNode) \ declare_c2_type(VectorTestNode, Node) \ From bda3c23af845d3abe38b81c37f275fc836cb539c Mon Sep 17 00:00:00 2001 From: "Jin,Zhonghui" Date: Fri, 3 Nov 2023 00:09:40 +0800 Subject: [PATCH 3/6] [Backport] 8265816: Handle new VectorMaskCast node for x86 Summary: 8265816: Handle new VectorMaskCast node for x86 Test Plan: ci jtreg Reviewed-by: JoshuaZhuwj Issue: https://github.com/dragonwell-project/dragonwell11/issues/597 --- src/hotspot/cpu/x86/x86.ad | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 3327b8497d5..6909fb9c9d8 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -7589,6 +7589,18 @@ instruct vstoreMask8B_evex(vec dst, vec src, immI_8 size) %{ ins_pipe( pipe_slow ); %} +instruct vmaskcast(vec dst) %{ + predicate((vector_length(n) == vector_length(n->in(1))) && + (vector_length_in_bytes(n) == vector_length_in_bytes(n->in(1)))); + match(Set dst (VectorMaskCast dst)); + ins_cost(0); + format %{ "vector_mask_cast $dst" %} + ins_encode %{ + // empty + %} + ins_pipe(empty); +%} + //-------------------------------- Load Iota Indices ---------------------------------- instruct loadIotaIndices(vec dst, immI_0 src, rRegP scratch) %{ From df1dd5904240defdff8469fbaa5f5e68bb303907 Mon Sep 17 00:00:00 2001 From: "Jin,Zhonghui" Date: Fri, 3 Nov 2023 00:28:34 +0800 Subject: [PATCH 4/6] [Backport] 8266180: compiler/vectorapi/TestVectorErgonomics should be run in driver mode Summary: 8266180: compiler/vectorapi/TestVectorErgonomics should be run in driver mode Test Plan: ci jtreg Reviewed-by: JoshuaZhuwj Issue: https://github.com/dragonwell-project/dragonwell11/issues/594 --- test/hotspot/jtreg/compiler/vectorapi/TestVectorErgonomics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hotspot/jtreg/compiler/vectorapi/TestVectorErgonomics.java b/test/hotspot/jtreg/compiler/vectorapi/TestVectorErgonomics.java index 594098e12bf..eb25472370f 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/TestVectorErgonomics.java +++ b/test/hotspot/jtreg/compiler/vectorapi/TestVectorErgonomics.java @@ -29,7 +29,7 @@ * @requires vm.compiler2.enabled * @summary Check ergonomics for Vector API * @library /test/lib - * @run main/othervm compiler.vectorapi.TestVectorErgonomics + * @run driver compiler.vectorapi.TestVectorErgonomics */ import jdk.test.lib.process.ProcessTools; From c275318ab11ca61da0fa87b6951dcf56d3095c49 Mon Sep 17 00:00:00 2001 From: "Jin,Zhonghui" Date: Fri, 3 Nov 2023 00:58:26 +0800 Subject: [PATCH 5/6] [Backport] 8265128: [REDO] Optimize Vector API slice and unslice operations Summary: 8265128: [REDO] Optimize Vector API slice and unslice operations Test Plan: ci jtreg Reviewed-by: JoshuaZhuwj Issue: https://github.com/dragonwell-project/dragonwell11/issues/598 --- .../jdk/incubator/vector/Byte128Vector.java | 20 +------ .../jdk/incubator/vector/Byte256Vector.java | 20 +------ .../jdk/incubator/vector/Byte512Vector.java | 20 +------ .../jdk/incubator/vector/Byte64Vector.java | 20 +------ .../jdk/incubator/vector/ByteMaxVector.java | 20 +------ .../jdk/incubator/vector/ByteVector.java | 58 +++++++++++-------- .../jdk/incubator/vector/Double128Vector.java | 20 +------ .../jdk/incubator/vector/Double256Vector.java | 20 +------ .../jdk/incubator/vector/Double512Vector.java | 20 +------ .../jdk/incubator/vector/Double64Vector.java | 20 +------ .../jdk/incubator/vector/DoubleMaxVector.java | 20 +------ .../jdk/incubator/vector/DoubleVector.java | 58 +++++++++++-------- .../jdk/incubator/vector/Float128Vector.java | 20 +------ .../jdk/incubator/vector/Float256Vector.java | 20 +------ .../jdk/incubator/vector/Float512Vector.java | 20 +------ .../jdk/incubator/vector/Float64Vector.java | 20 +------ .../jdk/incubator/vector/FloatMaxVector.java | 20 +------ .../jdk/incubator/vector/FloatVector.java | 58 +++++++++++-------- .../jdk/incubator/vector/Int128Vector.java | 20 +------ .../jdk/incubator/vector/Int256Vector.java | 20 +------ .../jdk/incubator/vector/Int512Vector.java | 20 +------ .../jdk/incubator/vector/Int64Vector.java | 20 +------ .../jdk/incubator/vector/IntMaxVector.java | 20 +------ .../jdk/incubator/vector/IntVector.java | 58 +++++++++++-------- .../jdk/incubator/vector/Long128Vector.java | 20 +------ .../jdk/incubator/vector/Long256Vector.java | 20 +------ .../jdk/incubator/vector/Long512Vector.java | 20 +------ .../jdk/incubator/vector/Long64Vector.java | 20 +------ .../jdk/incubator/vector/LongMaxVector.java | 20 +------ .../jdk/incubator/vector/LongVector.java | 58 +++++++++++-------- .../jdk/incubator/vector/Short128Vector.java | 20 +------ .../jdk/incubator/vector/Short256Vector.java | 20 +------ .../jdk/incubator/vector/Short512Vector.java | 20 +------ .../jdk/incubator/vector/Short64Vector.java | 20 +------ .../jdk/incubator/vector/ShortMaxVector.java | 20 +------ .../jdk/incubator/vector/ShortVector.java | 58 +++++++++++-------- .../incubator/vector/X-Vector.java.template | 58 +++++++++++-------- .../vector/X-VectorBits.java.template | 20 +------ .../vector/AbstractVectorConversionTest.java | 4 +- .../vector/Vector128ConversionTests.java | 11 ++-- .../vector/Vector256ConversionTests.java | 11 ++-- .../vector/Vector512ConversionTests.java | 11 ++-- .../vector/Vector64ConversionTests.java | 9 +-- 43 files changed, 363 insertions(+), 709 deletions(-) diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java index 4c422e1ec1c..e2baacff09d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Byte128Vector slice(int origin, Vector v) { @Override @ForceInline public Byte128Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Byte128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Byte128Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Byte128Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Byte128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((byte)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Byte128Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java index 7267b83cf0f..f100935e6f0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Byte256Vector slice(int origin, Vector v) { @Override @ForceInline public Byte256Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Byte256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Byte256Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Byte256Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Byte256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((byte)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Byte256Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java index a16f0e6055c..70a23a22fd1 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Byte512Vector slice(int origin, Vector v) { @Override @ForceInline public Byte512Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Byte512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Byte512Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Byte512Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Byte512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((byte)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Byte512Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java index ea7506489d9..e33be0fb12c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Byte64Vector slice(int origin, Vector v) { @Override @ForceInline public Byte64Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Byte64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Byte64Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Byte64Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Byte64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((byte)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Byte64Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java index 6288ffaf72d..19ec4bf57e6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public ByteMaxVector slice(int origin, Vector v) { @Override @ForceInline public ByteMaxVector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - ByteMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (ByteMaxVector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public ByteMaxVector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - ByteMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((byte)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (ByteMaxVector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java index de090ca9686..a50fb8123ff 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java @@ -1981,14 +1981,11 @@ public final ByteVector blend(long e, ByteVector sliceTemplate(int origin, Vector v1) { ByteVector that = (ByteVector) v1; that.check(this); - byte[] a0 = this.vec(); - byte[] a1 = that.vec(); - byte[] res = new byte[a0.length]; - int vlen = res.length; - int firstPart = vlen - origin; - System.arraycopy(a0, origin, res, 0, firstPart); - System.arraycopy(a1, 0, res, firstPart, origin); - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return that.rearrange(iota).blend(this.rearrange(iota), blendMask); } /** @@ -2010,6 +2007,17 @@ ByteVector slice(int origin, public abstract ByteVector slice(int origin); + /*package-private*/ + final + @ForceInline + ByteVector sliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + /** * {@inheritDoc} */ @@ -2024,21 +2032,12 @@ ByteVector slice(int origin, unsliceTemplate(int origin, Vector w, int part) { ByteVector that = (ByteVector) w; that.check(this); - byte[] slice = this.vec(); - byte[] res = that.vec().clone(); - int vlen = res.length; - int firstPart = vlen - origin; - switch (part) { - case 0: - System.arraycopy(slice, 0, res, origin, firstPart); - break; - case 1: - System.arraycopy(slice, firstPart, res, 0, origin); - break; - default: - throw wrongPartForSlice(part); - } - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT, + (broadcast((byte)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return that.blend(this.rearrange(iota), blendMask); } /*package-private*/ @@ -2068,6 +2067,19 @@ ByteVector slice(int origin, public abstract ByteVector unslice(int origin); + /*package-private*/ + final + @ForceInline + ByteVector + unsliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.GE, + (broadcast((byte)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + private ArrayIndexOutOfBoundsException wrongPartForSlice(int part) { String msg = String.format("bad part number %d for slice operation", diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index 6294f995ecb..6df75dec501 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public Double128Vector slice(int origin, Vector v) { @Override @ForceInline public Double128Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Double128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((double)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Double128Vector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public Double128Vector unslice(int origin, Vector w, int part, VectorMas @Override @ForceInline public Double128Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Double128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((double)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Double128Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index 27656383a72..d4058fb25f7 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public Double256Vector slice(int origin, Vector v) { @Override @ForceInline public Double256Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Double256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((double)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Double256Vector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public Double256Vector unslice(int origin, Vector w, int part, VectorMas @Override @ForceInline public Double256Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Double256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((double)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Double256Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index 4e9b3f6e996..04f27791871 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public Double512Vector slice(int origin, Vector v) { @Override @ForceInline public Double512Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Double512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((double)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Double512Vector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public Double512Vector unslice(int origin, Vector w, int part, VectorMas @Override @ForceInline public Double512Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Double512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((double)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Double512Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index 91234cbf04a..14672601009 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public Double64Vector slice(int origin, Vector v) { @Override @ForceInline public Double64Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Double64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((double)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Double64Vector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public Double64Vector unslice(int origin, Vector w, int part, VectorMask @Override @ForceInline public Double64Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Double64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((double)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Double64Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index 436eaaf7dde..54b5a9f4805 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public DoubleMaxVector slice(int origin, Vector v) { @Override @ForceInline public DoubleMaxVector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - DoubleMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((double)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (DoubleMaxVector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public DoubleMaxVector unslice(int origin, Vector w, int part, VectorMas @Override @ForceInline public DoubleMaxVector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - DoubleMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((double)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (DoubleMaxVector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java index dad2915fbf5..7db52e25e10 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java @@ -1894,14 +1894,11 @@ public final DoubleVector blend(long e, DoubleVector sliceTemplate(int origin, Vector v1) { DoubleVector that = (DoubleVector) v1; that.check(this); - double[] a0 = this.vec(); - double[] a1 = that.vec(); - double[] res = new double[a0.length]; - int vlen = res.length; - int firstPart = vlen - origin; - System.arraycopy(a0, origin, res, 0, firstPart); - System.arraycopy(a1, 0, res, firstPart, origin); - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((double)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return that.rearrange(iota).blend(this.rearrange(iota), blendMask); } /** @@ -1923,6 +1920,17 @@ DoubleVector slice(int origin, public abstract DoubleVector slice(int origin); + /*package-private*/ + final + @ForceInline + DoubleVector sliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((double)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + /** * {@inheritDoc} */ @@ -1937,21 +1945,12 @@ DoubleVector slice(int origin, unsliceTemplate(int origin, Vector w, int part) { DoubleVector that = (DoubleVector) w; that.check(this); - double[] slice = this.vec(); - double[] res = that.vec().clone(); - int vlen = res.length; - int firstPart = vlen - origin; - switch (part) { - case 0: - System.arraycopy(slice, 0, res, origin, firstPart); - break; - case 1: - System.arraycopy(slice, firstPart, res, 0, origin); - break; - default: - throw wrongPartForSlice(part); - } - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT, + (broadcast((double)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return that.blend(this.rearrange(iota), blendMask); } /*package-private*/ @@ -1981,6 +1980,19 @@ DoubleVector slice(int origin, public abstract DoubleVector unslice(int origin); + /*package-private*/ + final + @ForceInline + DoubleVector + unsliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.GE, + (broadcast((double)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + private ArrayIndexOutOfBoundsException wrongPartForSlice(int part) { String msg = String.format("bad part number %d for slice operation", diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index 61fc241e7a1..a1a9634f4e9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public Float128Vector slice(int origin, Vector v) { @Override @ForceInline public Float128Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Float128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((float)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Float128Vector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public Float128Vector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public Float128Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Float128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((float)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Float128Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index 88429843a02..69786cffc4c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public Float256Vector slice(int origin, Vector v) { @Override @ForceInline public Float256Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Float256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((float)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Float256Vector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public Float256Vector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public Float256Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Float256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((float)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Float256Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index 72c6f09a852..41b9222fffb 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public Float512Vector slice(int origin, Vector v) { @Override @ForceInline public Float512Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Float512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((float)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Float512Vector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public Float512Vector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public Float512Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Float512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((float)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Float512Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index df6ea0427f4..0c615a9bc9d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public Float64Vector slice(int origin, Vector v) { @Override @ForceInline public Float64Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Float64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((float)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Float64Vector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public Float64Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Float64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((float)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Float64Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index ef353d5e224..1a6f6c6c7ea 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -381,14 +381,7 @@ public FloatMaxVector slice(int origin, Vector v) { @Override @ForceInline public FloatMaxVector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - FloatMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((float)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (FloatMaxVector) super.sliceTemplate(origin); // specialize } @Override @@ -409,14 +402,7 @@ public FloatMaxVector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public FloatMaxVector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - FloatMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((float)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (FloatMaxVector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java index 2cad3dbea93..70b94067255 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java @@ -1906,14 +1906,11 @@ public final FloatVector blend(long e, FloatVector sliceTemplate(int origin, Vector v1) { FloatVector that = (FloatVector) v1; that.check(this); - float[] a0 = this.vec(); - float[] a1 = that.vec(); - float[] res = new float[a0.length]; - int vlen = res.length; - int firstPart = vlen - origin; - System.arraycopy(a0, origin, res, 0, firstPart); - System.arraycopy(a1, 0, res, firstPart, origin); - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((float)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return that.rearrange(iota).blend(this.rearrange(iota), blendMask); } /** @@ -1935,6 +1932,17 @@ FloatVector slice(int origin, public abstract FloatVector slice(int origin); + /*package-private*/ + final + @ForceInline + FloatVector sliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((float)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + /** * {@inheritDoc} */ @@ -1949,21 +1957,12 @@ FloatVector slice(int origin, unsliceTemplate(int origin, Vector w, int part) { FloatVector that = (FloatVector) w; that.check(this); - float[] slice = this.vec(); - float[] res = that.vec().clone(); - int vlen = res.length; - int firstPart = vlen - origin; - switch (part) { - case 0: - System.arraycopy(slice, 0, res, origin, firstPart); - break; - case 1: - System.arraycopy(slice, firstPart, res, 0, origin); - break; - default: - throw wrongPartForSlice(part); - } - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT, + (broadcast((float)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return that.blend(this.rearrange(iota), blendMask); } /*package-private*/ @@ -1993,6 +1992,19 @@ FloatVector slice(int origin, public abstract FloatVector unslice(int origin); + /*package-private*/ + final + @ForceInline + FloatVector + unsliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.GE, + (broadcast((float)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + private ArrayIndexOutOfBoundsException wrongPartForSlice(int part) { String msg = String.format("bad part number %d for slice operation", diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java index 80f6f79d592..d5d39fc36d9 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Int128Vector slice(int origin, Vector v) { @Override @ForceInline public Int128Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Int128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((int)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Int128Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Int128Vector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public Int128Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Int128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((int)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Int128Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java index 6b75d52b2d1..15f2361ebcd 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Int256Vector slice(int origin, Vector v) { @Override @ForceInline public Int256Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Int256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((int)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Int256Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Int256Vector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public Int256Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Int256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((int)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Int256Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java index fa252640675..1d6551e5379 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Int512Vector slice(int origin, Vector v) { @Override @ForceInline public Int512Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Int512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((int)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Int512Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Int512Vector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public Int512Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Int512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((int)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Int512Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java index 7ce0c09f8dc..d5ef529f11d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Int64Vector slice(int origin, Vector v) { @Override @ForceInline public Int64Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Int64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((int)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Int64Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Int64Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Int64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((int)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Int64Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java index babdab4ccf8..bb75b346db8 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public IntMaxVector slice(int origin, Vector v) { @Override @ForceInline public IntMaxVector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - IntMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((int)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (IntMaxVector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public IntMaxVector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public IntMaxVector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - IntMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((int)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (IntMaxVector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java index 037ab4996fc..f6fa9df9d4d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java @@ -1980,14 +1980,11 @@ public final IntVector blend(long e, IntVector sliceTemplate(int origin, Vector v1) { IntVector that = (IntVector) v1; that.check(this); - int[] a0 = this.vec(); - int[] a1 = that.vec(); - int[] res = new int[a0.length]; - int vlen = res.length; - int firstPart = vlen - origin; - System.arraycopy(a0, origin, res, 0, firstPart); - System.arraycopy(a1, 0, res, firstPart, origin); - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((int)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return that.rearrange(iota).blend(this.rearrange(iota), blendMask); } /** @@ -2009,6 +2006,17 @@ IntVector slice(int origin, public abstract IntVector slice(int origin); + /*package-private*/ + final + @ForceInline + IntVector sliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((int)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + /** * {@inheritDoc} */ @@ -2023,21 +2031,12 @@ IntVector slice(int origin, unsliceTemplate(int origin, Vector w, int part) { IntVector that = (IntVector) w; that.check(this); - int[] slice = this.vec(); - int[] res = that.vec().clone(); - int vlen = res.length; - int firstPart = vlen - origin; - switch (part) { - case 0: - System.arraycopy(slice, 0, res, origin, firstPart); - break; - case 1: - System.arraycopy(slice, firstPart, res, 0, origin); - break; - default: - throw wrongPartForSlice(part); - } - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT, + (broadcast((int)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return that.blend(this.rearrange(iota), blendMask); } /*package-private*/ @@ -2067,6 +2066,19 @@ IntVector slice(int origin, public abstract IntVector unslice(int origin); + /*package-private*/ + final + @ForceInline + IntVector + unsliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.GE, + (broadcast((int)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + private ArrayIndexOutOfBoundsException wrongPartForSlice(int part) { String msg = String.format("bad part number %d for slice operation", diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java index 1988465cee4..a3801a317c4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -377,14 +377,7 @@ public Long128Vector slice(int origin, Vector v) { @Override @ForceInline public Long128Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Long128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((long)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Long128Vector) super.sliceTemplate(origin); // specialize } @Override @@ -405,14 +398,7 @@ public Long128Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Long128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((long)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Long128Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java index 6182a0fbe2f..7f4526c0b33 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -377,14 +377,7 @@ public Long256Vector slice(int origin, Vector v) { @Override @ForceInline public Long256Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Long256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((long)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Long256Vector) super.sliceTemplate(origin); // specialize } @Override @@ -405,14 +398,7 @@ public Long256Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Long256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((long)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Long256Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java index 249fe7848ce..b940bd6f290 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -377,14 +377,7 @@ public Long512Vector slice(int origin, Vector v) { @Override @ForceInline public Long512Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Long512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((long)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Long512Vector) super.sliceTemplate(origin); // specialize } @Override @@ -405,14 +398,7 @@ public Long512Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Long512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((long)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Long512Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java index de24768a667..b38985b7a5f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -377,14 +377,7 @@ public Long64Vector slice(int origin, Vector v) { @Override @ForceInline public Long64Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Long64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((long)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Long64Vector) super.sliceTemplate(origin); // specialize } @Override @@ -405,14 +398,7 @@ public Long64Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Long64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((long)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Long64Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java index 5e61e621739..04b627e1c6a 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -377,14 +377,7 @@ public LongMaxVector slice(int origin, Vector v) { @Override @ForceInline public LongMaxVector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - LongMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((long)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (LongMaxVector) super.sliceTemplate(origin); // specialize } @Override @@ -405,14 +398,7 @@ public LongMaxVector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - LongMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((long)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (LongMaxVector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java index 76fcc0c4566..2283b3c8ec2 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java @@ -1851,14 +1851,11 @@ public final LongVector blend(long e, LongVector sliceTemplate(int origin, Vector v1) { LongVector that = (LongVector) v1; that.check(this); - long[] a0 = this.vec(); - long[] a1 = that.vec(); - long[] res = new long[a0.length]; - int vlen = res.length; - int firstPart = vlen - origin; - System.arraycopy(a0, origin, res, 0, firstPart); - System.arraycopy(a1, 0, res, firstPart, origin); - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((long)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return that.rearrange(iota).blend(this.rearrange(iota), blendMask); } /** @@ -1880,6 +1877,17 @@ LongVector slice(int origin, public abstract LongVector slice(int origin); + /*package-private*/ + final + @ForceInline + LongVector sliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((long)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + /** * {@inheritDoc} */ @@ -1894,21 +1902,12 @@ LongVector slice(int origin, unsliceTemplate(int origin, Vector w, int part) { LongVector that = (LongVector) w; that.check(this); - long[] slice = this.vec(); - long[] res = that.vec().clone(); - int vlen = res.length; - int firstPart = vlen - origin; - switch (part) { - case 0: - System.arraycopy(slice, 0, res, origin, firstPart); - break; - case 1: - System.arraycopy(slice, firstPart, res, 0, origin); - break; - default: - throw wrongPartForSlice(part); - } - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT, + (broadcast((long)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return that.blend(this.rearrange(iota), blendMask); } /*package-private*/ @@ -1938,6 +1937,19 @@ LongVector slice(int origin, public abstract LongVector unslice(int origin); + /*package-private*/ + final + @ForceInline + LongVector + unsliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.GE, + (broadcast((long)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + private ArrayIndexOutOfBoundsException wrongPartForSlice(int part) { String msg = String.format("bad part number %d for slice operation", diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java index 9d9f8417598..a1c702394ad 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Short128Vector slice(int origin, Vector v) { @Override @ForceInline public Short128Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Short128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((short)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Short128Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Short128Vector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public Short128Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Short128Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((short)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Short128Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java index 74f2c24aa66..1d552f35df4 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Short256Vector slice(int origin, Vector v) { @Override @ForceInline public Short256Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Short256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((short)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Short256Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Short256Vector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public Short256Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Short256Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((short)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Short256Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java index 1b535d3d68a..ccd6b8ccf62 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Short512Vector slice(int origin, Vector v) { @Override @ForceInline public Short512Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Short512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((short)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Short512Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Short512Vector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public Short512Vector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Short512Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((short)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Short512Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java index b05149e7220..c8fa39031c6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public Short64Vector slice(int origin, Vector v) { @Override @ForceInline public Short64Vector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Short64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((short)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Short64Vector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public Short64Vector unslice(int origin, Vector w, int part, VectorMask= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - Short64Shuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((short)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (Short64Vector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java index 263b3a251df..302fa04a7bc 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -387,14 +387,7 @@ public ShortMaxVector slice(int origin, Vector v) { @Override @ForceInline public ShortMaxVector slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - ShortMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((short)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (ShortMaxVector) super.sliceTemplate(origin); // specialize } @Override @@ -415,14 +408,7 @@ public ShortMaxVector unslice(int origin, Vector w, int part, VectorMask< @Override @ForceInline public ShortMaxVector unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - ShortMaxShuffle Iota = iotaShuffle(); - VectorMask BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((short)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return (ShortMaxVector) super.unsliceTemplate(origin); // specialize } @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java index f8a5e16d046..022653cefd7 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java @@ -1981,14 +1981,11 @@ public final ShortVector blend(long e, ShortVector sliceTemplate(int origin, Vector v1) { ShortVector that = (ShortVector) v1; that.check(this); - short[] a0 = this.vec(); - short[] a1 = that.vec(); - short[] res = new short[a0.length]; - int vlen = res.length; - int firstPart = vlen - origin; - System.arraycopy(a0, origin, res, 0, firstPart); - System.arraycopy(a1, 0, res, firstPart, origin); - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((short)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return that.rearrange(iota).blend(this.rearrange(iota), blendMask); } /** @@ -2010,6 +2007,17 @@ ShortVector slice(int origin, public abstract ShortVector slice(int origin); + /*package-private*/ + final + @ForceInline + ShortVector sliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast((short)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + /** * {@inheritDoc} */ @@ -2024,21 +2032,12 @@ ShortVector slice(int origin, unsliceTemplate(int origin, Vector w, int part) { ShortVector that = (ShortVector) w; that.check(this); - short[] slice = this.vec(); - short[] res = that.vec().clone(); - int vlen = res.length; - int firstPart = vlen - origin; - switch (part) { - case 0: - System.arraycopy(slice, 0, res, origin, firstPart); - break; - case 1: - System.arraycopy(slice, firstPart, res, 0, origin); - break; - default: - throw wrongPartForSlice(part); - } - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT, + (broadcast((short)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return that.blend(this.rearrange(iota), blendMask); } /*package-private*/ @@ -2068,6 +2067,19 @@ ShortVector slice(int origin, public abstract ShortVector unslice(int origin); + /*package-private*/ + final + @ForceInline + ShortVector + unsliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle iota = iotaShuffle(); + VectorMask blendMask = iota.toVector().compare(VectorOperators.GE, + (broadcast((short)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + private ArrayIndexOutOfBoundsException wrongPartForSlice(int part) { String msg = String.format("bad part number %d for slice operation", diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template index ec64f6d3ab0..26391abfb7e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template @@ -2250,14 +2250,11 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { $abstractvectortype$ sliceTemplate(int origin, Vector<$Boxtype$> v1) { $abstractvectortype$ that = ($abstractvectortype$) v1; that.check(this); - $type$[] a0 = this.vec(); - $type$[] a1 = that.vec(); - $type$[] res = new $type$[a0.length]; - int vlen = res.length; - int firstPart = vlen - origin; - System.arraycopy(a0, origin, res, 0, firstPart); - System.arraycopy(a1, 0, res, firstPart, origin); - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle<$Boxtype$> iota = iotaShuffle(); + VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return that.rearrange(iota).blend(this.rearrange(iota), blendMask); } /** @@ -2279,6 +2276,17 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { public abstract $abstractvectortype$ slice(int origin); + /*package-private*/ + final + @ForceInline + $abstractvectortype$ sliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle<$Boxtype$> iota = iotaShuffle(); + VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(length() - origin)))); + iota = iotaShuffle(origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + /** * {@inheritDoc} */ @@ -2293,21 +2301,12 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { unsliceTemplate(int origin, Vector<$Boxtype$> w, int part) { $abstractvectortype$ that = ($abstractvectortype$) w; that.check(this); - $type$[] slice = this.vec(); - $type$[] res = that.vec().clone(); - int vlen = res.length; - int firstPart = vlen - origin; - switch (part) { - case 0: - System.arraycopy(slice, 0, res, origin, firstPart); - break; - case 1: - System.arraycopy(slice, firstPart, res, 0, origin); - break; - default: - throw wrongPartForSlice(part); - } - return vectorFactory(res); + Objects.checkIndex(origin, length() + 1); + VectorShuffle<$Boxtype$> iota = iotaShuffle(); + VectorMask<$Boxtype$> blendMask = iota.toVector().compare((part == 0) ? VectorOperators.GE : VectorOperators.LT, + (broadcast(($type$)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return that.blend(this.rearrange(iota), blendMask); } /*package-private*/ @@ -2337,6 +2336,19 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> { public abstract $abstractvectortype$ unslice(int origin); + /*package-private*/ + final + @ForceInline + $abstractvectortype$ + unsliceTemplate(int origin) { + Objects.checkIndex(origin, length() + 1); + VectorShuffle<$Boxtype$> iota = iotaShuffle(); + VectorMask<$Boxtype$> blendMask = iota.toVector().compare(VectorOperators.GE, + (broadcast(($type$)(origin)))); + iota = iotaShuffle(-origin, 1, true); + return vspecies().zero().blend(this.rearrange(iota), blendMask); + } + private ArrayIndexOutOfBoundsException wrongPartForSlice(int part) { String msg = String.format("bad part number %d for slice operation", diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index 011ff621d61..634960e5167 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -393,14 +393,7 @@ final class $vectortype$ extends $abstractvectortype$ { @Override @ForceInline public $vectortype$ slice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - $shuffletype$ Iota = iotaShuffle(); - VectorMask<$Boxtype$> BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast(($type$)(VLENGTH-origin)))); - Iota = iotaShuffle(origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return ($vectortype$) super.sliceTemplate(origin); // specialize } @Override @@ -421,14 +414,7 @@ final class $vectortype$ extends $abstractvectortype$ { @Override @ForceInline public $vectortype$ unslice(int origin) { - if ((origin < 0) || (origin >= VLENGTH)) { - throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH); - } else { - $shuffletype$ Iota = iotaShuffle(); - VectorMask<$Boxtype$> BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast(($type$)(origin)))); - Iota = iotaShuffle(-origin, 1, true); - return ZERO.blend(this.rearrange(Iota), BlendMask); - } + return ($vectortype$) super.unsliceTemplate(origin); // specialize } @Override diff --git a/test/jdk/jdk/incubator/vector/AbstractVectorConversionTest.java b/test/jdk/jdk/incubator/vector/AbstractVectorConversionTest.java index 7404cb4e6b9..1921135b9a9 100644 --- a/test/jdk/jdk/incubator/vector/AbstractVectorConversionTest.java +++ b/test/jdk/jdk/incubator/vector/AbstractVectorConversionTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -48,7 +48,7 @@ public void getRunTime(ITestResult tr) { System.out.println(tr.getName() + " took " + time + " ms"); } - static final int INVOC_COUNT = Integer.getInteger("jdk.incubator.vector.test.loop-iterations", 1000); + static final int INVOC_COUNT = Integer.getInteger("jdk.incubator.vector.test.loop-iterations", 100); static IntFunction withToString(String s, IntFunction f) { return new IntFunction<>() { diff --git a/test/jdk/jdk/incubator/vector/Vector128ConversionTests.java b/test/jdk/jdk/incubator/vector/Vector128ConversionTests.java index 85191da8466..b93b5865ba5 100644 --- a/test/jdk/jdk/incubator/vector/Vector128ConversionTests.java +++ b/test/jdk/jdk/incubator/vector/Vector128ConversionTests.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,6 +40,7 @@ public class Vector128ConversionTests extends AbstractVectorConversionTest { static final VectorShape SHAPE = VectorShape.S_128_BIT; + static final int BUFFER_SIZE = Integer.getInteger("jdk.incubator.vector.test.buffer-size", 1024); @DataProvider public Object[][] fixedShapeXfixedShape() { @@ -53,25 +54,25 @@ public Object[][] fixedShapeXShape() { @Test(dataProvider = "fixedShapeXfixedShape") static void convert(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CONVERT); } @Test(dataProvider = "fixedShapeXShape") static void convertShape(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CONVERTSHAPE); } @Test(dataProvider = "fixedShapeXShape") static void castShape(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CASTSHAPE); } @Test(dataProvider = "fixedShapeXShape") static void reinterpret(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); reinterpret_kernel(src, dst, a); } } diff --git a/test/jdk/jdk/incubator/vector/Vector256ConversionTests.java b/test/jdk/jdk/incubator/vector/Vector256ConversionTests.java index ab73b469ae8..389ee537ecb 100644 --- a/test/jdk/jdk/incubator/vector/Vector256ConversionTests.java +++ b/test/jdk/jdk/incubator/vector/Vector256ConversionTests.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,6 +40,7 @@ public class Vector256ConversionTests extends AbstractVectorConversionTest { static final VectorShape SHAPE = VectorShape.S_256_BIT; + static final int BUFFER_SIZE = Integer.getInteger("jdk.incubator.vector.test.buffer-size", 1024); @DataProvider public Object[][] fixedShapeXfixedShape() { @@ -53,25 +54,25 @@ public Object[][] fixedShapeXShape() { @Test(dataProvider = "fixedShapeXfixedShape") static void convert(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CONVERT); } @Test(dataProvider = "fixedShapeXShape") static void convertShape(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CONVERTSHAPE); } @Test(dataProvider = "fixedShapeXShape") static void castShape(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CASTSHAPE); } @Test(dataProvider = "fixedShapeXShape") static void reinterpret(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); reinterpret_kernel(src, dst, a); } } diff --git a/test/jdk/jdk/incubator/vector/Vector512ConversionTests.java b/test/jdk/jdk/incubator/vector/Vector512ConversionTests.java index 3f802712b4c..c12481b27d4 100644 --- a/test/jdk/jdk/incubator/vector/Vector512ConversionTests.java +++ b/test/jdk/jdk/incubator/vector/Vector512ConversionTests.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,6 +40,7 @@ public class Vector512ConversionTests extends AbstractVectorConversionTest { static final VectorShape SHAPE = VectorShape.S_512_BIT; + static final int BUFFER_SIZE = Integer.getInteger("jdk.incubator.vector.test.buffer-size", 1024); @DataProvider public Object[][] fixedShapeXfixedShape() { @@ -53,25 +54,25 @@ public Object[][] fixedShapeXShape() { @Test(dataProvider = "fixedShapeXfixedShape") static void convert(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CONVERT); } @Test(dataProvider = "fixedShapeXShape") static void convertShape(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CONVERTSHAPE); } @Test(dataProvider = "fixedShapeXShape") static void castShape(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CASTSHAPE); } @Test(dataProvider = "fixedShapeXShape") static void reinterpret(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); reinterpret_kernel(src, dst, a); } } diff --git a/test/jdk/jdk/incubator/vector/Vector64ConversionTests.java b/test/jdk/jdk/incubator/vector/Vector64ConversionTests.java index 835a49f025f..b98bfc0a789 100644 --- a/test/jdk/jdk/incubator/vector/Vector64ConversionTests.java +++ b/test/jdk/jdk/incubator/vector/Vector64ConversionTests.java @@ -51,6 +51,7 @@ public class Vector64ConversionTests extends AbstractVectorConversionTest { static final VectorShape SHAPE = VectorShape.S_64_BIT; + static final int BUFFER_SIZE = Integer.getInteger("jdk.incubator.vector.test.buffer-size", 1024); @DataProvider public Object[][] fixedShapeXfixedShape() { @@ -64,25 +65,25 @@ public Object[][] fixedShapeXShape() { @Test(dataProvider = "fixedShapeXfixedShape") static void convert(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CONVERT); } @Test(dataProvider = "fixedShapeXShape") static void convertShape(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CONVERTSHAPE); } @Test(dataProvider = "fixedShapeXShape") static void castShape(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); conversion_kernel(src, dst, a, ConvAPI.CASTSHAPE); } @Test(dataProvider = "fixedShapeXShape") static void reinterpret(VectorSpecies src, VectorSpecies dst, IntFunction fa) { - Object a = fa.apply(1024); + Object a = fa.apply(BUFFER_SIZE); reinterpret_kernel(src, dst, a); } } From 9ab7cd90a9ae466b4597e60a6edf513172b539a1 Mon Sep 17 00:00:00 2001 From: "Jin,Zhonghui" Date: Fri, 3 Nov 2023 01:09:24 +0800 Subject: [PATCH 6/6] [Backport] 8265126: [REDO] unified handling for VectorMask object re-materialization during de-optimization Summary: 8265126: [REDO] unified handling for VectorMask object re-materialization during de-optimization Test Plan: ci jtreg Reviewed-by: JoshuaZhuwj Issue: https://github.com/dragonwell-project/dragonwell11/issues/599 --- src/hotspot/share/opto/vector.cpp | 36 +++++++++----- src/hotspot/share/prims/vectorSupport.cpp | 57 +++++++++-------------- src/hotspot/share/prims/vectorSupport.hpp | 2 +- 3 files changed, 47 insertions(+), 48 deletions(-) diff --git a/src/hotspot/share/opto/vector.cpp b/src/hotspot/share/opto/vector.cpp index 91f16b374c1..10bd14a1c8d 100644 --- a/src/hotspot/share/opto/vector.cpp +++ b/src/hotspot/share/opto/vector.cpp @@ -30,6 +30,15 @@ #include "opto/vector.hpp" #include "utilities/macros.hpp" +static bool is_vector_mask(ciKlass* klass) { + return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); +} + +static bool is_vector_shuffle(ciKlass* klass) { + return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); +} + + void PhaseVector::optimize_vector_boxes() { Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); @@ -236,13 +245,24 @@ void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) { } } + ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass(); + int n_fields = iklass->nof_nonstatic_fields(); + assert(n_fields == 1, "sanity"); + + // If a mask is feeding into safepoint[s], then its value should be + // packed into a boolean/byte vector first, this will simplify the + // re-materialization logic for both predicated and non-predicated + // targets. + bool is_mask = is_vector_mask(iklass); + if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) { + const TypeVect* vt = vec_value->bottom_type()->is_vect(); + BasicType bt = vt->element_basic_type(); + vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length())); + } + while (safepoints.size() > 0) { SafePointNode* sfpt = safepoints.pop()->as_SafePoint(); - ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass(); - int n_fields = iklass->nof_nonstatic_fields(); - assert(n_fields == 1, "sanity"); - uint first_ind = (sfpt->req() - sfpt->jvms()->scloff()); Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(), #ifdef ASSERT @@ -303,14 +323,6 @@ Node* PhaseVector::expand_vbox_node_helper(Node* vbox, } } -static bool is_vector_mask(ciKlass* klass) { - return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); -} - -static bool is_vector_shuffle(ciKlass* klass) { - return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); -} - Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc, Node* value, const TypeInstPtr* box_type, diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index b228f1e48ae..b0031afcdf8 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -62,6 +62,8 @@ BasicType VectorSupport::klass2bt(InstanceKlass* ik) { if (is_vector_shuffle(ik)) { return T_BYTE; + } else if (is_vector_mask(ik)) { + return T_BOOLEAN; } else { // vector and mask oop value = ik->java_mirror()->obj_field(fd.offset()); BasicType elem_bt = java_lang_Class::as_BasicType(value); @@ -83,48 +85,33 @@ jint VectorSupport::klass2length(InstanceKlass* ik) { return vlen; } -void VectorSupport::init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr) { - if (is_mask) { - // Masks require special handling: when boxed they are packed and stored in boolean - // arrays, but in scalarized form they have the same size as corresponding vectors. - // For example, Int512Mask is represented in memory as boolean[16], but - // occupies the whole 512-bit vector register when scalarized. - // (In generated code, the conversion is performed by VectorStoreMask.) - // - // TODO: revisit when predicate registers are fully supported. - switch (elem_bt) { - case T_BYTE: arr->bool_at_put(index, (*(jbyte*)addr) != 0); break; - case T_SHORT: arr->bool_at_put(index, (*(jshort*)addr) != 0); break; - case T_INT: // fall-through - case T_FLOAT: arr->bool_at_put(index, (*(jint*)addr) != 0); break; - case T_LONG: // fall-through - case T_DOUBLE: arr->bool_at_put(index, (*(jlong*)addr) != 0); break; - - default: fatal("unsupported: %s", type2name(elem_bt)); - } - } else { - switch (elem_bt) { - case T_BYTE: arr-> byte_at_put(index, *(jbyte*)addr); break; - case T_SHORT: arr-> short_at_put(index, *(jshort*)addr); break; - case T_INT: arr-> int_at_put(index, *(jint*)addr); break; - case T_FLOAT: arr-> float_at_put(index, *(jfloat*)addr); break; - case T_LONG: arr-> long_at_put(index, *(jlong*)addr); break; - case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break; - - default: fatal("unsupported: %s", type2name(elem_bt)); - } +// Masks require special handling: when boxed they are packed and stored in boolean +// arrays, but in scalarized form they have the same size as corresponding vectors. +// For example, Int512Mask is represented in memory as boolean[16], but +// occupies the whole 512-bit vector register when scalarized. +// During scalarization inserting a VectorStoreMask node between mask +// and safepoint node always ensures the existence of masks in a boolean array. + +void VectorSupport::init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr) { + switch (elem_bt) { + case T_BOOLEAN: arr->bool_at_put(index, *(jboolean*)addr); break; + case T_BYTE: arr->byte_at_put(index, *(jbyte*)addr); break; + case T_SHORT: arr->short_at_put(index, *(jshort*)addr); break; + case T_INT: arr->int_at_put(index, *(jint*)addr); break; + case T_FLOAT: arr->float_at_put(index, *(jfloat*)addr); break; + case T_LONG: arr->long_at_put(index, *(jlong*)addr); break; + case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break; + default: fatal("unsupported: %s", type2name(elem_bt)); } } Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS) { - bool is_mask = is_vector_mask(ik); - int num_elem = klass2length(ik); BasicType elem_bt = klass2bt(ik); int elem_size = type2aelembytes(elem_bt); // On-heap vector values are represented as primitive arrays. - TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(is_mask ? T_BOOLEAN : elem_bt)); + TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(elem_bt)); typeArrayOop arr = tak->allocate(num_elem, CHECK_NH); // safepoint @@ -137,13 +124,13 @@ Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* f int off = (i * elem_size) % VMRegImpl::stack_slot_size; address elem_addr = reg_map->location(vreg->next(vslot)) + off; - init_payload_element(arr, is_mask, elem_bt, i, elem_addr); + init_payload_element(arr, elem_bt, i, elem_addr); } } else { // Value was directly saved on the stack. address base_addr = ((address)fr->unextended_sp()) + location.stack_offset(); for (int i = 0; i < num_elem; i++) { - init_payload_element(arr, is_mask, elem_bt, i, base_addr + i * elem_size); + init_payload_element(arr, elem_bt, i, base_addr + i * elem_size); } } return Handle(THREAD, arr); diff --git a/src/hotspot/share/prims/vectorSupport.hpp b/src/hotspot/share/prims/vectorSupport.hpp index 19e885f9dcf..5debf65ac5b 100644 --- a/src/hotspot/share/prims/vectorSupport.hpp +++ b/src/hotspot/share/prims/vectorSupport.hpp @@ -42,7 +42,7 @@ class VectorSupport : AllStatic { static Handle allocate_vector_payload(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ScopeValue* payload, TRAPS); static Handle allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS); - static void init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr); + static void init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr); static BasicType klass2bt(InstanceKlass* ik); static jint klass2length(InstanceKlass* ik);