diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eddbcf90f60f..fab1eb1a2b7b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -149,6 +149,7 @@ jobs: cat common.json | jq -r '.deps.common.packages | to_entries[] | select(.key | startswith("pip:")) | (.key | split(":")[1]) + .value' | xargs sudo pip install + ${MX_PYTHON} -m pip install jsonschema==4.6.1 - name: Build GraalVM and run gate env: ${{ matrix.env }} run: | diff --git a/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Address.java b/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Address.java index badc9125ecf8..71772cd554f4 100644 --- a/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Address.java +++ b/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Address.java @@ -25,7 +25,7 @@ package org.graalvm.compiler.asm.amd64; import org.graalvm.compiler.asm.AbstractAddress; -import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.core.common.Stride; import jdk.vm.ci.code.CodeUtil; import jdk.vm.ci.code.Register; @@ -39,7 +39,7 @@ public final class AMD64Address extends AbstractAddress { private final Register base; private final Register index; - private final Scale scale; + private final Stride stride; private final int displacement; private final Object displacementAnnotation; @@ -56,7 +56,7 @@ public final class AMD64Address extends AbstractAddress { * @param base the base register */ public AMD64Address(Register base) { - this(base, Register.None, Scale.Times1, 0); + this(base, Register.None, Stride.S1, 0); } /** @@ -67,7 +67,7 @@ public AMD64Address(Register base) { * @param displacement the displacement */ public AMD64Address(Register base, int displacement) { - this(base, Register.None, Scale.Times1, displacement); + this(base, Register.None, Stride.S1, displacement); } /** @@ -76,10 +76,10 @@ public AMD64Address(Register base, int displacement) { * * @param base the base register * @param index the index register - * @param scale the scaling factor + * @param stride the scaling factor */ - public AMD64Address(Register base, Register index, Scale scale) { - this(base, index, scale, 0, null, -1); + public AMD64Address(Register base, Register index, Stride stride) { + this(base, index, stride, 0, null, -1); } /** @@ -88,105 +88,40 @@ public AMD64Address(Register base, Register index, Scale scale) { * * @param base the base register * @param index the index register - * @param scale the scaling factor + * @param stride the scaling factor * @param displacement the displacement */ - public AMD64Address(Register base, Register index, Scale scale, int displacement) { - this(base, index, scale, displacement, null, -1); + public AMD64Address(Register base, Register index, Stride stride, int displacement) { + this(base, index, stride, displacement, null, -1); } - public AMD64Address(Register base, Register index, Scale scale, int displacement, Object displacementAnnotation) { - this(base, index, scale, displacement, displacementAnnotation, -1); + public AMD64Address(Register base, Register index, Stride stride, int displacement, Object displacementAnnotation) { + this(base, index, stride, displacement, displacementAnnotation, -1); } - AMD64Address(Register base, Register index, Scale scale, int displacement, Object displacementAnnotation, int instructionStartPosition) { + AMD64Address(Register base, Register index, Stride stride, int displacement, Object displacementAnnotation, int instructionStartPosition) { this.base = base; this.index = index; - this.scale = scale; + this.stride = stride; this.displacement = displacement; this.displacementAnnotation = displacementAnnotation; this.instructionStartPosition = instructionStartPosition; - assert scale != null; + assert stride != null; } /** - * A scaling factor used in the SIB addressing mode. + * Determines if the scaling factor {@code scale} is supported. */ - public enum Scale { - Times1(1, 0), - Times2(2, 1), - Times4(4, 2), - Times8(8, 3); - - Scale(int value, int log2) { - this.value = value; - this.log2 = log2; - } - - /** - * The value (or multiplier) of this scale. - */ - public final int value; - - /** - * The {@linkplain #value value} of this scale log 2. - */ - public final int log2; - - /** - * Creates a {@link Scale} for the scaling factor in {@code scale}. - * - * @throws IllegalArgumentException if {@code scale} is an unsupported scaling factor - */ - public static Scale fromInt(int scale) { - switch (scale) { - case 1: - return Times1; - case 2: - return Times2; - case 4: - return Times4; - case 8: - return Times8; - default: - throw new IllegalArgumentException("Unsupported SIB addressing mode scaling factor: " + scale); - } - } - - /** - * Creates a {@link Scale} for the log2 scaling factor {@code shift}. - * - * @throws IllegalArgumentException if {@code shift} is an unsupported scaling factor - */ - public static Scale fromShift(int shift) { - switch (shift) { - case 0: - return Times1; - case 1: - return Times2; - case 2: - return Times4; - case 3: - return Times8; - default: - throw GraalError.shouldNotReachHere("Unsupported SIB addressing mode scaling factor: " + (1 << shift)); - } - } - - /** - * Determines if the scaling factor {@code scale} is supported. - */ - public static boolean isScaleSupported(int scale) { - return CodeUtil.isPowerOf2(scale) && scale <= 8; - } + public static boolean isScaleSupported(int scale) { + return CodeUtil.isPowerOf2(scale) && scale <= 8; + } - /** - * Determines if the log2 scaling factor {@code shift} is supported. - */ - public static boolean isScaleShiftSupported(int shift) { - return shift >= 0 && shift <= 3; - } + /** + * Determines if the log2 scaling factor {@code shift} is supported. + */ + public static boolean isScaleShiftSupported(int shift) { + return shift >= 0 && shift <= 3; } @Override @@ -233,8 +168,8 @@ public Register getIndex() { /** * @return Scaling factor for indexing, dependent on target operand size. */ - public Scale getScale() { - return scale; + public Stride getScale() { + return stride; } /** diff --git a/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java b/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java index e4ce8104420f..3248374eabd5 100644 --- a/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java +++ b/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java @@ -84,7 +84,7 @@ import java.util.EnumSet; import org.graalvm.compiler.asm.Label; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.common.calc.Condition; import org.graalvm.compiler.debug.GraalError; @@ -4764,7 +4764,7 @@ public AMD64Address makeAddress(Register base, int displacement) { @Override public AMD64Address getPlaceholder(int instructionStartPosition) { - return new AMD64Address(AMD64.rip, Register.None, Scale.Times1, 0, null, instructionStartPosition); + return new AMD64Address(AMD64.rip, Register.None, Stride.S1, 0, null, instructionStartPosition); } private void prefetchPrefix(AMD64Address src) { @@ -4861,7 +4861,7 @@ public void clflushopt(AMD64Address adr) { assert supportsCPUFeature("FLUSHOPT"); // adr should be base reg only with no index or offset assert adr.getIndex().equals(Register.None) : adr; - assert adr.getScale().equals(Scale.Times1) : adr; + assert adr.getScale().equals(Stride.S1) : adr; assert adr.getDisplacement() == 0 : adr; // instruction prefix is 0x66 emitByte(0x66); diff --git a/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java b/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java index 33f17cc196f1..1b4eaa59e483 100644 --- a/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java +++ b/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java @@ -55,7 +55,7 @@ import org.graalvm.collections.EconomicSet; import org.graalvm.compiler.asm.Assembler; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.debug.GraalError; @@ -686,7 +686,7 @@ private void emitOperandHelper(int reg, AMD64Address addr, boolean force4Byte, i Register base = addr.getBase(); Register index = addr.getIndex(); - Scale scale = addr.getScale(); + Stride stride = addr.getScale(); int disp = addr.getDisplacement(); Object dispAnnotation = addr.getDisplacementAnnotation(); @@ -710,7 +710,7 @@ private void emitOperandHelper(int reg, AMD64Address addr, boolean force4Byte, i // [00 reg 100][ss index base] assert !index.equals(rsp) : "illegal addressing mode"; emitByte(0x04 | regenc); - emitByte(scale.log2 << 6 | indexenc | baseenc); + emitByte(stride.log2 << 6 | indexenc | baseenc); } else { if (evexDisp8Scale > 1 && !overriddenForce4Byte) { if (disp % evexDisp8Scale == 0) { @@ -728,7 +728,7 @@ private void emitOperandHelper(int reg, AMD64Address addr, boolean force4Byte, i // [01 reg 100][ss index base] imm8 assert !index.equals(rsp) : "illegal addressing mode"; emitByte(0x44 | regenc); - emitByte(scale.log2 << 6 | indexenc | baseenc); + emitByte(stride.log2 << 6 | indexenc | baseenc); assert dispAnnotation == null; emitByte(disp & 0xFF); } else { @@ -736,7 +736,7 @@ private void emitOperandHelper(int reg, AMD64Address addr, boolean force4Byte, i // [10 reg 100][ss index base] disp32 assert !index.equals(rsp) : "illegal addressing mode"; emitByte(0x84 | regenc); - emitByte(scale.log2 << 6 | indexenc | baseenc); + emitByte(stride.log2 << 6 | indexenc | baseenc); emitDisplacementInt(disp, dispAnnotation); } } @@ -814,7 +814,7 @@ private void emitOperandHelper(int reg, AMD64Address addr, boolean force4Byte, i // [00 reg 100][ss index 101] disp32 assert !index.equals(rsp) : "illegal addressing mode"; emitByte(0x04 | regenc); - emitByte(scale.log2 << 6 | indexenc | 0x05); + emitByte(stride.log2 << 6 | indexenc | 0x05); emitDisplacementInt(disp, dispAnnotation); } else { // [disp] ABSOLUTE diff --git a/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64MacroAssembler.java b/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64MacroAssembler.java index 6d78c2191d2d..8652682d152e 100644 --- a/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64MacroAssembler.java +++ b/compiler/src/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64MacroAssembler.java @@ -44,6 +44,7 @@ import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.common.NumUtil; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.options.OptionValues; @@ -738,30 +739,30 @@ public enum ExtendMode { } public static void movSZx(AMD64MacroAssembler asm, OperandSize operandSize, ExtendMode extendMode, Register dst, AMD64Address src) { - movSZx(asm, AMD64Address.Scale.fromInt(operandSize.getBytes()), extendMode, dst, src); + movSZx(asm, Stride.fromInt(operandSize.getBytes()), extendMode, dst, src); } /** * Load one, two, four or eight bytes, according to {@code scaleSrc}, into {@code dst} and zero- * or sign-extend depending on {@code extendMode}. */ - public static void movSZx(AMD64MacroAssembler asm, AMD64Address.Scale scaleSrc, ExtendMode extendMode, Register dst, AMD64Address src) { - switch (scaleSrc) { - case Times1: + public static void movSZx(AMD64MacroAssembler asm, Stride strideSrc, ExtendMode extendMode, Register dst, AMD64Address src) { + switch (strideSrc) { + case S1: if (extendMode == ExtendMode.SIGN_EXTEND) { asm.movsbq(dst, src); } else { asm.movzbq(dst, src); } break; - case Times2: + case S2: if (extendMode == ExtendMode.SIGN_EXTEND) { asm.movswq(dst, src); } else { asm.movzwq(dst, src); } break; - case Times4: + case S4: if (extendMode == ExtendMode.SIGN_EXTEND) { asm.movslq(dst, src); } else { @@ -769,7 +770,7 @@ public static void movSZx(AMD64MacroAssembler asm, AMD64Address.Scale scaleSrc, asm.movl(dst, src); } break; - case Times8: + case S8: asm.movq(dst, src); break; default: @@ -777,9 +778,9 @@ public static void movSZx(AMD64MacroAssembler asm, AMD64Address.Scale scaleSrc, } } - public static void pmovSZx(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, AMD64Address.Scale scaleDst, Register src, AMD64Address.Scale scaleSrc, + public static void pmovSZx(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, Stride strideDst, Register src, Stride strideSrc, int displacement) { - pmovSZx(asm, size, extendMode, dst, scaleDst, src, scaleSrc, null, displacement); + pmovSZx(asm, size, extendMode, dst, strideDst, src, strideSrc, null, displacement); } /** @@ -788,69 +789,69 @@ public static void pmovSZx(AMD64MacroAssembler asm, AVXSize size, ExtendMode ext * * @param size vector size. May be {@link AVXSize#XMM} or {@link AVXSize#YMM}. * @param dst a XMM or YMM vector register. - * @param scaleDst target stride. Must be greater or equal to {@code scaleSrc}. + * @param strideDst target stride. Must be greater or equal to {@code scaleSrc}. * @param src the source address. - * @param scaleSrc source stride. Must be smaller or equal to {@code scaleDst}. + * @param strideSrc source stride. Must be smaller or equal to {@code scaleDst}. * @param index address index offset, scaled by {@code scaleSrc}. * @param displacement address displacement in bytes. If {@code scaleDst} is greater than * {@code scaleSrc}, this displacement is scaled by the ratio of the former and - * latter scales, e.g. if {@code scaleDst} is {@link AMD64Address.Scale#Times4} and - * {@code scaleSrc} is {@link AMD64Address.Scale#Times2}, the displacement is halved. + * latter scales, e.g. if {@code scaleDst} is {@link Stride#S4} and {@code scaleSrc} + * is {@link Stride#S2}, the displacement is halved. */ - public static void pmovSZx(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, AMD64Address.Scale scaleDst, Register src, AMD64Address.Scale scaleSrc, Register index, + public static void pmovSZx(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, Stride strideDst, Register src, Stride strideSrc, Register index, int displacement) { assert size == AVXSize.QWORD || size == AVXSize.XMM || size == AVXSize.YMM; - int scaledDisplacement = scaleDisplacement(scaleDst, scaleSrc, displacement); - AMD64Address address = index == null ? new AMD64Address(src, scaledDisplacement) : new AMD64Address(src, index, scaleSrc, scaledDisplacement); - pmovSZx(asm, size, extendMode, dst, scaleDst, address, scaleSrc); + int scaledDisplacement = scaleDisplacement(strideDst, strideSrc, displacement); + AMD64Address address = index == null ? new AMD64Address(src, scaledDisplacement) : new AMD64Address(src, index, strideSrc, scaledDisplacement); + pmovSZx(asm, size, extendMode, dst, strideDst, address, strideSrc); } - public static void pmovSZx(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, AMD64Address.Scale scaleDst, AMD64Address src, AMD64Address.Scale scaleSrc) { - if (scaleSrc.value < scaleDst.value) { + public static void pmovSZx(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, Stride strideDst, AMD64Address src, Stride strideSrc) { + if (strideSrc.value < strideDst.value) { if (size.getBytes() < AVXSize.XMM.getBytes()) { - movdqu(asm, pmovSZxGetSrcLoadSize(size, scaleDst, scaleSrc), dst, src); + movdqu(asm, pmovSZxGetSrcLoadSize(size, strideDst, strideSrc), dst, src); if (isAVX(asm)) { - loadAndExtendAVX(asm, size, extendMode, dst, scaleDst, dst, scaleSrc); + loadAndExtendAVX(asm, size, extendMode, dst, strideDst, dst, strideSrc); } else { - loadAndExtendSSE(asm, extendMode, dst, scaleDst, dst, scaleSrc); + loadAndExtendSSE(asm, extendMode, dst, strideDst, dst, strideSrc); } } else { if (isAVX(asm)) { - loadAndExtendAVX(asm, size, extendMode, dst, scaleDst, src, scaleSrc); + loadAndExtendAVX(asm, size, extendMode, dst, strideDst, src, strideSrc); } else { - loadAndExtendSSE(asm, extendMode, dst, scaleDst, src, scaleSrc); + loadAndExtendSSE(asm, extendMode, dst, strideDst, src, strideSrc); } } } else { - assert scaleSrc.value == scaleDst.value; + assert strideSrc.value == strideDst.value; movdqu(asm, size, dst, src); } } - public static void pmovSZx(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, AMD64Address.Scale scaleDst, Register src, AMD64Address.Scale scaleSrc) { - if (scaleSrc.value < scaleDst.value) { + public static void pmovSZx(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, Stride strideDst, Register src, Stride strideSrc) { + if (strideSrc.value < strideDst.value) { if (size.getBytes() < AVXSize.XMM.getBytes()) { - movdqu(asm, pmovSZxGetSrcLoadSize(size, scaleDst, scaleSrc), dst, src); + movdqu(asm, pmovSZxGetSrcLoadSize(size, strideDst, strideSrc), dst, src); if (isAVX(asm)) { - loadAndExtendAVX(asm, size, extendMode, dst, scaleDst, dst, scaleSrc); + loadAndExtendAVX(asm, size, extendMode, dst, strideDst, dst, strideSrc); } else { - loadAndExtendSSE(asm, extendMode, dst, scaleDst, dst, scaleSrc); + loadAndExtendSSE(asm, extendMode, dst, strideDst, dst, strideSrc); } } else { if (isAVX(asm)) { - loadAndExtendAVX(asm, size, extendMode, dst, scaleDst, src, scaleSrc); + loadAndExtendAVX(asm, size, extendMode, dst, strideDst, src, strideSrc); } else { - loadAndExtendSSE(asm, extendMode, dst, scaleDst, src, scaleSrc); + loadAndExtendSSE(asm, extendMode, dst, strideDst, src, strideSrc); } } } else { - assert scaleSrc.value == scaleDst.value; + assert strideSrc.value == strideDst.value; movdqu(asm, size, dst, src); } } - private static AVXSize pmovSZxGetSrcLoadSize(AVXSize size, AMD64Address.Scale scaleDst, AMD64Address.Scale scaleSrc) { - int srcBytes = size.getBytes() >> (scaleDst.log2 - scaleSrc.log2); + private static AVXSize pmovSZxGetSrcLoadSize(AVXSize size, Stride strideDst, Stride strideSrc) { + int srcBytes = size.getBytes() >> (strideDst.log2 - strideSrc.log2); switch (srcBytes) { case 4: return AVXSize.DWORD; @@ -939,7 +940,7 @@ private static VexMoveOp getVMOVOp(AVXSize size) { * Compares all packed bytes/words/dwords in {@code dst} to {@code src}. Matching values are set * to all ones (0xff, 0xffff, ...), non-matching values are set to zero. */ - public static void pcmpeq(AMD64MacroAssembler asm, AVXSize vectorSize, AMD64Address.Scale elementStride, Register dst, Register src) { + public static void pcmpeq(AMD64MacroAssembler asm, AVXSize vectorSize, Stride elementStride, Register dst, Register src) { pcmpeq(asm, vectorSize, elementStride.value, dst, src); } @@ -995,7 +996,7 @@ public static void pcmpeqd(AMD64MacroAssembler asm, AVXSize vectorSize, Register * Compares all packed bytes/words/dwords in {@code dst} to {@code src}. Matching values are set * to all ones (0xff, 0xffff, ...), non-matching values are set to zero. */ - public static void pcmpeq(AMD64MacroAssembler asm, AVXSize size, AMD64Address.Scale elementStride, Register dst, AMD64Address src) { + public static void pcmpeq(AMD64MacroAssembler asm, AVXSize size, Stride elementStride, Register dst, AMD64Address src) { pcmpeq(asm, size, elementStride.value, dst, src); } @@ -1063,69 +1064,69 @@ public static void pcmpgtd(AMD64MacroAssembler asm, AVXSize size, Register dst, } } - private static int scaleDisplacement(AMD64Address.Scale scaleDst, AMD64Address.Scale scaleSrc, int displacement) { - if (scaleSrc.value < scaleDst.value) { - assert (displacement & ((1 << (scaleDst.log2 - scaleSrc.log2)) - 1)) == 0; - return displacement >> (scaleDst.log2 - scaleSrc.log2); + private static int scaleDisplacement(Stride strideDst, Stride strideSrc, int displacement) { + if (strideSrc.value < strideDst.value) { + assert (displacement & ((1 << (strideDst.log2 - strideSrc.log2)) - 1)) == 0; + return displacement >> (strideDst.log2 - strideSrc.log2); } - assert scaleSrc.value == scaleDst.value; + assert strideSrc.value == strideDst.value; return displacement; } - public static void loadAndExtendAVX(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, AMD64Address.Scale scaleDst, Register src, AMD64Address.Scale scaleSrc) { - getAVXLoadAndExtendOp(scaleDst, scaleSrc, extendMode).emit(asm, size, dst, src); + public static void loadAndExtendAVX(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, Stride strideDst, Register src, Stride strideSrc) { + getAVXLoadAndExtendOp(strideDst, strideSrc, extendMode).emit(asm, size, dst, src); } - public static void loadAndExtendAVX(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, AMD64Address.Scale scaleDst, AMD64Address src, AMD64Address.Scale scaleSrc) { - getAVXLoadAndExtendOp(scaleDst, scaleSrc, extendMode).emit(asm, size, dst, src); + public static void loadAndExtendAVX(AMD64MacroAssembler asm, AVXSize size, ExtendMode extendMode, Register dst, Stride strideDst, AMD64Address src, Stride strideSrc) { + getAVXLoadAndExtendOp(strideDst, strideSrc, extendMode).emit(asm, size, dst, src); } - private static VexRMOp getAVXLoadAndExtendOp(AMD64Address.Scale scaleDst, AMD64Address.Scale scaleSrc, ExtendMode extendMode) { - switch (scaleSrc) { - case Times1: - switch (scaleDst) { - case Times2: + private static VexRMOp getAVXLoadAndExtendOp(Stride strideDst, Stride strideSrc, ExtendMode extendMode) { + switch (strideSrc) { + case S1: + switch (strideDst) { + case S2: return extendMode == ExtendMode.SIGN_EXTEND ? VexRMOp.VPMOVSXBW : VexRMOp.VPMOVZXBW; - case Times4: + case S4: return extendMode == ExtendMode.SIGN_EXTEND ? VexRMOp.VPMOVSXBD : VexRMOp.VPMOVZXBD; - case Times8: + case S8: return extendMode == ExtendMode.SIGN_EXTEND ? VexRMOp.VPMOVSXBQ : VexRMOp.VPMOVZXBQ; } throw GraalError.shouldNotReachHere(); - case Times2: - switch (scaleDst) { - case Times4: + case S2: + switch (strideDst) { + case S4: return extendMode == ExtendMode.SIGN_EXTEND ? VexRMOp.VPMOVSXWD : VexRMOp.VPMOVZXWD; - case Times8: + case S8: return extendMode == ExtendMode.SIGN_EXTEND ? VexRMOp.VPMOVSXWQ : VexRMOp.VPMOVZXWQ; } throw GraalError.shouldNotReachHere(); - case Times4: + case S4: return extendMode == ExtendMode.SIGN_EXTEND ? VexRMOp.VPMOVSXDQ : VexRMOp.VPMOVZXDQ; } throw GraalError.shouldNotReachHere(); } - public static void loadAndExtendSSE(AMD64MacroAssembler asm, ExtendMode extendMode, Register dst, AMD64Address.Scale scaleDst, AMD64Address src, AMD64Address.Scale scaleSrc) { + public static void loadAndExtendSSE(AMD64MacroAssembler asm, ExtendMode extendMode, Register dst, Stride strideDst, AMD64Address src, Stride strideSrc) { boolean signExtend = extendMode == ExtendMode.SIGN_EXTEND; - switch (scaleSrc) { - case Times1: - switch (scaleDst) { - case Times2: + switch (strideSrc) { + case S1: + switch (strideDst) { + case S2: if (signExtend) { asm.pmovsxbw(dst, src); } else { asm.pmovzxbw(dst, src); } return; - case Times4: + case S4: if (signExtend) { asm.pmovsxbd(dst, src); } else { asm.pmovzxbd(dst, src); } return; - case Times8: + case S8: if (signExtend) { asm.pmovsxbq(dst, src); } else { @@ -1134,16 +1135,16 @@ public static void loadAndExtendSSE(AMD64MacroAssembler asm, ExtendMode extendMo return; } throw GraalError.shouldNotReachHere(); - case Times2: - switch (scaleDst) { - case Times4: + case S2: + switch (strideDst) { + case S4: if (signExtend) { asm.pmovsxwd(dst, src); } else { asm.pmovzxwd(dst, src); } return; - case Times8: + case S8: if (signExtend) { asm.pmovsxwq(dst, src); } else { @@ -1152,7 +1153,7 @@ public static void loadAndExtendSSE(AMD64MacroAssembler asm, ExtendMode extendMo return; } throw GraalError.shouldNotReachHere(); - case Times4: + case S4: if (signExtend) { asm.pmovsxdq(dst, src); } else { @@ -1163,26 +1164,26 @@ public static void loadAndExtendSSE(AMD64MacroAssembler asm, ExtendMode extendMo throw GraalError.shouldNotReachHere(); } - public static void loadAndExtendSSE(AMD64MacroAssembler asm, ExtendMode extendMode, Register dst, AMD64Address.Scale scaleDst, Register src, AMD64Address.Scale scaleSrc) { + public static void loadAndExtendSSE(AMD64MacroAssembler asm, ExtendMode extendMode, Register dst, Stride strideDst, Register src, Stride strideSrc) { boolean signExtend = extendMode == ExtendMode.SIGN_EXTEND; - switch (scaleSrc) { - case Times1: - switch (scaleDst) { - case Times2: + switch (strideSrc) { + case S1: + switch (strideDst) { + case S2: if (signExtend) { asm.pmovsxbw(dst, src); } else { asm.pmovzxbw(dst, src); } return; - case Times4: + case S4: if (signExtend) { asm.pmovsxbd(dst, src); } else { asm.pmovzxbd(dst, src); } return; - case Times8: + case S8: if (signExtend) { asm.pmovsxbq(dst, src); } else { @@ -1191,16 +1192,16 @@ public static void loadAndExtendSSE(AMD64MacroAssembler asm, ExtendMode extendMo return; } throw GraalError.shouldNotReachHere(); - case Times2: - switch (scaleDst) { - case Times4: + case S2: + switch (strideDst) { + case S4: if (signExtend) { asm.pmovsxwd(dst, src); } else { asm.pmovzxwd(dst, src); } return; - case Times8: + case S8: if (signExtend) { asm.pmovsxwq(dst, src); } else { @@ -1209,7 +1210,7 @@ public static void loadAndExtendSSE(AMD64MacroAssembler asm, ExtendMode extendMo return; } throw GraalError.shouldNotReachHere(); - case Times4: + case S4: if (signExtend) { asm.pmovsxdq(dst, src); } else { diff --git a/compiler/src/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java b/compiler/src/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java index 4e9a4db25896..0241e81b1903 100644 --- a/compiler/src/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java +++ b/compiler/src/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java @@ -35,6 +35,7 @@ import org.graalvm.compiler.asm.aarch64.AArch64Address; import org.graalvm.compiler.asm.aarch64.AArch64Assembler.ConditionFlag; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.calc.Condition; import org.graalvm.compiler.core.common.memory.MemoryOrderMode; import org.graalvm.compiler.core.common.spi.LIRKindTool; @@ -504,41 +505,44 @@ public Variable emitByteSwap(Value input) { } @Override - public Variable emitArrayCompareTo(JavaKind kind1, JavaKind kind2, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value array2, Value length1, Value length2) { + public Variable emitArrayCompareTo(Stride strideA, Stride strideB, EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value lengthA, Value arrayB, Value lengthB) { LIRKind resultKind = LIRKind.value(AArch64Kind.DWORD); // DMS TODO: check calling conversion and registers used RegisterValue res = AArch64.r0.asValue(resultKind); - RegisterValue cnt1 = AArch64.r1.asValue(length1.getValueKind()); - RegisterValue cnt2 = AArch64.r2.asValue(length2.getValueKind()); - emitMove(cnt1, length1); - emitMove(cnt2, length2); - append(new AArch64ArrayCompareToOp(this, kind1, kind2, array1BaseOffset, array2BaseOffset, res, array1, array2, cnt1, cnt2)); + RegisterValue cntA = AArch64.r1.asValue(lengthA.getValueKind()); + RegisterValue cntB = AArch64.r2.asValue(lengthB.getValueKind()); + emitMove(cntA, lengthA); + emitMove(cntB, lengthB); + append(new AArch64ArrayCompareToOp(this, strideA, strideB, res, arrayA, cntA, arrayB, cntB)); Variable result = newVariable(resultKind); emitMove(result, res); return result; } @Override - public Variable emitArrayEquals(JavaKind kind, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value array2, Value length) { + public Variable emitArrayEquals(JavaKind kind, EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length) { + GraalError.guarantee(!kind.isNumericFloat(), "Float arrays comparison (bitwise_equal || both_NaN) isn't supported on AARCH64"); Variable result = newVariable(LIRKind.value(AArch64Kind.DWORD)); - append(new AArch64ArrayEqualsOp(this, kind, array1BaseOffset, array2BaseOffset, result, - asAllocatable(array1), null, asAllocatable(array2), null, asAllocatable(length))); + append(new AArch64ArrayEqualsOp(this, Stride.fromJavaKind(kind), result, + asAllocatable(arrayA), asAllocatable(offsetA), asAllocatable(arrayB), asAllocatable(offsetB), asAllocatable(length))); return result; } @Override - public Variable emitArrayEquals(JavaKind kind, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value offset1, Value array2, Value offset2, Value length) { + public Variable emitArrayEquals(Stride strideA, Stride strideB, EnumSet runtimeCheckedCPUFeatures, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length) { + if (strideA != strideB) { + throw GraalError.unimplemented("arrayEquals with different strides not yet implemented on AARCH64"); + } Variable result = newVariable(LIRKind.value(AArch64Kind.DWORD)); - append(new AArch64ArrayEqualsOp(this, kind, array1BaseOffset, array2BaseOffset, result, - asAllocatable(array1), asAllocatable(offset1), asAllocatable(array2), asAllocatable(offset2), asAllocatable(length))); + append(new AArch64ArrayEqualsOp(this, strideA, result, + asAllocatable(arrayA), asAllocatable(offsetA), asAllocatable(arrayB), asAllocatable(offsetB), asAllocatable(length))); return result; } @Override - public Variable emitArrayIndexOf(int arrayBaseOffset, JavaKind valueKind, boolean findTwoConsecutive, boolean withMask, EnumSet runtimeCheckedCPUFeatures, + public Variable emitArrayIndexOf(Stride stride, boolean findTwoConsecutive, boolean withMask, EnumSet runtimeCheckedCPUFeatures, Value arrayPointer, Value arrayOffset, Value arrayLength, Value fromIndex, Value... searchValues) { if (findTwoConsecutive) { GraalError.guarantee(searchValues.length == 2, "findTwoConsecutive requires exactly two search values"); @@ -553,7 +557,7 @@ public Variable emitArrayIndexOf(int arrayBaseOffset, JavaKind valueKind, boolea for (int i = 0; i < searchValues.length; i++) { allocatableSearchValues[i] = asAllocatable(searchValues[i]); } - append(new AArch64ArrayIndexOfOp(arrayBaseOffset, valueKind, findTwoConsecutive, this, result, asAllocatable(arrayPointer), asAllocatable(arrayOffset), asAllocatable(arrayLength), + append(new AArch64ArrayIndexOfOp(stride, findTwoConsecutive, this, result, asAllocatable(arrayPointer), asAllocatable(arrayOffset), asAllocatable(arrayLength), asAllocatable(fromIndex), allocatableSearchValues)); return result; } diff --git a/compiler/src/org.graalvm.compiler.core.amd64.test/src/org/graalvm/compiler/core/amd64/test/AMD64AddressLoweringTest.java b/compiler/src/org.graalvm.compiler.core.amd64.test/src/org/graalvm/compiler/core/amd64/test/AMD64AddressLoweringTest.java index a0796abf04c5..60ea53a9bc2d 100644 --- a/compiler/src/org.graalvm.compiler.core.amd64.test/src/org/graalvm/compiler/core/amd64/test/AMD64AddressLoweringTest.java +++ b/compiler/src/org.graalvm.compiler.core.amd64.test/src/org/graalvm/compiler/core/amd64/test/AMD64AddressLoweringTest.java @@ -26,7 +26,7 @@ import static org.junit.Assume.assumeTrue; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.amd64.AMD64AddressLowering; import org.graalvm.compiler.core.amd64.AMD64AddressNode; import org.graalvm.compiler.core.test.GraalCompilerTest; @@ -60,14 +60,14 @@ public void convertBaseAndIndexToDisplacement() { ValueNode base = graph.unique(const64(1000)); ValueNode index = graph.unique(const64(10)); AddressNode result = lowering.lower(base, index); - assertAddress(result, null, null, Scale.Times1, 1010); + assertAddress(result, null, null, Stride.S1, 1010); } @Test public void convertBaseToDisplacement() { ValueNode constantAddress = graph.addOrUniqueWithInputs(const64(1000)); AddressNode result = lowering.lower(constantAddress, null); - assertAddress(result, null, null, Scale.Times1, 1000); + assertAddress(result, null, null, Stride.S1, 1000); } @Test @@ -75,7 +75,7 @@ public void convertBaseAndShiftedIndexToDisplacement() { ValueNode base = graph.addOrUniqueWithInputs(const64(1000)); ValueNode index = graph.addOrUniqueWithInputs(new LeftShiftNode(const64(10), const32(1))); AddressNode result = lowering.lower(base, index); - assertAddress(result, null, null, Scale.Times2, 1020); + assertAddress(result, null, null, Stride.S2, 1020); } @Test @@ -83,7 +83,7 @@ public void convertBaseAndNegatedShiftedIndexToDisplacement() { ValueNode base = graph.addOrUniqueWithInputs(const64(1000)); ValueNode index = graph.addOrUniqueWithInputs(new NegateNode(new LeftShiftNode(const64(10), const32(2)))); AddressNode result = lowering.lower(base, index); - assertAddress(result, null, null, Scale.Times4, 960); + assertAddress(result, null, null, Stride.S4, 960); } @Test @@ -91,7 +91,7 @@ public void convertNegatedBaseAndNegatedShiftedIndexToDisplacement() { ValueNode base = graph.addOrUniqueWithInputs(new NegateNode(const64(1000))); ValueNode index = graph.addOrUniqueWithInputs(new NegateNode(new LeftShiftNode(const64(10), const32(2)))); AddressNode result = lowering.lower(base, index); - assertAddress(result, null, null, Scale.Times4, -1040); + assertAddress(result, null, null, Stride.S4, -1040); } @Test @@ -99,7 +99,7 @@ public void convertNegatedShiftedBaseAndNegatedIndexToDisplacement() { ValueNode base = graph.addOrUniqueWithInputs(new NegateNode(new LeftShiftNode(const64(10), const32(2)))); ValueNode index = graph.addOrUniqueWithInputs(new NegateNode(const64(1000))); AddressNode result = lowering.lower(base, index); - assertAddress(result, null, null, Scale.Times4, -1040); + assertAddress(result, null, null, Stride.S4, -1040); } @Test @@ -107,7 +107,7 @@ public void convertTwoLevelsOfNegatedShiftedBaseAndNegatedIndexToDisplacement() ValueNode base = graph.addOrUniqueWithInputs(new NegateNode(new LeftShiftNode(new NegateNode(new LeftShiftNode(const64(500), const32(1))), const32(1)))); ValueNode index = graph.addOrUniqueWithInputs(new NegateNode(new AddNode(new NegateNode(const64(13)), const64(3)))); AddressNode result = lowering.lower(base, index); - assertAddress(result, null, null, Scale.Times4, 2010); + assertAddress(result, null, null, Stride.S4, 2010); } private static ConstantNode const64(long value) { @@ -118,11 +118,11 @@ private static ConstantNode const32(long value) { return ConstantNode.forIntegerBits(Integer.SIZE, value); } - private static void assertAddress(AddressNode actual, ValueNode expectedBase, ValueNode expectedIndex, Scale expectedScale, int expectedDisplacement) { + private static void assertAddress(AddressNode actual, ValueNode expectedBase, ValueNode expectedIndex, Stride expectedStride, int expectedDisplacement) { AMD64AddressNode address = (AMD64AddressNode) actual; Assert.assertEquals(expectedBase, address.getBase()); Assert.assertEquals(expectedIndex, address.getIndex()); - Assert.assertEquals(expectedScale, address.getScale()); + Assert.assertEquals(expectedStride, address.getScale()); Assert.assertEquals(expectedDisplacement, address.getDisplacement()); } } diff --git a/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressLowering.java b/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressLowering.java index 9750a6471c64..e2711bc18d6e 100644 --- a/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressLowering.java +++ b/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressLowering.java @@ -25,7 +25,8 @@ package org.graalvm.compiler.core.amd64; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.core.common.type.AbstractPointerStamp; import org.graalvm.compiler.core.common.type.IntegerStamp; @@ -93,15 +94,15 @@ protected boolean improve(StructuredGraph graph, DebugContext debug, AMD64Addres LeftShiftNode shift = (LeftShiftNode) ret.getIndex(); if (shift.getY().isConstant()) { int amount = ret.getScale().log2 + shift.getY().asJavaConstant().asInt(); - if (Scale.isScaleShiftSupported(amount)) { + if (AMD64Address.isScaleShiftSupported(amount)) { ret.setIndex(shift.getX()); - ret.setScale(Scale.fromShift(amount)); + ret.setScale(Stride.fromLog2(amount)); return true; } } } - if (ret.getScale() == Scale.Times1) { + if (ret.getScale() == Stride.S1) { if (ret.getIndex() == null && ret.getBase() instanceof AddNode) { AddNode add = (AddNode) ret.getBase(); ret.setBase(add.getX()); diff --git a/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressNode.java b/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressNode.java index b96785fb051e..74b689d16975 100644 --- a/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressNode.java +++ b/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressNode.java @@ -25,7 +25,7 @@ package org.graalvm.compiler.core.amd64; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.type.IntegerStamp; import org.graalvm.compiler.graph.NodeClass; @@ -59,7 +59,7 @@ public class AMD64AddressNode extends AddressNode implements Simplifiable, LIRLo @OptionalInput private ValueNode base; @OptionalInput private ValueNode index; - private Scale scale; + private Stride stride; private int displacement; @@ -71,7 +71,7 @@ public AMD64AddressNode(ValueNode base, ValueNode index) { super(TYPE); this.base = base; this.index = index; - this.scale = Scale.Times1; + this.stride = Stride.S1; } public void canonicalizeIndex(SimplifierTool tool) { @@ -86,7 +86,7 @@ public void canonicalizeIndex(SimplifierTool tool) { ValueNode valY = add.getY(); if (valY instanceof ConstantNode) { int addBy = valY.asJavaConstant().asInt(); - displacement = displacement + scale.value * addBy; + displacement = displacement + stride.value * addBy; replaceFirstInput(index, phi); tool.addToWorkList(index); } @@ -107,7 +107,7 @@ public void generate(NodeLIRBuilderTool gen) { AllocatableValue indexReference; if (index == null) { indexReference = null; - } else if (scale.equals(Scale.Times1)) { + } else if (stride.equals(Stride.S1)) { indexReference = LIRKind.derivedBaseFromValue(indexValue); } else { if (LIRKind.isValue(indexValue)) { @@ -118,7 +118,7 @@ public void generate(NodeLIRBuilderTool gen) { } LIRKind kind = LIRKind.combineDerived(tool.getLIRKind(stamp(NodeView.DEFAULT)), baseReference, indexReference); - gen.setResult(this, new AMD64AddressValue(kind, baseValue, indexValue, scale, displacement)); + gen.setResult(this, new AMD64AddressValue(kind, baseValue, indexValue, stride, displacement)); } @Override @@ -147,12 +147,12 @@ public void setIndex(ValueNode index) { this.index = index; } - public Scale getScale() { - return scale; + public Stride getScale() { + return stride; } - public void setScale(Scale scale) { - this.scale = scale; + public void setScale(Stride stride) { + this.stride = stride; } public int getDisplacement() { diff --git a/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64CompressAddressLowering.java b/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64CompressAddressLowering.java index b766b057803b..31916c57f288 100644 --- a/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64CompressAddressLowering.java +++ b/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64CompressAddressLowering.java @@ -26,8 +26,8 @@ package org.graalvm.compiler.core.amd64; import jdk.vm.ci.code.Register; -import org.graalvm.compiler.asm.amd64.AMD64Address; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.debug.CounterKey; import org.graalvm.compiler.debug.DebugContext; @@ -53,7 +53,7 @@ protected final boolean improve(StructuredGraph graph, DebugContext debug, AMD64 return true; } - if (!isBaseNegated && !isIndexNegated && addr.getScale() == AMD64Address.Scale.Times1) { + if (!isBaseNegated && !isIndexNegated && addr.getScale() == Stride.S1) { ValueNode base = addr.getBase(); ValueNode index = addr.getIndex(); diff --git a/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java b/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java index 02d99c0bee37..a8ccb66eabf8 100644 --- a/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java +++ b/compiler/src/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java @@ -54,6 +54,7 @@ import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.NumUtil; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.calc.Condition; import org.graalvm.compiler.core.common.memory.MemoryOrderMode; import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; @@ -579,16 +580,17 @@ public Variable emitByteSwap(Value input) { @SuppressWarnings("unchecked") @Override - public Variable emitArrayCompareTo(JavaKind kind1, JavaKind kind2, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value array2, Value length1, Value length2) { + public Variable emitArrayCompareTo(Stride strideA, Stride strideB, EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value lengthA, + Value arrayB, Value lengthB) { LIRKind resultKind = LIRKind.value(AMD64Kind.DWORD); RegisterValue raxRes = AMD64.rax.asValue(resultKind); - RegisterValue cnt1 = AMD64.rcx.asValue(length1.getValueKind()); - RegisterValue cnt2 = AMD64.rdx.asValue(length2.getValueKind()); - emitMove(cnt1, length1); - emitMove(cnt2, length2); - append(new AMD64ArrayCompareToOp(this, getAVX3Threshold(), kind1, kind2, array1BaseOffset, array2BaseOffset, (EnumSet) runtimeCheckedCPUFeatures, - raxRes, array1, array2, cnt1, cnt2)); + RegisterValue cntA = AMD64.rcx.asValue(lengthA.getValueKind()); + RegisterValue cntB = AMD64.rdx.asValue(lengthB.getValueKind()); + emitMove(cntA, lengthA); + emitMove(cntB, lengthB); + append(new AMD64ArrayCompareToOp(this, getAVX3Threshold(), strideA, strideB, (EnumSet) runtimeCheckedCPUFeatures, + raxRes, arrayA, cntA, arrayB, cntB)); Variable result = newVariable(resultKind); emitMove(result, raxRes); return result; @@ -596,17 +598,17 @@ public Variable emitArrayCompareTo(JavaKind kind1, JavaKind kind2, int array1Bas @SuppressWarnings("unchecked") @Override - public Variable emitArrayRegionCompareTo(EnumSet runtimeCheckedCPUFeatures, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length, Value stride) { + public Variable emitArrayRegionCompareTo(EnumSet runtimeCheckedCPUFeatures, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length, Value dynamicStrides) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); append(AMD64ArrayRegionCompareToOp.movParamsAndCreate(this, null, null, (EnumSet) runtimeCheckedCPUFeatures, - result, arrayA, offsetA, arrayB, offsetB, length, stride, + result, arrayA, offsetA, arrayB, offsetB, length, dynamicStrides, ZERO_EXTEND)); return result; } @SuppressWarnings("unchecked") @Override - public Variable emitArrayRegionCompareTo(JavaKind strideA, JavaKind strideB, EnumSet runtimeCheckedCPUFeatures, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length) { + public Variable emitArrayRegionCompareTo(Stride strideA, Stride strideB, EnumSet runtimeCheckedCPUFeatures, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); append(AMD64ArrayRegionCompareToOp.movParamsAndCreate(this, strideA, strideB, (EnumSet) runtimeCheckedCPUFeatures, result, arrayA, offsetA, arrayB, offsetB, length, null, @@ -616,71 +618,90 @@ public Variable emitArrayRegionCompareTo(JavaKind strideA, JavaKind strideB, Enu @SuppressWarnings("unchecked") @Override - public Variable emitArrayEquals(JavaKind kind, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, Value array1, Value array2, Value length) { - Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); - append(AMD64ArrayEqualsOp.movParamsAndCreate(this, kind, kind, kind, array1BaseOffset, array2BaseOffset, 0, (EnumSet) runtimeCheckedCPUFeatures, - result, array1, null, array2, null, null, length, - ZERO_EXTEND)); - return result; - } - - @SuppressWarnings("unchecked") - @Override - public Variable emitArrayEquals(int baseOffsetA, int baseOffsetB, EnumSet runtimeCheckedCPUFeatures, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length, Value stride) { + public Variable emitArrayEquals(JavaKind commonElementKind, + EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, + Value arrayB, Value offsetB, + Value length) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); - append(AMD64ArrayEqualsOp.movParamsAndCreate(this, baseOffsetA, baseOffsetB, 0, (EnumSet) runtimeCheckedCPUFeatures, - result, arrayA, offsetA, arrayB, offsetB, null, length, stride, + Stride stride = Stride.fromJavaKind(commonElementKind); + append(AMD64ArrayEqualsOp.movParamsAndCreate(this, commonElementKind, stride, stride, stride, + (EnumSet) runtimeCheckedCPUFeatures, + result, arrayA, offsetA, arrayB, offsetB, null, length, null, ZERO_EXTEND)); return result; } @SuppressWarnings("unchecked") @Override - public Variable emitArrayEquals(JavaKind kind, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, Value array1, Value offset1, Value array2, Value offset2, + public Variable emitArrayEquals( + Stride strideA, Stride strideB, + EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, + Value arrayB, Value offsetB, Value length) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); - append(AMD64ArrayEqualsOp.movParamsAndCreate(this, kind, kind, kind, array1BaseOffset, array2BaseOffset, 0, (EnumSet) runtimeCheckedCPUFeatures, - result, array1, offset1, array2, offset2, null, length, + append(AMD64ArrayEqualsOp.movParamsAndCreate(this, strideA, strideB, strideB, + (EnumSet) runtimeCheckedCPUFeatures, + result, arrayA, offsetA, arrayB, offsetB, null, length, ZERO_EXTEND)); return result; } @SuppressWarnings("unchecked") @Override - public Variable emitArrayEquals(JavaKind kind1, JavaKind kind2, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, Value array1, Value offset1, Value array2, - Value offset2, Value length) { + public Variable emitArrayEqualsDynamicStrides( + EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, + Value arrayB, Value offsetB, + Value length, + Value dynamicStrides) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); - append(AMD64ArrayEqualsOp.movParamsAndCreate(this, kind1, kind2, kind2, array1BaseOffset, array2BaseOffset, 0, (EnumSet) runtimeCheckedCPUFeatures, - result, array1, offset1, array2, offset2, null, length, + append(AMD64ArrayEqualsOp.movParamsAndCreate(this, (EnumSet) runtimeCheckedCPUFeatures, + result, arrayA, offsetA, arrayB, offsetB, null, length, dynamicStrides, ZERO_EXTEND)); return result; } @SuppressWarnings("unchecked") @Override - public Variable emitArrayEquals(int baseOffsetA, int baseOffsetB, int baseOffsetMask, - EnumSet runtimeCheckedCPUFeatures, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value mask, Value length, Value stride) { + public Variable emitArrayEqualsWithMask( + Stride strideA, + Stride strideB, + Stride strideMask, + EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, + Value arrayB, Value offsetB, + Value mask, + Value length) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); - append(AMD64ArrayEqualsOp.movParamsAndCreate(this, baseOffsetA, baseOffsetB, baseOffsetMask, (EnumSet) runtimeCheckedCPUFeatures, - result, arrayA, offsetA, arrayB, offsetB, mask, length, stride, + append(AMD64ArrayEqualsOp.movParamsAndCreate(this, strideA, strideB, strideMask, + (EnumSet) runtimeCheckedCPUFeatures, + result, arrayA, offsetA, arrayB, offsetB, mask, length, ZERO_EXTEND)); return result; } @SuppressWarnings("unchecked") @Override - public Variable emitArrayEquals(JavaKind kind1, JavaKind kind2, JavaKind kindMask, int array1BaseOffset, int array2BaseOffset, int maskBaseOffset, - EnumSet runtimeCheckedCPUFeatures, Value array1, Value offset1, Value array2, Value offset2, Value mask, Value length) { + public Variable emitArrayEqualsWithMaskDynamicStrides( + EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, + Value arrayB, Value offsetB, + Value mask, + Value length, + Value dynamicStrides) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); - append(AMD64ArrayEqualsOp.movParamsAndCreate(this, kind1, kind2, kindMask, array1BaseOffset, array2BaseOffset, maskBaseOffset, (EnumSet) runtimeCheckedCPUFeatures, - result, array1, offset1, array2, offset2, mask, length, + append(AMD64ArrayEqualsOp.movParamsAndCreate(this, + (EnumSet) runtimeCheckedCPUFeatures, + result, arrayA, offsetA, arrayB, offsetB, mask, length, dynamicStrides, ZERO_EXTEND)); return result; } @SuppressWarnings("unchecked") @Override - public void emitArrayCopyWithConversion(JavaKind strideSrc, JavaKind strideDst, EnumSet runtimeCheckedCPUFeatures, + public void emitArrayCopyWithConversion(Stride strideSrc, Stride strideDst, EnumSet runtimeCheckedCPUFeatures, Value arraySrc, Value offsetSrc, Value arrayDst, Value offsetDst, Value length) { append(AMD64ArrayCopyWithConversionsOp.movParamsAndCreate(this, strideSrc, strideDst, (EnumSet) runtimeCheckedCPUFeatures, arraySrc, offsetSrc, arrayDst, offsetDst, length, @@ -690,8 +711,9 @@ public void emitArrayCopyWithConversion(JavaKind strideSrc, JavaKind strideDst, @SuppressWarnings("unchecked") @Override public void emitArrayCopyWithConversion(EnumSet runtimeCheckedCPUFeatures, - Value arraySrc, Value offsetSrc, Value arrayDst, Value offsetDst, Value length, Value stride) { - append(AMD64ArrayCopyWithConversionsOp.movParamsAndCreate(this, (EnumSet) runtimeCheckedCPUFeatures, arraySrc, offsetSrc, arrayDst, offsetDst, length, stride, ZERO_EXTEND)); + Value arraySrc, Value offsetSrc, Value arrayDst, Value offsetDst, Value length, Value dynamicStrides) { + append(AMD64ArrayCopyWithConversionsOp.movParamsAndCreate(this, (EnumSet) runtimeCheckedCPUFeatures, arraySrc, offsetSrc, arrayDst, offsetDst, length, dynamicStrides, + ZERO_EXTEND)); } @SuppressWarnings("unchecked") @@ -748,11 +770,10 @@ protected int getAVX3Threshold() { @SuppressWarnings("unchecked") @Override - public Variable emitArrayIndexOf(int arrayBaseOffset, JavaKind valueKind, boolean findTwoConsecutive, boolean withMask, EnumSet runtimeCheckedCPUFeatures, - Value arrayPointer, Value arrayOffset, Value arrayLength, Value fromIndex, - Value... searchValues) { + public Variable emitArrayIndexOf(Stride stride, boolean findTwoConsecutive, boolean withMask, EnumSet runtimeCheckedCPUFeatures, + Value arrayPointer, Value arrayOffset, Value arrayLength, Value fromIndex, Value... searchValues) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); - append(AMD64ArrayIndexOfOp.movParamsAndCreate(arrayBaseOffset, valueKind, findTwoConsecutive, withMask, this, (EnumSet) runtimeCheckedCPUFeatures, + append(AMD64ArrayIndexOfOp.movParamsAndCreate(stride, findTwoConsecutive, withMask, this, (EnumSet) runtimeCheckedCPUFeatures, result, arrayPointer, arrayOffset, arrayLength, fromIndex, searchValues)); return result; } diff --git a/compiler/src/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/Stride.java b/compiler/src/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/Stride.java new file mode 100644 index 000000000000..a383f475ef0a --- /dev/null +++ b/compiler/src/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/Stride.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.core.common; + +import org.graalvm.compiler.debug.GraalError; + +import jdk.vm.ci.meta.JavaKind; + +/** + * A byte stride. + */ +public enum Stride { + S1(1, 0), + S2(2, 1), + S4(4, 2), + S8(8, 3); + + Stride(int value, int log2) { + this.value = value; + this.log2 = log2; + } + + /** + * The value (or multiplier) of this stride. + */ + public final int value; + + /** + * The {@linkplain #value value} of this stride log 2. + */ + public final int log2; + + public int getBitCount() { + return value << 3; + } + + /** + * Creates a {@link Stride} for the scaling factor in {@code scale}. + */ + public static Stride fromInt(int scale) { + switch (scale) { + case 1: + return S1; + case 2: + return S2; + case 4: + return S4; + case 8: + return S8; + default: + throw GraalError.shouldNotReachHere("Unsupported stride: " + scale); + } + } + + /** + * Creates a {@link Stride} for the log2 scaling factor {@code shift}. + */ + public static Stride fromLog2(int shift) { + switch (shift) { + case 0: + return S1; + case 1: + return S2; + case 2: + return S4; + case 3: + return S8; + default: + throw GraalError.shouldNotReachHere("Unsupported stride: " + (1 << shift)); + } + } + + public static Stride fromJavaKind(JavaKind kind) { + return fromInt(kind.getByteCount()); + } + + public static Stride min(Stride a, Stride b) { + return a.value < b.value ? a : b; + } + + public static Stride max(Stride a, Stride b) { + return a.value > b.value ? a : b; + } +} diff --git a/compiler/src/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/StrideUtil.java b/compiler/src/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/StrideUtil.java index aac63ef65b11..2c2af43d8afc 100644 --- a/compiler/src/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/StrideUtil.java +++ b/compiler/src/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/StrideUtil.java @@ -24,14 +24,10 @@ */ package org.graalvm.compiler.core.common; -import jdk.vm.ci.meta.Value; -import org.graalvm.compiler.debug.GraalError; - -import jdk.vm.ci.code.CodeUtil; -import jdk.vm.ci.meta.JavaKind; - import static jdk.vm.ci.code.ValueUtil.isIllegal; +import jdk.vm.ci.meta.Value; + /** * This class provides utility methods for "stride-agnostic" intrinsic nodes such as * {@code org.graalvm.compiler.replacements.nodes.ArrayRegionCompareToNode}. These intrinsics may @@ -45,35 +41,22 @@ public final class StrideUtil { /** * Number of possible values of a single stride parameter. Currently, possible values are 0, 1, - * and 2. + * and 2. Note that {@link Stride#S8} is not supported. */ public static final int N_STRIDES = 3; - /** - * Short aliases for intrinsics that take {@link JavaKind} parameters to describe a stride in - * bytes. The naming is "S" for "stride", followed by the stride width in bytes. - */ - public static final JavaKind S1 = JavaKind.Byte; - public static final JavaKind S2 = JavaKind.Char; - public static final JavaKind S4 = JavaKind.Int; - /** - * Alias for intrinsics that take a {@link JavaKind} parameter to declare the type of an array - * parameter. This parameter "NONE" means that the array type is unknown or a native buffer has - * been used instead. - */ - public static final JavaKind NONE = JavaKind.Void; /** * Extract {@code strideA} from {@code directStubCallIndex}. */ - public static JavaKind getConstantStrideA(int directStubCallIndex) { - return log2ToStride(directStubCallIndex / N_STRIDES); + public static Stride getConstantStrideA(int directStubCallIndex) { + return Stride.fromLog2(directStubCallIndex / N_STRIDES); } /** * Extract {@code strideB} from {@code directStubCallIndex}. */ - public static JavaKind getConstantStrideB(int directStubCallIndex) { - return log2ToStride(directStubCallIndex % N_STRIDES); + public static Stride getConstantStrideB(int directStubCallIndex) { + return Stride.fromLog2(directStubCallIndex % N_STRIDES); } /** @@ -85,29 +68,6 @@ public static int getDirectStubCallIndex(int log2StrideA, int log2StrideB) { return (log2StrideA * N_STRIDES) + log2StrideB; } - /** - * Get the given stride's log2 form. - */ - public static int log2(JavaKind stride) { - return CodeUtil.log2(stride.getByteCount()); - } - - /** - * Convert the given log2 stride value to a {@link JavaKind} value. - */ - public static JavaKind log2ToStride(int log2Stride) { - switch (log2Stride) { - case 0: - return S1; - case 1: - return S2; - case 2: - return S4; - default: - throw GraalError.shouldNotReachHere(); - } - } - /** * Returns {@code true} if the {@code dynamicStride} parameter is unused, which implies constant * strides must be used instead. diff --git a/compiler/src/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/ReachabilityFenceCanonicalizationTest.java b/compiler/src/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/ReachabilityFenceCanonicalizationTest.java index f9b8476d0c4b..3b4ed70aa755 100644 --- a/compiler/src/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/ReachabilityFenceCanonicalizationTest.java +++ b/compiler/src/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/ReachabilityFenceCanonicalizationTest.java @@ -76,7 +76,7 @@ public void testUncompressOnlyUsedByReachabilityFences() { */ assumeTrue("skipping AMD64 specific test", getTarget().arch instanceof AMD64); assumeTrue("requires -XX:+UseCompressedOops", runtime().getVMConfig().useCompressedOops); - assumeTrue("skipping because of oop encoding", AMD64Address.Scale.isScaleShiftSupported(runtime().getVMConfig().getOopEncoding().getShift())); + assumeTrue("skipping because of oop encoding", AMD64Address.isScaleShiftSupported(runtime().getVMConfig().getOopEncoding().getShift())); Object[] inputs = new Payload[]{new Payload(), new Payload(), new Payload()}; test("uncompressOnlyUsedByReachabilityFences", inputs, false); diff --git a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayCompareToStub.java b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayCompareToStub.java index 9972fcc98556..c7a0e0ed95ce 100644 --- a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayCompareToStub.java +++ b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayCompareToStub.java @@ -24,10 +24,8 @@ */ package org.graalvm.compiler.hotspot.amd64; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; - import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.stubs.SnippetStub; @@ -42,22 +40,22 @@ public AMD64ArrayCompareToStub(OptionValues options, HotSpotProviders providers, } @Snippet - private static int byteArrayCompareToByteArray(Pointer array1, Pointer array2, int length1, int length2) { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S1, S1); + private static int byteArrayCompareToByteArray(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S1, Stride.S1); } @Snippet - private static int byteArrayCompareToCharArray(Pointer array1, Pointer array2, int length1, int length2) { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S1, S2); + private static int byteArrayCompareToCharArray(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S1, Stride.S2); } @Snippet - private static int charArrayCompareToByteArray(Pointer array1, Pointer array2, int length1, int length2) { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S2, S1); + private static int charArrayCompareToByteArray(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S2, Stride.S1); } @Snippet - private static int charArrayCompareToCharArray(Pointer array1, Pointer array2, int length1, int length2) { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S2, S2); + private static int charArrayCompareToCharArray(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S2, Stride.S2); } } diff --git a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayCopyWithConversionsStub.java b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayCopyWithConversionsStub.java index fd8c1bbecaff..75eda303a5ac 100644 --- a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayCopyWithConversionsStub.java +++ b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayCopyWithConversionsStub.java @@ -24,11 +24,8 @@ */ package org.graalvm.compiler.hotspot.amd64; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.stubs.SnippetStub; @@ -52,47 +49,47 @@ public AMD64ArrayCopyWithConversionsStub(OptionValues options, HotSpotProviders @Snippet private static void arrayCopyWithConversionsS1S1(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S1, S1); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S1, Stride.S1); } @Snippet private static void arrayCopyWithConversionsS1S2(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S1, S2); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S1, Stride.S2); } @Snippet private static void arrayCopyWithConversionsS1S4(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S1, S4); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S1, Stride.S4); } @Snippet private static void arrayCopyWithConversionsS2S1(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S2, S1); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S2, Stride.S1); } @Snippet private static void arrayCopyWithConversionsS2S2(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S2, S2); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S2, Stride.S2); } @Snippet private static void arrayCopyWithConversionsS2S4(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S2, S4); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S2, Stride.S4); } @Snippet private static void arrayCopyWithConversionsS4S1(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S4, S1); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S4, Stride.S1); } @Snippet private static void arrayCopyWithConversionsS4S2(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S4, S2); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S4, Stride.S2); } @Snippet private static void arrayCopyWithConversionsS4S4(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S4, S4); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S4, Stride.S4); } @Snippet diff --git a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayEqualsStub.java b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayEqualsStub.java index b6024b755595..3b47500a972f 100644 --- a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayEqualsStub.java +++ b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayEqualsStub.java @@ -24,11 +24,8 @@ */ package org.graalvm.compiler.hotspot.amd64; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.stubs.SnippetStub; @@ -46,43 +43,18 @@ public AMD64ArrayEqualsStub(OptionValues options, HotSpotProviders providers, Ho } @Snippet - private static boolean booleanArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Boolean); - } - - @Snippet - private static boolean byteArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Byte); - } - - @Snippet - private static boolean charArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Char); - } - - @Snippet - private static boolean shortArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Short); - } - - @Snippet - private static boolean intArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Int); - } - - @Snippet - private static boolean longArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Long); + private static boolean longArraysEquals(Pointer array1, long offset1, Pointer array2, long offset2, int length) { + return ArrayEqualsNode.equals(array1, offset1, array2, offset2, length, JavaKind.Long); } @Snippet - private static boolean floatArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Float); + private static boolean floatArraysEquals(Pointer array1, long offset1, Pointer array2, long offset2, int length) { + return ArrayEqualsNode.equals(array1, offset1, array2, offset2, length, JavaKind.Float); } @Snippet - private static boolean doubleArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Double); + private static boolean doubleArraysEquals(Pointer array1, long offset1, Pointer array2, long offset2, int length) { + return ArrayEqualsNode.equals(array1, offset1, array2, offset2, length, JavaKind.Double); } @Snippet @@ -92,46 +64,46 @@ private static boolean arrayRegionEqualsDynamicStrides(Object arrayA, long offse @Snippet private static boolean arrayRegionEqualsS1S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S1, S1); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S1); } @Snippet private static boolean arrayRegionEqualsS1S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S1, S2); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S2); } @Snippet private static boolean arrayRegionEqualsS1S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S1, S4); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S4); } @Snippet private static boolean arrayRegionEqualsS2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S2, S1); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S1); } @Snippet private static boolean arrayRegionEqualsS2S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S2, S2); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S2); } @Snippet private static boolean arrayRegionEqualsS2S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S2, S4); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S4); } @Snippet private static boolean arrayRegionEqualsS4S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S4, S1); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S1); } @Snippet private static boolean arrayRegionEqualsS4S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S4, S2); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S2); } @Snippet private static boolean arrayRegionEqualsS4S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S4, S4); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S4); } } diff --git a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayEqualsWithMaskStub.java b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayEqualsWithMaskStub.java index c878b56bf4b7..090e4a971822 100644 --- a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayEqualsWithMaskStub.java +++ b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayEqualsWithMaskStub.java @@ -24,17 +24,14 @@ */ package org.graalvm.compiler.hotspot.amd64; -import static org.graalvm.compiler.core.common.StrideUtil.NONE; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.stubs.SnippetStub; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.replacements.amd64.AMD64ArrayRegionEqualsWithMaskNode; +import org.graalvm.word.Pointer; public final class AMD64ArrayEqualsWithMaskStub extends SnippetStub { @@ -43,87 +40,62 @@ public AMD64ArrayEqualsWithMaskStub(OptionValues options, HotSpotProviders provi } @Snippet - private static boolean arrayRegionEqualsBS1S1S1(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S1, S1, S1); - } - - @Snippet - private static boolean arrayRegionEqualsBS1S2S1(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S1, S2, S1); - } - - @Snippet - private static boolean arrayRegionEqualsBS1S2S2(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S1, S2, S2); - } - - @Snippet - private static boolean arrayRegionEqualsBS2S1S1(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S2, S1, S1); - } - - @Snippet - private static boolean arrayRegionEqualsBS2S2S1(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S2, S2, S1); - } - - @Snippet - private static boolean arrayRegionEqualsBS2S2S2(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S2, S2, S2); + private static boolean arrayRegionEqualsS1S2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S2, Stride.S1); } @Snippet - private static boolean arrayRegionEqualsC(char[] arrayA, long offsetA, char[] arrayB, long offsetB, char[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S2, S2, S2, S2); + private static boolean arrayRegionEqualsS2S2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S2, Stride.S1); } @Snippet - private static boolean arrayRegionEqualsS1S1(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S1, S1, S1); + private static boolean arrayRegionEqualsS1S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S1, Stride.S1); } @Snippet - private static boolean arrayRegionEqualsS1S2(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S1, S2, S2); + private static boolean arrayRegionEqualsS1S2(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S2, Stride.S2); } @Snippet - private static boolean arrayRegionEqualsS1S4(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S1, S4, S4); + private static boolean arrayRegionEqualsS1S4(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S4, Stride.S4); } @Snippet - private static boolean arrayRegionEqualsS2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S2, S1, S1); + private static boolean arrayRegionEqualsS2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S1, Stride.S1); } @Snippet - private static boolean arrayRegionEqualsS2S2(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S2, S2, S2); + private static boolean arrayRegionEqualsS2S2(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S2, Stride.S2); } @Snippet - private static boolean arrayRegionEqualsS2S4(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S2, S4, S4); + private static boolean arrayRegionEqualsS2S4(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S4, Stride.S4); } @Snippet - private static boolean arrayRegionEqualsS4S1(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S4, S1, S1); + private static boolean arrayRegionEqualsS4S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S4, Stride.S1, Stride.S1); } @Snippet - private static boolean arrayRegionEqualsS4S2(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S4, S2, S2); + private static boolean arrayRegionEqualsS4S2(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S4, Stride.S2, Stride.S2); } @Snippet - private static boolean arrayRegionEqualsS4S4(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S4, S4, S4); + private static boolean arrayRegionEqualsS4S4(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S4, Stride.S4, Stride.S4); } @Snippet - private static boolean arrayRegionEqualsDynamicStrides(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length, int dynamicStrides) { + private static boolean arrayRegionEqualsDynamicStrides(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length, int dynamicStrides) { return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, dynamicStrides); } } diff --git a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayRegionCompareToStub.java b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayRegionCompareToStub.java index 8b5e3e8da0bb..11cce2e24f42 100644 --- a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayRegionCompareToStub.java +++ b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64ArrayRegionCompareToStub.java @@ -24,11 +24,8 @@ */ package org.graalvm.compiler.hotspot.amd64; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.stubs.SnippetStub; @@ -43,47 +40,47 @@ public AMD64ArrayRegionCompareToStub(OptionValues options, HotSpotProviders prov @Snippet private static int arrayRegionCompareToS1S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S1, S1); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S1); } @Snippet private static int arrayRegionCompareToS1S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S1, S2); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S2); } @Snippet private static int arrayRegionCompareToS1S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S1, S4); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S4); } @Snippet private static int arrayRegionCompareToS2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S2, S1); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S1); } @Snippet private static int arrayRegionCompareToS2S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S2, S2); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S2); } @Snippet private static int arrayRegionCompareToS2S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S2, S4); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S4); } @Snippet private static int arrayRegionCompareToS4S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S4, S1); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S1); } @Snippet private static int arrayRegionCompareToS4S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S4, S2); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S2); } @Snippet private static int arrayRegionCompareToS4S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S4, S4); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S4); } @Snippet diff --git a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java index 6d17d296fb8b..7b958f650493 100644 --- a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java +++ b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java @@ -26,7 +26,8 @@ package org.graalvm.compiler.hotspot.amd64; import org.graalvm.collections.EconomicMap; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.amd64.AMD64AddressNode; import org.graalvm.compiler.core.amd64.AMD64CompressAddressLowering; import org.graalvm.compiler.core.common.CompressEncoding; @@ -74,7 +75,7 @@ public AMD64HotSpotAddressLowering(GraalHotSpotVMConfig config, Register heapBas @Override protected final boolean improveUncompression(AMD64AddressNode addr, CompressionNode compression, ValueNode other) { CompressEncoding encoding = compression.getEncoding(); - if (!Scale.isScaleShiftSupported(encoding.getShift())) { + if (!AMD64Address.isScaleShiftSupported(encoding.getShift())) { return false; } @@ -95,8 +96,8 @@ protected final boolean improveUncompression(AMD64AddressNode addr, CompressionN addr.setBase(other); } - Scale scale = Scale.fromShift(encoding.getShift()); - addr.setScale(scale); + Stride stride = Stride.fromLog2(encoding.getShift()); + addr.setScale(stride); addr.setIndex(compression.getValue()); return true; } diff --git a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java index 55fbfe8161a1..87f709dbedba 100644 --- a/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java +++ b/compiler/src/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java @@ -30,7 +30,8 @@ import java.util.EnumSet; import java.util.List; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.amd64.AMD64ArithmeticLIRGenerator; import org.graalvm.compiler.core.amd64.AMD64LIRGenerator; @@ -569,11 +570,11 @@ public void emitNullCheck(Value address, LIRFrameState state) { CompressEncoding encoding = config.getOopEncoding(); Value uncompressed; int shift = encoding.getShift(); - if (Scale.isScaleShiftSupported(shift)) { + if (AMD64Address.isScaleShiftSupported(shift)) { LIRKind wordKind = LIRKind.unknownReference(target().arch.getWordKind()); RegisterValue heapBase = getProviders().getRegisters().getHeapBaseRegister().asValue(wordKind); - Scale scale = Scale.fromShift(shift); - uncompressed = new AMD64AddressValue(wordKind, heapBase, asAllocatable(address), scale, 0); + Stride stride = Stride.fromLog2(shift); + uncompressed = new AMD64AddressValue(wordKind, heapBase, asAllocatable(address), stride, 0); } else { uncompressed = emitUncompress(address, encoding, false); } diff --git a/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/ArrayIndexOfStub.java b/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/ArrayIndexOfStub.java index 1fe054037a09..43bc60789610 100644 --- a/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/ArrayIndexOfStub.java +++ b/compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/ArrayIndexOfStub.java @@ -24,12 +24,9 @@ */ package org.graalvm.compiler.hotspot; -import static org.graalvm.compiler.core.common.StrideUtil.NONE; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - +import jdk.vm.ci.meta.JavaKind; import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.stubs.SnippetStub; import org.graalvm.compiler.options.OptionValues; @@ -41,213 +38,108 @@ public ArrayIndexOfStub(OptionValues options, HotSpotProviders providers, HotSpo super(linkage.getDescriptor().getName(), options, providers, linkage); } - @Snippet - private static int indexOfTwoConsecutiveBS1(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, true, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Snippet - private static int indexOfTwoConsecutiveBS2(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, true, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Snippet - private static int indexOfTwoConsecutiveCS2(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, true, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Snippet - private static int indexOfB1S1(byte[] array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, array, offset, arrayLength, fromIndex, v1); - } - - @Snippet - private static int indexOfB2S1(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Snippet - private static int indexOfB3S1(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); - } - - @Snippet - private static int indexOfB4S1(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } - - @Snippet - private static int indexOfB1S2(byte[] array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, array, offset, arrayLength, fromIndex, v1); - } - - @Snippet - private static int indexOfB2S2(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Snippet - private static int indexOfB3S2(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); - } - - @Snippet - private static int indexOfB4S2(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } - - @Snippet - private static int indexOfC1S2(char[] array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, array, offset, arrayLength, fromIndex, v1); - } - - @Snippet - private static int indexOfC2S2(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Snippet - private static int indexOfC3S2(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); - } - - @Snippet - private static int indexOfC4S2(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } - @Snippet private static int indexOfTwoConsecutiveS1(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, true, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, true, false, array, offset, arrayLength, fromIndex, v1, v2); } @Snippet private static int indexOfTwoConsecutiveS2(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, true, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, true, false, array, offset, arrayLength, fromIndex, v1, v2); } @Snippet private static int indexOfTwoConsecutiveS4(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, true, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, true, false, array, offset, arrayLength, fromIndex, v1, v2); } @Snippet private static int indexOf1S1(Object array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, array, offset, arrayLength, fromIndex, v1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, array, offset, arrayLength, fromIndex, v1); } @Snippet private static int indexOf2S1(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, array, offset, arrayLength, fromIndex, v1, v2); } @Snippet private static int indexOf3S1(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); } @Snippet private static int indexOf4S1(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); } @Snippet private static int indexOf1S2(Object array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, array, offset, arrayLength, fromIndex, v1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, array, offset, arrayLength, fromIndex, v1); } @Snippet private static int indexOf2S2(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, array, offset, arrayLength, fromIndex, v1, v2); } @Snippet private static int indexOf3S2(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); } @Snippet private static int indexOf4S2(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); } @Snippet private static int indexOf1S4(Object array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, array, offset, arrayLength, fromIndex, v1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, array, offset, arrayLength, fromIndex, v1); } @Snippet private static int indexOf2S4(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, array, offset, arrayLength, fromIndex, v1, v2); } @Snippet private static int indexOf3S4(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); } @Snippet private static int indexOf4S4(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } - - @Snippet - private static int indexOfWithMaskBS1(byte[] array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, true, array, offset, length, fromIndex, v1, mask1); - } - - @Snippet - private static int indexOfWithMaskBS2(byte[] array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, true, array, offset, length, fromIndex, v1, mask1); - } - - @Snippet - private static int indexOfTwoConsecutiveWithMaskBS1(byte[] array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); - } - - @Snippet - private static int indexOfTwoConsecutiveWithMaskBS2(byte[] array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); - } - - @Snippet - private static int indexOfWithMaskCS2(char[] array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, true, array, offset, length, fromIndex, v1, mask1); - } - - @Snippet - private static int indexOfTwoConsecutiveWithMaskCS2(char[] array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); } @Snippet private static int indexOfWithMaskS1(Object array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, true, array, offset, length, fromIndex, v1, mask1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, true, array, offset, length, fromIndex, v1, mask1); } @Snippet private static int indexOfWithMaskS2(Object array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, true, array, offset, length, fromIndex, v1, mask1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, true, array, offset, length, fromIndex, v1, mask1); } @Snippet private static int indexOfWithMaskS4(Object array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, true, array, offset, length, fromIndex, v1, mask1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, true, array, offset, length, fromIndex, v1, mask1); } @Snippet private static int indexOfTwoConsecutiveWithMaskS1(Object array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); } @Snippet private static int indexOfTwoConsecutiveWithMaskS2(Object array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); } @Snippet private static int indexOfTwoConsecutiveWithMaskS4(Object array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); } } diff --git a/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayCompareToOp.java b/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayCompareToOp.java index 7bdd05f01204..292da4d630ad 100644 --- a/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayCompareToOp.java +++ b/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayCompareToOp.java @@ -35,6 +35,7 @@ import org.graalvm.compiler.asm.aarch64.AArch64Assembler.ConditionFlag; import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; @@ -43,7 +44,6 @@ import jdk.vm.ci.aarch64.AArch64Kind; import jdk.vm.ci.code.Register; import jdk.vm.ci.meta.AllocatableValue; -import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.Value; /** @@ -61,9 +61,6 @@ public final class AArch64ArrayCompareToOp extends AArch64LIRInstruction { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64ArrayCompareToOp.class); - private final int array1BaseOffset; - private final int array2BaseOffset; - private final boolean isLL; private final boolean isUU; private final boolean isLU; @@ -91,37 +88,33 @@ public final class AArch64ArrayCompareToOp extends AArch64LIRInstruction { @Temp({REG}) protected AllocatableValue vectorTemp4; @Temp({REG}) protected AllocatableValue vectorTemp5; - public AArch64ArrayCompareToOp(LIRGeneratorTool tool, JavaKind kind1, JavaKind kind2, int array1BaseOffset, int array2BaseOffset, Value result, Value array1, Value array2, Value length1, - Value length2) { + public AArch64ArrayCompareToOp(LIRGeneratorTool tool, Stride strideA, Stride strideB, Value result, + Value arrayA, Value lengthA, Value arrayB, Value lengthB) { super(TYPE); - assert array1.getPlatformKind() == AArch64Kind.QWORD && array1.getPlatformKind() == array2.getPlatformKind(); - assert length1.getPlatformKind() == AArch64Kind.DWORD && length1.getPlatformKind() == length2.getPlatformKind(); + assert arrayA.getPlatformKind() == AArch64Kind.QWORD && arrayA.getPlatformKind() == arrayB.getPlatformKind(); + assert lengthA.getPlatformKind() == AArch64Kind.DWORD && lengthA.getPlatformKind() == lengthB.getPlatformKind(); assert result.getPlatformKind() == AArch64Kind.DWORD; - assert kind1 == JavaKind.Byte || kind1 == JavaKind.Char; - assert kind2 == JavaKind.Byte || kind2 == JavaKind.Char; + assert strideA == Stride.S1 || strideA == Stride.S2; + assert strideB == Stride.S1 || strideB == Stride.S2; - this.isLL = (kind1 == kind2 && kind1 == JavaKind.Byte); - this.isUU = (kind1 == kind2 && kind1 == JavaKind.Char); - this.isLU = (kind1 != kind2 && kind1 == JavaKind.Byte); - this.isUL = (kind1 != kind2 && kind1 == JavaKind.Char); + this.isLL = (strideA == strideB && strideA == Stride.S1); + this.isUU = (strideA == strideB && strideA == Stride.S2); + this.isLU = (strideA != strideB && strideA == Stride.S1); + this.isUL = (strideA != strideB && strideA == Stride.S2); - this.array1BaseOffset = array1BaseOffset; - this.array2BaseOffset = array2BaseOffset; this.resultValue = result; - this.array1Value = array1; - this.array2Value = array2; + this.array1Value = arrayA; + this.array2Value = arrayB; /* * The length values are inputs but are also killed like temporaries so need both Use and * Temp annotations, which will only work with fixed registers. */ - this.length1Value = length1; - this.length2Value = length2; - this.length1ValueTemp = length1; - this.length2ValueTemp = length2; + this.length1Value = length1ValueTemp = lengthA; + this.length2Value = length2ValueTemp = lengthB; LIRKind archWordKind = LIRKind.unknownReference(tool.target().arch.getWordKind()); this.temp1 = tool.newVariable(archWordKind); @@ -153,8 +146,8 @@ protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm final Label simdImpl = new Label(); /* Load array base addresses. */ - masm.add(64, array1, asRegister(array1Value), array1BaseOffset); - masm.add(64, array2, asRegister(array2Value), array2BaseOffset); + masm.mov(64, array1, asRegister(array1Value)); + masm.mov(64, array2, asRegister(array2Value)); masm.prfm(AArch64Address.createBaseRegisterOnlyAddress(64, array1), AArch64Assembler.PrefetchMode.PLDL1STRM); masm.prfm(AArch64Address.createBaseRegisterOnlyAddress(64, array2), AArch64Assembler.PrefetchMode.PLDL1STRM); diff --git a/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayEqualsOp.java b/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayEqualsOp.java index 941de7030f07..eb994c6092de 100644 --- a/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayEqualsOp.java +++ b/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayEqualsOp.java @@ -37,7 +37,7 @@ import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.ScratchRegister; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.core.common.NumUtil; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; @@ -46,7 +46,6 @@ import jdk.vm.ci.aarch64.AArch64Kind; import jdk.vm.ci.code.Register; import jdk.vm.ci.meta.AllocatableValue; -import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.Value; /** @@ -57,10 +56,7 @@ public final class AArch64ArrayEqualsOp extends AArch64LIRInstruction { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64ArrayEqualsOp.class); - private final JavaKind kind; - private final int array1BaseOffset; - private final int array2BaseOffset; - private final int arrayIndexScale; + private final Stride stride; @Def({REG}) protected Value resultValue; @Alive({REG}) protected Value array1Value; @@ -78,11 +74,9 @@ public final class AArch64ArrayEqualsOp extends AArch64LIRInstruction { @Temp({REG}) protected AllocatableValue vectorTemp3; @Temp({REG}) protected AllocatableValue vectorTemp4; - public AArch64ArrayEqualsOp(LIRGeneratorTool tool, JavaKind kind, int array1BaseOffset, int array2BaseOffset, Value result, Value array1, Value offset1, Value array2, Value offset2, - Value length) { + public AArch64ArrayEqualsOp(LIRGeneratorTool tool, Stride stride, Value result, Value array1, Value offset1, Value array2, Value offset2, Value length) { super(TYPE); - - assert !kind.isNumericFloat() : "Float arrays comparison (bitwise_equal || both_NaN) isn't supported"; + this.stride = stride; assert result.getPlatformKind() == AArch64Kind.DWORD; assert array1.getPlatformKind() == AArch64Kind.QWORD && array1.getPlatformKind() == array2.getPlatformKind(); @@ -90,17 +84,6 @@ public AArch64ArrayEqualsOp(LIRGeneratorTool tool, JavaKind kind, int array1Base assert offset2 == null || offset2.getPlatformKind() == AArch64Kind.QWORD; assert length.getPlatformKind() == AArch64Kind.DWORD; - this.kind = kind; - - /* - * The arrays are expected to have the same kind and thus the same index scale. For - * primitive arrays, this will mean the same array base offset as well; but if we compare a - * regular array with a hybrid object, they may have two different offsets. - */ - this.array1BaseOffset = array1BaseOffset; - this.array2BaseOffset = array2BaseOffset; - this.arrayIndexScale = tool.getProviders().getMetaAccess().getArrayIndexScale(kind); - this.resultValue = result; this.array1Value = array1; this.offset1Value = offset1 == null ? Value.ILLEGAL : offset1; @@ -133,10 +116,9 @@ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { Register scratch = sc2.getRegister(); // Get array length in bytes and store as a 64-bit value. - int shiftAmt = NumUtil.log2Ceil(arrayIndexScale); masm.mov(32, byteArrayLength, asRegister(lengthValue)); - masm.lsl(64, byteArrayLength, byteArrayLength, shiftAmt); - masm.compare(32, asRegister(lengthValue), 32 / arrayIndexScale); + masm.lsl(64, byteArrayLength, byteArrayLength, stride.log2); + masm.compare(32, asRegister(lengthValue), 32 / stride.value); masm.branchConditionally(ConditionFlag.LE, scalarCompare); emitSIMDCompare(masm, byteArrayLength, hasMismatch, scratch, breakLabel); @@ -152,22 +134,11 @@ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { } private void loadArrayStart(AArch64MacroAssembler masm, Register array1, Register array2) { - if (!offset1Value.equals(Value.ILLEGAL)) { - // address start = pointer + baseOffset + offset - masm.add(64, array1, asRegister(array1Value), asRegister(offset1Value)); - masm.add(64, array1, array1, array1BaseOffset); - } else { - // address start = pointer + base offset - masm.add(64, array1, asRegister(array1Value), array1BaseOffset); - } - if (!offset2Value.equals(Value.ILLEGAL)) { - // address start = pointer + baseOffset + offset - masm.add(64, array2, asRegister(array2Value), asRegister(offset2Value)); - masm.add(64, array2, array2, array2BaseOffset); - } else { - // address start = pointer + base offset - masm.add(64, array2, asRegister(array2Value), array2BaseOffset); - } + // address start = pointer + offset + masm.add(64, array1, asRegister(array1Value), asRegister(offset1Value)); + // address start = pointer + offset + masm.add(64, array2, asRegister(array2Value), asRegister(offset2Value)); + } private void emitScalarCompare(AArch64MacroAssembler masm, Register byteArrayLength, Register hasMismatch, Register scratch, Label breakLabel) { @@ -241,7 +212,7 @@ private void emitTailCompares(AArch64MacroAssembler masm, Register result, Regis Register temp = asRegister(temp4); - if (kind.getByteCount() <= 4) { + if (stride.value <= 4) { // Compare trailing 4 bytes, if any. masm.ands(32, zr, result, 4); masm.branchConditionally(ConditionFlag.EQ, compare2Bytes); @@ -250,7 +221,7 @@ private void emitTailCompares(AArch64MacroAssembler masm, Register result, Regis masm.eor(32, rscratch1, temp, rscratch1); masm.cbnz(32, rscratch1, breakLabel); - if (kind.getByteCount() <= 2) { + if (stride.value <= 2) { // Compare trailing 2 bytes, if any. masm.bind(compare2Bytes); masm.ands(32, zr, result, 2); @@ -261,7 +232,7 @@ private void emitTailCompares(AArch64MacroAssembler masm, Register result, Regis masm.cbnz(32, rscratch1, breakLabel); // The one-byte tail compare is only required for boolean and byte arrays. - if (kind.getByteCount() <= 1) { + if (stride.value <= 1) { // Compare trailing byte, if any. masm.bind(compare1Byte); masm.ands(32, zr, result, 1); @@ -406,9 +377,9 @@ private void emitSIMDCompare(AArch64MacroAssembler masm, Register byteArrayLengt masm.mov(64, array1Address, refAddress1); if (!offset2Value.equals(Value.ILLEGAL)) { masm.add(64, array2Address, asRegister(array2Value), asRegister(offset2Value)); - masm.add(64, array2Address, array2Address, array2BaseOffset - 32); + masm.sub(64, array2Address, array2Address, 32); } else { - masm.add(64, array2Address, asRegister(array2Value), array2BaseOffset - 32); + masm.sub(64, array2Address, asRegister(array2Value), 32); } masm.add(64, array2Address, array2Address, byteArrayLength); masm.jmp(compareByChunkTail); diff --git a/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayIndexOfOp.java b/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayIndexOfOp.java index ccd94f6af263..b4ef53721f98 100644 --- a/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayIndexOfOp.java +++ b/compiler/src/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArrayIndexOfOp.java @@ -43,6 +43,7 @@ import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.ScratchRegister; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; @@ -52,7 +53,6 @@ import jdk.vm.ci.aarch64.AArch64Kind; import jdk.vm.ci.code.Register; import jdk.vm.ci.meta.AllocatableValue; -import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.Value; @Opcode("AArch64_ARRAY_INDEX_OF") @@ -60,7 +60,6 @@ public final class AArch64ArrayIndexOfOp extends AArch64LIRInstruction { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64ArrayIndexOfOp.class); private final boolean findTwoConsecutive; - private final int arrayBaseOffset; private final int elementByteSize; @Def({REG}) protected AllocatableValue resultValue; @@ -82,7 +81,7 @@ public final class AArch64ArrayIndexOfOp extends AArch64LIRInstruction { @Temp({REG, ILLEGAL}) protected AllocatableValue vectorTemp5; @Temp({REG, ILLEGAL}) protected AllocatableValue vectorTemp6; - public AArch64ArrayIndexOfOp(int arrayBaseOffset, JavaKind valueKind, boolean findTwoConsecutive, LIRGeneratorTool tool, + public AArch64ArrayIndexOfOp(Stride stride, boolean findTwoConsecutive, LIRGeneratorTool tool, AllocatableValue result, AllocatableValue arrayPtr, AllocatableValue arrayOffset, AllocatableValue arrayLength, AllocatableValue fromIndex, AllocatableValue[] searchValues) { super(TYPE); @@ -94,8 +93,7 @@ public AArch64ArrayIndexOfOp(int arrayBaseOffset, JavaKind valueKind, boolean fi assert fromIndex.getPlatformKind() == AArch64Kind.DWORD; assert Arrays.stream(searchValues).allMatch(sv -> sv.getPlatformKind() == AArch64Kind.DWORD); - this.arrayBaseOffset = arrayBaseOffset; - this.elementByteSize = tool.getProviders().getMetaAccess().getArrayIndexScale(valueKind); + this.elementByteSize = stride.value; this.findTwoConsecutive = findTwoConsecutive; resultValue = result; arrayPtrValue = arrayPtr; @@ -432,8 +430,7 @@ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { masm.cbz(32, arrayLength, done); /* Load address of first array element */ - masm.add(64, baseAddress, asRegister(arrayPtrValue), arrayBaseOffset); - masm.add(64, baseAddress, baseAddress, asRegister(arrayOffsetValue)); + masm.add(64, baseAddress, asRegister(arrayPtrValue), asRegister(arrayOffsetValue)); /* * Search element-by-element for small arrays (with search space size of less than 32 bytes, diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64AddressValue.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64AddressValue.java index 63b573d87738..f5831f37b7cc 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64AddressValue.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64AddressValue.java @@ -30,7 +30,7 @@ import java.util.EnumSet; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.CompositeValue; import org.graalvm.compiler.lir.InstructionValueConsumer; import org.graalvm.compiler.lir.InstructionValueProcedure; @@ -48,29 +48,29 @@ public final class AMD64AddressValue extends CompositeValue { @Component({REG, OperandFlag.ILLEGAL}) protected AllocatableValue base; @Component({REG, OperandFlag.ILLEGAL}) protected AllocatableValue index; - protected final Scale scale; + protected final Stride stride; protected final int displacement; private final Object displacementAnnotation; private static final EnumSet flags = EnumSet.of(OperandFlag.REG, OperandFlag.ILLEGAL); public AMD64AddressValue(ValueKind kind, AllocatableValue base, int displacement) { - this(kind, base, Value.ILLEGAL, Scale.Times1, displacement); + this(kind, base, Value.ILLEGAL, Stride.S1, displacement); } - public AMD64AddressValue(ValueKind kind, AllocatableValue base, AllocatableValue index, Scale scale, int displacement) { - this(kind, base, index, scale, displacement, null); + public AMD64AddressValue(ValueKind kind, AllocatableValue base, AllocatableValue index, Stride stride, int displacement) { + this(kind, base, index, stride, displacement, null); } - public AMD64AddressValue(ValueKind kind, AllocatableValue base, AllocatableValue index, Scale scale, int displacement, Object displacementAnnotation) { + public AMD64AddressValue(ValueKind kind, AllocatableValue base, AllocatableValue index, Stride stride, int displacement, Object displacementAnnotation) { super(kind); this.base = base; this.index = index; - this.scale = scale; + this.stride = stride; this.displacement = displacement; this.displacementAnnotation = displacementAnnotation; - assert scale != null; + assert stride != null; } public AllocatableValue getBase() { @@ -86,7 +86,7 @@ public CompositeValue forEachComponent(LIRInstruction inst, OperandMode mode, In AllocatableValue newBase = (AllocatableValue) proc.doValue(inst, base, mode, flags); AllocatableValue newIndex = (AllocatableValue) proc.doValue(inst, index, mode, flags); if (!base.identityEquals(newBase) || !index.identityEquals(newIndex)) { - return new AMD64AddressValue(getValueKind(), newBase, newIndex, scale, displacement, displacementAnnotation); + return new AMD64AddressValue(getValueKind(), newBase, newIndex, stride, displacement, displacementAnnotation); } return this; } @@ -98,7 +98,7 @@ protected void visitEachComponent(LIRInstruction inst, OperandMode mode, Instruc } public AMD64AddressValue withKind(ValueKind newKind) { - return new AMD64AddressValue(newKind, base, index, scale, displacement, displacementAnnotation); + return new AMD64AddressValue(newKind, base, index, stride, displacement, displacementAnnotation); } private static Register toRegister(AllocatableValue value) { @@ -111,7 +111,7 @@ private static Register toRegister(AllocatableValue value) { } public AMD64Address toAddress() { - return new AMD64Address(toRegister(base), toRegister(index), scale, displacement, displacementAnnotation); + return new AMD64Address(toRegister(base), toRegister(index), stride, displacement, displacementAnnotation); } @Override @@ -123,7 +123,7 @@ public String toString() { sep = " + "; } if (isLegal(index)) { - s.append(sep).append(index).append(" * ").append(scale.value); + s.append(sep).append(index).append(" * ").append(stride.value); sep = " + "; } if (displacement < 0) { @@ -146,13 +146,13 @@ public boolean isValidImplicitNullCheckFor(Value value, int implicitNullCheckLim public boolean equals(Object obj) { if (obj instanceof AMD64AddressValue) { AMD64AddressValue addr = (AMD64AddressValue) obj; - return getValueKind().equals(addr.getValueKind()) && displacement == addr.displacement && base.equals(addr.base) && scale == addr.scale && index.equals(addr.index); + return getValueKind().equals(addr.getValueKind()) && displacement == addr.displacement && base.equals(addr.base) && stride == addr.stride && index.equals(addr.index); } return false; } @Override public int hashCode() { - return base.hashCode() ^ index.hashCode() ^ (displacement << 4) ^ (scale.value << 8) ^ getValueKind().hashCode(); + return base.hashCode() ^ index.hashCode() ^ (displacement << 4) ^ (stride.value << 8) ^ getValueKind().hashCode(); } } diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayCompareToOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayCompareToOp.java index f1e5231f7abc..90a46dff8f25 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayCompareToOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayCompareToOp.java @@ -36,11 +36,11 @@ import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; @@ -50,7 +50,6 @@ import jdk.vm.ci.amd64.AMD64Kind; import jdk.vm.ci.code.CodeUtil; import jdk.vm.ci.code.Register; -import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.Value; /** @@ -61,49 +60,43 @@ public final class AMD64ArrayCompareToOp extends AMD64ComplexVectorOp { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64ArrayCompareToOp.class); - private final JavaKind kind1; - private final JavaKind kind2; - private final int array1BaseOffset; - private final int array2BaseOffset; - + private final Stride strideA; + private final Stride strideB; private final int useAVX3Threshold; @Def({REG}) protected Value resultValue; - @Alive({REG}) protected Value array1Value; - @Alive({REG}) protected Value array2Value; - @Use({REG}) protected Value length1Value; - @Use({REG}) protected Value length2Value; - @Temp({REG}) protected Value length1ValueTemp; - @Temp({REG}) protected Value length2ValueTemp; + @Alive({REG}) protected Value arrayAValue; + @Alive({REG}) protected Value arrayBValue; + @Use({REG}) protected Value lengthAValue; + @Use({REG}) protected Value lengthBValue; + @Temp({REG}) protected Value lengthAValueTemp; + @Temp({REG}) protected Value lengthBValueTemp; @Temp({REG}) protected Value temp1; @Temp({REG}) protected Value temp2; @Temp({REG, ILLEGAL}) protected Value vectorTemp1; - public AMD64ArrayCompareToOp(LIRGeneratorTool tool, int useAVX3Threshold, JavaKind kind1, JavaKind kind2, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value result, Value array1, Value array2, Value length1, Value length2) { + public AMD64ArrayCompareToOp(LIRGeneratorTool tool, int useAVX3Threshold, Stride strideA, Stride strideB, + EnumSet runtimeCheckedCPUFeatures, + Value result, + Value arrayA, Value lengthA, + Value arrayB, Value lengthB) { super(TYPE, tool, runtimeCheckedCPUFeatures, AVXSize.ZMM); assert CodeUtil.isPowerOf2(useAVX3Threshold) : "AVX3Threshold must be power of 2"; this.useAVX3Threshold = useAVX3Threshold; - this.kind1 = kind1; - this.kind2 = kind2; - - // Both offsets should be the same but better be safe than sorry. - this.array1BaseOffset = array1BaseOffset; - this.array2BaseOffset = array2BaseOffset; + this.strideA = strideA; + this.strideB = strideB; this.resultValue = result; - this.array1Value = array1; - this.array2Value = array2; + this.arrayAValue = arrayA; + this.arrayBValue = arrayB; /* * The length values are inputs but are also killed like temporaries so need both Use and * Temp annotations, which will only work with fixed registers. */ - this.length1Value = length1; - this.length2Value = length2; - this.length1ValueTemp = length1; - this.length2ValueTemp = length2; + this.lengthAValue = lengthAValueTemp = lengthA; + this.lengthBValue = lengthBValueTemp = lengthB; // Allocate some temporaries. this.temp1 = tool.newVariable(LIRKind.unknownReference(tool.target().arch.getWordKind())); @@ -124,32 +117,29 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { Register str2 = asRegister(temp2); // Load array base addresses. - masm.leaq(str1, new AMD64Address(asRegister(array1Value), array1BaseOffset)); - masm.leaq(str2, new AMD64Address(asRegister(array2Value), array2BaseOffset)); - Register cnt1 = asRegister(length1Value); - Register cnt2 = asRegister(length2Value); + masm.movq(str1, asRegister(arrayAValue)); + masm.movq(str2, asRegister(arrayBValue)); + Register cnt1 = asRegister(lengthAValue); + Register cnt2 = asRegister(lengthBValue); Label labelLengthDiff = new Label(); Label labelPop = new Label(); Label labelDone = new Label(); Label labelWhileHead = new Label(); Label labelCompareWideVectorsLoopFailed = new Label(); // used only _LP64 && AVX3 - int stride; - int stride2; - int adrStride = -1; - int adrStride1 = -1; - int adrStride2 = -1; int stride2x2 = 0x40; - AMD64Address.Scale scale = null; - AMD64Address.Scale scale1 = null; - AMD64Address.Scale scale2 = null; + Stride maxStride = Stride.max(strideA, strideB); + Stride scale1 = Stride.S1; + Stride scale2 = Stride.S2; + int elementsPerXMMVector = 16 >> maxStride.log2; + int elementsPerYMMVector = 32 >> maxStride.log2; - if (!(kind1 == JavaKind.Byte && kind2 == JavaKind.Byte)) { + if (!(strideA == Stride.S1 && strideB == Stride.S1)) { stride2x2 = 0x20; } - if (kind1 != kind2) { + if (strideA != strideB) { masm.shrl(cnt2, 1); } // Compute the minimum of the string lengths and the @@ -163,11 +153,11 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // Is the minimum length zero? masm.testlAndJcc(cnt2, cnt2, ConditionFlag.Zero, labelLengthDiff, false); - if (kind1 == JavaKind.Byte && kind2 == JavaKind.Byte) { + if (strideA == Stride.S1 && strideB == Stride.S1) { // Load first bytes masm.movzbl(result, new AMD64Address(str1, 0)); // result = str1[0] masm.movzbl(cnt1, new AMD64Address(str2, 0)); // cnt1 = str2[0] - } else if (kind1 == JavaKind.Char && kind2 == JavaKind.Char) { + } else if (strideA == Stride.S2 && strideB == Stride.S2) { // Load first characters masm.movzwl(result, new AMD64Address(str1, 0)); masm.movzwl(cnt1, new AMD64Address(str2, 0)); @@ -177,28 +167,15 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { } masm.sublAndJcc(result, cnt1, ConditionFlag.NotZero, labelPop, false); - if (kind1 == JavaKind.Char && kind2 == JavaKind.Char) { + if (strideA == Stride.S2 && strideB == Stride.S2) { // Divide length by 2 to get number of chars masm.shrl(cnt2, 1); } masm.cmplAndJcc(cnt2, 1, ConditionFlag.Equal, labelLengthDiff, false); // Check if the strings start at the same location and setup scale and stride - if (kind1 == kind2) { + if (strideA == strideB) { masm.cmpqAndJcc(str1, str2, ConditionFlag.Equal, labelLengthDiff, false); - - if (kind1 == JavaKind.Byte && kind2 == JavaKind.Byte) { - scale = AMD64Address.Scale.Times1; - stride = 16; - } else { - scale = AMD64Address.Scale.Times2; - stride = 8; - } - } else { - scale1 = AMD64Address.Scale.Times1; - scale2 = AMD64Address.Scale.Times2; - // scale not used - stride = 8; } if (supportsAVX2AndYMM() && masm.supports(CPUFeature.SSE4_2)) { @@ -216,33 +193,21 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { Label labelCompareWideVectorsLoopAVX3 = new Label(); // used only _LP64 && AVX3 int pcmpmask = 0x19; - if (kind1 == JavaKind.Byte && kind2 == JavaKind.Byte) { + if (strideA == Stride.S1 && strideB == Stride.S1) { pcmpmask &= ~0x01; } // Setup to compare 16-chars (32-bytes) vectors, // start from first character again because it has aligned address. - if (kind1 == JavaKind.Byte && kind2 == JavaKind.Byte) { - stride2 = 32; - } else { - stride2 = 16; - } - - if (kind1 == kind2) { - adrStride = stride << scale.log2; - } else { - adrStride1 = 8; // stride << scale1; - adrStride2 = 16; // stride << scale2; - } assert result.equals(rax) && cnt2.equals(rdx) && cnt1.equals(rcx) : "pcmpestri"; // rax and rdx are used by pcmpestri as elements counters masm.movl(result, cnt2); - masm.andlAndJcc(cnt2, ~(stride2 - 1), ConditionFlag.Zero, labelCompareTailLong, false); + masm.andlAndJcc(cnt2, ~(elementsPerYMMVector - 1), ConditionFlag.Zero, labelCompareTailLong, false); // fast path : compare first 2 8-char vectors. masm.bind(labelCompare16Chars); - if (kind1 == kind2) { + if (strideA == strideB) { masm.movdqu(vec1, new AMD64Address(str1, 0)); } else { masm.pmovzxbw(vec1, new AMD64Address(str1, 0)); @@ -250,33 +215,32 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.pcmpestri(vec1, new AMD64Address(str2, 0), pcmpmask); masm.jccb(ConditionFlag.Below, labelCompareIndexChar); - if (kind1 == kind2) { - masm.movdqu(vec1, new AMD64Address(str1, adrStride)); - masm.pcmpestri(vec1, new AMD64Address(str2, adrStride), pcmpmask); + if (strideA == strideB) { + masm.movdqu(vec1, new AMD64Address(str1, 16)); } else { - masm.pmovzxbw(vec1, new AMD64Address(str1, adrStride1)); - masm.pcmpestri(vec1, new AMD64Address(str2, adrStride2), pcmpmask); + masm.pmovzxbw(vec1, new AMD64Address(str1, 8)); } + masm.pcmpestri(vec1, new AMD64Address(str2, 16), pcmpmask); masm.jccb(ConditionFlag.AboveEqual, labelCompareWideVectors); - masm.addl(cnt1, stride); + masm.addl(cnt1, elementsPerXMMVector); // Compare the characters at index in cnt1 masm.bind(labelCompareIndexChar); // cnt1 has the offset of the mismatching character - loadNextElements(masm, result, cnt2, str1, str2, scale, scale1, scale2, cnt1); + loadNextElements(masm, result, cnt2, str1, str2, maxStride, scale1, scale2, cnt1); masm.subl(result, cnt2); masm.jmp(labelPop); // Setup the registers to start vector comparison loop masm.bind(labelCompareWideVectors); - if (kind1 == kind2) { - masm.leaq(str1, new AMD64Address(str1, result, scale)); - masm.leaq(str2, new AMD64Address(str2, result, scale)); + if (strideA == strideB) { + masm.leaq(str1, new AMD64Address(str1, result, maxStride)); + masm.leaq(str2, new AMD64Address(str2, result, maxStride)); } else { masm.leaq(str1, new AMD64Address(str1, result, scale1)); masm.leaq(str2, new AMD64Address(str2, result, scale2)); } - masm.subl(result, stride2); - masm.sublAndJcc(cnt2, stride2, ConditionFlag.Zero, labelCompareWideTail, false); + masm.subl(result, elementsPerYMMVector); + masm.sublAndJcc(cnt2, elementsPerYMMVector, ConditionFlag.Zero, labelCompareWideTail, false); masm.negq(result); // In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest) @@ -289,10 +253,10 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.testlAndJcc(cnt2, stride2x2 - 1, ConditionFlag.NotZero, labelCompareWideVectorsLoopAVX2, true); masm.bind(labelCompareWideVectorsLoopAVX3); // the hottest loop - if (kind1 == kind2) { - masm.evmovdqu64(vec1, new AMD64Address(str1, result, scale)); + if (strideA == strideB) { + masm.evmovdqu64(vec1, new AMD64Address(str1, result, maxStride)); // k7 == 11..11, if operands equal, otherwise k7 has some 0 - masm.evpcmpeqb(k7, vec1, new AMD64Address(str2, result, scale)); + masm.evpcmpeqb(k7, vec1, new AMD64Address(str2, result, maxStride)); } else { masm.evpmovzxbw(vec1, new AMD64Address(str1, result, scale1)); // k7 == 11..11, if operands equal, otherwise k7 has some 0 @@ -309,17 +273,17 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { } masm.bind(labelCompareWideVectorsLoopAVX2); - if (kind1 == kind2) { - masm.vmovdqu(vec1, new AMD64Address(str1, result, scale)); - masm.vpxor(vec1, vec1, new AMD64Address(str2, result, scale)); + if (strideA == strideB) { + masm.vmovdqu(vec1, new AMD64Address(str1, result, maxStride)); + masm.vpxor(vec1, vec1, new AMD64Address(str2, result, maxStride)); } else { masm.vpmovzxbw(vec1, new AMD64Address(str1, result, scale1)); masm.vpxor(vec1, vec1, new AMD64Address(str2, result, scale2)); } masm.vptest(vec1, vec1); masm.jcc(ConditionFlag.NotZero, labelVectorNotEqual); - masm.addq(result, stride2); - masm.sublAndJcc(cnt2, stride2, ConditionFlag.NotZero, labelCompareWideVectorsLoop, false); + masm.addq(result, elementsPerYMMVector); + masm.sublAndJcc(cnt2, elementsPerYMMVector, ConditionFlag.NotZero, labelCompareWideVectorsLoop, false); // clean upper bits of YMM registers masm.vpxor(vec1, vec1, vec1); @@ -327,7 +291,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.bind(labelCompareWideTail); masm.testqAndJcc(result, result, ConditionFlag.Zero, labelLengthDiff, false); - masm.movl(result, stride2); + masm.movl(result, elementsPerYMMVector); masm.movl(cnt2, result); masm.negq(result); masm.jmp(labelCompareWideVectorsLoopAVX2); @@ -336,9 +300,9 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.bind(labelVectorNotEqual); // clean upper bits of YMM registers masm.vpxor(vec1, vec1, vec1); - if (kind1 == kind2) { - masm.leaq(str1, new AMD64Address(str1, result, scale)); - masm.leaq(str2, new AMD64Address(str2, result, scale)); + if (strideA == strideB) { + masm.leaq(str1, new AMD64Address(str1, result, maxStride)); + masm.leaq(str2, new AMD64Address(str2, result, maxStride)); } else { masm.leaq(str1, new AMD64Address(str1, result, scale1)); masm.leaq(str2, new AMD64Address(str2, result, scale2)); @@ -348,19 +312,19 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // Compare tail chars, length between 1 to 15 chars masm.bind(labelCompareTailLong); masm.movl(cnt2, result); - masm.cmplAndJcc(cnt2, stride, ConditionFlag.Less, labelCompareSmallStr, false); + masm.cmplAndJcc(cnt2, elementsPerXMMVector, ConditionFlag.Less, labelCompareSmallStr, false); - if (kind1 == kind2) { + if (strideA == strideB) { masm.movdqu(vec1, new AMD64Address(str1, 0)); } else { masm.pmovzxbw(vec1, new AMD64Address(str1, 0)); } masm.pcmpestri(vec1, new AMD64Address(str2, 0), pcmpmask); masm.jcc(ConditionFlag.Below, labelCompareIndexChar); - masm.subqAndJcc(cnt2, stride, ConditionFlag.Zero, labelLengthDiff, false); - if (kind1 == kind2) { - masm.leaq(str1, new AMD64Address(str1, result, scale)); - masm.leaq(str2, new AMD64Address(str2, result, scale)); + masm.subqAndJcc(cnt2, elementsPerXMMVector, ConditionFlag.Zero, labelLengthDiff, false); + if (strideA == strideB) { + masm.leaq(str1, new AMD64Address(str1, result, maxStride)); + masm.leaq(str2, new AMD64Address(str2, result, maxStride)); } else { masm.leaq(str1, new AMD64Address(str1, result, scale1)); masm.leaq(str2, new AMD64Address(str2, result, scale2)); @@ -379,13 +343,13 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // Setup to compare 8-char (16-byte) vectors, // start from first character again because it has aligned address. masm.movl(result, cnt2); - if (kind1 == JavaKind.Byte && kind2 == JavaKind.Byte) { + if (strideA == Stride.S1 && strideB == Stride.S1) { pcmpmask &= ~0x01; } - masm.andlAndJcc(cnt2, ~(stride - 1), ConditionFlag.Zero, labelCompareTail, false); - if (kind1 == kind2) { - masm.leaq(str1, new AMD64Address(str1, result, scale)); - masm.leaq(str2, new AMD64Address(str2, result, scale)); + masm.andlAndJcc(cnt2, ~(elementsPerXMMVector - 1), ConditionFlag.Zero, labelCompareTail, false); + if (strideA == strideB) { + masm.leaq(str1, new AMD64Address(str1, result, maxStride)); + masm.leaq(str2, new AMD64Address(str2, result, maxStride)); } else { masm.leaq(str1, new AMD64Address(str1, result, scale1)); masm.leaq(str2, new AMD64Address(str2, result, scale2)); @@ -405,9 +369,9 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { assert result.equals(rax) && cnt2.equals(rdx) && cnt1.equals(rcx) : "pcmpestri"; masm.bind(labelCompareWideVectors); - if (kind1 == kind2) { - masm.movdqu(vec1, new AMD64Address(str1, result, scale)); - masm.pcmpestri(vec1, new AMD64Address(str2, result, scale), pcmpmask); + if (strideA == strideB) { + masm.movdqu(vec1, new AMD64Address(str1, result, maxStride)); + masm.pcmpestri(vec1, new AMD64Address(str2, result, maxStride), pcmpmask); } else { masm.pmovzxbw(vec1, new AMD64Address(str1, result, scale1)); masm.pcmpestri(vec1, new AMD64Address(str2, result, scale2), pcmpmask); @@ -415,18 +379,18 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // After pcmpestri cnt1(rcx) contains mismatched element index masm.jccb(ConditionFlag.Below, labelVectorNotEqual); // CF==1 - masm.addq(result, stride); - masm.subqAndJcc(cnt2, stride, ConditionFlag.NotZero, labelCompareWideVectors, true); + masm.addq(result, elementsPerXMMVector); + masm.subqAndJcc(cnt2, elementsPerXMMVector, ConditionFlag.NotZero, labelCompareWideVectors, true); // compare wide vectors tail masm.testqAndJcc(result, result, ConditionFlag.Zero, labelLengthDiff, false); - masm.movl(cnt2, stride); - masm.movl(result, stride); + masm.movl(cnt2, elementsPerXMMVector); + masm.movl(result, elementsPerXMMVector); masm.negq(result); - if (kind1 == kind2) { - masm.movdqu(vec1, new AMD64Address(str1, result, scale)); - masm.pcmpestri(vec1, new AMD64Address(str2, result, scale), pcmpmask); + if (strideA == strideB) { + masm.movdqu(vec1, new AMD64Address(str1, result, maxStride)); + masm.pcmpestri(vec1, new AMD64Address(str2, result, maxStride), pcmpmask); } else { masm.pmovzxbw(vec1, new AMD64Address(str1, result, scale1)); masm.pcmpestri(vec1, new AMD64Address(str2, result, scale2), pcmpmask); @@ -436,7 +400,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // Mismatched characters in the vectors masm.bind(labelVectorNotEqual); masm.addq(cnt1, result); - loadNextElements(masm, result, cnt2, str1, str2, scale, scale1, scale2, cnt1); + loadNextElements(masm, result, cnt2, str1, str2, maxStride, scale1, scale2, cnt1); masm.subl(result, cnt2); masm.jmpb(labelPop); @@ -446,9 +410,9 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { } // Shift str2 and str1 to the end of the arrays, negate min - if (kind1 == kind2) { - masm.leaq(str1, new AMD64Address(str1, cnt2, scale)); - masm.leaq(str2, new AMD64Address(str2, cnt2, scale)); + if (strideA == strideB) { + masm.leaq(str1, new AMD64Address(str1, cnt2, maxStride)); + masm.leaq(str2, new AMD64Address(str2, cnt2, maxStride)); } else { masm.leaq(str1, new AMD64Address(str1, cnt2, scale1)); masm.leaq(str2, new AMD64Address(str2, cnt2, scale2)); @@ -458,14 +422,14 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // Compare the rest of the elements masm.bind(labelWhileHead); - loadNextElements(masm, result, cnt1, str1, str2, scale, scale1, scale2, cnt2); + loadNextElements(masm, result, cnt1, str1, str2, maxStride, scale1, scale2, cnt2); masm.sublAndJcc(result, cnt1, ConditionFlag.NotZero, labelPop, true); masm.incqAndJcc(cnt2, ConditionFlag.NotZero, labelWhileHead, true); // Strings are equal up to min length. Return the length difference. masm.bind(labelLengthDiff); masm.pop(result); - if (kind1 == JavaKind.Char && kind2 == JavaKind.Char) { + if (strideA == Stride.S2 && strideB == Stride.S2) { // Divide diff by 2 to get number of chars masm.sarl(result, 1); } @@ -478,17 +442,17 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.notq(cnt1); masm.bsfq(cnt2, cnt1); // if (ae != StrIntrinsicNode::LL) { - if (!(kind1 == JavaKind.Byte && kind2 == JavaKind.Byte)) { + if (!(strideA == Stride.S1 && strideB == Stride.S1)) { // Divide diff by 2 to get number of chars masm.sarl(cnt2, 1); } masm.addq(result, cnt2); - if (kind1 == JavaKind.Byte && kind2 == JavaKind.Byte) { - masm.movzbl(cnt1, new AMD64Address(str2, result, Scale.Times1)); - masm.movzbl(result, new AMD64Address(str1, result, Scale.Times1)); - } else if (kind1 == JavaKind.Char && kind2 == JavaKind.Char) { - masm.movzwl(cnt1, new AMD64Address(str2, result, scale)); - masm.movzwl(result, new AMD64Address(str1, result, scale)); + if (strideA == Stride.S1 && strideB == Stride.S1) { + masm.movzbl(cnt1, new AMD64Address(str2, result, Stride.S1)); + masm.movzbl(result, new AMD64Address(str1, result, Stride.S1)); + } else if (strideA == Stride.S2 && strideB == Stride.S2) { + masm.movzwl(cnt1, new AMD64Address(str2, result, maxStride)); + masm.movzwl(result, new AMD64Address(str1, result, maxStride)); } else { masm.movzwl(cnt1, new AMD64Address(str2, result, scale2)); masm.movzbl(result, new AMD64Address(str1, result, scale1)); @@ -503,23 +467,23 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // That's it masm.bind(labelDone); - if (kind1 == JavaKind.Char && kind2 == JavaKind.Byte) { + if (strideA == Stride.S2 && strideB == Stride.S1) { masm.negl(result); } } private void loadNextElements(AMD64MacroAssembler masm, Register elem1, Register elem2, Register str1, Register str2, - AMD64Address.Scale scale, AMD64Address.Scale scale1, - AMD64Address.Scale scale2, Register index) { - if (kind1 == JavaKind.Byte && kind2 == JavaKind.Byte) { - masm.movzbl(elem1, new AMD64Address(str1, index, scale, 0)); - masm.movzbl(elem2, new AMD64Address(str2, index, scale, 0)); - } else if (kind1 == JavaKind.Char && kind2 == JavaKind.Char) { - masm.movzwl(elem1, new AMD64Address(str1, index, scale, 0)); - masm.movzwl(elem2, new AMD64Address(str2, index, scale, 0)); + Stride stride, Stride stride1, + Stride stride2, Register index) { + if (strideA == Stride.S1 && strideB == Stride.S1) { + masm.movzbl(elem1, new AMD64Address(str1, index, stride, 0)); + masm.movzbl(elem2, new AMD64Address(str2, index, stride, 0)); + } else if (strideA == Stride.S2 && strideB == Stride.S2) { + masm.movzwl(elem1, new AMD64Address(str1, index, stride, 0)); + masm.movzwl(elem2, new AMD64Address(str2, index, stride, 0)); } else { - masm.movzbl(elem1, new AMD64Address(str1, index, scale1, 0)); - masm.movzwl(elem2, new AMD64Address(str2, index, scale2, 0)); + masm.movzbl(elem1, new AMD64Address(str1, index, stride1, 0)); + masm.movzwl(elem2, new AMD64Address(str2, index, stride2, 0)); } } } diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayCopyWithConversionsOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayCopyWithConversionsOp.java index 25bbc41b9f71..6e24abfcd68a 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayCopyWithConversionsOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayCopyWithConversionsOp.java @@ -43,17 +43,16 @@ import java.util.Arrays; import java.util.EnumSet; -import java.util.Objects; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRMIOp; import org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.StrideUtil; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; @@ -97,8 +96,8 @@ private enum Op { private static final Register REG_LENGTH = rdx; private static final Register REG_STRIDE = r8; - private final Scale scaleSrcConst; - private final Scale scaleDstConst; + private final Stride strideSrcConst; + private final Stride strideDstConst; private final AMD64MacroAssembler.ExtendMode extendMode; @Use({REG}) private Value arraySrc; @@ -134,7 +133,7 @@ private enum Op { * @param extendMode sign- or zero-extend array elements when inflating to a bigger stride. * @param dynamicStrides dynamic stride dispatch as described in {@link StrideUtil}. */ - private AMD64ArrayCopyWithConversionsOp(LIRGeneratorTool tool, JavaKind strideSrc, JavaKind strideDst, EnumSet runtimeCheckedCPUFeatures, + private AMD64ArrayCopyWithConversionsOp(LIRGeneratorTool tool, Stride strideSrc, Stride strideDst, EnumSet runtimeCheckedCPUFeatures, Value arraySrc, Value offsetSrc, Value arrayDst, Value offsetDst, Value length, Value dynamicStrides, AMD64MacroAssembler.ExtendMode extendMode) { super(TYPE, tool, runtimeCheckedCPUFeatures, YMM); this.extendMode = extendMode; @@ -149,13 +148,12 @@ private AMD64ArrayCopyWithConversionsOp(LIRGeneratorTool tool, JavaKind strideSr this.dynamicStridesTmp = this.dynamicStrides = dynamicStrides; if (StrideUtil.useConstantStrides(dynamicStrides)) { - assert strideSrc.isNumericInteger() && strideDst.isNumericInteger(); - this.scaleSrcConst = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(strideSrc))); - this.scaleDstConst = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(strideDst))); - this.vectorTemp = new Value[getNumberOfRequiredVectorRegisters(getOp(scaleDstConst, scaleSrcConst))]; + this.strideSrcConst = strideSrc; + this.strideDstConst = strideDst; + this.vectorTemp = new Value[getNumberOfRequiredVectorRegisters(getOp(strideDstConst, strideSrcConst))]; } else { - scaleSrcConst = null; - scaleDstConst = null; + strideSrcConst = null; + strideDstConst = null; this.vectorTemp = new Value[5]; } for (int i = 0; i < vectorTemp.length; i++) { @@ -163,7 +161,7 @@ private AMD64ArrayCopyWithConversionsOp(LIRGeneratorTool tool, JavaKind strideSr } } - public static AMD64ArrayCopyWithConversionsOp movParamsAndCreate(LIRGeneratorTool tool, JavaKind strideSrc, JavaKind strideDst, + public static AMD64ArrayCopyWithConversionsOp movParamsAndCreate(LIRGeneratorTool tool, Stride strideSrc, Stride strideDst, EnumSet runtimeCheckedCPUFeatures, Value arraySrc, Value offsetSrc, Value arrayDst, Value offsetDst, Value length, AMD64MacroAssembler.ExtendMode extendMode) { return movParamsAndCreate(tool, strideSrc, strideDst, runtimeCheckedCPUFeatures, arraySrc, offsetSrc, arrayDst, offsetDst, length, Value.ILLEGAL, extendMode); } @@ -173,7 +171,7 @@ public static AMD64ArrayCopyWithConversionsOp movParamsAndCreate(LIRGeneratorToo return movParamsAndCreate(tool, null, null, runtimeCheckedCPUFeatures, arraySrc, offsetSrc, arrayDst, offsetDst, length, stride, extendMode); } - private static AMD64ArrayCopyWithConversionsOp movParamsAndCreate(LIRGeneratorTool tool, JavaKind strideSrc, JavaKind strideDst, + private static AMD64ArrayCopyWithConversionsOp movParamsAndCreate(LIRGeneratorTool tool, Stride strideSrc, Stride strideDst, EnumSet runtimeCheckedCPUFeatures, Value arraySrc, Value offsetSrc, Value arrayDst, Value offsetDst, Value length, Value dynamicStrides, AMD64MacroAssembler.ExtendMode extendMode) { RegisterValue regArraySrc = REG_ARRAY_SRC.asValue(arraySrc.getValueKind()); @@ -197,20 +195,20 @@ private static AMD64ArrayCopyWithConversionsOp movParamsAndCreate(LIRGeneratorTo extendMode); } - private static Op getOp(Scale scaleDst, Scale scaleSrc) { - if (scaleDst.value == scaleSrc.value) { + private static Op getOp(Stride strideDst, Stride strideSrc) { + if (strideDst.value == strideSrc.value) { return Op.copy; } - if (scaleDst.value < scaleSrc.value) { - switch (scaleSrc) { - case Times2: - assert scaleDst == Scale.Times1; + if (strideDst.value < strideSrc.value) { + switch (strideSrc) { + case S2: + assert strideDst == Stride.S1; return Op.compressCharToByte; - case Times4: - switch (scaleDst) { - case Times1: + case S4: + switch (strideDst) { + case S1: return Op.compressIntToByte; - case Times2: + case S2: return Op.compressIntToChar; default: throw new UnsupportedOperationException(); @@ -219,18 +217,18 @@ private static Op getOp(Scale scaleDst, Scale scaleSrc) { throw new UnsupportedOperationException(); } } - switch (scaleSrc) { - case Times1: - switch (scaleDst) { - case Times2: + switch (strideSrc) { + case S1: + switch (strideDst) { + case S2: return Op.inflateByteToChar; - case Times4: + case S4: return Op.inflateByteToInt; default: throw new UnsupportedOperationException(); } - case Times2: - assert scaleDst == Scale.Times4; + case S2: + assert strideDst == Stride.S4; return Op.inflateCharToInt; default: throw new UnsupportedOperationException(); @@ -257,11 +255,11 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { Register dso = asRegister(offsetDst); Register len = asRegister(length); - asm.leaq(src, new AMD64Address(src, sro, Scale.Times1)); - asm.leaq(dst, new AMD64Address(dst, dso, Scale.Times1)); + asm.leaq(src, new AMD64Address(src, sro, Stride.S1)); + asm.leaq(dst, new AMD64Address(dst, dso, Stride.S1)); if (isIllegal(dynamicStrides)) { - emitOp(crb, asm, this.scaleSrcConst, this.scaleDstConst, src, sro, dst, len); + emitOp(crb, asm, this.strideSrcConst, this.strideDstConst, src, sro, dst, len); } else { Label[] variants = new Label[9]; Label end = new Label(); @@ -273,24 +271,24 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { // use the 1-byte-1-byte stride variant for the 2-2 and 4-4 cases by simply shifting the // length asm.align(crb.target.wordSize * 2); - asm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Scale.Times4, Scale.Times4)]); + asm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Stride.S4, Stride.S4)]); asm.shll(len, 1); asm.align(crb.target.wordSize * 2); - asm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Scale.Times2, Scale.Times2)]); + asm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Stride.S2, Stride.S2)]); asm.shll(len, 1); asm.align(crb.target.wordSize * 2); - asm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Scale.Times1, Scale.Times1)]); - emitOp(crb, asm, Scale.Times1, Scale.Times1, src, sro, dst, len); + asm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Stride.S1, Stride.S1)]); + emitOp(crb, asm, Stride.S1, Stride.S1, src, sro, dst, len); asm.jmp(end); - for (Scale scaleSrc : new Scale[]{Scale.Times1, Scale.Times2, Scale.Times4}) { - for (Scale scaleDst : new Scale[]{Scale.Times1, Scale.Times2, Scale.Times4}) { - if (scaleSrc == scaleDst) { + for (Stride strideSrc : new Stride[]{Stride.S1, Stride.S2, Stride.S4}) { + for (Stride strideDst : new Stride[]{Stride.S1, Stride.S2, Stride.S4}) { + if (strideSrc == strideDst) { continue; } asm.align(crb.target.wordSize * 2); - asm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(scaleSrc, scaleDst)]); - emitOp(crb, asm, scaleSrc, scaleDst, src, sro, dst, len); + asm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(strideSrc, strideDst)]); + emitOp(crb, asm, strideSrc, strideDst, src, sro, dst, len); asm.jmp(end); } } @@ -298,13 +296,13 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { } } - private void emitOp(CompilationResultBuilder crb, AMD64MacroAssembler asm, Scale scaleSrc, Scale scaleDst, Register src, Register sro, Register dst, Register len) { - if (scaleSrc.value < scaleDst.value) { - emitInflate(crb, asm, scaleSrc, scaleDst, src, dst, len, sro); - } else if (scaleSrc.value == scaleDst.value) { - emitCopy(crb, asm, scaleSrc, scaleDst, src, dst, len, sro); + private void emitOp(CompilationResultBuilder crb, AMD64MacroAssembler asm, Stride strideSrc, Stride strideDst, Register src, Register sro, Register dst, Register len) { + if (strideSrc.value < strideDst.value) { + emitInflate(crb, asm, strideSrc, strideDst, src, dst, len, sro); + } else if (strideSrc.value == strideDst.value) { + emitCopy(crb, asm, strideSrc, strideDst, src, dst, len, sro); } else { - emitCompress(crb, asm, scaleSrc, scaleDst, src, dst, len, sro); + emitCompress(crb, asm, strideSrc, strideDst, src, dst, len, sro); } } @@ -314,13 +312,13 @@ private void emitOp(CompilationResultBuilder crb, AMD64MacroAssembler asm, Scale * {@code src} are zero. Breaking this assumption will lead to incorrect results. */ private void emitCompress(CompilationResultBuilder crb, AMD64MacroAssembler asm, - Scale scaleSrc, - Scale scaleDst, + Stride strideSrc, + Stride strideDst, Register src, Register dst, Register len, Register tmp) { - Op op = getOp(scaleDst, scaleSrc); + Op op = getOp(strideDst, strideSrc); Label labelScalarLoop = new Label(); Label labelDone = new Label(); @@ -330,7 +328,7 @@ private void emitCompress(CompilationResultBuilder crb, AMD64MacroAssembler asm, Label labelPack8Bytes = new Label(); Label labelCopyTail = new Label(); - int vectorLength = vectorSize.getBytes() / scaleDst.value; + int vectorLength = vectorSize.getBytes() / strideDst.value; asm.movl(tmp, len); @@ -352,38 +350,38 @@ private void emitCompress(CompilationResultBuilder crb, AMD64MacroAssembler asm, }); } - asm.leaq(src, new AMD64Address(src, len, scaleSrc)); - asm.leaq(dst, new AMD64Address(dst, len, scaleDst)); + asm.leaq(src, new AMD64Address(src, len, strideSrc)); + asm.leaq(dst, new AMD64Address(dst, len, strideDst)); asm.negq(len); // vectorized loop asm.align(crb.target.wordSize * 2); asm.bind(labelPackVectorLoop); - packVector(asm, vectorSize, op, scaleSrc, scaleDst, src, dst, len, 0, false); + packVector(asm, vectorSize, op, strideSrc, strideDst, src, dst, len, 0, false); asm.addqAndJcc(len, vectorLength, ConditionFlag.NotZero, labelPackVectorLoop, true); // vectorized tail - packVector(asm, vectorSize, op, scaleSrc, scaleDst, src, dst, tmp, -vectorSize.getBytes(), false); + packVector(asm, vectorSize, op, strideSrc, strideDst, src, dst, tmp, -vectorSize.getBytes(), false); asm.jmp(labelDone); if (supportsAVX2AndYMM()) { asm.bind(labelPack16Bytes); - int vectorSizeXMM = XMM.getBytes() / scaleDst.value; + int vectorSizeXMM = XMM.getBytes() / strideDst.value; asm.cmplAndJcc(tmp, vectorSizeXMM, ConditionFlag.Less, labelPack8Bytes, true); - packVector(asm, XMM, op, scaleSrc, scaleDst, src, dst, len, 0, true); - packVector(asm, XMM, op, scaleSrc, scaleDst, src, dst, tmp, -XMM.getBytes(), false); + packVector(asm, XMM, op, strideSrc, strideDst, src, dst, len, 0, true); + packVector(asm, XMM, op, strideSrc, strideDst, src, dst, tmp, -XMM.getBytes(), false); asm.jmpb(labelDone); } asm.bind(labelPack8Bytes); - int vectorSizeQ = 8 / scaleDst.value; + int vectorSizeQ = 8 / strideDst.value; asm.cmplAndJcc(tmp, vectorSizeQ, ConditionFlag.Less, labelCopyTail, true); // array is too small for vectorized loop, use half vectors // pack aligned to beginning - pack8Bytes(asm, op, scaleSrc, scaleDst, src, dst, len, 0, true); + pack8Bytes(asm, op, strideSrc, strideDst, src, dst, len, 0, true); // pack aligned to end - pack8Bytes(asm, op, scaleSrc, scaleDst, src, dst, tmp, -8, false); + pack8Bytes(asm, op, strideSrc, strideDst, src, dst, tmp, -8, false); asm.jmpb(labelDone); asm.bind(labelCopyTail); @@ -391,24 +389,24 @@ private void emitCompress(CompilationResultBuilder crb, AMD64MacroAssembler asm, } asm.testlAndJcc(len, len, ConditionFlag.Zero, labelDone, true); - asm.leaq(src, new AMD64Address(src, len, scaleSrc)); - asm.leaq(dst, new AMD64Address(dst, len, scaleDst)); + asm.leaq(src, new AMD64Address(src, len, strideSrc)); + asm.leaq(dst, new AMD64Address(dst, len, strideDst)); asm.negq(len); // scalar loop asm.bind(labelScalarLoop); switch (op) { case compressCharToByte: - asm.movzwl(tmp, new AMD64Address(src, len, scaleSrc)); - asm.movb(new AMD64Address(dst, len, scaleDst), tmp); + asm.movzwl(tmp, new AMD64Address(src, len, strideSrc)); + asm.movb(new AMD64Address(dst, len, strideDst), tmp); break; case compressIntToChar: - asm.movl(tmp, new AMD64Address(src, len, scaleSrc)); - asm.movw(new AMD64Address(dst, len, scaleDst), tmp); + asm.movl(tmp, new AMD64Address(src, len, strideSrc)); + asm.movw(new AMD64Address(dst, len, strideDst), tmp); break; case compressIntToByte: - asm.movl(tmp, new AMD64Address(src, len, scaleSrc)); - asm.movb(new AMD64Address(dst, len, scaleDst), tmp); + asm.movl(tmp, new AMD64Address(src, len, strideSrc)); + asm.movb(new AMD64Address(dst, len, strideDst), tmp); break; } asm.incqAndJcc(len, ConditionFlag.NotZero, labelScalarLoop, true); @@ -420,12 +418,12 @@ private void loadMask(CompilationResultBuilder crb, AMD64MacroAssembler asm, Reg movdqu(asm, vectorSize, vecMask, (AMD64Address) crb.recordDataReferenceInCode(mask, align)); } - private void packVector(AMD64MacroAssembler asm, AVXSize vecSize, Op op, Scale scaleSrc, Scale scaleDst, Register src, Register dst, Register index, int displacement, boolean direct) { - int displacementSrc = displacement << scaleSrc.log2 - scaleDst.log2; + private void packVector(AMD64MacroAssembler asm, AVXSize vecSize, Op op, Stride strideSrc, Stride strideDst, Register src, Register dst, Register index, int displacement, boolean direct) { + int displacementSrc = displacement << strideSrc.log2 - strideDst.log2; Register vec1 = asRegister(vectorTemp[0]); Register vec2 = asRegister(vectorTemp[1]); - movdqu(asm, vecSize, vec1, indexAddressOrDirect(scaleSrc, src, index, displacementSrc, 0, direct)); - movdqu(asm, vecSize, vec2, indexAddressOrDirect(scaleSrc, src, index, displacementSrc, vecSize.getBytes(), direct)); + movdqu(asm, vecSize, vec1, indexAddressOrDirect(strideSrc, src, index, displacementSrc, 0, direct)); + movdqu(asm, vecSize, vec2, indexAddressOrDirect(strideSrc, src, index, displacementSrc, vecSize.getBytes(), direct)); switch (op) { case compressCharToByte: packuswb(asm, vecSize, vec1, vec2); @@ -436,8 +434,8 @@ private void packVector(AMD64MacroAssembler asm, AVXSize vecSize, Op op, Scale s case compressIntToByte: Register vec3 = asRegister(vectorTemp[2]); Register vec4 = asRegister(vectorTemp[3]); - movdqu(asm, vecSize, vec3, indexAddressOrDirect(scaleSrc, src, index, displacementSrc, vecSize.getBytes() * 2, direct)); - movdqu(asm, vecSize, vec4, indexAddressOrDirect(scaleSrc, src, index, displacementSrc, vecSize.getBytes() * 3, direct)); + movdqu(asm, vecSize, vec3, indexAddressOrDirect(strideSrc, src, index, displacementSrc, vecSize.getBytes() * 2, direct)); + movdqu(asm, vecSize, vec4, indexAddressOrDirect(strideSrc, src, index, displacementSrc, vecSize.getBytes() * 3, direct)); packusdw(asm, vecSize, vec1, vec2); packusdw(asm, vecSize, vec3, vec4); packuswb(asm, vecSize, vec1, vec3); @@ -450,14 +448,14 @@ private void packVector(AMD64MacroAssembler asm, AVXSize vecSize, Op op, Scale s VexRMIOp.VPERMQ.emit(asm, vecSize, vec1, vec1, 0b11011000); } } - movdqu(asm, vecSize, new AMD64Address(dst, index, scaleDst, displacement), vec1); + movdqu(asm, vecSize, new AMD64Address(dst, index, strideDst, displacement), vec1); } - private void pack8Bytes(AMD64MacroAssembler masm, Op op, Scale scaleSrc, Scale scaleDst, Register src, Register dst, Register index, int displacement, boolean direct) { + private void pack8Bytes(AMD64MacroAssembler masm, Op op, Stride strideSrc, Stride strideDst, Register src, Register dst, Register index, int displacement, boolean direct) { Register vec1 = asRegister(vectorTemp[0]); Register vec2 = asRegister(vectorTemp[1]); - int displacementSrc = displacement << scaleSrc.log2 - scaleDst.log2; - masm.movdqu(vec1, indexAddressOrDirect(scaleSrc, src, index, displacementSrc, 0, direct)); + int displacementSrc = displacement << strideSrc.log2 - strideDst.log2; + masm.movdqu(vec1, indexAddressOrDirect(strideSrc, src, index, displacementSrc, 0, direct)); switch (op) { case compressCharToByte: masm.pxor(vec2, vec2); @@ -468,26 +466,26 @@ private void pack8Bytes(AMD64MacroAssembler masm, Op op, Scale scaleSrc, Scale s masm.packusdw(vec1, vec2); break; case compressIntToByte: - masm.movdqu(vec2, indexAddressOrDirect(scaleSrc, src, index, displacementSrc, 16, direct)); + masm.movdqu(vec2, indexAddressOrDirect(strideSrc, src, index, displacementSrc, 16, direct)); masm.packusdw(vec1, vec2); masm.packuswb(vec1, vec2); break; } - masm.movq(new AMD64Address(dst, index, scaleDst, displacement), vec1); + masm.movq(new AMD64Address(dst, index, strideDst, displacement), vec1); } - private static AMD64Address indexAddressOrDirect(Scale scaleSrc, Register array, Register index, int baseOffset, int displacement, boolean direct) { - return direct ? new AMD64Address(array, displacement) : new AMD64Address(array, index, scaleSrc, baseOffset + displacement); + private static AMD64Address indexAddressOrDirect(Stride strideSrc, Register array, Register index, int baseOffset, int displacement, boolean direct) { + return direct ? new AMD64Address(array, displacement) : new AMD64Address(array, index, strideSrc, baseOffset + displacement); } private void emitInflate(CompilationResultBuilder crb, AMD64MacroAssembler asm, - Scale scaleSrc, - Scale scaleDst, + Stride strideSrc, + Stride strideDst, Register src, Register dst, Register len, Register tmp) { - Op op = getOp(scaleDst, scaleSrc); + Op op = getOp(strideDst, strideSrc); Register vec = asRegister(vectorTemp[0]); Label labelScalarLoop = new Label(); Label labelDone = new Label(); @@ -500,28 +498,28 @@ private void emitInflate(CompilationResultBuilder crb, AMD64MacroAssembler asm, Label labelXMMTail = new Label(); Label labelTail = new Label(); - int vectorLength = vectorSize.getBytes() / scaleDst.value; - int vectorLengthXMM = XMM.getBytes() / scaleDst.value; + int vectorLength = vectorSize.getBytes() / strideDst.value; + int vectorLengthXMM = XMM.getBytes() / strideDst.value; - int scaleDelta = scaleDst.log2 - scaleSrc.log2; + int scaleDelta = strideDst.log2 - strideSrc.log2; asm.andl(tmp, vectorLength - 1); asm.andlAndJcc(len, -vectorLength, ConditionFlag.Zero, supportsAVX2AndYMM() ? labelXMMTail : labelTail, true); - asm.leaq(src, new AMD64Address(src, len, scaleSrc)); - asm.leaq(dst, new AMD64Address(dst, len, scaleDst)); + asm.leaq(src, new AMD64Address(src, len, strideSrc)); + asm.leaq(dst, new AMD64Address(dst, len, strideDst)); asm.negq(len); // vectorized loop asm.align(crb.target.wordSize * 2); asm.bind(labelMainLoop); - pmovSZx(asm, vectorSize, extendMode, vec, scaleDst, new AMD64Address(src, len, scaleSrc), scaleSrc); - movdqu(asm, vectorSize, new AMD64Address(dst, len, scaleDst), vec); + pmovSZx(asm, vectorSize, extendMode, vec, strideDst, new AMD64Address(src, len, strideSrc), strideSrc); + movdqu(asm, vectorSize, new AMD64Address(dst, len, strideDst), vec); asm.addqAndJcc(len, vectorLength, ConditionFlag.NotZero, labelMainLoop, true); // vectorized tail - pmovSZx(asm, vectorSize, extendMode, vec, scaleDst, new AMD64Address(src, tmp, scaleSrc, -vectorSize.getBytes() >> scaleDelta), scaleSrc); - movdqu(asm, vectorSize, new AMD64Address(dst, tmp, scaleDst, -vectorSize.getBytes()), vec); + pmovSZx(asm, vectorSize, extendMode, vec, strideDst, new AMD64Address(src, tmp, strideSrc, -vectorSize.getBytes() >> scaleDelta), strideSrc); + movdqu(asm, vectorSize, new AMD64Address(dst, tmp, strideDst, -vectorSize.getBytes()), vec); asm.jmpb(labelDone); if (supportsAVX2AndYMM()) { @@ -529,12 +527,12 @@ private void emitInflate(CompilationResultBuilder crb, AMD64MacroAssembler asm, asm.cmplAndJcc(tmp, vectorLengthXMM, ConditionFlag.Less, labelTail, true); // half vector size - pmovSZx(asm, XMM, extendMode, vec, scaleDst, new AMD64Address(src), scaleSrc); + pmovSZx(asm, XMM, extendMode, vec, strideDst, new AMD64Address(src), strideSrc); asm.movdqu(new AMD64Address(dst), vec); // half vector size tail - pmovSZx(asm, XMM, extendMode, vec, scaleDst, new AMD64Address(src, tmp, scaleSrc, -16 >> scaleDelta), scaleSrc); - asm.movdqu(new AMD64Address(dst, tmp, scaleDst, -16), vec); + pmovSZx(asm, XMM, extendMode, vec, strideDst, new AMD64Address(src, tmp, strideSrc, -16 >> scaleDelta), strideSrc); + asm.movdqu(new AMD64Address(dst, tmp, strideDst, -16), vec); asm.jmpb(labelDone); } @@ -544,15 +542,15 @@ private void emitInflate(CompilationResultBuilder crb, AMD64MacroAssembler asm, // xmm half vector size if (op != Op.inflateByteToInt) { assert scaleDelta == 1; - asm.cmplAndJcc(len, 4 >> scaleSrc.log2, ConditionFlag.Less, labelSkipXMMHalf, true); + asm.cmplAndJcc(len, 4 >> strideSrc.log2, ConditionFlag.Less, labelSkipXMMHalf, true); asm.movdl(vec, new AMD64Address(src)); - pmovSZx(asm, XMM, extendMode, vec, scaleDst, vec, scaleSrc); + pmovSZx(asm, XMM, extendMode, vec, strideDst, vec, strideSrc); asm.movq(new AMD64Address(dst), vec); - asm.movdl(vec, new AMD64Address(src, len, scaleSrc, -4)); - pmovSZx(asm, XMM, extendMode, vec, scaleDst, vec, scaleSrc); - asm.movq(new AMD64Address(dst, len, scaleDst, -8), vec); + asm.movdl(vec, new AMD64Address(src, len, strideSrc, -4)); + pmovSZx(asm, XMM, extendMode, vec, strideDst, vec, strideSrc); + asm.movq(new AMD64Address(dst, len, strideDst, -8), vec); asm.jmpb(labelDone); } @@ -560,24 +558,24 @@ private void emitInflate(CompilationResultBuilder crb, AMD64MacroAssembler asm, } asm.testlAndJcc(len, len, ConditionFlag.Zero, labelDone, true); - asm.leaq(src, new AMD64Address(src, len, scaleSrc)); - asm.leaq(dst, new AMD64Address(dst, len, scaleDst)); + asm.leaq(src, new AMD64Address(src, len, strideSrc)); + asm.leaq(dst, new AMD64Address(dst, len, strideDst)); asm.negq(len); // scalar loop asm.bind(labelScalarLoop); switch (op) { case inflateByteToChar: - asm.movzbl(tmp, new AMD64Address(src, len, scaleSrc)); - asm.movw(new AMD64Address(dst, len, scaleDst), tmp); + asm.movzbl(tmp, new AMD64Address(src, len, strideSrc)); + asm.movw(new AMD64Address(dst, len, strideDst), tmp); break; case inflateByteToInt: - asm.movzbl(tmp, new AMD64Address(src, len, scaleSrc)); - asm.movl(new AMD64Address(dst, len, scaleDst), tmp); + asm.movzbl(tmp, new AMD64Address(src, len, strideSrc)); + asm.movl(new AMD64Address(dst, len, strideDst), tmp); break; case inflateCharToInt: - asm.movzwl(tmp, new AMD64Address(src, len, scaleSrc)); - asm.movl(new AMD64Address(dst, len, scaleDst), tmp); + asm.movzwl(tmp, new AMD64Address(src, len, strideSrc)); + asm.movl(new AMD64Address(dst, len, strideDst), tmp); break; } asm.incqAndJcc(len, ConditionFlag.NotZero, labelScalarLoop, true); @@ -586,8 +584,8 @@ private void emitInflate(CompilationResultBuilder crb, AMD64MacroAssembler asm, } private void emitCopy(CompilationResultBuilder crb, AMD64MacroAssembler masm, - Scale scaleSrc, - Scale scaleDst, + Stride strideSrc, + Stride strideDst, Register src, Register dst, Register len, @@ -600,15 +598,15 @@ private void emitCopy(CompilationResultBuilder crb, AMD64MacroAssembler masm, Label labelTailBYTE = new Label(); Label labelDone = new Label(); - int vectorLength = vectorSize.getBytes() / scaleDst.value; + int vectorLength = vectorSize.getBytes() / strideDst.value; masm.movl(tmp, len); masm.andl(tmp, vectorLength - 1); masm.andlAndJcc(len, -vectorLength, ConditionFlag.Zero, supportsAVX2AndYMM() ? labelTailXMM : labelTailQWORD, true); - masm.leaq(src, new AMD64Address(src, len, scaleSrc)); - masm.leaq(dst, new AMD64Address(dst, len, scaleDst)); + masm.leaq(src, new AMD64Address(src, len, strideSrc)); + masm.leaq(dst, new AMD64Address(dst, len, strideDst)); masm.negq(len); Label labelYMMLoop = new Label(); @@ -618,37 +616,37 @@ private void emitCopy(CompilationResultBuilder crb, AMD64MacroAssembler masm, // vectorized loop masm.align(crb.target.wordSize * 2); masm.bind(labelYMMLoop); - masm.vmovdqu(vec, new AMD64Address(src, len, scaleSrc)); - masm.vmovdqu(new AMD64Address(dst, len, scaleDst), vec); + masm.vmovdqu(vec, new AMD64Address(src, len, strideSrc)); + masm.vmovdqu(new AMD64Address(dst, len, strideDst), vec); masm.addqAndJcc(len, vectorLength, ConditionFlag.NotZero, labelYMMLoop, true); // vectorized tail - masm.vmovdqu(vec, new AMD64Address(src, tmp, scaleSrc, -32)); - masm.vmovdqu(new AMD64Address(dst, tmp, scaleDst, -32), vec); + masm.vmovdqu(vec, new AMD64Address(src, tmp, strideSrc, -32)); + masm.vmovdqu(new AMD64Address(dst, tmp, strideDst, -32), vec); masm.jmpb(labelDone); // half vector size masm.bind(labelTailXMM); - masm.cmplAndJcc(tmp, 16 / scaleDst.value, ConditionFlag.Less, labelTailQWORD, true); + masm.cmplAndJcc(tmp, 16 / strideDst.value, ConditionFlag.Less, labelTailQWORD, true); masm.movdqu(vec, new AMD64Address(src)); masm.movdqu(new AMD64Address(dst), vec); // half vector size tail - masm.movdqu(vec, new AMD64Address(src, tmp, scaleSrc, -16)); - masm.movdqu(new AMD64Address(dst, tmp, scaleDst, -16), vec); + masm.movdqu(vec, new AMD64Address(src, tmp, strideSrc, -16)); + masm.movdqu(new AMD64Address(dst, tmp, strideDst, -16), vec); masm.jmpb(labelDone); } else { // xmm vectorized loop masm.align(crb.target.wordSize * 2); masm.bind(labelXMMLoop); - masm.movdqu(vec, new AMD64Address(src, len, scaleSrc)); - masm.movdqu(new AMD64Address(dst, len, scaleDst), vec); + masm.movdqu(vec, new AMD64Address(src, len, strideSrc)); + masm.movdqu(new AMD64Address(dst, len, strideDst), vec); masm.addqAndJcc(len, vectorLength, ConditionFlag.NotZero, labelXMMLoop, true); // xmm vectorized tail - masm.movdqu(vec, new AMD64Address(src, tmp, scaleSrc, -16)); - masm.movdqu(new AMD64Address(dst, tmp, scaleDst, -16), vec); + masm.movdqu(vec, new AMD64Address(src, tmp, strideSrc, -16)); + masm.movdqu(new AMD64Address(dst, tmp, strideDst, -16), vec); masm.jmpb(labelDone); } @@ -662,39 +660,39 @@ private void emitCopy(CompilationResultBuilder crb, AMD64MacroAssembler masm, */ masm.bind(labelTailQWORD); - masm.cmplAndJcc(tmp, 8 / scaleDst.value, ConditionFlag.Less, labelTailDWORD, true); + masm.cmplAndJcc(tmp, 8 / strideDst.value, ConditionFlag.Less, labelTailDWORD, true); masm.movq(len, new AMD64Address(src)); masm.movq(new AMD64Address(dst), len); - masm.movq(len, new AMD64Address(src, tmp, scaleSrc, -8)); - masm.movq(new AMD64Address(dst, tmp, scaleDst, -8), len); + masm.movq(len, new AMD64Address(src, tmp, strideSrc, -8)); + masm.movq(new AMD64Address(dst, tmp, strideDst, -8), len); masm.jmpb(labelDone); masm.bind(labelTailDWORD); - if (scaleDst.value < 4) { - masm.cmplAndJcc(tmp, 4 / scaleDst.value, ConditionFlag.Less, labelTailWORD, true); + if (strideDst.value < 4) { + masm.cmplAndJcc(tmp, 4 / strideDst.value, ConditionFlag.Less, labelTailWORD, true); } else { masm.testlAndJcc(tmp, tmp, ConditionFlag.Zero, labelDone, true); } masm.movl(len, new AMD64Address(src)); masm.movl(new AMD64Address(dst), len); - masm.movl(len, new AMD64Address(src, tmp, scaleSrc, -4)); - masm.movl(new AMD64Address(dst, tmp, scaleDst, -4), len); + masm.movl(len, new AMD64Address(src, tmp, strideSrc, -4)); + masm.movl(new AMD64Address(dst, tmp, strideDst, -4), len); - if (scaleDst.value < 4) { + if (strideDst.value < 4) { masm.jmpb(labelDone); masm.bind(labelTailWORD); - if (scaleDst.value < 2) { - masm.cmplAndJcc(tmp, 2 / scaleDst.value, ConditionFlag.Less, labelTailBYTE, true); + if (strideDst.value < 2) { + masm.cmplAndJcc(tmp, 2 / strideDst.value, ConditionFlag.Less, labelTailBYTE, true); } else { masm.testlAndJcc(tmp, tmp, ConditionFlag.Zero, labelDone, true); } masm.movw(len, new AMD64Address(src)); masm.movw(new AMD64Address(dst), len); - masm.movw(len, new AMD64Address(src, tmp, scaleSrc, -2)); - masm.movw(new AMD64Address(dst, tmp, scaleDst, -2), len); + masm.movw(len, new AMD64Address(src, tmp, strideSrc, -2)); + masm.movw(new AMD64Address(dst, tmp, strideDst, -2), len); } - if (scaleDst.value < 2) { + if (strideDst.value < 2) { masm.jmpb(labelDone); masm.bind(labelTailBYTE); masm.testlAndJcc(tmp, tmp, ConditionFlag.Zero, labelDone, true); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java index 9a9f60dd3a1e..7ab1511a0a98 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java @@ -44,11 +44,9 @@ import java.util.Arrays; import java.util.EnumSet; -import java.util.Objects; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; import org.graalvm.compiler.asm.amd64.AMD64Assembler; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64Assembler.SSEOp; @@ -56,6 +54,7 @@ import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.StrideUtil; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; @@ -92,15 +91,10 @@ public final class AMD64ArrayEqualsOp extends AMD64ComplexVectorOp { private static final Register REG_STRIDE = r9; private final JavaKind elementKind; - private final int baseOffsetA; - private final int baseOffsetB; - private final int baseOffsetMask; - private final int constOffsetA; - private final int constOffsetB; private final int constLength; - private final Scale argScaleA; - private final Scale argScaleB; - private final Scale argScaleMask; + private final Stride argStrideA; + private final Stride argStrideB; + private final Stride argStrideMask; private final AMD64MacroAssembler.ExtendMode extendMode; private final boolean canGenerateConstantLengthCompare; @@ -125,53 +119,38 @@ public final class AMD64ArrayEqualsOp extends AMD64ComplexVectorOp { @Temp({REG}) private Value[] vectorTemp; - private AMD64ArrayEqualsOp(LIRGeneratorTool tool, JavaKind kindA, JavaKind kindB, JavaKind kindMask, int baseOffsetA, int baseOffsetB, int baseOffsetMask, + private AMD64ArrayEqualsOp(LIRGeneratorTool tool, JavaKind elementKind, Stride strideA, Stride strideB, Stride strideMask, EnumSet runtimeCheckedCPUFeatures, Value result, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value mask, Value length, Value dynamicStrides, - AMD64MacroAssembler.ExtendMode extendMode, - int constOffsetA, int constOffsetB, int constLength) { + AMD64MacroAssembler.ExtendMode extendMode, int constLength) { super(TYPE, tool, runtimeCheckedCPUFeatures, AVXSize.YMM); this.extendMode = extendMode; - - this.constOffsetA = constOffsetA; - this.constOffsetB = constOffsetB; this.constLength = constLength; - - this.baseOffsetA = baseOffsetA; - this.baseOffsetB = baseOffsetB; - this.baseOffsetMask = baseOffsetMask; - + this.elementKind = elementKind; if (StrideUtil.useConstantStrides(dynamicStrides)) { - assert kindA.isNumericInteger() && kindB.isNumericInteger() || kindA == kindB; - this.elementKind = kindA; - this.argScaleA = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(kindA))); - this.argScaleB = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(kindB))); - this.argScaleMask = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(kindMask))); + assert elementKind.isNumericInteger() || strideA == strideB; + this.argStrideA = strideA; + this.argStrideB = strideB; + this.argStrideMask = strideMask; } else { - this.elementKind = JavaKind.Byte; - this.argScaleA = null; - this.argScaleB = null; - this.argScaleMask = null; + this.argStrideA = null; + this.argStrideB = null; + this.argStrideMask = null; } - this.canGenerateConstantLengthCompare = canGenerateConstantLengthCompare(tool.target(), runtimeCheckedCPUFeatures, kindA, kindB, constLength, dynamicStrides, vectorSize); + this.canGenerateConstantLengthCompare = canGenerateConstantLengthCompare(tool.target(), runtimeCheckedCPUFeatures, elementKind, strideA, strideB, constLength, dynamicStrides, vectorSize); this.resultValue = result; this.arrayAValue = this.arrayAValueTemp = arrayA; - // if constOffset is 0, no offset parameter was used, but we still need the register as a - // temp register, so we set the @Use property to ILLEGAL but keep the register in the @Temp - // property - this.offsetAValue = constOffsetA == 0 ? Value.ILLEGAL : offsetA; - this.offsetAValueTemp = offsetA; + this.offsetAValue = this.offsetAValueTemp = offsetA; this.arrayBValue = this.arrayBValueTemp = arrayB; - this.offsetBValue = constOffsetB == 0 ? Value.ILLEGAL : offsetB; - this.offsetBValueTemp = offsetB; + this.offsetBValue = this.offsetBValueTemp = offsetB; this.arrayMaskValue = this.arrayMaskValueTemp = mask; this.lengthValue = this.lengthValueTemp = length; this.dynamicStridesValue = this.dynamicStrideValueTemp = dynamicStrides; - if (kindA == JavaKind.Float) { + if (elementKind == JavaKind.Float) { this.tempXMM = tool.newVariable(LIRKind.value(AMD64Kind.SINGLE)); - } else if (kindA == JavaKind.Double) { + } else if (elementKind == JavaKind.Double) { this.tempXMM = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); } else { this.tempXMM = Value.ILLEGAL; @@ -189,20 +168,43 @@ private AMD64ArrayEqualsOp(LIRGeneratorTool tool, JavaKind kindA, JavaKind kindB } } - public static AMD64ArrayEqualsOp movParamsAndCreate(LIRGeneratorTool tool, - int baseOffsetA, int baseOffsetB, int baseOffsetMask, + public static AMD64ArrayEqualsOp movParamsAndCreate( + LIRGeneratorTool tool, EnumSet runtimeCheckedCPUFeatures, - Value result, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value mask, Value length, Value dynamicStrides, + Value result, + Value arrayA, Value offsetA, + Value arrayB, Value offsetB, + Value mask, + Value length, + Value dynamicStrides, AMD64MacroAssembler.ExtendMode extendMode) { - return movParamsAndCreate(tool, null, null, null, baseOffsetA, baseOffsetB, baseOffsetMask, runtimeCheckedCPUFeatures, result, arrayA, offsetA, arrayB, offsetB, mask, length, dynamicStrides, + return movParamsAndCreate(tool, null, null, null, null, + runtimeCheckedCPUFeatures, + result, + arrayA, offsetA, + arrayB, offsetB, + mask, + length, + dynamicStrides, extendMode); } - public static AMD64ArrayEqualsOp movParamsAndCreate(LIRGeneratorTool tool, JavaKind strideA, JavaKind strideB, JavaKind strideMask, int baseOffsetA, int baseOffsetB, int baseOffsetMask, + public static AMD64ArrayEqualsOp movParamsAndCreate(LIRGeneratorTool tool, + Stride strideA, Stride strideB, Stride strideMask, EnumSet runtimeCheckedCPUFeatures, - Value result, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value mask, Value length, + Value result, + Value arrayA, Value offsetA, + Value arrayB, Value offsetB, + Value mask, + Value length, AMD64MacroAssembler.ExtendMode extendMode) { - return movParamsAndCreate(tool, strideA, strideB, strideMask, baseOffsetA, baseOffsetB, baseOffsetMask, runtimeCheckedCPUFeatures, result, arrayA, offsetA, arrayB, offsetB, mask, length, null, + return movParamsAndCreate(tool, null, strideA, strideB, strideMask, runtimeCheckedCPUFeatures, + result, + arrayA, offsetA, + arrayB, offsetB, + mask, + length, + null, extendMode); } @@ -212,6 +214,11 @@ public static AMD64ArrayEqualsOp movParamsAndCreate(LIRGeneratorTool tool, JavaK * {@code arrayMask} is not {@code null}, it is OR-ed with {@code arrayA} before comparison with * {@code arrayB}. * + * @param elementKind Array element kind. This is only relevant when comparing values of + * {@link JavaKind#Float} or {@link JavaKind#Double}. In this case, all strides must + * be equal to the element kind's {@link JavaKind#getByteCount() byte count}. If no + * floating-point comparison should be done, this parameter should be set to + * {@link JavaKind#Byte}. * @param strideA element size of {@code arrayA}. May be {@code null} if {@code dynamicStrides} * is used. * @param strideB element size of {@code arrayB}. May be {@code null} if {@code dynamicStrides} @@ -225,13 +232,18 @@ public static AMD64ArrayEqualsOp movParamsAndCreate(LIRGeneratorTool tool, JavaK * @param dynamicStrides dynamic stride dispatch as described in {@link StrideUtil}. * @param extendMode integer extension mode for the array with the smaller element size. */ - public static AMD64ArrayEqualsOp movParamsAndCreate(LIRGeneratorTool tool, JavaKind strideA, JavaKind strideB, JavaKind strideMask, int baseOffsetA, int baseOffsetB, int baseOffsetMask, - EnumSet runtimeCheckedCPUFeatures, Value result, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value arrayMask, Value length, Value dynamicStrides, + public static AMD64ArrayEqualsOp movParamsAndCreate(LIRGeneratorTool tool, + JavaKind elementKind, + Stride strideA, Stride strideB, Stride strideMask, + EnumSet runtimeCheckedCPUFeatures, + Value result, + Value arrayA, Value offsetA, + Value arrayB, Value offsetB, Value arrayMask, Value length, Value dynamicStrides, AMD64MacroAssembler.ExtendMode extendMode) { RegisterValue regArrayA = REG_ARRAY_A.asValue(arrayA.getValueKind()); - RegisterValue regOffsetA = REG_OFFSET_A.asValue(offsetA == null ? LIRKind.value(AMD64Kind.QWORD) : offsetA.getValueKind()); + RegisterValue regOffsetA = REG_OFFSET_A.asValue(offsetA.getValueKind()); RegisterValue regArrayB = REG_ARRAY_B.asValue(arrayB.getValueKind()); - RegisterValue regOffsetB = REG_OFFSET_B.asValue(offsetB == null ? LIRKind.value(AMD64Kind.QWORD) : offsetB.getValueKind()); + RegisterValue regOffsetB = REG_OFFSET_B.asValue(offsetB.getValueKind()); Value regMask = arrayMask == null ? Value.ILLEGAL : REG_MASK.asValue(arrayMask.getValueKind()); RegisterValue regLength = REG_LENGTH.asValue(length.getValueKind()); Value regStride = dynamicStrides == null ? Value.ILLEGAL : REG_STRIDE.asValue(dynamicStrides.getValueKind()); @@ -239,47 +251,31 @@ public static AMD64ArrayEqualsOp movParamsAndCreate(LIRGeneratorTool tool, JavaK tool.emitConvertNullToZero(regArrayA, arrayA); tool.emitConvertNullToZero(regArrayB, arrayB); tool.emitMove(regLength, length); - if (offsetA != null) { - tool.emitMove(regOffsetA, offsetA); - } - if (offsetB != null) { - tool.emitMove(regOffsetB, offsetB); - } + tool.emitMove(regOffsetA, offsetA); + tool.emitMove(regOffsetB, offsetB); if (arrayMask != null) { tool.emitMove((RegisterValue) regMask, arrayMask); } if (dynamicStrides != null) { tool.emitMove((RegisterValue) regStride, dynamicStrides); } - return new AMD64ArrayEqualsOp(tool, strideA, strideB, strideMask, baseOffsetA, baseOffsetB, baseOffsetMask, + return new AMD64ArrayEqualsOp(tool, elementKind == null ? JavaKind.Byte : elementKind, strideA, strideB, strideMask, runtimeCheckedCPUFeatures, result, regArrayA, regOffsetA, regArrayB, regOffsetB, regMask, regLength, regStride, - extendMode, constOffset(offsetA), constOffset(offsetB), LIRValueUtil.isJavaConstant(length) ? LIRValueUtil.asJavaConstant(length).asInt() : -1); + extendMode, LIRValueUtil.isJavaConstant(length) ? LIRValueUtil.asJavaConstant(length).asInt() : -1); } - private static int constOffset(Value offset) { - if (offset == null) { - // no offset parameter (e.g. in Arrays.equal) - return 0; - } - if (LIRValueUtil.isJavaConstant(offset) && LIRValueUtil.asJavaConstant(offset).asLong() <= Integer.MAX_VALUE) { - // constant offset parameter - return (int) LIRValueUtil.asJavaConstant(offset).asLong(); - } - // non-constant offset parameter - return -1; + private static boolean canGenerateConstantLengthCompare(TargetDescription target, EnumSet runtimeCheckedCPUFeatures, + JavaKind elementKind, Stride strideA, Stride strideB, int constantLength, Value stride, AVXSize vectorSize) { + return isIllegal(stride) && constantLength >= 0 && canGenerateConstantLengthCompare(target, runtimeCheckedCPUFeatures, elementKind, strideA, strideB, constantLength, vectorSize); } - private static boolean canGenerateConstantLengthCompare(TargetDescription target, EnumSet runtimeCheckedCPUFeatures, JavaKind kindA, JavaKind kindB, int constantLength, Value stride, - AVXSize vectorSize) { - return isIllegal(stride) && constantLength >= 0 && canGenerateConstantLengthCompare(target, runtimeCheckedCPUFeatures, kindA, kindB, constantLength, vectorSize); - } - - public static boolean canGenerateConstantLengthCompare(TargetDescription target, EnumSet runtimeCheckedCPUFeatures, JavaKind kindA, JavaKind kindB, int constantLength, - AVXSize vectorSize) { - int elementSize = Math.max(kindA.getByteCount(), kindB.getByteCount()); + public static boolean canGenerateConstantLengthCompare(TargetDescription target, EnumSet runtimeCheckedCPUFeatures, + JavaKind elementKind, Stride strideA, Stride strideB, int constantLength, AVXSize vectorSize) { + int elementSize = Math.max(strideA.value, strideB.value); int minVectorSize = AVXSize.XMM.getBytes() / elementSize; int maxVectorSize = vectorSize.getBytes() / elementSize; - return supports(target, runtimeCheckedCPUFeatures, CPUFeature.SSE4_1) && kindA.isNumericInteger() && (kindA == kindB || minVectorSize <= constantLength) && constantLength <= maxVectorSize * 2; + return supports(target, runtimeCheckedCPUFeatures, CPUFeature.SSE4_1) && elementKind.isNumericInteger() && (strideA == strideB || minVectorSize <= constantLength) && + constantLength <= maxVectorSize * 2; } private boolean isLengthConstant() { @@ -309,11 +305,8 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { Register arrayB = asRegister(arrayBValue); Register mask = withMask() ? asRegister(arrayMaskValue) : null; // Load array base addresses. - loadBaseAddress(masm, arrayA, baseOffsetA, constOffsetA, offsetAValue); - loadBaseAddress(masm, arrayB, baseOffsetB, constOffsetB, offsetBValue); - if (withMask()) { - masm.leaq(mask, new AMD64Address(mask, baseOffsetMask)); - } + masm.addq(arrayA, asRegister(offsetAValue)); + masm.addq(arrayB, asRegister(offsetBValue)); if (canGenerateConstantLengthCompare) { emitConstantLengthArrayCompareBytes(masm, result); } else { @@ -330,35 +323,35 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // use the 1-byte-1-byte stride variant for the 2-2 and 4-4 cases by simply shifting // the length masm.align(crb.target.wordSize * 2); - masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Scale.Times4, Scale.Times4)]); + masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Stride.S4, Stride.S4)]); masm.shll(length, 1); masm.align(crb.target.wordSize * 2); - masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Scale.Times2, Scale.Times2)]); + masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Stride.S2, Stride.S2)]); masm.shll(length, 1); masm.align(crb.target.wordSize * 2); - masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Scale.Times1, Scale.Times1)]); - emitArrayCompare(crb, masm, Scale.Times1, Scale.Times1, Scale.Times1, result, arrayA, arrayB, mask, length, done, false); + masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(Stride.S1, Stride.S1)]); + emitArrayCompare(crb, masm, Stride.S1, Stride.S1, Stride.S1, result, arrayA, arrayB, mask, length, done, false); masm.jmp(done); - for (Scale scaleA : new Scale[]{Scale.Times1, Scale.Times2, Scale.Times4}) { - for (Scale scaleB : new Scale[]{Scale.Times1, Scale.Times2, Scale.Times4}) { - if (scaleA.log2 <= scaleB.log2) { + for (Stride strideA : new Stride[]{Stride.S1, Stride.S2, Stride.S4}) { + for (Stride strideB : new Stride[]{Stride.S1, Stride.S2, Stride.S4}) { + if (strideA.log2 <= strideB.log2) { continue; } masm.align(crb.target.wordSize * 2); // use the same implementation for e.g. stride 1-2 and 2-1 by swapping the // arguments in one variant - masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(scaleB, scaleA)]); + masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(strideB, strideA)]); masm.movq(tmp, arrayA); masm.movq(arrayA, arrayB); masm.movq(arrayB, tmp); - masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(scaleA, scaleB)]); - emitArrayCompare(crb, masm, scaleA, scaleB, scaleB, result, arrayA, arrayB, mask, length, done, false); + masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(strideA, strideB)]); + emitArrayCompare(crb, masm, strideA, strideB, strideB, result, arrayA, arrayB, mask, length, done, false); masm.jmp(done); } } } else { - emitArrayCompare(crb, masm, argScaleA, argScaleB, argScaleMask, result, arrayA, arrayB, mask, length, done, true); + emitArrayCompare(crb, masm, argStrideA, argStrideB, argStrideMask, result, arrayA, arrayB, mask, length, done, true); } } masm.bind(done); @@ -375,29 +368,21 @@ private static void emitReturnValue(AMD64MacroAssembler masm, Register result, L masm.xorl(result, result); } - private static void loadBaseAddress(AMD64MacroAssembler masm, Register array, int baseOffset, int constantOffset, Value dynamicOffset) { - if (constantOffset >= 0) { - masm.leaq(array, new AMD64Address(array, constantOffset + baseOffset)); - } else { - masm.leaq(array, new AMD64Address(array, asRegister(dynamicOffset), Scale.Times1, baseOffset)); - } - } - private void emitArrayCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, - Scale scaleA, Scale scaleB, Scale scaleMask, + Stride strideA, Stride strideB, Stride strideMask, Register result, Register array1, Register array2, Register mask, Register length, Label done, boolean shortJmp) { Label trueLabel = new Label(); Label falseLabel = new Label(); masm.movl(result, length); if (masm.supports(CPUFeature.SSE4_1)) { - emitVectorCompare(crb, masm, scaleA, scaleB, scaleMask, result, array1, array2, mask, length, trueLabel, falseLabel); + emitVectorCompare(crb, masm, strideA, strideB, strideMask, result, array1, array2, mask, length, trueLabel, falseLabel); } - if (scaleA == scaleB && scaleA == scaleMask) { - emit8ByteCompare(crb, masm, scaleA, scaleB, scaleMask, result, array1, array2, mask, length, trueLabel, falseLabel); - emitTailCompares(masm, scaleA, scaleB, scaleMask, result, array1, array2, mask, length, trueLabel, falseLabel); + if (strideA == strideB && strideA == strideMask) { + emit8ByteCompare(crb, masm, strideA, strideB, strideMask, result, array1, array2, mask, length, trueLabel, falseLabel); + emitTailCompares(masm, strideA, strideB, strideMask, result, array1, array2, mask, length, trueLabel, falseLabel); } else { - emitDifferentKindsElementWiseCompare(crb, masm, scaleA, scaleB, scaleMask, result, array1, array2, mask, length, trueLabel, falseLabel); + emitDifferentKindsElementWiseCompare(crb, masm, strideA, strideB, strideMask, result, array1, array2, mask, length, trueLabel, falseLabel); } emitReturnValue(masm, result, trueLabel, falseLabel, done, shortJmp); } @@ -406,17 +391,17 @@ private void emitArrayCompare(CompilationResultBuilder crb, AMD64MacroAssembler * Emits code that uses SSE4.1/AVX1 128-bit (16-byte) or AVX2 256-bit (32-byte) vector compares. */ private void emitVectorCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, - Scale scaleA, Scale scaleB, Scale scaleMask, + Stride strideA, Stride strideB, Stride strideMask, Register result, Register arrayA, Register arrayB, Register mask, Register length, Label trueLabel, Label falseLabel) { assert masm.supports(CPUFeature.SSE4_1); - Scale maxScale = max(scaleA, scaleB); + Stride maxStride = Stride.max(strideA, strideB); Register vector1 = asRegister(vectorTemp[0]); Register vector2 = asRegister(vectorTemp[1]); Register vector3 = withMask() ? asRegister(vectorTemp[2]) : null; - int elementsPerVector = getElementsPerVector(vectorSize, maxScale); + int elementsPerVector = getElementsPerVector(vectorSize, maxStride); Label loop = new Label(); Label compareTail = new Label(); @@ -429,20 +414,20 @@ private void emitVectorCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm.andl(result, elementsPerVector - 1); // tail count masm.andlAndJcc(length, ~(elementsPerVector - 1), ConditionFlag.Zero, compareTail, false); - masm.leaq(arrayA, new AMD64Address(arrayA, length, scaleA, 0)); - masm.leaq(arrayB, new AMD64Address(arrayB, length, scaleB, 0)); + masm.leaq(arrayA, new AMD64Address(arrayA, length, strideA, 0)); + masm.leaq(arrayB, new AMD64Address(arrayB, length, strideB, 0)); if (withMask()) { - masm.leaq(mask, new AMD64Address(mask, length, scaleMask, 0)); + masm.leaq(mask, new AMD64Address(mask, length, strideMask, 0)); } masm.negq(length); // Align the main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); - pmovSZx(masm, vectorSize, vector1, maxScale, arrayA, length, 0, scaleA); - pmovSZx(masm, vectorSize, vector2, maxScale, arrayB, length, 0, scaleB); + pmovSZx(masm, vectorSize, vector1, maxStride, arrayA, length, 0, strideA); + pmovSZx(masm, vectorSize, vector2, maxStride, arrayB, length, 0, strideB); if (withMask()) { - pmovSZx(masm, vectorSize, vector3, maxScale, mask, length, 0, scaleMask); + pmovSZx(masm, vectorSize, vector3, maxStride, mask, length, 0, strideMask); por(masm, vectorSize, vector1, vector3); } emitVectorCmp(masm, vector1, vector2, vectorSize); @@ -458,7 +443,7 @@ private void emitVectorCompare(CompilationResultBuilder crb, AMD64MacroAssembler Label unalignedCheck = new Label(); masm.jmpb(unalignedCheck); masm.bind(nanCheck); - emitFloatCompareWithinRange(crb, masm, scaleA, scaleB, arrayA, arrayB, length, 0, falseLabel, elementsPerVector); + emitFloatCompareWithinRange(crb, masm, strideA, strideB, arrayA, arrayB, length, 0, falseLabel, elementsPerVector); masm.jmpb(loopCheck); masm.bind(unalignedCheck); } @@ -467,17 +452,17 @@ private void emitVectorCompare(CompilationResultBuilder crb, AMD64MacroAssembler * Compare the remaining bytes with an unaligned memory load aligned to the end of the * array. */ - pmovSZx(masm, vectorSize, vector1, maxScale, arrayA, result, -vectorSize.getBytes(), scaleA); - pmovSZx(masm, vectorSize, vector2, maxScale, arrayB, result, -vectorSize.getBytes(), scaleB); + pmovSZx(masm, vectorSize, vector1, maxStride, arrayA, result, -vectorSize.getBytes(), strideA); + pmovSZx(masm, vectorSize, vector2, maxStride, arrayB, result, -vectorSize.getBytes(), strideB); if (withMask()) { - pmovSZx(masm, vectorSize, vector3, maxScale, mask, result, -vectorSize.getBytes(), scaleMask); + pmovSZx(masm, vectorSize, vector3, maxStride, mask, result, -vectorSize.getBytes(), strideMask); por(masm, vectorSize, vector1, vector3); } emitVectorCmp(masm, vector1, vector2, vectorSize); if (requiresNaNCheck) { assert !withMask(); masm.jcc(ConditionFlag.Zero, trueLabel); - emitFloatCompareWithinRange(crb, masm, scaleA, scaleB, arrayA, arrayB, result, -vectorSize.getBytes(), falseLabel, elementsPerVector); + emitFloatCompareWithinRange(crb, masm, strideA, strideB, arrayA, arrayB, result, -vectorSize.getBytes(), falseLabel, elementsPerVector); } else { masm.jcc(ConditionFlag.NotZero, falseLabel); } @@ -487,16 +472,16 @@ private void emitVectorCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm.movl(length, result); } - private static int getElementsPerVector(AVXSize vSize, Scale maxScale) { - return vSize.getBytes() >> maxScale.log2; + private static int getElementsPerVector(AVXSize vSize, Stride maxStride) { + return vSize.getBytes() >> maxStride.log2; } - private void pmovSZx(AMD64MacroAssembler asm, AVXSize size, Register dst, Scale maxScale, Register src, int displacement, Scale scale) { - pmovSZx(asm, size, dst, maxScale, src, Register.None, displacement, scale); + private void pmovSZx(AMD64MacroAssembler asm, AVXSize size, Register dst, Stride maxStride, Register src, int displacement, Stride stride) { + pmovSZx(asm, size, dst, maxStride, src, Register.None, displacement, stride); } - private void pmovSZx(AMD64MacroAssembler asm, AVXSize size, Register dst, Scale maxScale, Register src, Register index, int displacement, Scale scale) { - AMD64MacroAssembler.pmovSZx(asm, size, extendMode, dst, maxScale, src, scale, index, displacement); + private void pmovSZx(AMD64MacroAssembler asm, AVXSize size, Register dst, Stride maxStride, Register src, Register index, int displacement, Stride stride) { + AMD64MacroAssembler.pmovSZx(asm, size, extendMode, dst, maxStride, src, stride, index, displacement); } private static void emitVectorCmp(AMD64MacroAssembler masm, Register vector1, Register vector2, AVXSize size) { @@ -513,13 +498,13 @@ private static void emitVectorCmp(AMD64MacroAssembler masm, Register vector1, Re * Emits code that uses 8-byte vector compares. */ private void emit8ByteCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, - Scale scaleA, Scale scaleB, Scale scaleMask, + Stride strideA, Stride strideB, Stride strideMask, Register result, Register arrayA, Register arrayB, Register mask, Register length, Label trueLabel, Label falseLabel) { - assert scaleA == scaleB && scaleA == scaleMask; + assert strideA == strideB && strideA == strideMask; Label loop = new Label(); Label compareTail = new Label(); - int elementsPerVector = 8 >> scaleA.log2; + int elementsPerVector = 8 >> strideA.log2; boolean requiresNaNCheck = elementKind.isNumericFloat(); Label loopCheck = new Label(); @@ -530,21 +515,21 @@ private void emit8ByteCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm.andl(result, elementsPerVector - 1); // tail count masm.andlAndJcc(length, ~(elementsPerVector - 1), ConditionFlag.Zero, compareTail, false); - masm.leaq(arrayA, new AMD64Address(arrayA, length, scaleA, 0)); - masm.leaq(arrayB, new AMD64Address(arrayB, length, scaleB, 0)); + masm.leaq(arrayA, new AMD64Address(arrayA, length, strideA, 0)); + masm.leaq(arrayB, new AMD64Address(arrayB, length, strideB, 0)); if (withMask()) { - masm.leaq(mask, new AMD64Address(mask, length, scaleMask, 0)); + masm.leaq(mask, new AMD64Address(mask, length, strideMask, 0)); } masm.negq(length); // Align the main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); - masm.movq(temp, new AMD64Address(arrayA, length, scaleA, 0)); + masm.movq(temp, new AMD64Address(arrayA, length, strideA, 0)); if (withMask()) { - masm.orq(temp, new AMD64Address(mask, length, scaleMask, 0)); + masm.orq(temp, new AMD64Address(mask, length, strideMask, 0)); } - masm.cmpqAndJcc(temp, new AMD64Address(arrayB, length, scaleB, 0), ConditionFlag.NotEqual, requiresNaNCheck ? nanCheck : falseLabel, requiresNaNCheck); + masm.cmpqAndJcc(temp, new AMD64Address(arrayB, length, strideB, 0), ConditionFlag.NotEqual, requiresNaNCheck ? nanCheck : falseLabel, requiresNaNCheck); masm.bind(loopCheck); masm.addqAndJcc(length, elementsPerVector, ConditionFlag.NotZero, loop, true); @@ -558,8 +543,8 @@ private void emit8ByteCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm.jmpb(unalignedCheck); masm.bind(nanCheck); // At most two iterations, unroll in the emitted code. - for (int offset = 0; offset < VECTOR_SIZE; offset += scaleA.value) { - emitFloatCompare(masm, scaleA, scaleB, arrayA, arrayB, length, offset, falseLabel, scaleA.value == VECTOR_SIZE); + for (int offset = 0; offset < VECTOR_SIZE; offset += strideA.value) { + emitFloatCompare(masm, strideA, strideB, arrayA, arrayB, length, offset, falseLabel, strideA.value == VECTOR_SIZE); } masm.jmpb(loopCheck); masm.bind(unalignedCheck); @@ -569,19 +554,19 @@ private void emit8ByteCompare(CompilationResultBuilder crb, AMD64MacroAssembler * Compare the remaining bytes with an unaligned memory load aligned to the end of the * array. */ - masm.movq(temp, new AMD64Address(arrayA, result, scaleA, -VECTOR_SIZE)); + masm.movq(temp, new AMD64Address(arrayA, result, strideA, -VECTOR_SIZE)); if (requiresNaNCheck) { assert !withMask(); - masm.cmpqAndJcc(temp, new AMD64Address(arrayB, result, scaleB, -VECTOR_SIZE), ConditionFlag.Equal, trueLabel, false); + masm.cmpqAndJcc(temp, new AMD64Address(arrayB, result, strideB, -VECTOR_SIZE), ConditionFlag.Equal, trueLabel, false); // At most two iterations, unroll in the emitted code. - for (int offset = 0; offset < VECTOR_SIZE; offset += scaleA.value) { - emitFloatCompare(masm, scaleA, scaleB, arrayA, arrayB, result, -VECTOR_SIZE + offset, falseLabel, scaleA.value == VECTOR_SIZE); + for (int offset = 0; offset < VECTOR_SIZE; offset += strideA.value) { + emitFloatCompare(masm, strideA, strideB, arrayA, arrayB, result, -VECTOR_SIZE + offset, falseLabel, strideA.value == VECTOR_SIZE); } } else { if (withMask()) { - masm.orq(temp, new AMD64Address(mask, result, scaleMask, -VECTOR_SIZE)); + masm.orq(temp, new AMD64Address(mask, result, strideMask, -VECTOR_SIZE)); } - masm.cmpqAndJcc(temp, new AMD64Address(arrayB, result, scaleB, -VECTOR_SIZE), ConditionFlag.NotEqual, falseLabel, true); + masm.cmpqAndJcc(temp, new AMD64Address(arrayB, result, strideB, -VECTOR_SIZE), ConditionFlag.NotEqual, falseLabel, true); } masm.jmpb(trueLabel); @@ -593,22 +578,22 @@ private void emit8ByteCompare(CompilationResultBuilder crb, AMD64MacroAssembler * Emits code to compare the remaining 1 to 4 bytes. */ private void emitTailCompares(AMD64MacroAssembler masm, - Scale scaleA, Scale scaleB, Scale scaleMask, + Stride strideA, Stride strideB, Stride strideMask, Register result, Register arrayA, Register arrayB, Register mask, Register length, Label trueLabel, Label falseLabel) { - assert scaleA == scaleB && scaleA == scaleMask; + assert strideA == strideB && strideA == strideMask; Label compare2Bytes = new Label(); Label compare1Byte = new Label(); Register temp = asRegister(offsetAValueTemp); - if (scaleA.value <= 4) { + if (strideA.value <= 4) { // Compare trailing 4 bytes, if any. - masm.testlAndJcc(result, 4 >> scaleA.log2, ConditionFlag.Zero, compare2Bytes, true); + masm.testlAndJcc(result, 4 >> strideA.log2, ConditionFlag.Zero, compare2Bytes, true); masm.movl(temp, new AMD64Address(arrayA, 0)); if (elementKind == JavaKind.Float) { assert !withMask(); masm.cmplAndJcc(temp, new AMD64Address(arrayB, 0), ConditionFlag.Equal, trueLabel, true); - emitFloatCompare(masm, scaleA, scaleB, arrayA, arrayB, Register.None, 0, falseLabel, true); + emitFloatCompare(masm, strideA, strideB, arrayA, arrayB, Register.None, 0, falseLabel, true); masm.jmpb(trueLabel); } else { if (withMask()) { @@ -616,7 +601,7 @@ private void emitTailCompares(AMD64MacroAssembler masm, } masm.cmplAndJcc(temp, new AMD64Address(arrayB, 0), ConditionFlag.NotEqual, falseLabel, true); } - if (scaleA.value <= 2) { + if (strideA.value <= 2) { // Move array pointers forward. masm.leaq(arrayA, new AMD64Address(arrayA, 4)); masm.leaq(arrayB, new AMD64Address(arrayB, 4)); @@ -625,7 +610,7 @@ private void emitTailCompares(AMD64MacroAssembler masm, } // Compare trailing 2 bytes, if any. masm.bind(compare2Bytes); - masm.testlAndJcc(result, 2 >> scaleA.log2, ConditionFlag.Zero, compare1Byte, true); + masm.testlAndJcc(result, 2 >> strideA.log2, ConditionFlag.Zero, compare1Byte, true); masm.movzwl(temp, new AMD64Address(arrayA, 0)); if (withMask()) { masm.movzwl(length, new AMD64Address(mask, 0)); @@ -635,7 +620,7 @@ private void emitTailCompares(AMD64MacroAssembler masm, masm.cmplAndJcc(temp, length, ConditionFlag.NotEqual, falseLabel, true); // The one-byte tail compare is only required for boolean and byte arrays. - if (scaleA.value <= 1) { + if (strideA.value <= 1) { // Move array pointers forward. masm.leaq(arrayA, new AMD64Address(arrayA, 2)); masm.leaq(arrayB, new AMD64Address(arrayB, 2)); @@ -663,9 +648,9 @@ private void emitTailCompares(AMD64MacroAssembler masm, } private void emitDifferentKindsElementWiseCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, - Scale scaleA, Scale scaleB, Scale scaleMask, + Stride strideA, Stride strideB, Stride strideMask, Register result, Register array1, Register array2, Register mask, Register length, Label trueLabel, Label falseLabel) { - assert scaleA != scaleB || scaleA != scaleMask; + assert strideA != strideB || strideA != strideMask; assert elementKind.isNumericInteger(); Label loop = new Label(); Label compareTail = new Label(); @@ -678,10 +663,10 @@ private void emitDifferentKindsElementWiseCompare(CompilationResultBuilder crb, masm.andl(result, elementsPerLoopIteration - 1); // tail count masm.andlAndJcc(length, ~(elementsPerLoopIteration - 1), ConditionFlag.Zero, compareTail, true); - masm.leaq(array1, new AMD64Address(array1, length, scaleA, 0)); - masm.leaq(array2, new AMD64Address(array2, length, scaleB, 0)); + masm.leaq(array1, new AMD64Address(array1, length, strideA, 0)); + masm.leaq(array2, new AMD64Address(array2, length, strideB, 0)); if (withMask()) { - masm.leaq(mask, new AMD64Address(mask, length, scaleMask, 0)); + masm.leaq(mask, new AMD64Address(mask, length, strideMask, 0)); } masm.negq(length); @@ -689,12 +674,12 @@ private void emitDifferentKindsElementWiseCompare(CompilationResultBuilder crb, masm.align(crb.target.wordSize * 2); masm.bind(loop); for (int i = 0; i < elementsPerLoopIteration; i++) { - movSZx(masm, scaleA, extendMode, tmp1, new AMD64Address(array1, length, scaleA, i << scaleA.log2)); + movSZx(masm, strideA, extendMode, tmp1, new AMD64Address(array1, length, strideA, i << strideA.log2)); if (withMask()) { - movSZx(masm, scaleMask, extendMode, tmp2, new AMD64Address(mask, length, scaleMask, i << scaleMask.log2)); + movSZx(masm, strideMask, extendMode, tmp2, new AMD64Address(mask, length, strideMask, i << strideMask.log2)); masm.orq(tmp1, tmp2); } - movSZx(masm, scaleB, extendMode, tmp2, new AMD64Address(array2, length, scaleB, i << scaleB.log2)); + movSZx(masm, strideB, extendMode, tmp2, new AMD64Address(array2, length, strideB, i << strideB.log2)); masm.cmpqAndJcc(tmp1, tmp2, ConditionFlag.NotEqual, falseLabel, true); } masm.addqAndJcc(length, elementsPerLoopIteration, ConditionFlag.NotZero, loop, true); @@ -702,12 +687,12 @@ private void emitDifferentKindsElementWiseCompare(CompilationResultBuilder crb, masm.bind(compareTail); masm.testlAndJcc(result, result, ConditionFlag.Zero, trueLabel, true); for (int i = 0; i < elementsPerLoopIteration - 1; i++) { - movSZx(masm, scaleA, extendMode, tmp1, new AMD64Address(array1, length, scaleA, 0)); + movSZx(masm, strideA, extendMode, tmp1, new AMD64Address(array1, length, strideA, 0)); if (withMask()) { - movSZx(masm, scaleMask, extendMode, tmp2, new AMD64Address(mask, length, scaleMask, 0)); + movSZx(masm, strideMask, extendMode, tmp2, new AMD64Address(mask, length, strideMask, 0)); masm.orq(tmp1, tmp2); } - movSZx(masm, scaleB, extendMode, tmp2, new AMD64Address(array2, length, scaleB, 0)); + movSZx(masm, strideB, extendMode, tmp2, new AMD64Address(array2, length, strideB, 0)); masm.cmpqAndJcc(tmp1, tmp2, ConditionFlag.NotEqual, falseLabel, true); if (i < elementsPerLoopIteration - 2) { masm.incrementq(length, 1); @@ -737,11 +722,11 @@ private void emitNaNCheck(AMD64MacroAssembler masm, AMD64Address src, Label bran * Emits code to compare if two floats are bitwise equal or both NaN. */ private void emitFloatCompare(AMD64MacroAssembler masm, - Scale scaleA, Scale scaleB, + Stride strideA, Stride strideB, Register arrayA, Register arrayB, Register index, int offset, Label falseLabel, boolean skipBitwiseCompare) { - AMD64Address address1 = new AMD64Address(arrayA, index, scaleA, offset); - AMD64Address address2 = new AMD64Address(arrayB, index, scaleB, offset); + AMD64Address address1 = new AMD64Address(arrayA, index, strideA, offset); + AMD64Address address2 = new AMD64Address(arrayB, index, strideB, offset); Label bitwiseEqual = new Label(); @@ -768,7 +753,7 @@ private void emitFloatCompare(AMD64MacroAssembler masm, * Emits code to compare float equality within a range. */ private void emitFloatCompareWithinRange(CompilationResultBuilder crb, AMD64MacroAssembler masm, - Scale scaleA, Scale scaleB, Register arrayA, Register arrayB, Register index, int offset, Label falseLabel, int range) { + Stride strideA, Stride strideB, Register arrayA, Register arrayB, Register index, int offset, Label falseLabel, int range) { assert elementKind.isNumericFloat(); Label loop = new Label(); Register i = asRegister(offsetBValueTemp); @@ -778,7 +763,7 @@ private void emitFloatCompareWithinRange(CompilationResultBuilder crb, AMD64Macr // Align the main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); - emitFloatCompare(masm, scaleA, scaleB, arrayA, arrayB, index, offset, falseLabel, range == 1); + emitFloatCompare(masm, strideA, strideB, arrayA, arrayB, index, offset, falseLabel, range == 1); masm.incrementq(index, 1); masm.incqAndJcc(i, ConditionFlag.NotZero, loop, true); // Floats within the range are equal, revert change to the register index @@ -794,7 +779,7 @@ private void emitConstantLengthArrayCompareBytes(AMD64MacroAssembler asm, Regist // do nothing return; } - Scale maxScale = max(argScaleA, argScaleB); + Stride maxStride = Stride.max(argStrideA, argStrideB); Register arrayA = asRegister(arrayAValue); Register arrayB = asRegister(arrayBValue); Register mask = withMask() ? asRegister(arrayMaskValue) : null; @@ -803,50 +788,50 @@ private void emitConstantLengthArrayCompareBytes(AMD64MacroAssembler asm, Regist Register vector3 = asRegister(vectorTemp[2]); Register vector4 = withMask() ? asRegister(vectorTemp[3]) : null; Register tmp = asRegister(lengthValue); - GraalError.guarantee(constantLength() <= getElementsPerVector(vectorSize, maxScale) * 2, "constant length too long for specialized arrayEquals!"); + GraalError.guarantee(constantLength() <= getElementsPerVector(vectorSize, maxStride) * 2, "constant length too long for specialized arrayEquals!"); AVXSize vSize = vectorSize; - if (constantLength() < getElementsPerVector(vectorSize, maxScale)) { + if (constantLength() < getElementsPerVector(vectorSize, maxStride)) { vSize = AVXSize.XMM; } - int elementsPerVector = getElementsPerVector(vSize, maxScale); + int elementsPerVector = getElementsPerVector(vSize, maxStride); if (elementsPerVector > constantLength()) { - assert argScaleA == argScaleB && argScaleA == argScaleMask; - int byteLength = constantLength() << argScaleA.log2; + assert argStrideA == argStrideB && argStrideA == argStrideMask; + int byteLength = constantLength() << argStrideA.log2; // array is shorter than any vector register, use regular XOR instructions - Scale movScale = (byteLength < 2) ? Scale.Times1 : ((byteLength < 4) ? Scale.Times2 : ((byteLength < 8) ? Scale.Times4 : Scale.Times8)); - movSZx(asm, movScale, extendMode, tmp, new AMD64Address(arrayA)); + Stride movStride = (byteLength < 2) ? Stride.S1 : ((byteLength < 4) ? Stride.S2 : ((byteLength < 8) ? Stride.S4 : Stride.S8)); + movSZx(asm, movStride, extendMode, tmp, new AMD64Address(arrayA)); if (withMask()) { - emitOrBytes(asm, tmp, new AMD64Address(mask, 0), movScale); + emitOrBytes(asm, tmp, new AMD64Address(mask, 0), movStride); } - if (byteLength > movScale.value) { - emitXorBytes(asm, tmp, new AMD64Address(arrayB), movScale); - movSZx(asm, movScale, extendMode, arrayA, new AMD64Address(arrayA, byteLength - movScale.value)); + if (byteLength > movStride.value) { + emitXorBytes(asm, tmp, new AMD64Address(arrayB), movStride); + movSZx(asm, movStride, extendMode, arrayA, new AMD64Address(arrayA, byteLength - movStride.value)); if (withMask()) { - emitOrBytes(asm, arrayA, new AMD64Address(mask, byteLength - movScale.value), movScale); + emitOrBytes(asm, arrayA, new AMD64Address(mask, byteLength - movStride.value), movStride); } - emitXorBytes(asm, arrayA, new AMD64Address(arrayB, byteLength - movScale.value), movScale); + emitXorBytes(asm, arrayA, new AMD64Address(arrayB, byteLength - movStride.value), movStride); asm.xorq(arrayB, arrayB); asm.orq(tmp, arrayA); asm.cmovl(AMD64Assembler.ConditionFlag.NotZero, result, arrayB); } else { asm.xorq(arrayA, arrayA); - emitXorBytes(asm, tmp, new AMD64Address(arrayB), movScale); + emitXorBytes(asm, tmp, new AMD64Address(arrayB), movStride); asm.cmovl(AMD64Assembler.ConditionFlag.NotZero, result, arrayA); } } else { - pmovSZx(asm, vSize, vector1, maxScale, arrayA, 0, argScaleA); - pmovSZx(asm, vSize, vector2, maxScale, arrayB, 0, argScaleB); + pmovSZx(asm, vSize, vector1, maxStride, arrayA, 0, argStrideA); + pmovSZx(asm, vSize, vector2, maxStride, arrayB, 0, argStrideB); if (withMask()) { - pmovSZx(asm, vSize, vector4, maxScale, mask, 0, argScaleMask); + pmovSZx(asm, vSize, vector4, maxStride, mask, 0, argStrideMask); por(asm, vSize, vector1, vector4); } pxor(asm, vSize, vector1, vector2); if (constantLength() > elementsPerVector) { - int endOffset = (constantLength() << maxScale.log2) - vSize.getBytes(); - pmovSZx(asm, vSize, vector3, maxScale, arrayA, endOffset, argScaleA); - pmovSZx(asm, vSize, vector2, maxScale, arrayB, endOffset, argScaleB); + int endOffset = (constantLength() << maxStride.log2) - vSize.getBytes(); + pmovSZx(asm, vSize, vector3, maxStride, arrayA, endOffset, argStrideA); + pmovSZx(asm, vSize, vector2, maxStride, arrayB, endOffset, argStrideB); if (withMask()) { - pmovSZx(asm, vSize, vector4, maxScale, mask, endOffset, argScaleMask); + pmovSZx(asm, vSize, vector4, maxStride, mask, endOffset, argStrideMask); por(asm, vSize, vector3, vector4); } pxor(asm, vSize, vector3, vector2); @@ -858,25 +843,25 @@ private void emitConstantLengthArrayCompareBytes(AMD64MacroAssembler asm, Regist } } - private static void emitOrBytes(AMD64MacroAssembler asm, Register dst, AMD64Address src, Scale scale) { - OperandSize opSize = getOperandSize(scale); + private static void emitOrBytes(AMD64MacroAssembler asm, Register dst, AMD64Address src, Stride stride) { + OperandSize opSize = getOperandSize(stride); OR.getRMOpcode(opSize).emit(asm, opSize, dst, src); } - private static void emitXorBytes(AMD64MacroAssembler asm, Register dst, AMD64Address src, Scale scale) { - OperandSize opSize = getOperandSize(scale); + private static void emitXorBytes(AMD64MacroAssembler asm, Register dst, AMD64Address src, Stride stride) { + OperandSize opSize = getOperandSize(stride); XOR.getRMOpcode(opSize).emit(asm, opSize, dst, src); } - private static OperandSize getOperandSize(Scale size) { + private static OperandSize getOperandSize(Stride size) { switch (size) { - case Times1: + case S1: return OperandSize.BYTE; - case Times2: + case S2: return OperandSize.WORD; - case Times4: + case S4: return OperandSize.DWORD; - case Times8: + case S8: return OperandSize.QWORD; default: throw new IllegalStateException(); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayIndexOfOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayIndexOfOp.java index 02d08e54c9b9..8866d746cdcf 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayIndexOfOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayIndexOfOp.java @@ -44,11 +44,9 @@ import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; import java.util.EnumSet; -import java.util.Objects; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; import org.graalvm.compiler.asm.amd64.AMD64Assembler; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64Assembler.VexMoveOp; @@ -58,6 +56,7 @@ import org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.ConstantValue; import org.graalvm.compiler.lir.LIRInstructionClass; @@ -70,7 +69,6 @@ import jdk.vm.ci.code.Register; import jdk.vm.ci.code.RegisterValue; import jdk.vm.ci.meta.JavaConstant; -import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.Value; /** @@ -86,13 +84,12 @@ public final class AMD64ArrayIndexOfOp extends AMD64ComplexVectorOp { private static final Register REG_SEARCH_VALUE_1 = rcx; private static final Register REG_SEARCH_VALUE_2 = r8; - private final JavaKind valueKind; + private final Stride stride; private final int nValues; private final boolean findTwoConsecutive; private final boolean withMask; private final AMD64Kind vectorKind; - private final Scale arrayIndexScale; - private final int arrayBaseOffset; + private final Stride arrayIndexStride; private final int constOffset; @Def({REG}) Value resultValue; @@ -117,19 +114,18 @@ public final class AMD64ArrayIndexOfOp extends AMD64ComplexVectorOp { @Temp({REG}) Value[] vectorCompareVal; @Temp({REG}) Value[] vectorArray; - private AMD64ArrayIndexOfOp(int arrayBaseOffset, JavaKind valueKind, boolean findTwoConsecutive, boolean withMask, int constOffset, int nValues, LIRGeneratorTool tool, + private AMD64ArrayIndexOfOp(Stride stride, boolean findTwoConsecutive, boolean withMask, int constOffset, int nValues, LIRGeneratorTool tool, EnumSet runtimeCheckedCPUFeatures, Value result, Value arrayPtr, Value arrayOffset, Value arrayLength, Value fromIndex, Value searchValue1, Value searchValue2, Value searchValue3, Value searchValue4) { super(TYPE, tool, runtimeCheckedCPUFeatures, AVXSize.YMM); - this.valueKind = valueKind; - this.arrayIndexScale = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(valueKind))); - this.arrayBaseOffset = arrayBaseOffset; + this.stride = stride; + this.arrayIndexStride = stride; this.findTwoConsecutive = findTwoConsecutive; this.withMask = withMask; this.constOffset = constOffset; this.nValues = nValues; GraalError.guarantee(0 < nValues && nValues <= 4, "only 1 - 4 values supported"); - GraalError.guarantee(valueKind == JavaKind.Byte || valueKind == JavaKind.Char || valueKind == JavaKind.Int, "supported strides are 1, 2 and 4 bytes"); + GraalError.guarantee(stride.value <= 4, "supported strides are 1, 2 and 4 bytes"); GraalError.guarantee(supports(tool.target(), runtimeCheckedCPUFeatures, CPUFeature.SSE2), "needs at least SSE2 support"); GraalError.guarantee(!(!withMask && findTwoConsecutive) || nValues == 2, "findTwoConsecutive mode requires exactly 2 search values"); GraalError.guarantee(!(withMask && findTwoConsecutive) || nValues == 4, "findTwoConsecutive mode with mask requires exactly 4 search values"); @@ -145,9 +141,9 @@ private AMD64ArrayIndexOfOp(int arrayBaseOffset, JavaKind valueKind, boolean fin this.searchValue3 = searchValue3; this.searchValue4 = searchValue4; - this.vectorKind = getVectorKind(valueKind); - this.vectorCompareVal = allocateVectorRegisters(tool, valueKind, nValues); - this.vectorArray = allocateVectorRegisters(tool, valueKind, 4); + this.vectorKind = getVectorKind(stride); + this.vectorCompareVal = allocateVectorRegisters(tool, stride, nValues); + this.vectorArray = allocateVectorRegisters(tool, stride, 4); } private static Register[] asRegisters(Value[] values) { @@ -158,7 +154,7 @@ private static Register[] asRegisters(Value[] values) { return registers; } - public static AMD64ArrayIndexOfOp movParamsAndCreate(int arrayBaseOffset, JavaKind valueKind, boolean findTwoConsecutive, boolean withMask, LIRGeneratorTool tool, + public static AMD64ArrayIndexOfOp movParamsAndCreate(Stride stride, boolean findTwoConsecutive, boolean withMask, LIRGeneratorTool tool, EnumSet runtimeCheckedCPUFeatures, Value result, Value arrayPtr, Value arrayOffset, Value arrayLength, Value fromIndex, Value... searchValues) { int nValues = searchValues.length; @@ -185,7 +181,7 @@ public static AMD64ArrayIndexOfOp movParamsAndCreate(int arrayBaseOffset, JavaKi } else { constOffset = -1; } - return new AMD64ArrayIndexOfOp(arrayBaseOffset, valueKind, findTwoConsecutive, withMask, constOffset, nValues, tool, + return new AMD64ArrayIndexOfOp(stride, findTwoConsecutive, withMask, constOffset, nValues, tool, runtimeCheckedCPUFeatures, result, regArray, regOffset, regLength, regFromIndex, regSearchValue1, regSearchValue2, regSearchValue3, regSearchValue4); } @@ -228,9 +224,9 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { int bulkSize = vectorLength * nVectors; if (useConstantOffset()) { - asm.leaq(arrayPtr, new AMD64Address(arrayPtr, constOffset + arrayBaseOffset)); + asm.leaq(arrayPtr, new AMD64Address(arrayPtr, constOffset)); } else { - asm.leaq(arrayPtr, new AMD64Address(arrayPtr, asRegister(offsetReg), Scale.Times1, arrayBaseOffset)); + asm.leaq(arrayPtr, new AMD64Address(arrayPtr, asRegister(offsetReg), Stride.S1)); } // index = fromIndex + vectorLength (+1 if findTwoConsecutive) @@ -256,10 +252,10 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { // check if vector vector load is in bounds asm.cmpqAndJcc(index, arrayLength, ConditionFlag.Greater, qWordWise, false); // do one vector comparison from fromIndex - emitVectorCompare(asm, valueKind, AVXSize.XMM, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, xmmFound, true); + emitVectorCompare(asm, AVXSize.XMM, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, xmmFound, true); // and one aligned to the array end asm.movq(index, arrayLength); - emitVectorCompare(asm, valueKind, AVXSize.XMM, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, xmmFound, true); + emitVectorCompare(asm, AVXSize.XMM, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, xmmFound, true); // no match, return negative asm.jmp(elementWiseNotFound); // match found, adjust index by XMM offset @@ -270,7 +266,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { } asm.bind(qWordWise); - int vectorLengthQWord = AVXSize.QWORD.getBytes() / valueKind.getByteCount(); + int vectorLengthQWord = AVXSize.QWORD.getBytes() / stride.value; // region is too short for XMM vectors, try QWORD Label[] qWordFound = {new Label()}; // index = fromIndex (+ 1 if findTwoConsecutive) + QWORD vector size @@ -278,10 +274,10 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { // check if vector vector load is in bounds asm.cmpqAndJcc(index, arrayLength, ConditionFlag.Greater, elementWise, false); // do one vector comparison from fromIndex - emitVectorCompare(asm, valueKind, AVXSize.QWORD, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, qWordFound, true); + emitVectorCompare(asm, AVXSize.QWORD, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, qWordFound, true); // and one aligned to the array end asm.movq(index, arrayLength); - emitVectorCompare(asm, valueKind, AVXSize.QWORD, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, qWordFound, true); + emitVectorCompare(asm, AVXSize.QWORD, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, qWordFound, true); // no match, return negative asm.jmpb(elementWiseNotFound); // match found, adjust index by QWORD offset @@ -297,10 +293,10 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { // check if enough array slots remain asm.cmpqAndJcc(index, arrayLength, ConditionFlag.GreaterEqual, elementWiseNotFound, true); - OperandSize valueSize = getOpSize(valueKind); + OperandSize valueSize = getOpSize(stride); // combine consecutive search values into one register if (findTwoConsecutive) { - asm.shlq(asRegister(searchValue[1]), valueKind.getBitCount()); + asm.shlq(asRegister(searchValue[1]), stride.getBitCount()); asm.orq(asRegister(searchValue[0]), asRegister(searchValue[1])); if (withMask) { if (isStackSlot(searchValue[3])) { @@ -308,7 +304,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { } else { asm.movq(asRegister(searchValue[1]), asRegister(searchValue[3])); } - asm.shlq(asRegister(searchValue[1]), valueKind.getBitCount()); + asm.shlq(asRegister(searchValue[1]), stride.getBitCount()); if (isStackSlot(searchValue[2])) { movSZx(asm, valueSize, ZERO_EXTEND, cmpResult, (AMD64Address) crb.asAddress(searchValue[2])); asm.orq(asRegister(searchValue[1]), cmpResult); @@ -321,18 +317,18 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { // compare one-by-one asm.bind(elementWiseLoop); if (findTwoConsecutive) { - AMD64Address arrayAddr = new AMD64Address(arrayPtr, index, arrayIndexScale, -valueKind.getByteCount()); + AMD64Address arrayAddr = new AMD64Address(arrayPtr, index, arrayIndexStride, -stride.value); if (withMask) { - movSZx(asm, getDoubleOpSize(valueKind), ZERO_EXTEND, cmpResult, arrayAddr); + movSZx(asm, getDoubleOpSize(stride), ZERO_EXTEND, cmpResult, arrayAddr); asm.orq(cmpResult, asRegister(searchValue[1])); asm.cmpqAndJcc(cmpResult, asRegister(searchValue[0]), AMD64Assembler.ConditionFlag.Equal, elementWiseFound, true); } else { - asm.cmpAndJcc(getDoubleOpSize(valueKind), asRegister(searchValue[0]), arrayAddr, AMD64Assembler.ConditionFlag.Equal, elementWiseFound, true); + asm.cmpAndJcc(getDoubleOpSize(stride), asRegister(searchValue[0]), arrayAddr, AMD64Assembler.ConditionFlag.Equal, elementWiseFound, true); } } else { // check for match // address = findTwoConsecutive ? array[index - 1] : array[index] - AMD64Address arrayAddr = new AMD64Address(arrayPtr, index, arrayIndexScale); + AMD64Address arrayAddr = new AMD64Address(arrayPtr, index, arrayIndexStride); boolean valuesOnStack = searchValuesOnStack(searchValue); if (withMask) { assert !valuesOnStack; @@ -376,12 +372,12 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { asm.bind(runVectorized); // do one unaligned vector comparison pass and adjust alignment afterwards - emitVectorCompare(asm, valueKind, vectorSize, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, vectorFound, false); + emitVectorCompare(asm, vectorSize, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, vectorFound, false); // adjust index to vector size alignment asm.movl(cmpResult, arrayPtr); - if (valueKind.getByteCount() > 1) { - asm.shrl(cmpResult, strideAsPowerOf2()); + if (stride.value > 1) { + asm.shrl(cmpResult, stride.log2); } asm.addq(index, cmpResult); // adjust to next lower multiple of vector size @@ -396,7 +392,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { emitAlign(crb, asm); asm.bind(bulkVectorLoop); // memory-aligned bulk comparison - emitVectorCompare(asm, valueKind, vectorSize, nVectors, arrayPtr, index, vecCmp, vecArray, cmpResult, vectorFound, false); + emitVectorCompare(asm, vectorSize, nVectors, arrayPtr, index, vecCmp, vecArray, cmpResult, vectorFound, false); // adjust index asm.addq(index, bulkSize); // check if there are enough array slots remaining for the bulk loop @@ -407,7 +403,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { // do last load from end of array asm.movq(index, arrayLength); // compare - emitVectorCompare(asm, valueKind, vectorSize, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, vectorFound, true); + emitVectorCompare(asm, vectorSize, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, vectorFound, true); } else { // remove bulk offset asm.subq(index, bulkSize); @@ -421,7 +417,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { // if load would be over bounds, set the load to the end of the array asm.cmovq(AMD64Assembler.ConditionFlag.Greater, index, arrayLength); // compare - emitVectorCompare(asm, valueKind, vectorSize, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, vectorFound, true); + emitVectorCompare(asm, vectorSize, 1, arrayPtr, index, vecCmp, vecArray, cmpResult, vectorFound, true); // check if there are enough array slots remaining for the loop asm.cmpqAndJcc(index, arrayLength, ConditionFlag.Less, singleVectorLoop, true); } @@ -439,9 +435,9 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler asm) { asm.bind(bsfAdd); // find offset bsfq(asm, cmpResult, cmpResult); - if (valueKind.getByteCount() > 1) { + if (stride.value > 1) { // convert byte offset to chars - asm.shrq(cmpResult, strideAsPowerOf2()); + asm.shrq(cmpResult, stride.log2); } // add offset to index asm.addq(index, cmpResult); @@ -464,7 +460,7 @@ private int getResultIndexDelta(int i) { private int getVectorOffset(int i, int j, AVXSize targetVectorSize) { if (findTwoConsecutive) { - return -(((i + 1) * targetVectorSize.getBytes() + ((j ^ 1) * valueKind.getByteCount()))); + return -(((i + 1) * targetVectorSize.getBytes() + ((j ^ 1) * stride.value))); } return -(((i + 1) * targetVectorSize.getBytes())); } @@ -472,7 +468,7 @@ private int getVectorOffset(int i, int j, AVXSize targetVectorSize) { private void broadcastSearchValue(CompilationResultBuilder crb, AMD64MacroAssembler asm, Register dst, Value srcVal, Register tmpReg, Register tmpVector) { Register src = asRegOrTmpReg(crb, asm, srcVal, tmpReg); AMD64MacroAssembler.movdl(asm, dst, src); - emitBroadcast(asm, valueKind, dst, tmpVector, vectorSize); + emitBroadcast(asm, stride, dst, tmpVector, vectorSize); } private static boolean isConstant(Value val) { @@ -504,9 +500,9 @@ private static void emitAlign(CompilationResultBuilder crb, AMD64MacroAssembler /** * Fills {@code vecDst} with copies of its lowest byte, word or dword. */ - private static void emitBroadcast(AMD64MacroAssembler asm, JavaKind kind, Register vecDst, Register vecTmp, AVXSize targetVectorSize) { - switch (kind) { - case Byte: + private static void emitBroadcast(AMD64MacroAssembler asm, Stride stride, Register vecDst, Register vecTmp, AVXSize targetVectorSize) { + switch (stride) { + case S1: if (asm.supports(CPUFeature.AVX2)) { VexRMOp.VPBROADCASTB.emit(asm, targetVectorSize, vecDst, vecDst); } else if (asm.supports(CPUFeature.AVX)) { @@ -521,8 +517,7 @@ private static void emitBroadcast(AMD64MacroAssembler asm, JavaKind kind, Regist asm.pshufd(vecDst, vecDst, 0); } break; - case Short: - case Char: + case S2: if (asm.supports(CPUFeature.AVX2)) { VexRMOp.VPBROADCASTW.emit(asm, targetVectorSize, vecDst, vecDst); } else if (asm.supports(CPUFeature.AVX)) { @@ -533,7 +528,7 @@ private static void emitBroadcast(AMD64MacroAssembler asm, JavaKind kind, Regist asm.pshufd(vecDst, vecDst, 0); } break; - case Int: + case S4: if (asm.supports(CPUFeature.AVX2)) { VexRMOp.VPBROADCASTD.emit(asm, targetVectorSize, vecDst, vecDst); } else if (asm.supports(CPUFeature.AVX)) { @@ -548,7 +543,6 @@ private static void emitBroadcast(AMD64MacroAssembler asm, JavaKind kind, Regist } private void emitVectorCompare(AMD64MacroAssembler asm, - JavaKind kind, AVXSize targetVectorSize, int nVectors, Register arrayPtr, @@ -573,8 +567,8 @@ private void emitVectorCompare(AMD64MacroAssembler asm, por(asm, targetVectorSize, vecArray[i], vecCmp[2]); por(asm, targetVectorSize, vecArray[i + 1], vecCmp[3]); } - pcmpeq(asm, targetVectorSize, kind, vecArray[i], vecCmp[0]); - pcmpeq(asm, targetVectorSize, kind, vecArray[i + 1], vecCmp[1]); + pcmpeq(asm, targetVectorSize, stride, vecArray[i], vecCmp[0]); + pcmpeq(asm, targetVectorSize, stride, vecArray[i + 1], vecCmp[1]); pand(asm, targetVectorSize, vecArray[i], vecArray[i + 1]); pmovmsk(asm, targetVectorSize, cmpResult, vecArray[i]); emitVectorCompareCheckVectorFound(asm, targetVectorSize, cmpResult, vectorFound[nVectors - ((i / 2) + 1)], shortJmp); @@ -582,14 +576,14 @@ private void emitVectorCompare(AMD64MacroAssembler asm, } else if (withMask) { assert nValues == 2 && nVectors == 1; por(asm, targetVectorSize, vecArray[0], vecCmp[1]); - pcmpeq(asm, targetVectorSize, kind, vecArray[0], vecCmp[0]); + pcmpeq(asm, targetVectorSize, stride, vecArray[0], vecCmp[0]); pmovmsk(asm, targetVectorSize, cmpResult, vecArray[0]); emitVectorCompareCheckVectorFound(asm, targetVectorSize, cmpResult, vectorFound[0], shortJmp); } else { for (int i = 0; i < nVectors; i++) { int base = i * nValues; for (int j = 0; j < nValues; j++) { - pcmpeq(asm, targetVectorSize, kind, vecArray[base + j], vecCmp[j]); + pcmpeq(asm, targetVectorSize, stride, vecArray[base + j], vecCmp[j]); if ((j & 1) == 1) { por(asm, targetVectorSize, vecArray[base + j - 1], vecArray[base + j]); } @@ -623,7 +617,7 @@ private static void emitVectorCompareCheckVectorFound(AMD64MacroAssembler asm, A @SuppressWarnings("fallthrough") private void emitArrayLoad(AMD64MacroAssembler asm, AVXSize targetVectorSize, Register vecDst, Register array, Register index, int displacement) { - AMD64Address src = new AMD64Address(array, index, arrayIndexScale, displacement); + AMD64Address src = new AMD64Address(array, index, arrayIndexStride, displacement); if (asm.supports(CPUFeature.AVX)) { switch (targetVectorSize) { case DWORD: @@ -659,34 +653,28 @@ private void emitArrayLoad(AMD64MacroAssembler asm, AVXSize targetVectorSize, Re } } - private static OperandSize getOpSize(JavaKind kind) { - switch (kind) { - case Byte: + private static OperandSize getOpSize(Stride stride) { + switch (stride) { + case S1: return OperandSize.BYTE; - case Short: - case Char: + case S2: return OperandSize.WORD; - case Int: + case S4: return OperandSize.DWORD; default: return OperandSize.QWORD; } } - private static OperandSize getDoubleOpSize(JavaKind kind) { - switch (kind) { - case Byte: + private static OperandSize getDoubleOpSize(Stride stride) { + switch (stride) { + case S1: return OperandSize.WORD; - case Short: - case Char: + case S2: return OperandSize.DWORD; default: - assert kind.equals(JavaKind.Int); + assert stride == Stride.S4; return OperandSize.QWORD; } } - - private int strideAsPowerOf2() { - return Integer.numberOfTrailingZeros(valueKind.getByteCount()); - } } diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayRegionCompareToOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayRegionCompareToOp.java index bbf286e28309..8b8a07931e1b 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayRegionCompareToOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayRegionCompareToOp.java @@ -45,15 +45,14 @@ import java.util.Arrays; import java.util.EnumSet; -import java.util.Objects; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; import org.graalvm.compiler.asm.amd64.AMD64Assembler; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.StrideUtil; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; @@ -99,8 +98,8 @@ public final class AMD64ArrayRegionCompareToOp extends AMD64ComplexVectorOp { private static final int ONES_16 = 0xffff; private static final int ONES_32 = 0xffffffff; - private final Scale argScaleA; - private final Scale argScaleB; + private final Stride argStrideA; + private final Stride argStrideB; private final AMD64MacroAssembler.ExtendMode extendMode; @Def({REG}) private Value resultValue; @@ -120,19 +119,19 @@ public final class AMD64ArrayRegionCompareToOp extends AMD64ComplexVectorOp { @Temp({REG}) Value[] vectorTemp; - private AMD64ArrayRegionCompareToOp(LIRGeneratorTool tool, JavaKind strideA, JavaKind strideB, + private AMD64ArrayRegionCompareToOp(LIRGeneratorTool tool, Stride strideA, Stride strideB, EnumSet runtimeCheckedCPUFeatures, Value result, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length, Value dynamicStrides, AMD64MacroAssembler.ExtendMode extendMode) { super(TYPE, tool, runtimeCheckedCPUFeatures, YMM); this.extendMode = extendMode; if (strideA == null) { - this.argScaleA = null; - this.argScaleB = null; + this.argStrideA = null; + this.argStrideB = null; } else { - GraalError.guarantee(strideA == JavaKind.Byte || strideA == JavaKind.Char || strideA == JavaKind.Int, "unsupported strideA"); - GraalError.guarantee(strideB == JavaKind.Byte || strideB == JavaKind.Char || strideB == JavaKind.Int, "unsupported strideB"); - this.argScaleA = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(strideA))); - this.argScaleB = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(strideB))); + GraalError.guarantee(strideA.value <= 4, "unsupported strideA"); + GraalError.guarantee(strideB.value <= 4, "unsupported strideB"); + this.argStrideA = strideA; + this.argStrideB = strideB; } this.resultValue = result; this.arrayAValue = this.arrayAValueTemp = arrayA; @@ -142,7 +141,7 @@ private AMD64ArrayRegionCompareToOp(LIRGeneratorTool tool, JavaKind strideA, Jav this.lengthValue = this.lengthValueTemp = length; this.dynamicStridesValue = this.dynamicStridesValueTemp = dynamicStrides; - this.vectorTemp = allocateVectorRegisters(tool, JavaKind.Byte, isVectorCompareSupported(tool.target(), runtimeCheckedCPUFeatures, argScaleA, argScaleB) ? 4 : 0); + this.vectorTemp = allocateVectorRegisters(tool, JavaKind.Byte, isVectorCompareSupported(tool.target(), runtimeCheckedCPUFeatures, argStrideA, argStrideB) ? 4 : 0); } /** @@ -155,7 +154,6 @@ private AMD64ArrayRegionCompareToOp(LIRGeneratorTool tool, JavaKind strideA, Jav * @param strideB element size of {@code arrayB}. May be one of {@link JavaKind#Byte}, * {@link JavaKind#Char} or {@link JavaKind#Int}. {@code strideB} must be less than * or equal to {@code strideA}! - * @param runtimeCheckedCPUFeatures * @param offsetA byte offset to be added to {@code arrayA}. Must include the array base offset! * @param offsetB byte offset to be added to {@code arrayB}. Must include the array base offset! * @param length length (number of array slots in respective array's stride) of the region to @@ -163,7 +161,7 @@ private AMD64ArrayRegionCompareToOp(LIRGeneratorTool tool, JavaKind strideA, Jav * @param dynamicStrides dynamic stride dispatch as described in {@link StrideUtil}. * @param extendMode integer extension mode for {@code arrayB}. */ - public static AMD64ArrayRegionCompareToOp movParamsAndCreate(LIRGeneratorTool tool, JavaKind strideA, JavaKind strideB, + public static AMD64ArrayRegionCompareToOp movParamsAndCreate(LIRGeneratorTool tool, Stride strideA, Stride strideB, EnumSet runtimeCheckedCPUFeatures, Value result, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length, Value dynamicStrides, AMD64MacroAssembler.ExtendMode extendMode) { @@ -194,11 +192,11 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { Register tmp2 = asRegister(offsetBValue); // add byte offset to array pointers - masm.leaq(arrayA, new AMD64Address(arrayA, asRegister(offsetAValue), Scale.Times1)); - masm.leaq(arrayB, new AMD64Address(arrayB, asRegister(offsetBValue), Scale.Times1)); + masm.leaq(arrayA, new AMD64Address(arrayA, asRegister(offsetAValue), Stride.S1)); + masm.leaq(arrayB, new AMD64Address(arrayB, asRegister(offsetBValue), Stride.S1)); if (isIllegal(dynamicStridesValue)) { - emitArrayCompare(crb, masm, argScaleA, argScaleB, result, arrayA, arrayB, length, tmp1, tmp2); + emitArrayCompare(crb, masm, argStrideA, argStrideB, result, arrayA, arrayB, length, tmp1, tmp2); } else { masm.xorq(tmp2, tmp2); Label[] variants = new Label[9]; @@ -207,11 +205,11 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { variants[i] = new Label(); } AMD64ControlFlow.RangeTableSwitchOp.emitJumpTable(crb, masm, tmp1, asRegister(dynamicStridesValue), 0, 8, Arrays.stream(variants)); - for (Scale scaleA : new Scale[]{Scale.Times1, Scale.Times2, Scale.Times4}) { - for (Scale scaleB : new Scale[]{Scale.Times1, Scale.Times2, Scale.Times4}) { + for (Stride strideA : new Stride[]{Stride.S1, Stride.S2, Stride.S4}) { + for (Stride strideB : new Stride[]{Stride.S1, Stride.S2, Stride.S4}) { masm.align(crb.target.wordSize * 2); - masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(scaleA, scaleB)]); - emitArrayCompare(crb, masm, scaleA, scaleB, result, arrayA, arrayB, length, tmp1, tmp2); + masm.bind(variants[AMD64StrideUtil.getDirectStubCallIndex(strideA, strideB)]); + emitArrayCompare(crb, masm, strideA, strideB, result, arrayA, arrayB, length, tmp1, tmp2); masm.jmp(done); } } @@ -219,20 +217,20 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { } } - private static boolean isVectorCompareSupported(TargetDescription target, EnumSet runtimeCheckedCPUFeatures, Scale scaleA, Scale scaleB) { + private static boolean isVectorCompareSupported(TargetDescription target, EnumSet runtimeCheckedCPUFeatures, Stride strideA, Stride strideB) { // if strides are not equal, we need the SSE4.1 pmovzx instruction, otherwise SSE2 is // enough. - return scaleA == scaleB || supports(target, runtimeCheckedCPUFeatures, CPUFeature.SSE4_1); + return strideA == strideB || supports(target, runtimeCheckedCPUFeatures, CPUFeature.SSE4_1); } private void emitArrayCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, - Scale scaleA, Scale scaleB, + Stride strideA, Stride strideB, Register result, Register arrayA, Register arrayB, Register length, Register tmp1, Register tmp2) { Label returnLabel = new Label(); - if (isVectorCompareSupported(crb.target, runtimeCheckedCPUFeatures, scaleA, scaleB)) { - emitVectorLoop(crb, masm, scaleA, scaleB, result, arrayA, arrayB, length, tmp1, tmp2, returnLabel); + if (isVectorCompareSupported(crb.target, runtimeCheckedCPUFeatures, strideA, strideB)) { + emitVectorLoop(crb, masm, strideA, strideB, result, arrayA, arrayB, length, tmp1, tmp2, returnLabel); } - emitScalarLoop(crb, masm, scaleA, scaleB, result, arrayA, arrayB, length, tmp1, returnLabel); + emitScalarLoop(crb, masm, strideA, strideB, result, arrayA, arrayB, length, tmp1, returnLabel); masm.bind(returnLabel); } @@ -241,15 +239,15 @@ private void emitArrayCompare(CompilationResultBuilder crb, AMD64MacroAssembler * compares. Underlying algorithm: check vectors for equality with PCMPEQ, identify the first * differing elements with PMOVMSK and BSF and return their scalar difference. */ - private void emitVectorLoop(CompilationResultBuilder crb, AMD64MacroAssembler masm, Scale scaleA, Scale scaleB, + private void emitVectorLoop(CompilationResultBuilder crb, AMD64MacroAssembler masm, Stride strideA, Stride strideB, Register result, Register arrayA, Register arrayB, Register length, Register tmp1, Register tmp2, Label returnLabel) { - Scale maxScale = max(scaleA, scaleB); + Stride maxStride = Stride.max(strideA, strideB); Register vector1 = asRegister(vectorTemp[0]); Register vector2 = asRegister(vectorTemp[1]); Register vector3 = asRegister(vectorTemp[2]); Register vector4 = asRegister(vectorTemp[3]); - int elementsPerVector = getElementsPerVector(vectorSize, maxScale); + int elementsPerVector = getElementsPerVector(vectorSize, maxStride); Label loop = new Label(); Label qwordTail = new Label(); @@ -258,8 +256,8 @@ private void emitVectorLoop(CompilationResultBuilder crb, AMD64MacroAssembler ma Label diffFound = new Label(); // fast path: check first element as scalar - movSZx(masm, scaleA, extendMode, result, new AMD64Address(arrayA)); - movSZx(masm, scaleB, extendMode, tmp1, new AMD64Address(arrayB)); + movSZx(masm, strideA, extendMode, result, new AMD64Address(arrayA)); + movSZx(masm, strideB, extendMode, tmp1, new AMD64Address(arrayB)); masm.subqAndJcc(result, tmp1, ConditionFlag.NotZero, returnLabel, false); masm.movl(result, length); @@ -267,18 +265,18 @@ private void emitVectorLoop(CompilationResultBuilder crb, AMD64MacroAssembler ma masm.andl(result, elementsPerVector - 1); // tail count masm.andlAndJcc(length, -elementsPerVector, ConditionFlag.Zero, tail, false); - masm.leaq(arrayA, new AMD64Address(arrayA, length, scaleA)); - masm.leaq(arrayB, new AMD64Address(arrayB, length, scaleB)); + masm.leaq(arrayA, new AMD64Address(arrayA, length, strideA)); + masm.leaq(arrayB, new AMD64Address(arrayB, length, strideB)); masm.negq(length); // main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); // load and extend elements of arrayB to match the stride of arrayA - pmovSZx(masm, vectorSize, extendMode, vector1, maxScale, arrayA, scaleA, length, 0); - pmovSZx(masm, vectorSize, extendMode, vector2, maxScale, arrayB, scaleB, length, 0); + pmovSZx(masm, vectorSize, extendMode, vector1, maxStride, arrayA, strideA, length, 0); + pmovSZx(masm, vectorSize, extendMode, vector2, maxStride, arrayB, strideB, length, 0); // compare elements of arrayA and arrayB - pcmpeq(masm, vectorSize, maxScale, vector1, vector2); + pcmpeq(masm, vectorSize, maxStride, vector1, vector2); // convert result to bitmask pmovmsk(masm, vectorSize, tmp1, vector1); // invert bit mask. if the result is non-zero, compared regions are not equal @@ -290,11 +288,11 @@ private void emitVectorLoop(CompilationResultBuilder crb, AMD64MacroAssembler ma masm.testlAndJcc(result, result, ConditionFlag.Zero, returnLabel, false); // tail: compare the remaining bytes with a vector load aligned to the end of the array. - pmovSZx(masm, vectorSize, extendMode, vector1, maxScale, arrayA, scaleA, result, -vectorSize.getBytes()); - pmovSZx(masm, vectorSize, extendMode, vector2, maxScale, arrayB, scaleB, result, -vectorSize.getBytes()); + pmovSZx(masm, vectorSize, extendMode, vector1, maxStride, arrayA, strideA, result, -vectorSize.getBytes()); + pmovSZx(masm, vectorSize, extendMode, vector2, maxStride, arrayB, strideB, result, -vectorSize.getBytes()); // adjust "length" for diffFound - masm.leaq(length, new AMD64Address(length, result, Scale.Times1, -elementsPerVector)); - pcmpeq(masm, vectorSize, maxScale, vector1, vector2); + masm.leaq(length, new AMD64Address(length, result, Stride.S1, -elementsPerVector)); + pcmpeq(masm, vectorSize, maxStride, vector1, vector2); pmovmsk(masm, vectorSize, tmp1, vector1); masm.xorlAndJcc(tmp1, vectorSize == XMM ? ONES_16 : ONES_32, ConditionFlag.NotZero, diffFound, true); // all elements are equal, return 0 @@ -305,35 +303,35 @@ private void emitVectorLoop(CompilationResultBuilder crb, AMD64MacroAssembler ma // different elements found in the current region, find the byte index of the first // non-equal elements bsfq(masm, tmp2, tmp1); - if (maxScale.value > 1) { + if (maxStride.value > 1) { // convert byte index to stride - masm.shrq(tmp2, maxScale.log2); + masm.shrq(tmp2, maxStride.log2); } // add to current vector loop index masm.addq(tmp2, length); // load differing elements and return difference - movSZx(masm, scaleA, extendMode, result, new AMD64Address(arrayA, tmp2, scaleA)); - movSZx(masm, scaleB, extendMode, tmp1, new AMD64Address(arrayB, tmp2, scaleB)); + movSZx(masm, strideA, extendMode, result, new AMD64Address(arrayA, tmp2, strideA)); + movSZx(masm, strideB, extendMode, tmp1, new AMD64Address(arrayB, tmp2, strideB)); masm.subq(result, tmp1); masm.jmp(returnLabel); - boolean canUseQWORD = !(maxScale == Scale.Times4 && min(scaleA, scaleB) == Scale.Times1); + boolean canUseQWORD = !(maxStride == Stride.S4 && Stride.min(strideA, strideB) == Stride.S1); masm.bind(tail); masm.movl(length, result); if (supportsAVX2AndYMM()) { // region is too small for YMM vectors, try XMM - emitVectorizedTail(masm, scaleA, scaleB, - result, arrayA, arrayB, length, tmp1, tmp2, returnLabel, maxScale, + emitVectorizedTail(masm, strideA, strideB, + result, arrayA, arrayB, length, tmp1, tmp2, returnLabel, maxStride, vector1, vector2, vector3, vector4, canUseQWORD ? qwordTail : scalarTail, XMM, YMM); } if (canUseQWORD) { masm.bind(qwordTail); // region is too small for XMM vectors, try QWORD - emitVectorizedTail(masm, scaleA, scaleB, - result, arrayA, arrayB, length, tmp1, tmp2, returnLabel, maxScale, + emitVectorizedTail(masm, strideA, strideB, + result, arrayA, arrayB, length, tmp1, tmp2, returnLabel, maxStride, vector1, vector2, vector3, vector4, scalarTail, QWORD, XMM); } @@ -341,16 +339,16 @@ private void emitVectorLoop(CompilationResultBuilder crb, AMD64MacroAssembler ma masm.bind(scalarTail); } - private void emitVectorizedTail(AMD64MacroAssembler masm, Scale scaleA, Scale scaleB, - Register result, Register arrayA, Register arrayB, Register length, Register tmp1, Register tmp2, Label returnLabel, Scale maxScale, + private void emitVectorizedTail(AMD64MacroAssembler masm, Stride strideA, Stride strideB, + Register result, Register arrayA, Register arrayB, Register length, Register tmp1, Register tmp2, Label returnLabel, Stride maxStride, Register vector1, Register vector2, Register vector3, Register vector4, Label nextTail, AVXSize loadSize, AVXSize cmpSize) { assert cmpSize.getBytes() == loadSize.getBytes() * 2; assert cmpSize == YMM || cmpSize == XMM; - masm.cmplAndJcc(length, getElementsPerVector(loadSize, maxScale), ConditionFlag.Less, nextTail, false); - pmovSZx(masm, loadSize, extendMode, vector1, maxScale, arrayA, scaleA, 0); - pmovSZx(masm, loadSize, extendMode, vector2, maxScale, arrayB, scaleB, 0); - pmovSZx(masm, loadSize, extendMode, vector3, maxScale, arrayA, scaleA, length, -loadSize.getBytes()); - pmovSZx(masm, loadSize, extendMode, vector4, maxScale, arrayB, scaleB, length, -loadSize.getBytes()); + masm.cmplAndJcc(length, getElementsPerVector(loadSize, maxStride), ConditionFlag.Less, nextTail, false); + pmovSZx(masm, loadSize, extendMode, vector1, maxStride, arrayA, strideA, 0); + pmovSZx(masm, loadSize, extendMode, vector2, maxStride, arrayB, strideB, 0); + pmovSZx(masm, loadSize, extendMode, vector3, maxStride, arrayA, strideA, length, -loadSize.getBytes()); + pmovSZx(masm, loadSize, extendMode, vector4, maxStride, arrayB, strideB, length, -loadSize.getBytes()); if (cmpSize == YMM) { AMD64Assembler.VexRVMIOp.VPERM2I128.emit(masm, cmpSize, vector1, vector3, vector1, 0x02); AMD64Assembler.VexRVMIOp.VPERM2I128.emit(masm, cmpSize, vector2, vector4, vector2, 0x02); @@ -358,42 +356,42 @@ private void emitVectorizedTail(AMD64MacroAssembler masm, Scale scaleA, Scale sc movlhps(masm, vector1, vector3); movlhps(masm, vector2, vector4); } - pcmpeq(masm, cmpSize, maxScale, vector1, vector2); + pcmpeq(masm, cmpSize, maxStride, vector1, vector2); pmovmsk(masm, cmpSize, result, vector1); masm.xorlAndJcc(result, cmpSize == XMM ? ONES_16 : ONES_32, ConditionFlag.Zero, returnLabel, false); bsfq(masm, tmp2, result); - if (maxScale.value > 1) { + if (maxStride.value > 1) { // convert byte index to stride - masm.shrq(tmp2, maxScale.log2); + masm.shrq(tmp2, maxStride.log2); } - masm.leaq(tmp1, new AMD64Address(tmp2, length, Scale.Times1, -getElementsPerVector(cmpSize, maxScale))); - masm.cmpq(tmp2, getElementsPerVector(loadSize, maxScale)); + masm.leaq(tmp1, new AMD64Address(tmp2, length, Stride.S1, -getElementsPerVector(cmpSize, maxStride))); + masm.cmpq(tmp2, getElementsPerVector(loadSize, maxStride)); // add to current vector loop index masm.cmovq(ConditionFlag.Greater, tmp2, tmp1); // load differing elements and return difference - movSZx(masm, scaleA, extendMode, result, new AMD64Address(arrayA, tmp2, scaleA)); - movSZx(masm, scaleB, extendMode, tmp1, new AMD64Address(arrayB, tmp2, scaleB)); + movSZx(masm, strideA, extendMode, result, new AMD64Address(arrayA, tmp2, strideA)); + movSZx(masm, strideB, extendMode, tmp1, new AMD64Address(arrayB, tmp2, strideB)); masm.subq(result, tmp1); masm.jmp(returnLabel); } - private static int getElementsPerVector(AVXSize vSize, Scale maxScale) { - return vSize.getBytes() >> maxScale.log2; + private static int getElementsPerVector(AVXSize vSize, Stride maxStride) { + return vSize.getBytes() >> maxStride.log2; } - private void emitScalarLoop(CompilationResultBuilder crb, AMD64MacroAssembler masm, Scale scaleA, Scale scaleB, + private void emitScalarLoop(CompilationResultBuilder crb, AMD64MacroAssembler masm, Stride strideA, Stride strideB, Register result, Register arrayA, Register arrayB, Register length, Register tmp, Label returnLabel) { Label loop = new Label(); - masm.leaq(arrayA, new AMD64Address(arrayA, length, scaleA)); - masm.leaq(arrayB, new AMD64Address(arrayB, length, scaleB)); + masm.leaq(arrayA, new AMD64Address(arrayA, length, strideA)); + masm.leaq(arrayB, new AMD64Address(arrayB, length, strideB)); masm.negq(length); masm.align(crb.target.wordSize * 2); masm.bind(loop); - movSZx(masm, scaleA, extendMode, result, new AMD64Address(arrayA, length, scaleA)); - movSZx(masm, scaleB, extendMode, tmp, new AMD64Address(arrayB, length, scaleB)); + movSZx(masm, strideA, extendMode, result, new AMD64Address(arrayA, length, strideA)); + movSZx(masm, strideB, extendMode, tmp, new AMD64Address(arrayB, length, strideB)); masm.subqAndJcc(result, tmp, ConditionFlag.NotZero, returnLabel, true); masm.incqAndJcc(length, ConditionFlag.NotZero, loop, true); } diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64CalcStringAttributesOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64CalcStringAttributesOp.java index a8346d82fb80..8afc938cf4f6 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64CalcStringAttributesOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64CalcStringAttributesOp.java @@ -68,7 +68,7 @@ import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.asm.amd64.AMD64Assembler; import org.graalvm.compiler.asm.amd64.AMD64BaseAssembler; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; @@ -182,7 +182,7 @@ public enum Op { private final Op op; - private final Scale scale; + private final Stride stride; private final int vectorLength; /** * If true, assume the string to be encoded correctly. @@ -209,7 +209,7 @@ private AMD64CalcStringAttributesOp(LIRGeneratorTool tool, Op op, EnumSet */ private void emitUTF8(CompilationResultBuilder crb, AMD64MacroAssembler asm, Register arr, Register len, Register lengthTail, Register ret, Register vecArray) { - assert scale.log2 == 0; + assert stride.log2 == 0; Register tmp = asRegister(temp[0]); Register vecMask = asRegister(vectorTemp[1]); Register vecMaskCB = asRegister(vectorTemp[2]); @@ -727,13 +727,13 @@ private void emitUTF8(CompilationResultBuilder crb, AMD64MacroAssembler asm, Reg // ascii loop: check if all bytes are |<| 0x80 with PTEST alignLoopHead(crb, asm); asm.bind(asciiLoop); - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, scale)); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, stride)); ptest(asm, vectorSize, vecArray, vecMask); asm.jccb(NotZero, labelMultiByteEntry); asm.addqAndJcc(len, vectorLength, NotZero, asciiLoop, true); // ascii tail - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, lengthTail, scale, -vectorSize.getBytes())); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, lengthTail, stride, -vectorSize.getBytes())); ptest(asm, vectorSize, vecArray, vecMask); asm.jcc(NotZero, labelMultiByteTail, assumeValid); asm.jmp(returnAscii); @@ -742,7 +742,7 @@ private void emitUTF8(CompilationResultBuilder crb, AMD64MacroAssembler asm, Reg // multibyte loop: at least one byte is not ascii, calculate the codepoint length alignLoopHead(crb, asm); asm.bind(labelMultiByteLoop); - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, scale)); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, stride)); asm.bind(labelMultiByteEntry); utf8SubtractContinuationBytes(asm, ret, vecArray, tmp, vecMask, vecMaskCB); asm.addqAndJcc(len, vectorLength, NotZero, labelMultiByteLoop, true); @@ -751,12 +751,12 @@ private void emitUTF8(CompilationResultBuilder crb, AMD64MacroAssembler asm, Reg // overlap with the last loop iteration asm.testlAndJcc(lengthTail, lengthTail, Zero, returnValid, false); // load tail vector - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, lengthTail, scale, -vectorSize.getBytes())); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, lengthTail, stride, -vectorSize.getBytes())); asm.bind(labelMultiByteTail); // load tail mask address asm.leaq(tmp, (AMD64Address) crb.recordDataSectionReference(validMaskTail)); // remove overlapping bytes - pandU(asm, vectorSize, vecArray, new AMD64Address(tmp, lengthTail, scale), vecTmp1); + pandU(asm, vectorSize, vecArray, new AMD64Address(tmp, lengthTail, stride), vecTmp1); // identify continuation bytes utf8SubtractContinuationBytes(asm, ret, vecArray, tmp, vecMask, vecMaskCB); asm.jmp(returnValid); @@ -806,7 +806,7 @@ private void emitUTF8(CompilationResultBuilder crb, AMD64MacroAssembler asm, Reg // multibyte loop: at least one byte is not ascii, calculate the codepoint length alignLoopHead(crb, asm); asm.bind(labelMultiByteLoop); - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, scale)); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, stride)); // fast path: check if current vector is ascii asm.bind(labelMultiByteTailLoopEntry); ptest(asm, vectorSize, vecArray, vecMask); @@ -924,14 +924,14 @@ private void emitUTF8(CompilationResultBuilder crb, AMD64MacroAssembler asm, Reg } asm.bind(labelScalarTail); - asm.leaq(arr, new AMD64Address(arr, lengthTail, scale)); + asm.leaq(arr, new AMD64Address(arr, lengthTail, stride)); asm.testqAndJcc(lengthTail, lengthTail, Zero, returnAscii, false); asm.negq(lengthTail); // scalar ascii loop alignLoopHead(crb, asm); asm.bind(labelScalarAsciiLoop); - asm.movzbl(tmp, new AMD64Address(arr, lengthTail, scale)); + asm.movzbl(tmp, new AMD64Address(arr, lengthTail, stride)); asm.testlAndJcc(tmp, 0x80, NotZero, labelScalarMultiByteLoopEntry, true); asm.incqAndJcc(lengthTail, NotZero, labelScalarAsciiLoop, true); asm.jmpb(returnAscii); @@ -941,7 +941,7 @@ private void emitUTF8(CompilationResultBuilder crb, AMD64MacroAssembler asm, Reg if (assumeValid) { alignLoopHead(crb, asm); asm.bind(labelScalarMultiByteLoop); - asm.movzbq(tmp, new AMD64Address(arr, lengthTail, scale)); + asm.movzbq(tmp, new AMD64Address(arr, lengthTail, stride)); asm.andl(tmp, 0xc0); asm.cmplAndJcc(tmp, 0x80, NotEqual, labelScalarMultiByteLoopSkipDec, true); asm.decl(ret); @@ -957,11 +957,11 @@ private void emitUTF8(CompilationResultBuilder crb, AMD64MacroAssembler asm, Reg asm.xorq(state, state); alignLoopHead(crb, asm); asm.bind(labelScalarMultiByteLoop); - asm.movzbq(tmp, new AMD64Address(arr, lengthTail, scale)); - asm.movzbq(type, new AMD64Address(len, tmp, scale)); + asm.movzbq(tmp, new AMD64Address(arr, lengthTail, stride)); + asm.movzbq(type, new AMD64Address(len, tmp, stride)); asm.andl(tmp, 0xc0); asm.addq(type, state); - asm.movzbq(state, new AMD64Address(len, type, scale, 256)); + asm.movzbq(state, new AMD64Address(len, type, stride, 256)); asm.cmplAndJcc(tmp, 0x80, NotEqual, labelScalarMultiByteLoopSkipDec, true); asm.decl(ret); asm.bind(labelScalarMultiByteLoopSkipDec); @@ -1012,7 +1012,7 @@ private void loadTailIntoYMMOrdered(CompilationResultBuilder crb, AMD64MacroAsse Label done = new Label(); asm.leaq(tmp, (AMD64Address) crb.recordDataSectionReference(xmmTailShuffleMask)); // load last 16 bytes - movdqu(asm, XMM, vecArray, new AMD64Address(arr, lengthTail, scale, -XMM.getBytes())); + movdqu(asm, XMM, vecArray, new AMD64Address(arr, lengthTail, stride, -XMM.getBytes())); // if we're using YMM vectors and the tail is greater than 16 bytes, load the tail's // first 16 bytes and prepend them to the shifted vector asm.cmpqAndJcc(lengthTail, elementsPerVector(XMM), Less, lessThan16, true); @@ -1020,7 +1020,7 @@ private void loadTailIntoYMMOrdered(CompilationResultBuilder crb, AMD64MacroAsse asm.negq(lengthTail); // load shuffle mask into a tmp vector, because pshufb doesn't support misaligned // memory parameters - movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, scale, XMM.getBytes() * 2)); + movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, stride, XMM.getBytes() * 2)); // shuffle the tail vector such that its content effectively gets left-shifted by // 16 - lengthTail bytes pshufb(asm, XMM, vecArray, vecTmp2); @@ -1030,7 +1030,7 @@ private void loadTailIntoYMMOrdered(CompilationResultBuilder crb, AMD64MacroAsse asm.negq(lengthTail); // load shuffle mask into a tmp vector, because pshufb doesn't support misaligned // memory parameters - movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, scale, XMM.getBytes())); + movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, stride, XMM.getBytes())); // shuffle the tail vector such that its content effectively gets right-shifted by // 16 - lengthTail bytes pshufb(asm, XMM, vecArray, vecTmp2); @@ -1038,11 +1038,11 @@ private void loadTailIntoYMMOrdered(CompilationResultBuilder crb, AMD64MacroAsse } else { asm.leaq(tmp, (AMD64Address) crb.recordDataSectionReference(xmmTailShuffleMask)); // load last 16 bytes - movdqu(asm, XMM, vecArray, new AMD64Address(arr, lengthTail, scale, -XMM.getBytes())); + movdqu(asm, XMM, vecArray, new AMD64Address(arr, lengthTail, stride, -XMM.getBytes())); asm.negq(lengthTail); // load shuffle mask into a tmp vector, because pshufb doesn't support misaligned // memory parameters - movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, scale, XMM.getBytes())); + movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, stride, XMM.getBytes())); // shuffle the tail vector such that its content effectively gets right-shifted by // 16 - lengthTail bytes pshufb(asm, XMM, vecArray, vecTmp2); @@ -1056,12 +1056,12 @@ private void loadLessThan32IntoYMMOrdered(CompilationResultBuilder crb, AMD64Mac Register arr, Register lengthTail, Register tmp, Register vecArray, Register vecTmp1, Register vecTmp2) { // array is between 16 and 31 bytes long, load it into a YMM register via two XMM loads movdqu(asm, XMM, vecTmp1, new AMD64Address(arr)); - movdqu(asm, XMM, vecArray, new AMD64Address(arr, lengthTail, scale, -XMM.getBytes())); + movdqu(asm, XMM, vecArray, new AMD64Address(arr, lengthTail, stride, -XMM.getBytes())); asm.leaq(tmp, (AMD64Address) crb.recordDataSectionReference(xmmTailShuffleMask)); asm.negq(lengthTail); // load shuffle mask into a tmp vector, because pshufb doesn't support misaligned // memory parameters - movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, scale, XMM.getBytes() * 2)); + movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, stride, XMM.getBytes() * 2)); // shuffle the tail vector such that its content effectively gets right-shifted by // 16 - lengthTail bytes pshufb(asm, XMM, vecArray, vecTmp2); @@ -1081,9 +1081,9 @@ private void loadLessThan32IntoYMMOrdered(CompilationResultBuilder crb, AMD64Mac private void loadLessThan32IntoYMMUnordered(CompilationResultBuilder crb, AMD64MacroAssembler asm, DataSection.Data maskTail, Register arr, Register lengthTail, Register tmp, Register vecArray, Register vecTmp1, Register vecTmp2) { movdqu(asm, XMM, vecArray, new AMD64Address(arr)); - movdqu(asm, XMM, vecTmp1, new AMD64Address(arr, lengthTail, scale, -XMM.getBytes())); + movdqu(asm, XMM, vecTmp1, new AMD64Address(arr, lengthTail, stride, -XMM.getBytes())); asm.leaq(tmp, (AMD64Address) crb.recordDataSectionReference(maskTail)); - pandU(asm, vectorSize, vecTmp1, new AMD64Address(tmp, lengthTail, scale), vecTmp2); + pandU(asm, vectorSize, vecTmp1, new AMD64Address(tmp, lengthTail, stride), vecTmp2); AMD64Assembler.VexRVMIOp.VPERM2I128.emit(asm, vectorSize, vecArray, vecArray, vecTmp1, 0x02); } @@ -1094,10 +1094,10 @@ private void loadLessThan16IntoXMMOrdered(CompilationResultBuilder crb, AMD64Mac Register arr, Register lengthTail, Register tmp, Register vecArray, Register vecTmp1, Register vecTmp2) { // array is between 8 and 15 bytes long, load it into a YMM register via two QWORD loads asm.movdq(vecArray, new AMD64Address(arr)); - asm.movdq(vecTmp1, new AMD64Address(arr, lengthTail, scale, -8)); + asm.movdq(vecTmp1, new AMD64Address(arr, lengthTail, stride, -8)); asm.leaq(tmp, getMaskOnce(crb, createXMMTailShuffleMask(8), XMM.getBytes() * 2)); asm.negq(lengthTail); - movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, scale, XMM.getBytes())); + movdqu(asm, XMM, vecTmp2, new AMD64Address(tmp, lengthTail, stride, XMM.getBytes())); pshufb(asm, XMM, vecTmp1, vecTmp2); movlhps(asm, vecArray, vecTmp1); } @@ -1109,9 +1109,9 @@ private void loadLessThan16IntoXMMUnordered(CompilationResultBuilder crb, AMD64M Register arr, Register lengthTail, Register tmp, Register vecArray, Register vecTmp1, Register vecTmp2) { // array is between 8 and 15 bytes long, load it into a YMM register via two QWORD loads asm.movdq(vecArray, new AMD64Address(arr)); - asm.movdq(vecTmp1, new AMD64Address(arr, lengthTail, scale, -8)); + asm.movdq(vecTmp1, new AMD64Address(arr, lengthTail, stride, -8)); asm.leaq(tmp, (AMD64Address) crb.recordDataSectionReference(maskTail)); - pandU(asm, vectorSize, vecTmp1, new AMD64Address(tmp, lengthTail, scale, supportsAVX2AndYMM() ? XMM.getBytes() : 0), vecTmp2); + pandU(asm, vectorSize, vecTmp1, new AMD64Address(tmp, lengthTail, stride, supportsAVX2AndYMM() ? XMM.getBytes() : 0), vecTmp2); movlhps(asm, vecArray, vecTmp1); } @@ -1119,14 +1119,14 @@ private void loadLessThan16IntoXMMUnordered(CompilationResultBuilder crb, AMD64M * Load an array of 4-7 bytes into a XMM register. */ private void loadLessThan8IntoXMMOrdered(AMD64MacroAssembler asm, Register arr, Register lengthTail, Register vecArray, Register tmp, Register tmp2) { - assert scale.log2 < 2; + assert stride.log2 < 2; // array is between 4 and 7 bytes long, load it into a YMM register via two DWORD loads asm.movl(tmp, new AMD64Address(arr)); - asm.movl(tmp2, new AMD64Address(arr, lengthTail, scale, -4)); + asm.movl(tmp2, new AMD64Address(arr, lengthTail, stride, -4)); // compute tail length - asm.andq(lengthTail, 3 >> scale.log2); + asm.andq(lengthTail, 3 >> stride.log2); // convert byte count to bit count - asm.shlq(lengthTail, 3 + scale.log2); + asm.shlq(lengthTail, 3 + stride.log2); assert lengthTail.equals(rcx); // shift second vector to the left by tailCount bits asm.shlq(tmp2); @@ -1144,8 +1144,8 @@ private void loadLessThan8IntoXMMUnordered(CompilationResultBuilder crb, AMD64Ma Register arr, Register lengthTail, Register vecArray, Register tmp, Register tmp2) { // array is between 4 and 7 bytes long, load it into a YMM register via two DWORD loads asm.leaq(tmp, (AMD64Address) crb.recordDataSectionReference(maskTail)); - asm.movl(tmp2, new AMD64Address(arr, lengthTail, scale, -4)); - asm.andq(tmp2, new AMD64Address(tmp, lengthTail, scale, (supportsAVX2AndYMM() ? XMM.getBytes() : 0) + 8)); + asm.movl(tmp2, new AMD64Address(arr, lengthTail, stride, -4)); + asm.andq(tmp2, new AMD64Address(tmp, lengthTail, stride, (supportsAVX2AndYMM() ? XMM.getBytes() : 0) + 8)); asm.movl(tmp, new AMD64Address(arr)); asm.shlq(tmp2, 32); asm.orq(tmp, tmp2); @@ -1205,7 +1205,7 @@ private static void prev(AMD64MacroAssembler asm, AVXKind.AVXSize size, Register } private void emitUTF16(CompilationResultBuilder crb, AMD64MacroAssembler asm, Register arr, Register len, Register lengthTail, Register ret, Register vecArray) { - assert scale.log2 == 1; + assert stride.log2 == 1; Register vecArrayTail = asRegister(vectorTemp[1]); Register vecMaskAscii = asRegister(vectorTemp[2]); Register vecMaskLatin = asRegister(vectorTemp[3]); @@ -1253,7 +1253,7 @@ private void emitUTF16(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re vectorLoopPrologue(asm, arr, len, lengthTail, tailLessThan32, tailLessThan16, false); - movdqu(asm, vectorSize, vecArrayTail, new AMD64Address(arr, lengthTail, scale, -vectorSize.getBytes())); + movdqu(asm, vectorSize, vecArrayTail, new AMD64Address(arr, lengthTail, stride, -vectorSize.getBytes())); // ascii loop: check if all chars are |<| 0x80 with VPTEST mask 0xff80 emitPTestLoop(crb, asm, arr, len, vecArray, vecMaskAscii, latin1Entry); emitPTestTail(asm, vectorSize, arr, lengthTail, vecArray, vecMaskAscii, null, latin1Tail, returnAscii, false); @@ -1266,7 +1266,7 @@ private void emitUTF16(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re // bmp loop: search for UTF-16 surrogate characters alignLoopHead(crb, asm); asm.bind(bmpLoop); - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, scale)); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, stride)); utf16FindSurrogatesAndTest(asm, vecArray, vecArray, vecMaskSurrogate); asm.jccb(NotZero, assumeValid ? labelSurrogateLoop : labelSurrogateEntry); asm.addqAndJcc(len, vectorLength, NotZero, bmpLoop, true); @@ -1284,7 +1284,7 @@ private void emitUTF16(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re // (assuming the string is encoded correctly) alignLoopHead(crb, asm); asm.bind(labelSurrogateLoop); - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, scale)); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, stride)); utf16MatchSurrogates(asm, vecArray, vecMaskSurrogate); utf16SubtractMatchedChars(asm, ret, vecArray, tmp); asm.addqAndJcc(len, vectorLength, NotZero, labelSurrogateLoop, true); @@ -1292,7 +1292,7 @@ private void emitUTF16(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re asm.testlAndJcc(lengthTail, lengthTail, Zero, returnValidOrBroken, false); // surrogate tail asm.leaq(tmp, (AMD64Address) crb.recordDataSectionReference(maskTail)); - pandU(asm, vectorSize, vecArrayTail, new AMD64Address(tmp, lengthTail, scale), vecTmp); + pandU(asm, vectorSize, vecArrayTail, new AMD64Address(tmp, lengthTail, stride), vecTmp); utf16MatchSurrogates(asm, vecArrayTail, vecMaskSurrogate); utf16SubtractMatchedChars(asm, ret, vecArrayTail, tmp); asm.jmp(returnValidOrBroken); @@ -1306,7 +1306,7 @@ private void emitUTF16(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re asm.cmplAndJcc(ret, vectorLength, Equal, surrogateExactlyVectorSize, false); // corner case: check if the first char is a low surrogate, and if so, set code // range to BROKEN - asm.movzwl(tmp, new AMD64Address(arr, len, scale)); + asm.movzwl(tmp, new AMD64Address(arr, len, stride)); asm.shrl(tmp, 10); asm.cmpl(tmp, 0x37); asm.movl(tmp, CR_BROKEN_MULTIBYTE); @@ -1331,9 +1331,9 @@ private void emitUTF16(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re alignLoopHead(crb, asm); asm.bind(labelSurrogateLoop); // load at current index - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, scale)); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, stride)); // load at current index + 1 - movdqu(asm, vectorSize, vecArrayTail, new AMD64Address(arr, len, scale, 2)); + movdqu(asm, vectorSize, vecArrayTail, new AMD64Address(arr, len, stride, 2)); utf16ValidateSurrogates(asm, ret, vecArray, vecArrayTail, vecMaskSurrogate, vecMaskAscii, vecTmp, vecResult, tmp); asm.addqAndJcc(len, vectorLength, NotZero, labelSurrogateLoop, true); @@ -1341,16 +1341,16 @@ private void emitUTF16(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re asm.bind(labelSurrogateTail); asm.leaq(tmp, (AMD64Address) crb.recordDataSectionReference(maskTail)); // load at up to array end - 1 - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, lengthTail, scale, -(vectorSize.getBytes() + 2))); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, lengthTail, stride, -(vectorSize.getBytes() + 2))); // load at up to array end - movdqu(asm, vectorSize, vecArrayTail, new AMD64Address(arr, lengthTail, scale, -vectorSize.getBytes())); + movdqu(asm, vectorSize, vecArrayTail, new AMD64Address(arr, lengthTail, stride, -vectorSize.getBytes())); // remove elements overlapping with the last loop vector - pandU(asm, vectorSize, vecArray, new AMD64Address(tmp, lengthTail, scale, -2), vecTmp); - pandU(asm, vectorSize, vecArrayTail, new AMD64Address(tmp, lengthTail, scale, -2), vecTmp); + pandU(asm, vectorSize, vecArray, new AMD64Address(tmp, lengthTail, stride, -2), vecTmp); + pandU(asm, vectorSize, vecArrayTail, new AMD64Address(tmp, lengthTail, stride, -2), vecTmp); utf16ValidateSurrogates(asm, ret, vecArray, vecArrayTail, vecMaskSurrogate, vecMaskAscii, vecTmp, vecResult, tmp); // corner case: check if last char is a high surrogate - asm.movzwl(tmp, new AMD64Address(arr, lengthTail, scale, -2)); + asm.movzwl(tmp, new AMD64Address(arr, lengthTail, stride, -2)); asm.shrl(tmp, 10); // if last char is a high surrogate, return BROKEN asm.cmpl(tmp, 0x36); @@ -1497,7 +1497,7 @@ private void utf16SubtractMatchedChars(AMD64MacroAssembler asm, Register ret, Re } private void emitUTF32(CompilationResultBuilder crb, AMD64MacroAssembler asm, Register arr, Register len, Register lengthTail, Register ret, Register vecArray) { - assert scale.log2 == 2; + assert stride.log2 == 2; Register vecMaskAscii = asRegister(vectorTemp[1]); Register vecMaskLatin1 = asRegister(vectorTemp[2]); Register vecMaskBMP = asRegister(vectorTemp[3]); @@ -1536,7 +1536,7 @@ private void emitUTF32(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re vectorLoopPrologue(asm, arr, len, lengthTail, tailLessThan32, tailLessThan16, false); - movdqu(asm, vectorSize, vecArrayTail, new AMD64Address(arr, lengthTail, scale, -vectorSize.getBytes())); + movdqu(asm, vectorSize, vecArrayTail, new AMD64Address(arr, lengthTail, stride, -vectorSize.getBytes())); // ascii loop: check if all codepoints are |<| 0x80 with VPTEST mask 0xffffff80 emitPTestLoop(crb, asm, arr, len, vecArray, vecMaskAscii, labelLatin1Entry); @@ -1551,7 +1551,7 @@ private void emitUTF32(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re // bmp loop: check if all codepoints are |<| 0x10000 with VPTEST mask 0xffff0000 alignLoopHead(crb, asm); asm.bind(labelBMPLoop); - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, scale)); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, stride)); ptest(asm, vectorSize, vecArray, vecMaskBMP); asm.jccb(NotZero, labelAstralLoop); // check if any codepoints are in the forbidden UTF-16 surrogate range; if so, break @@ -1573,7 +1573,7 @@ private void emitUTF32(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re // if so, break immediately and return BROKEN alignLoopHead(crb, asm); asm.bind(labelAstralLoop); - movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, scale)); + movdqu(asm, vectorSize, vecArray, new AMD64Address(arr, len, stride)); utf32CheckInvalid(asm, vecArray, vecArray, vecArrayTmp, vecMaskSurrogate, vecMaskOutOfRange, returnBroken, true); asm.addqAndJcc(len, vectorLength, NotZero, labelAstralLoop, true); @@ -1589,7 +1589,7 @@ private void emitUTF32(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re asm.cmplAndJcc(lengthTail, elementsPerVector(XMM), Less, tailLessThan16, true); movdqu(asm, XMM, vecArray, new AMD64Address(arr)); - movdqu(asm, XMM, vecArrayTail, new AMD64Address(arr, lengthTail, scale, -XMM.getBytes())); + movdqu(asm, XMM, vecArrayTail, new AMD64Address(arr, lengthTail, stride, -XMM.getBytes())); AMD64Assembler.VexRVMIOp.VPERM2I128.emit(asm, vectorSize, vecArray, vecArray, vecArrayTail, 0x02); asm.jmpb(tailSingleVector); } @@ -1598,7 +1598,7 @@ private void emitUTF32(CompilationResultBuilder crb, AMD64MacroAssembler asm, Re asm.cmplAndJcc(lengthTail, elementsPerVector(QWORD), Less, tailLessThan8, true); // array is between 8 and 12 bytes long, load it into a vector register via two QWORD loads asm.movdq(vecArray, new AMD64Address(arr)); - asm.movdq(vecArrayTail, new AMD64Address(arr, lengthTail, scale, -QWORD.getBytes())); + asm.movdq(vecArrayTail, new AMD64Address(arr, lengthTail, stride, -QWORD.getBytes())); movlhps(asm, vecArray, vecArrayTail); asm.jmpb(tailSingleVector); @@ -1685,7 +1685,7 @@ private DataSection.Data createMask(CompilationResultBuilder crb, int value) { private DataSection.Data createTailMask(CompilationResultBuilder crb) { byte[] mask = new byte[vectorSize.getBytes() * 2]; for (int i = vectorLength; i < vectorLength * 2; i++) { - writeValue(mask, scale, i, ~0); + writeValue(mask, stride, i, ~0); } return writeToDataSection(crb, mask); } @@ -1714,7 +1714,7 @@ private static byte[] createXMMTailShuffleMask(int length) { private byte[] createMaskBytes(int value) { byte[] mask = new byte[vectorSize.getBytes()]; for (int i = 0; i < vectorLength; i++) { - writeValue(mask, scale, i, value); + writeValue(mask, stride, i, value); } return mask; } @@ -1763,14 +1763,14 @@ private static void emitExitMultiByteAtEnd(AMD64MacroAssembler asm, Register ret asm.bind(end); } - private static void writeValue(byte[] array, Scale stride, int index, int value) { + private static void writeValue(byte[] array, Stride stride, int index, int value) { int i = index << stride.log2; - if (stride == Scale.Times1) { + if (stride == Stride.S1) { array[i] = (byte) value; return; } if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) { - if (stride == Scale.Times2) { + if (stride == Stride.S2) { array[i] = (byte) value; array[i + 1] = (byte) (value >> 8); } else { @@ -1780,7 +1780,7 @@ private static void writeValue(byte[] array, Scale stride, int index, int value) array[i + 3] = (byte) (value >> 24); } } else { - if (stride == Scale.Times2) { + if (stride == Stride.S2) { array[i] = (byte) (value >> 8); array[i + 1] = (byte) value; } else { diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ComplexVectorOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ComplexVectorOp.java index 9db32c6ff8d8..1ff7bca427f5 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ComplexVectorOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ComplexVectorOp.java @@ -24,14 +24,15 @@ */ package org.graalvm.compiler.lir.amd64; +import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp.TZCNT; +import static org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize.QWORD; + import java.util.EnumSet; -import jdk.vm.ci.code.Register; -import jdk.vm.ci.meta.Value; -import org.graalvm.compiler.asm.amd64.AMD64Address; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; @@ -39,11 +40,10 @@ import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.amd64.AMD64.CPUFeature; import jdk.vm.ci.amd64.AMD64Kind; +import jdk.vm.ci.code.Register; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.meta.JavaKind; - -import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp.TZCNT; -import static org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize.QWORD; +import jdk.vm.ci.meta.Value; /** * Base class for AMD64 LIR instruction using AVX CPU features. @@ -74,14 +74,6 @@ private static boolean isXMMOrGreater(AVXSize size) { return size == AVXSize.XMM || size == AVXSize.YMM || size == AVXSize.ZMM; } - public static AMD64Address.Scale min(AMD64Address.Scale a, AMD64Address.Scale b) { - return a.value < b.value ? a : b; - } - - public static AMD64Address.Scale max(AMD64Address.Scale a, AMD64Address.Scale b) { - return a.value > b.value ? a : b; - } - protected AMD64Kind getVectorKind(JavaKind valueKind) { switch (vectorSize) { case XMM: @@ -140,8 +132,61 @@ protected AMD64Kind getVectorKind(JavaKind valueKind) { } } + protected AMD64Kind getVectorKind(Stride stride) { + switch (vectorSize) { + case XMM: + switch (stride) { + case S1: + return AMD64Kind.V128_BYTE; + case S2: + return AMD64Kind.V128_WORD; + case S4: + return AMD64Kind.V128_DWORD; + case S8: + return AMD64Kind.V128_QWORD; + default: + throw GraalError.shouldNotReachHere("Unsupported base value kind."); + } + case YMM: + switch (stride) { + case S1: + return AMD64Kind.V256_BYTE; + case S2: + return AMD64Kind.V256_WORD; + case S4: + return AMD64Kind.V256_DWORD; + case S8: + return AMD64Kind.V256_QWORD; + default: + throw GraalError.shouldNotReachHere("Unsupported base value kind."); + } + case ZMM: + switch (stride) { + case S1: + return AMD64Kind.V512_BYTE; + case S2: + return AMD64Kind.V512_WORD; + case S4: + return AMD64Kind.V512_DWORD; + case S8: + return AMD64Kind.V512_QWORD; + default: + throw GraalError.shouldNotReachHere("Unsupported base value kind."); + } + default: + throw GraalError.shouldNotReachHere("Unsupported vector size."); + } + } + protected Value[] allocateVectorRegisters(LIRGeneratorTool tool, JavaKind valueKind, int n) { - LIRKind kind = LIRKind.value(getVectorKind(valueKind)); + return allocateVectorRegisters(tool, LIRKind.value(getVectorKind(valueKind)), n); + } + + protected Value[] allocateVectorRegisters(LIRGeneratorTool tool, Stride stride, int n) { + return allocateVectorRegisters(tool, LIRKind.value(getVectorKind(stride)), n); + } + + protected Value[] allocateVectorRegisters(LIRGeneratorTool tool, LIRKind kind, int n) { Value[] vectors = new Value[n]; for (int i = 0; i < vectors.length; i++) { vectors[i] = tool.newVariable(kind); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java index 072e2ee80f2d..55823dc729d0 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java @@ -44,7 +44,7 @@ import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; @@ -694,7 +694,7 @@ public static void emitJumpTable(CompilationResultBuilder crb, AMD64MacroAssembl final int afterLea = masm.position(); // Load jump table entry into scratch and jump to it - masm.movslq(idxScratchReg, new AMD64Address(scratchReg, idxScratchReg, Scale.Times4, 0)); + masm.movslq(idxScratchReg, new AMD64Address(scratchReg, idxScratchReg, Stride.S4, 0)); masm.addq(scratchReg, idxScratchReg); masm.jmp(scratchReg); @@ -764,7 +764,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { if (defaultTarget != null) { // Move the table entry (two DWORDs) into a QWORD - masm.movq(entryScratchReg, new AMD64Address(scratchReg, indexReg, Scale.Times8, 0)); + masm.movq(entryScratchReg, new AMD64Address(scratchReg, indexReg, Stride.S8, 0)); // Jump to the default target if the first DWORD (original key) doesn't match the // current key. Accounts for hash collisions with unknown keys @@ -776,7 +776,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // The jump table has a single DWORD with the label address if there's no // default target - masm.movslq(entryScratchReg, new AMD64Address(scratchReg, indexReg, Scale.Times4, 0)); + masm.movslq(entryScratchReg, new AMD64Address(scratchReg, indexReg, Stride.S4, 0)); } masm.addq(scratchReg, entryScratchReg); masm.jmp(scratchReg); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64EncodeArrayOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64EncodeArrayOp.java index f2f32093c400..7a38727b3c8a 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64EncodeArrayOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64EncodeArrayOp.java @@ -36,7 +36,7 @@ import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.core.common.LIRKind; @@ -136,8 +136,8 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.movl(result, len); - masm.leaq(src, new AMD64Address(src, len, Scale.Times2)); - masm.leaq(dst, new AMD64Address(dst, len, Scale.Times1)); + masm.leaq(src, new AMD64Address(src, len, Stride.S2)); + masm.leaq(dst, new AMD64Address(dst, len, Stride.S1)); masm.negq(len); if (supportsAVX2AndYMM() || masm.supports(CPUFeature.SSE4_2)) { @@ -158,14 +158,14 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.jmp(labelChars32Check); masm.bind(labelCopy32Chars); - masm.vmovdqu(vectorTemp3, new AMD64Address(src, len, Scale.Times2, -64)); - masm.vmovdqu(vectorTemp4, new AMD64Address(src, len, Scale.Times2, -32)); + masm.vmovdqu(vectorTemp3, new AMD64Address(src, len, Stride.S2, -64)); + masm.vmovdqu(vectorTemp4, new AMD64Address(src, len, Stride.S2, -32)); masm.emit(VPOR, vectorTemp2, vectorTemp3, vectorTemp4, YMM); masm.vptest(vectorTemp2, vectorTemp1); masm.jcc(ConditionFlag.NotZero, labelCopy32CharsExit, true); masm.emit(VPACKUSWB, vectorTemp3, vectorTemp3, vectorTemp4, YMM); masm.emit(VPERMQ, vectorTemp4, vectorTemp3, 0xD8, YMM); - masm.vmovdqu(new AMD64Address(dst, len, Scale.Times1, -32), vectorTemp4); + masm.vmovdqu(new AMD64Address(dst, len, Stride.S1, -32), vectorTemp4); masm.bind(labelChars32Check); masm.addqAndJcc(len, 32, ConditionFlag.LessEqual, labelCopy32Chars, false); @@ -183,20 +183,20 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.bind(labelCopy16Chars); if (supportsAVX2AndYMM()) { - masm.vmovdqu(vectorTemp2, new AMD64Address(src, len, Scale.Times2, -32)); + masm.vmovdqu(vectorTemp2, new AMD64Address(src, len, Stride.S2, -32)); masm.vptest(vectorTemp2, vectorTemp1); masm.jcc(ConditionFlag.NotZero, labelCopy16CharsExit); masm.emit(VPACKUSWB, vectorTemp2, vectorTemp2, vectorTemp1, YMM); masm.emit(VPERMQ, vectorTemp3, vectorTemp2, 0xD8, YMM); } else { if (masm.supports(CPUFeature.AVX)) { - masm.movdqu(vectorTemp3, new AMD64Address(src, len, Scale.Times2, -32)); - masm.movdqu(vectorTemp4, new AMD64Address(src, len, Scale.Times2, -16)); + masm.movdqu(vectorTemp3, new AMD64Address(src, len, Stride.S2, -32)); + masm.movdqu(vectorTemp4, new AMD64Address(src, len, Stride.S2, -16)); masm.emit(VPOR, vectorTemp2, vectorTemp3, vectorTemp4, XMM); } else { - masm.movdqu(vectorTemp3, new AMD64Address(src, len, Scale.Times2, -32)); + masm.movdqu(vectorTemp3, new AMD64Address(src, len, Stride.S2, -32)); masm.por(vectorTemp2, vectorTemp3); - masm.movdqu(vectorTemp4, new AMD64Address(src, len, Scale.Times2, -16)); + masm.movdqu(vectorTemp4, new AMD64Address(src, len, Stride.S2, -16)); masm.por(vectorTemp2, vectorTemp4); } @@ -205,7 +205,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.packuswb(vectorTemp3, vectorTemp4); } - masm.movdqu(new AMD64Address(dst, len, Scale.Times1, -16), vectorTemp3); + masm.movdqu(new AMD64Address(dst, len, Stride.S1, -16), vectorTemp3); masm.bind(labelChars16Check); masm.addqAndJcc(len, 16, ConditionFlag.LessEqual, labelCopy16Chars, false); @@ -214,12 +214,12 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.subqAndJcc(len, 8, ConditionFlag.Greater, labelCopy8CharsExit, true); masm.bind(labelCopy8Chars); - masm.movdqu(vectorTemp3, new AMD64Address(src, len, Scale.Times2, -16)); + masm.movdqu(vectorTemp3, new AMD64Address(src, len, Stride.S2, -16)); masm.ptest(vectorTemp3, vectorTemp1); masm.jccb(ConditionFlag.NotZero, labelCopy8CharsExit); masm.packuswb(vectorTemp3, vectorTemp1); - masm.movq(new AMD64Address(dst, len, Scale.Times1, -8), vectorTemp3); + masm.movq(new AMD64Address(dst, len, Stride.S1, -8), vectorTemp3); masm.addqAndJcc(len, 8, ConditionFlag.LessEqual, labelCopy8Chars, true); @@ -228,10 +228,10 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { } masm.bind(labelCopy1Char); - masm.movzwl(temp5, new AMD64Address(src, len, Scale.Times2, 0)); + masm.movzwl(temp5, new AMD64Address(src, len, Stride.S2, 0)); masm.testlAndJcc(temp5, shortMask, ConditionFlag.NotZero, labelCopy1CharExit, true); - masm.movb(new AMD64Address(dst, len, AMD64Address.Scale.Times1, 0), temp5); + masm.movb(new AMD64Address(dst, len, Stride.S1, 0), temp5); masm.addqAndJcc(len, 1, ConditionFlag.Less, labelCopy1Char, true); masm.bind(labelCopy1CharExit); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64HasNegativesOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64HasNegativesOp.java index de0ec2a44bd4..1c15e233c0bb 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64HasNegativesOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64HasNegativesOp.java @@ -37,7 +37,7 @@ import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.core.common.LIRKind; @@ -134,12 +134,12 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // vector count (in chars) masm.andlAndJcc(len, ~(64 - 1), ConditionFlag.Zero, labelTestTail, true); - masm.leaq(ary1, new AMD64Address(ary1, len, Scale.Times1)); + masm.leaq(ary1, new AMD64Address(ary1, len, Stride.S1)); masm.negq(len); masm.bind(labelTest64Loop); // Check whether our 64 elements of size byte contain negatives - masm.evpcmpgtb(mask1, vec2, new AMD64Address(ary1, len, Scale.Times1)); + masm.evpcmpgtb(mask1, vec2, new AMD64Address(ary1, len, Stride.S1)); masm.kortestq(mask1, mask1); masm.jcc(ConditionFlag.NotZero, labelTrue); @@ -174,7 +174,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // vector count (in bytes) masm.andlAndJcc(len, 0xffffffe0, ConditionFlag.Zero, labelCompareTail, true); - masm.leaq(ary1, new AMD64Address(ary1, len, Scale.Times1)); + masm.leaq(ary1, new AMD64Address(ary1, len, Stride.S1)); masm.negq(len); masm.movl(tmp1, 0x80808080); // create mask to test for Unicode chars in vector @@ -182,14 +182,14 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.emit(VPBROADCASTD, vec2, vec2, YMM); masm.bind(labelCompareWideVectors); - masm.vmovdqu(vec1, new AMD64Address(ary1, len, Scale.Times1)); + masm.vmovdqu(vec1, new AMD64Address(ary1, len, Stride.S1)); masm.vptest(vec1, vec2); masm.jcc(ConditionFlag.NotZero, labelTrue); masm.addqAndJcc(len, 32, ConditionFlag.NotZero, labelCompareWideVectors, false); masm.testlAndJcc(result, result, ConditionFlag.Zero, labelFalse, false); - masm.vmovdqu(vec1, new AMD64Address(ary1, result, Scale.Times1, -32)); + masm.vmovdqu(vec1, new AMD64Address(ary1, result, Stride.S1, -32)); masm.vptest(vec1, vec2); masm.jccb(ConditionFlag.NotZero, labelTrue); masm.jmp(labelFalse); @@ -208,7 +208,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // vector count (in bytes) masm.andlAndJcc(len, 0xfffffff0, ConditionFlag.Zero, labelCompareTail, false); - masm.leaq(ary1, new AMD64Address(ary1, len, Scale.Times1)); + masm.leaq(ary1, new AMD64Address(ary1, len, Stride.S1)); masm.negq(len); masm.movl(tmp1, 0x80808080); @@ -216,14 +216,14 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.pshufd(vec2, vec2, 0); masm.bind(labelCompareWideVectors); - masm.movdqu(vec1, new AMD64Address(ary1, len, Scale.Times1)); + masm.movdqu(vec1, new AMD64Address(ary1, len, Stride.S1)); masm.ptest(vec1, vec2); masm.jcc(ConditionFlag.NotZero, labelTrue); masm.addqAndJcc(len, 16, ConditionFlag.NotZero, labelCompareWideVectors, false); masm.testlAndJcc(result, result, ConditionFlag.Zero, labelFalse, false); - masm.movdqu(vec1, new AMD64Address(ary1, result, Scale.Times1, -16)); + masm.movdqu(vec1, new AMD64Address(ary1, result, Stride.S1, -16)); masm.ptest(vec1, vec2); masm.jccb(ConditionFlag.NotZero, labelTrue); masm.jmp(labelFalse); @@ -237,11 +237,11 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { // vector count (in bytes) masm.andlAndJcc(len, 0xfffffffc, ConditionFlag.Zero, labelCompareChar, true); - masm.leaq(ary1, new AMD64Address(ary1, len, Scale.Times1)); + masm.leaq(ary1, new AMD64Address(ary1, len, Stride.S1)); masm.negq(len); masm.bind(labelCompareVectors); - masm.movl(tmp1, new AMD64Address(ary1, len, Scale.Times1)); + masm.movl(tmp1, new AMD64Address(ary1, len, Stride.S1)); masm.andlAndJcc(tmp1, 0x80808080, ConditionFlag.NotZero, labelTrue, true); masm.addqAndJcc(len, 4, ConditionFlag.NotZero, labelCompareVectors, false); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathExpOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathExpOp.java index 151a00f889f5..6e234aa2b631 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathExpOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathExpOp.java @@ -46,6 +46,7 @@ import org.graalvm.compiler.asm.amd64.AMD64Address; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.StubPort; import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; @@ -306,7 +307,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.psllq(xmm7, 46); masm.subpd(xmm0, xmm3); masm.leaq(r11, recordExternalAddress(crb, tblAddr)); - masm.movdqu(xmm2, new AMD64Address(rcx, r11, AMD64Address.Scale.Times1)); + masm.movdqu(xmm2, new AMD64Address(rcx, r11, Stride.S1)); masm.mulpd(xmm4, xmm0); masm.movapd(xmm6, xmm0); masm.movapd(xmm1, xmm0); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLog10Op.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLog10Op.java index 99904c9ce5d0..51061a6866ed 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLog10Op.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLog10Op.java @@ -48,6 +48,7 @@ import org.graalvm.compiler.asm.amd64.AMD64Assembler; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.StubPort; import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; @@ -310,7 +311,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.subsd(xmm5, xmm2); masm.andl(rdx, 16711680); masm.shrl(rdx, 12); - masm.movdqu(xmm0, new AMD64Address(r11, rdx, AMD64Address.Scale.Times1, -1504)); + masm.movdqu(xmm0, new AMD64Address(r11, rdx, Stride.S1, -1504)); masm.movdqu(xmm4, recordExternalAddress(crb, coeff16)); // 0x385593b1, 0xc025c917, // 0xdc963467, 0x3ffc6a02 masm.addsd(xmm1, xmm5); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLogOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLogOp.java index 2cf8c42ebf34..b19658ce20cb 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLogOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLogOp.java @@ -48,6 +48,7 @@ import org.graalvm.compiler.asm.amd64.AMD64Assembler; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.StubPort; import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; @@ -286,7 +287,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.subsd(xmm5, xmm2); masm.andl(rdx, 16711680); masm.shrl(rdx, 12); - masm.movdqu(xmm0, new AMD64Address(r11, rdx, AMD64Address.Scale.Times1)); + masm.movdqu(xmm0, new AMD64Address(r11, rdx, Stride.S1)); masm.movdqu(xmm4, recordExternalAddress(crb, coeff16)); // 0x3d6fb175, 0xbfc5555e, // 0x55555555, 0x3fd55555 masm.addsd(xmm1, xmm5); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java index a4b218d6d012..140ae18f3290 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java @@ -49,6 +49,7 @@ import org.graalvm.compiler.asm.amd64.AMD64Address; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.StubPort; import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; @@ -1032,7 +1033,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.addsd(xmm3, xmm5); masm.andl(rdx, 16760832); masm.shrl(rdx, 10); - masm.addpd(xmm5, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times1, -3648)); + masm.addpd(xmm5, new AMD64Address(tmp4, rdx, Stride.S1, -3648)); masm.movdqu(xmm0, recordExternalAddress(crb, coeff48)); // 0x486ececc, 0x3fc4635e, // 0x161bb241, 0xbf5dabe1 masm.pshufd(xmm2, xmm3, 68); @@ -1079,7 +1080,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.movl(rcx, rdx); masm.andl(rdx, 255); masm.addl(rdx, rdx); - masm.movdqu(xmm5, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times8, 0)); + masm.movdqu(xmm5, new AMD64Address(tmp4, rdx, Stride.S8, 0)); masm.addsd(xmm4, xmm1); masm.mulsd(xmm2, xmm0); masm.movdqu(xmm7, recordExternalAddress(crb, eCoeff)); // 0xe78a6731, 0x3f55d87f, @@ -1257,7 +1258,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.movl(rcx, rax); masm.andl(rax, 255); masm.addl(rax, rax); - masm.movdqu(xmm5, new AMD64Address(tmp4, rax, AMD64Address.Scale.Times8, 0)); + masm.movdqu(xmm5, new AMD64Address(tmp4, rax, Stride.S8, 0)); masm.addsd(xmm2, xmm4); masm.sarl(rcx, 8); masm.movl(rax, rcx); @@ -1570,7 +1571,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.addsd(xmm3, xmm0); masm.leaq(tmp4, recordExternalAddress(crb, log2)); // 0xfefa39ef, 0x3fe62e42, // 0xfefa39ef, 0xbfe62e42 - masm.movq(xmm4, new AMD64Address(tmp4, tmp1, AMD64Address.Scale.Times8, 0)); + masm.movq(xmm4, new AMD64Address(tmp4, tmp1, Stride.S8, 0)); masm.mulsd(xmm1, xmm3); masm.xorpd(xmm0, xmm0); masm.movl(rax, 16368); @@ -1755,7 +1756,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.movq(xmm0, recordExternalAddress(crb, coeffH8)); // 0x00000000, 0xbf5dabe1 masm.andl(rdx, 16760832); masm.shrl(rdx, 10); - masm.addpd(xmm7, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times1, -3648)); + masm.addpd(xmm7, new AMD64Address(tmp4, rdx, Stride.S1, -3648)); masm.mulsd(xmm4, xmm5); masm.mulsd(xmm0, xmm5); masm.mulsd(xmm6, xmm2); @@ -1856,7 +1857,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.movl(rcx, rdx); masm.andl(rdx, 255); masm.addl(rdx, rdx); - masm.movdqu(xmm5, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times8, 0)); + masm.movdqu(xmm5, new AMD64Address(tmp4, rdx, Stride.S8, 0)); masm.addsd(xmm4, xmm1); masm.pextrw(rdx, xmm6, 3); masm.shrl(rcx, 8); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java index e0dae7291858..d7878f258dcc 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java @@ -43,7 +43,7 @@ import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MIOp; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MOp; import org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize; @@ -955,9 +955,9 @@ public static void emitUncompressWithBaseRegister(AMD64MacroAssembler masm, Regi */ public static void emitUncompressWithBaseRegister(AMD64MacroAssembler masm, Register resultReg, Register baseReg, Register inputReg, int shift, boolean preserveFlagsRegister) { assert !baseReg.equals(Register.None) || shift != 0 : "compression not enabled"; - if (Scale.isScaleShiftSupported(shift)) { - AMD64Address.Scale scale = AMD64Address.Scale.fromShift(shift); - masm.leaq(resultReg, new AMD64Address(baseReg, inputReg, scale)); + if (AMD64Address.isScaleShiftSupported(shift)) { + Stride stride = Stride.fromLog2(shift); + masm.leaq(resultReg, new AMD64Address(baseReg, inputReg, stride)); } else { if (preserveFlagsRegister) { throw GraalError.shouldNotReachHere("No valid flag-effect-free instruction available to uncompress oop"); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StrideUtil.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StrideUtil.java index 3c95ee2cea55..3b9ed423825d 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StrideUtil.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StrideUtil.java @@ -24,7 +24,7 @@ */ package org.graalvm.compiler.lir.amd64; -import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.StrideUtil; /** @@ -36,7 +36,7 @@ public class AMD64StrideUtil { /** * Compute the jump table index for two given strides {@code strideA} and {@code strideB}. */ - public static int getDirectStubCallIndex(AMD64Address.Scale strideA, AMD64Address.Scale strideB) { + public static int getDirectStubCallIndex(Stride strideA, Stride strideB) { return StrideUtil.getDirectStubCallIndex(strideA.log2, strideB.log2); } } diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StringLatin1InflateOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StringLatin1InflateOp.java index 43867d73158f..49129d27d37c 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StringLatin1InflateOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StringLatin1InflateOp.java @@ -37,6 +37,7 @@ import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; @@ -144,15 +145,15 @@ private void byteArrayInflate(AMD64MacroAssembler masm, Register src, Register d // The vector count (in chars). masm.andlAndJcc(len, -32, ConditionFlag.Zero, labelCopyTail, true); - masm.leaq(src, new AMD64Address(src, len, AMD64Address.Scale.Times1)); - masm.leaq(dst, new AMD64Address(dst, len, AMD64Address.Scale.Times2)); + masm.leaq(src, new AMD64Address(src, len, Stride.S1)); + masm.leaq(dst, new AMD64Address(dst, len, Stride.S2)); masm.negq(len); // Inflate 32 chars per iteration, reading 256-bit compact vectors // and writing 512-bit inflated ditto. masm.bind(labelCopy32Loop); - masm.evpmovzxbw(tmp1, new AMD64Address(src, len, AMD64Address.Scale.Times1)); - masm.evmovdqu16(new AMD64Address(dst, len, AMD64Address.Scale.Times2), tmp1); + masm.evpmovzxbw(tmp1, new AMD64Address(src, len, Stride.S1)); + masm.evmovdqu16(new AMD64Address(dst, len, Stride.S2), tmp1); masm.addqAndJcc(len, 32, ConditionFlag.NotZero, labelCopy32Loop, false); masm.bind(labelCopyTail); @@ -189,14 +190,14 @@ private void byteArrayInflate(AMD64MacroAssembler masm, Register src, Register d } // vectored inflation - masm.leaq(src, new AMD64Address(src, len, AMD64Address.Scale.Times1)); - masm.leaq(dst, new AMD64Address(dst, len, AMD64Address.Scale.Times2)); + masm.leaq(src, new AMD64Address(src, len, Stride.S1)); + masm.leaq(dst, new AMD64Address(dst, len, Stride.S2)); masm.negq(len); if (masm.supports(CPUFeature.AVX2)) { masm.bind(labelCopy16Loop); - masm.vpmovzxbw(tmp1, new AMD64Address(src, len, AMD64Address.Scale.Times1)); - masm.vmovdqu(new AMD64Address(dst, len, AMD64Address.Scale.Times2), tmp1); + masm.vpmovzxbw(tmp1, new AMD64Address(src, len, Stride.S1)); + masm.vmovdqu(new AMD64Address(dst, len, Stride.S2), tmp1); masm.addqAndJcc(len, 16, ConditionFlag.NotZero, labelCopy16Loop, false); // The avx512 logic may branch here. We assume that avx2 is supported when we use @@ -219,8 +220,8 @@ private void byteArrayInflate(AMD64MacroAssembler masm, Register src, Register d // Inflate 8 bytes (chars) per iteration, reading 64-bit compact vectors // and writing 128-bit inflated ditto. masm.bind(labelCopy8Loop); - masm.pmovzxbw(tmp1, new AMD64Address(src, len, AMD64Address.Scale.Times1)); - masm.movdqu(new AMD64Address(dst, len, AMD64Address.Scale.Times2), tmp1); + masm.pmovzxbw(tmp1, new AMD64Address(src, len, Stride.S1)); + masm.movdqu(new AMD64Address(dst, len, Stride.S2), tmp1); masm.addqAndJcc(len, 8, ConditionFlag.NotZero, labelCopy8Loop, false); masm.bind(labelCopyTail); @@ -243,14 +244,14 @@ private void byteArrayInflate(AMD64MacroAssembler masm, Register src, Register d // Inflate any remaining characters (bytes) using a vanilla implementation. masm.testlAndJcc(len, len, ConditionFlag.Zero, labelDone, true); - masm.leaq(src, new AMD64Address(src, len, AMD64Address.Scale.Times1)); - masm.leaq(dst, new AMD64Address(dst, len, AMD64Address.Scale.Times2)); + masm.leaq(src, new AMD64Address(src, len, Stride.S1)); + masm.leaq(dst, new AMD64Address(dst, len, Stride.S2)); masm.negq(len); // Inflate a single byte (char) per iteration. masm.bind(labelCopyCharsLoop); - masm.movzbl(tmp2, new AMD64Address(src, len, AMD64Address.Scale.Times1)); - masm.movw(new AMD64Address(dst, len, AMD64Address.Scale.Times2), tmp2); + masm.movzbl(tmp2, new AMD64Address(src, len, Stride.S1)); + masm.movw(new AMD64Address(dst, len, Stride.S2), tmp2); masm.incqAndJcc(len, ConditionFlag.NotZero, labelCopyCharsLoop, false); masm.bind(labelDone); diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StringUTF16CompressOp.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StringUTF16CompressOp.java index 5621341f8b99..be0923dc4743 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StringUTF16CompressOp.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64StringUTF16CompressOp.java @@ -41,6 +41,7 @@ import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; @@ -200,21 +201,21 @@ private void charArrayCompress(AMD64MacroAssembler masm, Register src, Register // The vector count (in chars). masm.andlAndJcc(len, ~(32 - 1), ConditionFlag.Zero, labelCopyLoopTail, false); - masm.leaq(src, new AMD64Address(src, len, AMD64Address.Scale.Times2)); - masm.leaq(dst, new AMD64Address(dst, len, AMD64Address.Scale.Times1)); + masm.leaq(src, new AMD64Address(src, len, Stride.S2)); + masm.leaq(dst, new AMD64Address(dst, len, Stride.S1)); masm.negq(len); // Test and compress 32 chars per iteration, reading 512-bit vectors and // writing 256-bit compressed ditto. masm.bind(labelCopy32Loop); - masm.evmovdqu16(tmp1Reg, new AMD64Address(src, len, AMD64Address.Scale.Times2)); + masm.evmovdqu16(tmp1Reg, new AMD64Address(src, len, Stride.S2)); masm.evpcmpuw(k2, tmp1Reg, tmp2Reg, EVEXComparisonPredicate.LE); masm.kortestd(k2, k2); masm.jcc(ConditionFlag.CarryClear, labelReturnZero); // All 32 chars in the current vector (chunk) are valid for compression, // write truncated byte elements to memory. - masm.evpmovwb(new AMD64Address(dst, len, AMD64Address.Scale.Times1), tmp1Reg); + masm.evpmovwb(new AMD64Address(dst, len, Stride.S1), tmp1Reg); masm.addqAndJcc(len, 32, ConditionFlag.NotZero, labelCopy32Loop, false); masm.bind(labelCopyLoopTail); @@ -261,23 +262,23 @@ private void charArrayCompress(AMD64MacroAssembler masm, Register src, Register masm.pshufd(tmp1Reg, tmp1Reg, 0); // Store Unicode mask in 'vtmp1'. masm.pxor(tmp4Reg, tmp4Reg); - masm.leaq(src, new AMD64Address(src, len, AMD64Address.Scale.Times2)); - masm.leaq(dst, new AMD64Address(dst, len, AMD64Address.Scale.Times1)); + masm.leaq(src, new AMD64Address(src, len, Stride.S2)); + masm.leaq(dst, new AMD64Address(dst, len, Stride.S1)); masm.negq(len); // Test and compress 16 chars per iteration, reading 128-bit vectors and // writing 64-bit compressed ditto. masm.bind(labelCopy32Loop); // load 1st 8 characters - masm.movdqu(tmp2Reg, new AMD64Address(src, len, AMD64Address.Scale.Times2)); + masm.movdqu(tmp2Reg, new AMD64Address(src, len, Stride.S2)); masm.por(tmp4Reg, tmp2Reg); // load next 8 characters - masm.movdqu(tmp3Reg, new AMD64Address(src, len, AMD64Address.Scale.Times2, 16)); + masm.movdqu(tmp3Reg, new AMD64Address(src, len, Stride.S2, 16)); masm.por(tmp4Reg, tmp3Reg); masm.ptest(tmp4Reg, tmp1Reg); // Check for Unicode chars in vector. masm.jcc(ConditionFlag.NotZero, labelReturnZero); masm.packuswb(tmp2Reg, tmp3Reg); // Only ASCII chars; compress each to a byte. - masm.movdqu(new AMD64Address(dst, len, AMD64Address.Scale.Times1), tmp2Reg); + masm.movdqu(new AMD64Address(dst, len, Stride.S1), tmp2Reg); masm.addqAndJcc(len, 16, ConditionFlag.NotZero, labelCopy32Loop, false); // Test and compress another 8 chars before final tail copy. @@ -305,17 +306,17 @@ private void charArrayCompress(AMD64MacroAssembler masm, Register src, Register // Compress any remaining characters using a vanilla implementation. masm.testlAndJcc(len, len, ConditionFlag.Zero, labelReturnLength, true); - masm.leaq(src, new AMD64Address(src, len, AMD64Address.Scale.Times2)); - masm.leaq(dst, new AMD64Address(dst, len, AMD64Address.Scale.Times1)); + masm.leaq(src, new AMD64Address(src, len, Stride.S2)); + masm.leaq(dst, new AMD64Address(dst, len, Stride.S1)); masm.negq(len); // Compress a single character per iteration. masm.bind(labelCopyCharsLoop); - masm.movzwl(result, new AMD64Address(src, len, AMD64Address.Scale.Times2)); + masm.movzwl(result, new AMD64Address(src, len, Stride.S2)); // Check if Unicode character. masm.testlAndJcc(result, 0xff00, ConditionFlag.NotZero, labelReturnZero, true); // An ASCII character; compress to a byte. - masm.movb(new AMD64Address(dst, len, AMD64Address.Scale.Times1), result); + masm.movb(new AMD64Address(dst, len, Stride.S1), result); masm.incqAndJcc(len, ConditionFlag.NotZero, labelCopyCharsLoop, false); // If compression succeeded, return the length. diff --git a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/vector/AMD64VectorGather.java b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/vector/AMD64VectorGather.java index 26c4fc5a56de..5648cb82bf2d 100644 --- a/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/vector/AMD64VectorGather.java +++ b/compiler/src/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/vector/AMD64VectorGather.java @@ -32,6 +32,7 @@ import org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.EVEXPrefixConfig; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.amd64.AMD64LIRInstruction; @@ -77,7 +78,7 @@ public VexVectorGatherOp(AMD64Assembler.VexGatherOp opcode, AVXKind.AVXSize size @Override public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { - AMD64Address address = new AMD64Address(asRegister(base), asRegister(index), AMD64Address.Scale.Times1); + AMD64Address address = new AMD64Address(asRegister(base), asRegister(index), Stride.S1); opcode.emit(masm, size, asRegister(result), address, asRegister(mask)); } @@ -104,7 +105,7 @@ public EvexVectorGatherOp(AMD64Assembler.EvexGatherOp opcode, AVXKind.AVXSize si @Override public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { - AMD64Address address = new AMD64Address(asRegister(base), asRegister(index), AMD64Address.Scale.Times1); + AMD64Address address = new AMD64Address(asRegister(base), asRegister(index), Stride.S1); opcode.emit(masm, size, asRegister(result), address, asRegister(mask), EVEXPrefixConfig.Z0, EVEXPrefixConfig.B0); } diff --git a/compiler/src/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java b/compiler/src/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java index 533423bf3f2b..0446e41a02a1 100644 --- a/compiler/src/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java +++ b/compiler/src/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java @@ -29,6 +29,7 @@ import org.graalvm.compiler.asm.VectorSize; import org.graalvm.compiler.core.common.CompressEncoding; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.calc.Condition; import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.core.common.memory.MemoryOrderMode; @@ -205,68 +206,61 @@ default AllocatableValue addressAsAllocatableInteger(Value value) { Variable emitByteSwap(Value operand); @SuppressWarnings("unused") - default Variable emitArrayCompareTo(JavaKind kind1, JavaKind kind2, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value array2, Value length1, Value length2) { + default Variable emitArrayCompareTo(Stride strideA, Stride strideB, EnumSet runtimeCheckedCPUFeatures, Value arrayA, Value lengthA, Value arrayB, Value lengthB) { throw GraalError.unimplemented("String.compareTo substitution is not implemented on this architecture"); } @SuppressWarnings("unused") default Variable emitArrayRegionCompareTo(EnumSet runtimeCheckedCPUFeatures, - Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length, Value stride) { + Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length, Value dynamicStrides) { throw GraalError.unimplemented("String.compareTo substitution is not implemented on this architecture"); } @SuppressWarnings("unused") - default Variable emitArrayRegionCompareTo(JavaKind strideA, JavaKind strideB, EnumSet runtimeCheckedCPUFeatures, + default Variable emitArrayRegionCompareTo(Stride strideA, Stride strideB, EnumSet runtimeCheckedCPUFeatures, Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length) { throw GraalError.unimplemented("String.compareTo substitution is not implemented on this architecture"); } @SuppressWarnings("unused") - default Variable emitArrayEquals(JavaKind kind, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value array2, Value length) { + default Variable emitArrayEquals(JavaKind commonElementKind, EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length) { throw GraalError.unimplemented("Array.equals substitution is not implemented on this architecture"); } @SuppressWarnings("unused") - default Variable emitArrayEquals(int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value offset1, Value array2, Value offset2, Value length, Value stride) { - throw GraalError.unimplemented("Array.equals with offset substitution is not implemented on this architecture"); - } - - @SuppressWarnings("unused") - default Variable emitArrayEquals(JavaKind kind, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value offset1, Value array2, Value offset2, Value length) { - throw GraalError.unimplemented("Array.equals with offset substitution is not implemented on this architecture"); + default Variable emitArrayEquals(Stride strideA, Stride strideB, EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length) { + throw GraalError.unimplemented("Array.equals with different types with offset substitution is not implemented on this architecture"); } @SuppressWarnings("unused") - default Variable emitArrayEquals(JavaKind kind1, JavaKind kind2, int array1BaseOffset, int array2BaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value offset1, Value array2, Value offset2, Value length) { + default Variable emitArrayEqualsDynamicStrides(EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value length, Value dynamicStrides) { throw GraalError.unimplemented("Array.equals with different types with offset substitution is not implemented on this architecture"); } @SuppressWarnings("unused") - default Variable emitArrayEquals(int array1BaseOffset, int array2BaseOffset, int maskBaseOffset, - EnumSet runtimeCheckedCPUFeatures, Value array1, Value offset1, Value array2, Value offset2, Value mask, Value length, Value stride) { + default Variable emitArrayEqualsWithMask(Stride strideA, Stride strideB, Stride strideMask, EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value mask, Value length) { throw GraalError.unimplemented("Array.equals with different types with offset substitution is not implemented on this architecture"); } @SuppressWarnings("unused") - default Variable emitArrayEquals(JavaKind kind1, JavaKind kind2, JavaKind kindMask, int array1BaseOffset, int array2BaseOffset, int maskBaseOffset, EnumSet runtimeCheckedCPUFeatures, - Value array1, Value offset1, Value array2, Value offset2, Value mask, Value length) { + default Variable emitArrayEqualsWithMaskDynamicStrides(EnumSet runtimeCheckedCPUFeatures, + Value arrayA, Value offsetA, Value arrayB, Value offsetB, Value mask, Value length, Value dynamicStrides) { throw GraalError.unimplemented("Array.equals with different types with offset substitution is not implemented on this architecture"); } @SuppressWarnings("unused") - default void emitArrayCopyWithConversion(JavaKind strideSrc, JavaKind strideDst, EnumSet runtimeCheckedCPUFeatures, + default void emitArrayCopyWithConversion(Stride strideSrc, Stride strideDst, EnumSet runtimeCheckedCPUFeatures, Value arraySrc, Value offsetSrc, Value arrayDst, Value offsetDst, Value length) { throw GraalError.unimplemented("Array.copy with variable stride substitution is not implemented on this architecture"); } @SuppressWarnings("unused") default void emitArrayCopyWithConversion(EnumSet runtimeCheckedCPUFeatures, - Value arraySrc, Value offsetSrc, Value arrayDst, Value offsetDst, Value length, Value stride) { + Value arraySrc, Value offsetSrc, Value arrayDst, Value offsetDst, Value length, Value dynamicStrides) { throw GraalError.unimplemented("Array.copy with variable stride substitution is not implemented on this architecture"); } @@ -277,7 +271,7 @@ default Variable emitCalcStringAttributes(Object op, EnumSet runtimeCheckedCP } @SuppressWarnings("unused") - default Variable emitArrayIndexOf(int arrayBaseOffset, JavaKind valueKind, boolean findTwoConsecutive, boolean withMask, EnumSet runtimeCheckedCPUFeatures, + default Variable emitArrayIndexOf(Stride stride, boolean findTwoConsecutive, boolean withMask, EnumSet runtimeCheckedCPUFeatures, Value array, Value offset, Value length, Value fromIndex, Value... searchValues) { throw GraalError.unimplemented("String.indexOf substitution is not implemented on this architecture"); } diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/loop/LoopEx.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/loop/LoopEx.java index 65ac5f0b6702..7d955c18f476 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/loop/LoopEx.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/loop/LoopEx.java @@ -542,34 +542,48 @@ public void deleteUnusedNodes() { } } + public static boolean canDuplicateLoopNode(Node node) { + /* + * Control flow anchored nodes must not be duplicated. + */ + if (node instanceof ControlFlowAnchored) { + return false; + } + if (node instanceof FrameState) { + FrameState frameState = (FrameState) node; + /* + * Exception handling frame states can cause problems when they are duplicated and one + * needs to create a framestate at the duplication merge. + */ + if (frameState.isExceptionHandlingBCI()) { + return false; + } + } + return true; + } + + public static boolean canStripMineLoopNode(Node node) { + if (node instanceof NeverStripMineNode) { + return false; + } + return true; + } + /** * @return true if all nodes in the loop can be duplicated. */ public boolean canDuplicateLoop() { for (Node node : inside().nodes()) { - /* - * Control flow anchored nodes must not be duplicated. - */ - if (node instanceof ControlFlowAnchored) { + if (!canDuplicateLoopNode(node)) { return false; } - if (node instanceof FrameState) { - FrameState frameState = (FrameState) node; - /* - * Exception handling frame states can cause problems when they are duplicated and - * one needs to create a framestate at the duplication merge. - */ - if (frameState.isExceptionHandlingBCI()) { - return false; - } - } } return true; } public boolean canStripMine() { for (Node node : inside().nodes()) { - if (node instanceof NeverStripMineNode) { + if (!canStripMineLoopNode(node)) { return false; } } diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/ConstantReflectionUtil.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/ConstantReflectionUtil.java index 198097b8a92e..6b78fcdf3b5d 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/ConstantReflectionUtil.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/ConstantReflectionUtil.java @@ -26,15 +26,17 @@ import java.nio.ByteOrder; +import org.graalvm.compiler.core.common.Stride; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.spi.CanonicalizerTool; + import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaType; -import org.graalvm.compiler.nodes.ConstantNode; -import org.graalvm.compiler.nodes.NodeView; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.spi.CanonicalizerTool; /** * Helper functions to access complex objects on the application heap via the @@ -77,24 +79,22 @@ public static int[] loadIntArrayConstant(ConstantReflectionProvider crp, JavaCon * @param array a constant java byte[], char[] or int[] array. * @param arrayKind the array's element kind ({@link JavaKind#Byte} for byte[], * {@link JavaKind#Char} for char[] and {@link JavaKind#Int} for int[]). - * @param stride the element width to read from the array ({@link JavaKind#Byte} == 1 byte, - * {@link JavaKind#Char} == 2 bytes and {@link JavaKind#Int} == 4 bytes). The stride - * must not be smaller than the array element kind, e.g. reading a byte from an int[] - * array is not supported. + * @param stride the element width to read from the array. The stride must not be smaller than + * the array element kind, e.g. reading a byte from an int[] array is not supported. * @param index the index to read from, scaled to {@code stride}. * @return the bytes read, zero-extended to an int value. */ - public static int readTypePunned(ConstantReflectionProvider provider, JavaConstant array, JavaKind arrayKind, JavaKind stride, int index) { + public static int readTypePunned(ConstantReflectionProvider provider, JavaConstant array, JavaKind arrayKind, Stride stride, int index) { assert arrayKind == JavaKind.Byte || arrayKind == JavaKind.Char || arrayKind == JavaKind.Int; - assert stride == JavaKind.Byte || stride == JavaKind.Char || stride == JavaKind.Int; - assert stride.getByteCount() >= arrayKind.getByteCount(); + assert stride.value <= 4; + assert stride.value >= arrayKind.getByteCount(); if (arrayKind == JavaKind.Byte) { - int i = index * stride.getByteCount(); - if (stride == JavaKind.Byte) { + int i = index * stride.value; + if (stride == Stride.S1) { return provider.readArrayElement(array, i).asInt() & 0xff; } if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) { - if (stride == JavaKind.Char) { + if (stride == Stride.S2) { return (provider.readArrayElement(array, i).asInt() & 0xff) | ((provider.readArrayElement(array, i + 1).asInt() & 0xff) << 8); } else { @@ -104,7 +104,7 @@ public static int readTypePunned(ConstantReflectionProvider provider, JavaConsta ((provider.readArrayElement(array, i + 3).asInt() & 0xff) << 24); } } else { - if (stride == JavaKind.Char) { + if (stride == Stride.S2) { return (provider.readArrayElement(array, i + 1).asInt() & 0xff) | ((provider.readArrayElement(array, i).asInt() & 0xff) << 8); } else { @@ -115,10 +115,10 @@ public static int readTypePunned(ConstantReflectionProvider provider, JavaConsta } } } else if (arrayKind == JavaKind.Char) { - if (stride == JavaKind.Char) { + if (stride == Stride.S2) { return provider.readArrayElement(array, index).asInt() & 0xffff; } else { - assert stride == JavaKind.Int; + assert stride == Stride.S4; int i = index * 2; if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) { return (provider.readArrayElement(array, i).asInt() & 0xffff) | @@ -129,13 +129,13 @@ public static int readTypePunned(ConstantReflectionProvider provider, JavaConsta } } } else { - assert stride == JavaKind.Int; + assert stride == Stride.S4; return provider.readArrayElement(array, index).asInt(); } } public interface ArrayBaseOffsetProvider { - int getArrayBaseOffset(MetaAccessProvider metaAccess, ValueNode array, JavaKind elementKind); + int getArrayBaseOffset(MetaAccessProvider metaAccess, ValueNode array, JavaKind arrayKind); } /** @@ -143,17 +143,13 @@ public interface ArrayBaseOffsetProvider { * {@code len} in-bounds constant reads starting at byte offset {@code offset}. Return * {@code false} if not everything constant or if we would try to read out of bounds. */ - public static boolean canFoldReads(CanonicalizerTool tool, ValueNode array, ValueNode offset, JavaKind stride, int len, ArrayBaseOffsetProvider arrayBaseOffsetProvider) { - if (array.isJavaConstant() && ((ConstantNode) array).getStableDimension() >= 1 && (offset == null || offset.isJavaConstant())) { + public static boolean canFoldReads(CanonicalizerTool tool, ValueNode array, ValueNode offset, Stride stride, int len, ArrayBaseOffsetProvider arrayBaseOffsetProvider) { + if (array.isJavaConstant() && ((ConstantNode) array).getStableDimension() >= 1 && offset.isJavaConstant()) { ResolvedJavaType arrayType = array.stamp(NodeView.DEFAULT).javaType(tool.getMetaAccess()); if (arrayType.isArray()) { Integer arrayLength = tool.getConstantReflection().readArrayLength(array.asJavaConstant()); - if (offset == null) { - return arrayLength != null && len <= arrayLength; - } else { - Integer index = startIndex(tool, array, offset.asJavaConstant(), stride, arrayBaseOffsetProvider); - return arrayLength != null && index != null && index >= 0 && index + len <= arrayLength; - } + Integer index = startIndex(tool, array, offset.asJavaConstant(), stride, arrayBaseOffsetProvider); + return arrayLength != null && index != null && index >= 0 && index + len <= arrayLength; } } return false; @@ -163,12 +159,12 @@ public static boolean canFoldReads(CanonicalizerTool tool, ValueNode array, Valu * Compute an element index from a byte offset from the start of the array object. Returns * {@code null} if the given offset is not aligned correctly for the element kind's stride. */ - public static Integer startIndex(CanonicalizerTool tool, ValueNode array, JavaConstant offset, JavaKind stride, ArrayBaseOffsetProvider arrayBaseOffsetProvider) { + public static Integer startIndex(CanonicalizerTool tool, ValueNode array, JavaConstant offset, Stride stride, ArrayBaseOffsetProvider arrayBaseOffsetProvider) { JavaKind arrayKind = array.stamp(NodeView.DEFAULT).javaType(tool.getMetaAccess()).getComponentType().getJavaKind(); long elementOffset = offset.asLong() - arrayBaseOffsetProvider.getArrayBaseOffset(tool.getMetaAccess(), array, arrayKind); - if (elementOffset % stride.getByteCount() != 0) { + if (elementOffset % stride.value != 0) { return null; } - return (int) (elementOffset / stride.getByteCount()); + return (int) (elementOffset / stride.value); } } diff --git a/compiler/src/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java b/compiler/src/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java index 72b59ad07729..dbce67185332 100644 --- a/compiler/src/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java +++ b/compiler/src/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java @@ -36,6 +36,7 @@ import java.lang.reflect.Type; import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.calc.Condition; import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.ConstantNode; @@ -62,7 +63,6 @@ import org.graalvm.compiler.nodes.memory.address.IndexAddressNode; import org.graalvm.compiler.nodes.spi.Replacements; import org.graalvm.compiler.options.OptionValues; -import org.graalvm.compiler.replacements.nodes.ArrayIndexOfNode; import org.graalvm.compiler.replacements.InvocationPluginHelper; import org.graalvm.compiler.replacements.SnippetSubstitutionInvocationPlugin; import org.graalvm.compiler.replacements.SnippetTemplate; @@ -73,6 +73,7 @@ import org.graalvm.compiler.replacements.StringUTF16Snippets; import org.graalvm.compiler.replacements.TargetGraphBuilderPlugins; import org.graalvm.compiler.replacements.nodes.ArrayCompareToNode; +import org.graalvm.compiler.replacements.nodes.ArrayIndexOfNode; import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode; import org.graalvm.compiler.replacements.nodes.CountLeadingZerosNode; import org.graalvm.compiler.replacements.nodes.CountTrailingZerosNode; @@ -232,36 +233,41 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec } private static final class ArrayCompareToPlugin extends InvocationPlugin { - private final JavaKind valueKind; - private final JavaKind otherKind; + private final Stride strideA; + private final Stride strideB; private final boolean swapped; - private ArrayCompareToPlugin(JavaKind valueKind, JavaKind otherKind, boolean swapped, String name, Type... argumentTypes) { + private ArrayCompareToPlugin(Stride strideA, Stride strideB, boolean swapped, String name, Type... argumentTypes) { super(name, argumentTypes); - this.valueKind = valueKind; - this.otherKind = otherKind; + this.strideA = strideA; + this.strideB = strideB; this.swapped = swapped; } - private ArrayCompareToPlugin(JavaKind valueKind, JavaKind otherKind, String name, Type... argumentTypes) { - this(valueKind, otherKind, false, name, argumentTypes); + private ArrayCompareToPlugin(Stride strideA, Stride strideB, String name, Type... argumentTypes) { + this(strideA, strideB, false, name, argumentTypes); } @Override - public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value, ValueNode other) { - ValueNode nonNullValue = b.nullCheckedValue(value); - ValueNode nonNullOther = b.nullCheckedValue(other); - - ValueNode valueLength = b.add(new ArrayLengthNode(nonNullValue)); - ValueNode otherLength = b.add(new ArrayLengthNode(nonNullOther)); - if (swapped) { - /* - * Swapping array arguments because intrinsic expects order to be byte[]/char[] but - * kind arguments stay in original order. - */ - b.addPush(JavaKind.Int, new ArrayCompareToNode(nonNullOther, nonNullValue, otherLength, valueLength, valueKind, otherKind)); - } else { - b.addPush(JavaKind.Int, new ArrayCompareToNode(nonNullValue, nonNullOther, valueLength, otherLength, valueKind, otherKind)); + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arrayA, ValueNode arrayB) { + try (InvocationPluginHelper helper = new InvocationPluginHelper(b, targetMethod)) { + ValueNode nonNullA = b.nullCheckedValue(arrayA); + ValueNode nonNullB = b.nullCheckedValue(arrayB); + + ValueNode lengthA = b.add(new ArrayLengthNode(nonNullA)); + ValueNode lengthB = b.add(new ArrayLengthNode(nonNullB)); + + ValueNode startA = helper.arrayStart(nonNullA, JavaKind.Byte); + ValueNode startB = helper.arrayStart(nonNullB, JavaKind.Byte); + if (swapped) { + /* + * Swapping array arguments because intrinsic expects order to be byte[]/char[] + * but kind arguments stay in original order. + */ + b.addPush(JavaKind.Int, new ArrayCompareToNode(startB, lengthB, startA, lengthA, strideA, strideB)); + } else { + b.addPush(JavaKind.Int, new ArrayCompareToNode(startA, lengthA, startB, lengthB, strideA, strideB)); + } } return true; } @@ -270,8 +276,8 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec private static void registerStringLatin1Plugins(InvocationPlugins plugins, Replacements replacements) { Registration r = new Registration(plugins, "java.lang.StringLatin1", replacements); r.setAllowOverwrite(true); - r.register(new ArrayCompareToPlugin(JavaKind.Byte, JavaKind.Byte, "compareTo", byte[].class, byte[].class)); - r.register(new ArrayCompareToPlugin(JavaKind.Byte, JavaKind.Char, "compareToUTF16", byte[].class, byte[].class)); + r.register(new ArrayCompareToPlugin(Stride.S1, Stride.S1, "compareTo", byte[].class, byte[].class)); + r.register(new ArrayCompareToPlugin(Stride.S1, Stride.S2, "compareToUTF16", byte[].class, byte[].class)); r.register(new InvocationPlugin("inflate", byte[].class, int.class, byte[].class, int.class, int.class) { @SuppressWarnings("try") @Override @@ -361,7 +367,7 @@ public SnippetTemplate.SnippetInfo getSnippet(StringLatin1Snippets.Templates tem r.register(new InvocationPlugin("indexOfChar", byte[].class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value, ValueNode ch, ValueNode fromIndex, ValueNode max) { - b.addPush(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(JavaKind.Byte, JavaKind.Byte, value, max, fromIndex, ch)); + b.addPush(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(b, JavaKind.Byte, Stride.S1, value, max, fromIndex, ch)); return true; } }); @@ -371,8 +377,8 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec private static void registerStringUTF16Plugins(InvocationPlugins plugins, Replacements replacements) { Registration r = new Registration(plugins, "java.lang.StringUTF16", replacements); r.setAllowOverwrite(true); - r.register(new ArrayCompareToPlugin(JavaKind.Char, JavaKind.Char, "compareTo", byte[].class, byte[].class)); - r.register(new ArrayCompareToPlugin(JavaKind.Char, JavaKind.Byte, true, "compareToLatin1", byte[].class, byte[].class)); + r.register(new ArrayCompareToPlugin(Stride.S2, Stride.S2, "compareTo", byte[].class, byte[].class)); + r.register(new ArrayCompareToPlugin(Stride.S2, Stride.S1, true, "compareToLatin1", byte[].class, byte[].class)); r.register(new InvocationPlugin("compress", byte[].class, int.class, byte[].class, int.class, int.class) { @SuppressWarnings("try") @Override @@ -459,7 +465,7 @@ public SnippetTemplate.SnippetInfo getSnippet(StringUTF16Snippets.Templates temp @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value, ValueNode ch, ValueNode fromIndex, ValueNode max) { ZeroExtendNode toChar = b.add(new ZeroExtendNode(b.add(new NarrowNode(ch, JavaKind.Char.getBitCount())), JavaKind.Int.getBitCount())); - b.addPush(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(JavaKind.Byte, JavaKind.Char, value, max, fromIndex, toChar)); + b.addPush(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(b, JavaKind.Byte, Stride.S2, value, max, fromIndex, toChar)); return true; } }); diff --git a/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayEqualsForeignCalls.java b/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayEqualsForeignCalls.java index ac9372bfac16..1b13f30d70dc 100644 --- a/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayEqualsForeignCalls.java +++ b/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayEqualsForeignCalls.java @@ -27,6 +27,7 @@ import java.util.EnumSet; import org.graalvm.compiler.asm.amd64.AVXKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.StrideUtil; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.debug.GraalError; @@ -44,10 +45,10 @@ public final class AMD64ArrayEqualsForeignCalls { @SuppressWarnings("unchecked") public static ForeignCallDescriptor getArrayEqualsStub(ArrayEqualsNode arrayEqualsNode, LIRGenerator gen) { - JavaKind stride = arrayEqualsNode.getKind(); + Stride stride = Stride.fromJavaKind(arrayEqualsNode.getKind()); ValueNode length = arrayEqualsNode.getLength(); if (length.isJavaConstant() && AMD64ArrayEqualsOp.canGenerateConstantLengthCompare(gen.target(), - (EnumSet) arrayEqualsNode.getRuntimeCheckedCPUFeatures(), stride, stride, + (EnumSet) arrayEqualsNode.getRuntimeCheckedCPUFeatures(), arrayEqualsNode.getKind(), stride, stride, length.asJavaConstant().asInt(), (AVXKind.AVXSize) gen.getMaxVectorSize(arrayEqualsNode.getRuntimeCheckedCPUFeatures()))) { // Yield constant-length arrays comparison assembly return null; @@ -63,6 +64,7 @@ public static ForeignCallDescriptor getRegionEqualsStub(ArrayRegionEqualsNode re if (directStubCallIndex >= 0 && length.isJavaConstant() && AMD64ArrayEqualsOp.canGenerateConstantLengthCompare(gen.target(), (EnumSet) regionEqualsNode.getRuntimeCheckedCPUFeatures(), + JavaKind.Byte, StrideUtil.getConstantStrideA(directStubCallIndex), StrideUtil.getConstantStrideB(directStubCallIndex), length.asJavaConstant().asInt(), (AVXKind.AVXSize) gen.getMaxVectorSize(regionEqualsNode.getRuntimeCheckedCPUFeatures()))) { diff --git a/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayEqualsWithMaskForeignCalls.java b/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayEqualsWithMaskForeignCalls.java index ae9187c79094..7897914f2d61 100644 --- a/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayEqualsWithMaskForeignCalls.java +++ b/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayEqualsWithMaskForeignCalls.java @@ -24,12 +24,10 @@ */ package org.graalvm.compiler.replacements.amd64; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; - import java.util.EnumSet; import org.graalvm.compiler.asm.amd64.AVXKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.StrideUtil; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.debug.GraalError; @@ -38,122 +36,71 @@ import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.replacements.NodeStrideUtil; import org.graalvm.compiler.replacements.nodes.ForeignCalls; +import org.graalvm.word.Pointer; import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.meta.JavaKind; public final class AMD64ArrayEqualsWithMaskForeignCalls { - private static final ForeignCallDescriptor STUB_REGION_EQUALS_B_S1_S1_S1 = foreignCallDescriptorByteArray("arrayRegionEqualsBS1S1S1"); - private static final ForeignCallDescriptor STUB_REGION_EQUALS_B_S1_S2_S1 = foreignCallDescriptorByteArray("arrayRegionEqualsBS1S2S1"); - private static final ForeignCallDescriptor STUB_REGION_EQUALS_B_S1_S2_S2 = foreignCallDescriptorByteArray("arrayRegionEqualsBS1S2S2"); - private static final ForeignCallDescriptor STUB_REGION_EQUALS_B_S2_S1_S1 = foreignCallDescriptorByteArray("arrayRegionEqualsBS2S1S1"); - private static final ForeignCallDescriptor STUB_REGION_EQUALS_B_S2_S2_S1 = foreignCallDescriptorByteArray("arrayRegionEqualsBS2S2S1"); - private static final ForeignCallDescriptor STUB_REGION_EQUALS_B_S2_S2_S2 = foreignCallDescriptorByteArray("arrayRegionEqualsBS2S2S2"); - private static final ForeignCallDescriptor STUB_REGION_EQUALS_C = ForeignCalls.pureFunctionForeignCallDescriptor( - "arrayRegionEqualsC", boolean.class, char[].class, long.class, char[].class, long.class, char[].class, int.class); + private static final ForeignCallDescriptor STUB_REGION_EQUALS_S1_S2_S1 = foreignCallDescriptor("arrayRegionEqualsS1S2S1"); + private static final ForeignCallDescriptor STUB_REGION_EQUALS_S2_S2_S1 = foreignCallDescriptor("arrayRegionEqualsS2S2S1"); private static final ForeignCallDescriptor STUB_REGION_EQUALS_DYNAMIC_STRIDES = ForeignCalls.pureFunctionForeignCallDescriptor( - "arrayRegionEqualsDynamicStrides", boolean.class, Object.class, long.class, Object.class, long.class, byte[].class, int.class, int.class); + "arrayRegionEqualsDynamicStrides", boolean.class, Object.class, long.class, Object.class, long.class, Pointer.class, int.class, int.class); /** * CAUTION: the ordering here is important: entries 0-9 must match the indices generated by - * {@link NodeStrideUtil#getDirectStubCallIndex(ValueNode, JavaKind, JavaKind)}. + * {@link NodeStrideUtil#getDirectStubCallIndex(ValueNode, Stride, Stride)}. * * @see #getStub(AMD64ArrayRegionEqualsWithMaskNode, LIRGenerator) */ public static final ForeignCallDescriptor[] STUBS = { - foreignCallDescriptorObject("arrayRegionEqualsS1S1"), - foreignCallDescriptorObject("arrayRegionEqualsS1S2"), - foreignCallDescriptorObject("arrayRegionEqualsS1S4"), - foreignCallDescriptorObject("arrayRegionEqualsS2S1"), - foreignCallDescriptorObject("arrayRegionEqualsS2S2"), - foreignCallDescriptorObject("arrayRegionEqualsS2S4"), - foreignCallDescriptorObject("arrayRegionEqualsS4S1"), - foreignCallDescriptorObject("arrayRegionEqualsS4S2"), - foreignCallDescriptorObject("arrayRegionEqualsS4S4"), + foreignCallDescriptor("arrayRegionEqualsS1S1"), + foreignCallDescriptor("arrayRegionEqualsS1S2"), + foreignCallDescriptor("arrayRegionEqualsS1S4"), + foreignCallDescriptor("arrayRegionEqualsS2S1"), + foreignCallDescriptor("arrayRegionEqualsS2S2"), + foreignCallDescriptor("arrayRegionEqualsS2S4"), + foreignCallDescriptor("arrayRegionEqualsS4S1"), + foreignCallDescriptor("arrayRegionEqualsS4S2"), + foreignCallDescriptor("arrayRegionEqualsS4S4"), STUB_REGION_EQUALS_DYNAMIC_STRIDES, - STUB_REGION_EQUALS_B_S1_S1_S1, - STUB_REGION_EQUALS_B_S1_S2_S1, - STUB_REGION_EQUALS_B_S1_S2_S2, - STUB_REGION_EQUALS_B_S2_S1_S1, - STUB_REGION_EQUALS_B_S2_S2_S1, - STUB_REGION_EQUALS_B_S2_S2_S2, - - STUB_REGION_EQUALS_C, + STUB_REGION_EQUALS_S1_S2_S1, + STUB_REGION_EQUALS_S2_S2_S1, }; - private static ForeignCallDescriptor foreignCallDescriptorObject(String name) { - return ForeignCalls.pureFunctionForeignCallDescriptor(name, boolean.class, Object.class, long.class, Object.class, long.class, byte[].class, int.class); - } - - private static ForeignCallDescriptor foreignCallDescriptorByteArray(String name) { - return ForeignCalls.pureFunctionForeignCallDescriptor(name, boolean.class, byte[].class, long.class, byte[].class, long.class, byte[].class, int.class); + private static ForeignCallDescriptor foreignCallDescriptor(String name) { + return ForeignCalls.pureFunctionForeignCallDescriptor(name, boolean.class, Object.class, long.class, Object.class, long.class, Pointer.class, int.class); } @SuppressWarnings("unchecked") public static ForeignCallDescriptor getStub(AMD64ArrayRegionEqualsWithMaskNode node, LIRGenerator gen) { - JavaKind strideA = node.getStrideA(); - JavaKind strideB = node.getStrideB(); - JavaKind strideM = node.getStrideMask(); + Stride strideA = node.getStrideA(); + Stride strideB = node.getStrideB(); + Stride strideM = node.getStrideMask(); ValueNode length = node.getLength(); int directStubCallIndex = node.getDirectStubCallIndex(); GraalError.guarantee(-1 <= directStubCallIndex && directStubCallIndex < 9, "invalid direct stub call index"); if (directStubCallIndex >= 0 && length.isJavaConstant() && AMD64ArrayEqualsOp.canGenerateConstantLengthCompare(gen.target(), (EnumSet) node.getRuntimeCheckedCPUFeatures(), + JavaKind.Byte, StrideUtil.getConstantStrideA(directStubCallIndex), StrideUtil.getConstantStrideB(directStubCallIndex), length.asJavaConstant().asInt(), (AVXKind.AVXSize) gen.getMaxVectorSize(node.getRuntimeCheckedCPUFeatures()))) { // Yield constant-length arrays comparison assembly return null; } - switch (node.getArrayKind()) { - case Byte: - switch (strideA) { - case Byte: - switch (strideB) { - case Byte: - GraalError.guarantee(strideM == S1, "mask stride must match strideA and strideB"); - return STUB_REGION_EQUALS_B_S1_S1_S1; - case Char: - switch (strideM) { - case Byte: - return STUB_REGION_EQUALS_B_S1_S2_S1; - case Char: - return STUB_REGION_EQUALS_B_S1_S2_S2; - default: - throw GraalError.shouldNotReachHere(); - } - default: - throw GraalError.shouldNotReachHere(); - } - case Char: - switch (strideB) { - case Byte: - GraalError.guarantee(strideM == S1, "mask stride must match strideA and strideB"); - return STUB_REGION_EQUALS_B_S2_S1_S1; - case Char: - switch (strideM) { - case Byte: - return STUB_REGION_EQUALS_B_S2_S2_S1; - case Char: - return STUB_REGION_EQUALS_B_S2_S2_S2; - default: - throw GraalError.shouldNotReachHere(); - } - default: - throw GraalError.shouldNotReachHere(); - } - default: - throw GraalError.shouldNotReachHere(); - } - case Char: - GraalError.guarantee(strideA == S2 && strideB == S2 && strideM == S2, "only stride2 allowed for char arrays"); - return STUB_REGION_EQUALS_C; - case Void: - return directStubCallIndex < 0 ? STUB_REGION_EQUALS_DYNAMIC_STRIDES : STUBS[directStubCallIndex]; - default: - throw GraalError.shouldNotReachHere(); + if (strideB != null && strideB != strideM) { + if (strideA == Stride.S1) { + GraalError.guarantee(strideB == Stride.S2 && strideM == Stride.S1, "unsupported strides"); + return STUB_REGION_EQUALS_S1_S2_S1; + } else { + GraalError.guarantee(strideA == Stride.S2 && strideB == Stride.S2 && strideM == Stride.S1, "unsupported strides"); + return STUB_REGION_EQUALS_S2_S2_S1; + } } + GraalError.guarantee(strideB == strideM, "unsupported strides"); + return directStubCallIndex < 0 ? STUB_REGION_EQUALS_DYNAMIC_STRIDES : STUBS[directStubCallIndex]; } } diff --git a/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayRegionEqualsWithMaskNode.java b/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayRegionEqualsWithMaskNode.java index 0f93e957f9c0..75120f097993 100644 --- a/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayRegionEqualsWithMaskNode.java +++ b/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64ArrayRegionEqualsWithMaskNode.java @@ -25,22 +25,19 @@ package org.graalvm.compiler.replacements.amd64; import static org.graalvm.compiler.core.common.GraalOptions.UseGraalStubs; -import static org.graalvm.compiler.core.common.StrideUtil.NONE; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; import java.util.EnumSet; import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.nodeinfo.NodeCycles; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodeinfo.NodeSize; +import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.ConstantNode; -import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.spi.Canonicalizable; @@ -64,36 +61,29 @@ public final class AMD64ArrayRegionEqualsWithMaskNode extends PureFunctionStubIn public static final NodeClass TYPE = NodeClass.create(AMD64ArrayRegionEqualsWithMaskNode.class); - /** - * {@link JavaKind} of the arrays {@link #arrayA}, {@link #arrayB}, and {@link #arrayMask}, e.g. - * if this is {@link JavaKind#Byte}, all arrays must be {@code byte[]} arrays, and the intrinsic - * will use the corresponding array base offset when reading array elements. This value can also - * be set to {@link JavaKind#Void}, which changes the operation's behavior in the following way: - * {@link #arrayA} and {@link #arrayB} may be any type, and their respective base offset must be - * added to {@link #offsetA} and {@link #offsetB}; {@link #arrayMask} must be a {@code byte[]} - * array. - */ - private final JavaKind arrayKind; /** * Stride for reading type punned array elements of {@link #arrayA}. Allowed values are - * {@link JavaKind#Byte}, {@link JavaKind#Char} and {@link JavaKind#Int}. + * {@link Stride#S1}, {@link Stride#S2} and {@link Stride#S4}. */ - private final JavaKind strideA; + private final Stride strideA; /** * Stride for reading type punned array elements of {@link #arrayB}. Allowed values are - * {@link JavaKind#Byte}, {@link JavaKind#Char} and {@link JavaKind#Int}. + * {@link Stride#S1}, {@link Stride#S2} and {@link Stride#S4}. */ - private final JavaKind strideB; + private final Stride strideB; /** * Stride for reading type punned array elements of {@link #arrayMask}. Allowed values are - * {@link JavaKind#Byte}, {@link JavaKind#Char} and {@link JavaKind#Int}. + * {@link Stride#S1}, {@link Stride#S2} and {@link Stride#S4}. */ - private final JavaKind strideMask; + private final Stride strideMask; @Input protected ValueNode arrayA; @Input protected ValueNode offsetA; @Input protected ValueNode arrayB; @Input protected ValueNode offsetB; + /** + * Direct pointer to the memory region to be OR-ed to arrayA. + */ @Input protected ValueNode arrayMask; @Input protected ValueNode length; @@ -111,12 +101,10 @@ public AMD64ArrayRegionEqualsWithMaskNode( ValueNode offsetB, ValueNode arrayMask, ValueNode length, - @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind strideA, - @ConstantNodeParameter JavaKind strideB, - @ConstantNodeParameter JavaKind strideMask) { - this(arrayA, offsetA, arrayB, offsetB, arrayMask, length, null, arrayKind, strideA, strideB, strideMask, null, - defaultLocationIdentity(arrayKind, strideA, strideB, strideMask)); + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, + @ConstantNodeParameter Stride strideMask) { + this(arrayA, offsetA, arrayB, offsetB, arrayMask, length, null, strideA, strideB, strideMask, null, LocationIdentity.ANY_LOCATION); } public AMD64ArrayRegionEqualsWithMaskNode( @@ -126,13 +114,11 @@ public AMD64ArrayRegionEqualsWithMaskNode( ValueNode offsetB, ValueNode arrayMask, ValueNode length, - @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind strideA, - @ConstantNodeParameter JavaKind strideB, - @ConstantNodeParameter JavaKind strideMask, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, + @ConstantNodeParameter Stride strideMask, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { - this(arrayA, offsetA, arrayB, offsetB, arrayMask, length, null, arrayKind, strideA, strideB, strideMask, runtimeCheckedCPUFeatures, - defaultLocationIdentity(arrayKind, strideA, strideB, strideMask)); + this(arrayA, offsetA, arrayB, offsetB, arrayMask, length, null, strideA, strideB, strideMask, runtimeCheckedCPUFeatures, LocationIdentity.ANY_LOCATION); } public AMD64ArrayRegionEqualsWithMaskNode( @@ -142,7 +128,7 @@ public AMD64ArrayRegionEqualsWithMaskNode( ValueNode offsetB, ValueNode arrayMask, ValueNode length, ValueNode dynamicStrides) { - this(arrayA, offsetA, arrayB, offsetB, arrayMask, length, dynamicStrides, NONE, null, null, null, null, LocationIdentity.ANY_LOCATION); + this(arrayA, offsetA, arrayB, offsetB, arrayMask, length, dynamicStrides, null, null, null, null, LocationIdentity.ANY_LOCATION); } public AMD64ArrayRegionEqualsWithMaskNode( @@ -153,7 +139,7 @@ public AMD64ArrayRegionEqualsWithMaskNode( ValueNode arrayMask, ValueNode length, ValueNode dynamicStrides, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { - this(arrayA, offsetA, arrayB, offsetB, arrayMask, length, dynamicStrides, NONE, null, null, null, runtimeCheckedCPUFeatures, LocationIdentity.ANY_LOCATION); + this(arrayA, offsetA, arrayB, offsetB, arrayMask, length, dynamicStrides, null, null, null, runtimeCheckedCPUFeatures, LocationIdentity.ANY_LOCATION); } public AMD64ArrayRegionEqualsWithMaskNode( @@ -164,17 +150,15 @@ public AMD64ArrayRegionEqualsWithMaskNode( ValueNode arrayMask, ValueNode length, ValueNode dynamicStrides, - JavaKind arrayKind, - JavaKind strideA, - JavaKind strideB, - JavaKind strideMask, + Stride strideA, + Stride strideB, + Stride strideMask, EnumSet runtimeCheckedCPUFeatures, LocationIdentity locationIdentity) { super(TYPE, StampFactory.forKind(JavaKind.Boolean), runtimeCheckedCPUFeatures, locationIdentity); assert validStride(strideA); assert validStride(strideB); assert validStride(strideMask); - this.arrayKind = arrayKind; this.strideA = strideA; this.strideB = strideB; this.strideMask = strideMask; @@ -187,23 +171,19 @@ public AMD64ArrayRegionEqualsWithMaskNode( this.dynamicStrides = dynamicStrides; } - private static boolean validStride(JavaKind stride) { - return stride == null || stride == S1 || stride == S2 || stride == S4; - } - - public JavaKind getArrayKind() { - return arrayKind; + private static boolean validStride(Stride stride) { + return stride == null || stride.value <= 4; } - public JavaKind getStrideA() { + public Stride getStrideA() { return strideA; } - public JavaKind getStrideB() { + public Stride getStrideB() { return strideB; } - public JavaKind getStrideMask() { + public Stride getStrideMask() { return strideMask; } @@ -215,14 +195,6 @@ public ValueNode getLength() { return length; } - private static boolean sameKinds(JavaKind arrayKind, JavaKind strideA, JavaKind strideB, JavaKind strideMask) { - return strideA == arrayKind && strideB == arrayKind && strideMask == arrayKind; - } - - private static LocationIdentity defaultLocationIdentity(JavaKind arrayKind, JavaKind strideA, JavaKind strideB, JavaKind strideMask) { - return !sameKinds(arrayKind, strideA, strideB, strideMask) || arrayKind == NONE ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(arrayKind); - } - public int getDirectStubCallIndex() { return NodeStrideUtil.getDirectStubCallIndex(dynamicStrides, strideA, strideB); } @@ -259,18 +231,16 @@ public void generate(NodeLIRBuilderTool gen) { } @Override - public int getArrayBaseOffset(MetaAccessProvider metaAccess, @SuppressWarnings("unused") ValueNode array, JavaKind elementKind) { - return metaAccess.getArrayBaseOffset(elementKind); + public int getArrayBaseOffset(MetaAccessProvider metaAccess, @SuppressWarnings("unused") ValueNode array, JavaKind arrayKind) { + return metaAccess.getArrayBaseOffset(arrayKind); } private void generateArrayRegionEquals(NodeLIRBuilderTool gen) { final Value result; - MetaAccessProvider metaAccess = gen.getLIRGeneratorTool().getMetaAccess(); - int maskBaseOffset = metaAccess.getArrayBaseOffset(arrayKind == NONE ? JavaKind.Byte : arrayKind); if (getDirectStubCallIndex() < 0) { - result = gen.getLIRGeneratorTool().emitArrayEquals( - 0, 0, maskBaseOffset, - getRuntimeCheckedCPUFeatures(), gen.operand(arrayA), + result = gen.getLIRGeneratorTool().emitArrayEqualsWithMaskDynamicStrides( + getRuntimeCheckedCPUFeatures(), + gen.operand(arrayA), gen.operand(offsetA), gen.operand(arrayB), gen.operand(offsetB), @@ -278,11 +248,10 @@ private void generateArrayRegionEquals(NodeLIRBuilderTool gen) { gen.operand(length), gen.operand(dynamicStrides)); } else { - result = gen.getLIRGeneratorTool().emitArrayEquals( + result = gen.getLIRGeneratorTool().emitArrayEqualsWithMask( NodeStrideUtil.getConstantStrideA(dynamicStrides, strideA), NodeStrideUtil.getConstantStrideB(dynamicStrides, strideB), NodeStrideUtil.getConstantStrideB(dynamicStrides, strideMask), - 0, 0, maskBaseOffset, getRuntimeCheckedCPUFeatures(), gen.operand(arrayA), gen.operand(offsetA), gen.operand(arrayB), @@ -295,17 +264,15 @@ private void generateArrayRegionEquals(NodeLIRBuilderTool gen) { @NodeIntrinsic public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, Object mask, int length, - @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind kindA, - @ConstantNodeParameter JavaKind kindB, - @ConstantNodeParameter JavaKind kindMask); + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, + @ConstantNodeParameter Stride strideMask); @NodeIntrinsic public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, Object mask, int length, - @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind kindA, - @ConstantNodeParameter JavaKind kindB, - @ConstantNodeParameter JavaKind kindMask, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, + @ConstantNodeParameter Stride strideMask, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures); @NodeIntrinsic @@ -320,25 +287,28 @@ public ValueNode canonical(CanonicalizerTool tool) { if (tool.allUsagesAvailable() && hasNoUsages()) { return null; } - if ((dynamicStrides == null || dynamicStrides.isJavaConstant()) && length.isJavaConstant()) { + if ((dynamicStrides == null || dynamicStrides.isJavaConstant()) && length.isJavaConstant() && arrayMask instanceof ComputeObjectAddressNode) { int len = length.asJavaConstant().asInt(); - JavaKind constStrideA = NodeStrideUtil.getConstantStrideA(dynamicStrides, strideA); - JavaKind constStrideB = NodeStrideUtil.getConstantStrideB(dynamicStrides, strideB); - JavaKind constStrideMask = NodeStrideUtil.getConstantStrideB(dynamicStrides, strideMask); - if (len * Math.max(constStrideA.getByteCount(), constStrideB.getByteCount()) < GraalOptions.ArrayRegionEqualsConstantLimit.getValue(tool.getOptions()) && + Stride constStrideA = NodeStrideUtil.getConstantStrideA(dynamicStrides, strideA); + Stride constStrideB = NodeStrideUtil.getConstantStrideB(dynamicStrides, strideB); + Stride constStrideMask = NodeStrideUtil.getConstantStrideB(dynamicStrides, strideMask); + ValueNode arrayMaskNode = ((ComputeObjectAddressNode) arrayMask).getObject(); + ValueNode offsetMaskNode = ((ComputeObjectAddressNode) arrayMask).getOffset(); + if (len * Math.max(constStrideA.value, constStrideB.value) < GraalOptions.ArrayRegionEqualsConstantLimit.getValue(tool.getOptions()) && ConstantReflectionUtil.canFoldReads(tool, arrayA, offsetA, constStrideA, len, this) && ConstantReflectionUtil.canFoldReads(tool, arrayB, offsetB, constStrideB, len, this) && - ConstantReflectionUtil.canFoldReads(tool, arrayMask, null, constStrideMask, len, this)) { + ConstantReflectionUtil.canFoldReads(tool, arrayMaskNode, offsetMaskNode, constStrideMask, len, this)) { Integer startIndexA = ConstantReflectionUtil.startIndex(tool, arrayA, offsetA.asJavaConstant(), constStrideA, this); Integer startIndexB = ConstantReflectionUtil.startIndex(tool, arrayB, offsetB.asJavaConstant(), constStrideB, this); - return ConstantNode.forBoolean(constantFold(tool, arrayA, startIndexA, arrayB, startIndexB, arrayMask, len, constStrideA, constStrideB, constStrideMask)); + Integer startIndexMask = ConstantReflectionUtil.startIndex(tool, arrayMaskNode, offsetMaskNode.asJavaConstant(), constStrideMask, this); + return ConstantNode.forBoolean(constantFold(tool, arrayA, startIndexA, arrayB, startIndexB, arrayMaskNode, startIndexMask, len, constStrideA, constStrideB, constStrideMask)); } } return this; } - private static boolean constantFold(CanonicalizerTool tool, ValueNode a, int startIndexA, ValueNode b, int startIndexB, ValueNode mask, int len, - JavaKind constStrideA, JavaKind constStrideB, JavaKind constStrideMask) { + private static boolean constantFold(CanonicalizerTool tool, ValueNode a, int startIndexA, ValueNode b, int startIndexB, ValueNode mask, int startIndexMask, int len, + Stride constStrideA, Stride constStrideB, Stride constStrideMask) { JavaKind arrayKindA = a.stamp(NodeView.DEFAULT).javaType(tool.getMetaAccess()).getComponentType().getJavaKind(); JavaKind arrayKindB = b.stamp(NodeView.DEFAULT).javaType(tool.getMetaAccess()).getComponentType().getJavaKind(); JavaKind arrayKindM = mask.stamp(NodeView.DEFAULT).javaType(tool.getMetaAccess()).getComponentType().getJavaKind(); @@ -346,7 +316,7 @@ private static boolean constantFold(CanonicalizerTool tool, ValueNode a, int sta for (int i = 0; i < len; i++) { int valueA = ConstantReflectionUtil.readTypePunned(constantReflection, a.asJavaConstant(), arrayKindA, constStrideA, startIndexA + i); int valueB = ConstantReflectionUtil.readTypePunned(constantReflection, b.asJavaConstant(), arrayKindB, constStrideB, startIndexB + i); - int valueM = ConstantReflectionUtil.readTypePunned(constantReflection, mask.asJavaConstant(), arrayKindM, constStrideMask, i); + int valueM = ConstantReflectionUtil.readTypePunned(constantReflection, mask.asJavaConstant(), arrayKindM, constStrideMask, startIndexMask + i); if ((valueA | valueM) != valueB) { return false; } diff --git a/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java b/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java index f00a24748d00..b0ceea7f15f2 100644 --- a/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java +++ b/compiler/src/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java @@ -38,6 +38,7 @@ import java.util.Arrays; import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.calc.Condition; import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.ConstantNode; @@ -273,36 +274,41 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec } private static final class ArrayCompareToPlugin extends InvocationPlugin { - private final JavaKind valueKind; - private final JavaKind otherKind; + private final Stride strideA; + private final Stride strideB; private final boolean swapped; - private ArrayCompareToPlugin(JavaKind valueKind, JavaKind otherKind, boolean swapped, String name, Type... argumentTypes) { + private ArrayCompareToPlugin(Stride strideA, Stride strideB, boolean swapped, String name, Type... argumentTypes) { super(name, argumentTypes); - this.valueKind = valueKind; - this.otherKind = otherKind; + this.strideA = strideA; + this.strideB = strideB; this.swapped = swapped; } - private ArrayCompareToPlugin(JavaKind valueKind, JavaKind otherKind, String name, Type... argumentTypes) { - this(valueKind, otherKind, false, name, argumentTypes); + private ArrayCompareToPlugin(Stride strideA, Stride strideB, String name, Type... argumentTypes) { + this(strideA, strideB, false, name, argumentTypes); } @Override - public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value, ValueNode other) { - ValueNode nonNullValue = b.nullCheckedValue(value); - ValueNode nonNullOther = b.nullCheckedValue(other); - - ValueNode valueLength = b.add(new ArrayLengthNode(nonNullValue)); - ValueNode otherLength = b.add(new ArrayLengthNode(nonNullOther)); - if (swapped) { - /* - * Swapping array arguments because intrinsic expects order to be byte[]/char[] but - * kind arguments stay in original order. - */ - b.addPush(JavaKind.Int, new ArrayCompareToNode(nonNullOther, nonNullValue, otherLength, valueLength, valueKind, otherKind)); - } else { - b.addPush(JavaKind.Int, new ArrayCompareToNode(nonNullValue, nonNullOther, valueLength, otherLength, valueKind, otherKind)); + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arrayA, ValueNode arrayB) { + try (InvocationPluginHelper helper = new InvocationPluginHelper(b, targetMethod)) { + ValueNode nonNullA = b.nullCheckedValue(arrayA); + ValueNode nonNullB = b.nullCheckedValue(arrayB); + + ValueNode lengthA = b.add(new ArrayLengthNode(nonNullA)); + ValueNode lengthB = b.add(new ArrayLengthNode(nonNullB)); + + ValueNode startA = helper.arrayStart(nonNullA, JavaKind.Byte); + ValueNode startB = helper.arrayStart(nonNullB, JavaKind.Byte); + if (swapped) { + /* + * Swapping array arguments because intrinsic expects order to be byte[]/char[] + * but kind arguments stay in original order. + */ + b.addPush(JavaKind.Int, new ArrayCompareToNode(startB, lengthB, startA, lengthA, strideA, strideB)); + } else { + b.addPush(JavaKind.Int, new ArrayCompareToNode(startA, lengthA, startB, lengthB, strideA, strideB)); + } } return true; } @@ -311,8 +317,8 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec private static void registerStringLatin1Plugins(InvocationPlugins plugins, Replacements replacements) { Registration r = new Registration(plugins, "java.lang.StringLatin1", replacements); r.setAllowOverwrite(true); - r.register(new ArrayCompareToPlugin(JavaKind.Byte, JavaKind.Byte, "compareTo", byte[].class, byte[].class)); - r.register(new ArrayCompareToPlugin(JavaKind.Byte, JavaKind.Char, "compareToUTF16", byte[].class, byte[].class)); + r.register(new ArrayCompareToPlugin(Stride.S1, Stride.S1, "compareTo", byte[].class, byte[].class)); + r.register(new ArrayCompareToPlugin(Stride.S1, Stride.S2, "compareToUTF16", byte[].class, byte[].class)); r.register(new InvocationPlugin("inflate", byte[].class, int.class, byte[].class, int.class, int.class) { @SuppressWarnings("try") @Override @@ -402,7 +408,7 @@ public SnippetTemplate.SnippetInfo getSnippet(StringLatin1Snippets.Templates tem r.register(new InvocationPlugin("indexOfChar", byte[].class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value, ValueNode ch, ValueNode fromIndex, ValueNode max) { - b.addPush(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(JavaKind.Byte, JavaKind.Byte, value, max, fromIndex, ch)); + b.addPush(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(b, JavaKind.Byte, Stride.S1, value, max, fromIndex, ch)); return true; } }); @@ -413,8 +419,8 @@ private static void registerStringUTF16Plugins(InvocationPlugins plugins, Replac Registration r = new Registration(plugins, "java.lang.StringUTF16", replacements); r.setAllowOverwrite(true); - r.register(new ArrayCompareToPlugin(JavaKind.Char, JavaKind.Char, "compareTo", byte[].class, byte[].class)); - r.register(new ArrayCompareToPlugin(JavaKind.Char, JavaKind.Byte, true, "compareToLatin1", byte[].class, byte[].class)); + r.register(new ArrayCompareToPlugin(Stride.S2, Stride.S2, "compareTo", byte[].class, byte[].class)); + r.register(new ArrayCompareToPlugin(Stride.S2, Stride.S1, true, "compareToLatin1", byte[].class, byte[].class)); r.register(new InvocationPlugin("compress", byte[].class, int.class, byte[].class, int.class, int.class) { @SuppressWarnings("try") @Override @@ -509,7 +515,7 @@ public SnippetTemplate.SnippetInfo getSnippet(StringUTF16Snippets.Templates temp @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value, ValueNode ch, ValueNode fromIndex, ValueNode max) { ZeroExtendNode toChar = b.add(new ZeroExtendNode(b.add(new NarrowNode(ch, JavaKind.Char.getBitCount())), JavaKind.Int.getBitCount())); - b.addPush(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(JavaKind.Byte, JavaKind.Char, value, max, fromIndex, toChar)); + b.addPush(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(b, JavaKind.Byte, Stride.S2, value, max, fromIndex, toChar)); return true; } }); diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/NodeStrideUtil.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/NodeStrideUtil.java index 73aff2946951..fd18cc36ed2f 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/NodeStrideUtil.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/NodeStrideUtil.java @@ -24,11 +24,10 @@ */ package org.graalvm.compiler.replacements; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.StrideUtil; import org.graalvm.compiler.nodes.ValueNode; -import jdk.vm.ci.meta.JavaKind; - /** * This class provides utility methods for "stride-agnostic" intrinsic nodes such as * {@link org.graalvm.compiler.replacements.nodes.ArrayRegionCompareToNode}. These intrinsics may @@ -45,7 +44,7 @@ public final class NodeStrideUtil { * If the constant stride parameter {@code strideA} is non-null, return it. Otherwise, extract * {@code strideA} from the constant direct stub call index value {@code dynamicStrides}. */ - public static JavaKind getConstantStrideA(ValueNode dynamicStrides, JavaKind strideA) { + public static Stride getConstantStrideA(ValueNode dynamicStrides, Stride strideA) { if (strideA != null) { return strideA; } @@ -56,7 +55,7 @@ public static JavaKind getConstantStrideA(ValueNode dynamicStrides, JavaKind str * If the constant stride parameter {@code strideB} is non-null, return it. Otherwise, extract * {@code strideB} from the constant direct stub call index value {@code dynamicStrides}. */ - public static JavaKind getConstantStrideB(ValueNode dynamicStrides, JavaKind strideB) { + public static Stride getConstantStrideB(ValueNode dynamicStrides, Stride strideB) { if (strideB != null) { return strideB; } @@ -69,9 +68,9 @@ public static JavaKind getConstantStrideB(ValueNode dynamicStrides, JavaKind str * {@code dynamicStrides}, if it is constant. If {@code dynamicStrides} is not constant, return * {@code -1}. */ - public static int getDirectStubCallIndex(ValueNode dynamicStrides, JavaKind strideA, JavaKind strideB) { + public static int getDirectStubCallIndex(ValueNode dynamicStrides, Stride strideA, Stride strideB) { if (strideA != null && strideB != null) { - return StrideUtil.getDirectStubCallIndex(StrideUtil.log2(strideA), StrideUtil.log2(strideB)); + return StrideUtil.getDirectStubCallIndex(strideA.log2, strideB.log2); } if (dynamicStrides != null && dynamicStrides.isJavaConstant()) { return dynamicStrides.asJavaConstant().asInt(); diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java index 84f57e101694..b5824705ffbb 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java @@ -43,6 +43,7 @@ import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.calc.CanonicalCondition; import org.graalvm.compiler.core.common.calc.Condition; import org.graalvm.compiler.core.common.calc.Condition.CanonicalizedCondition; @@ -329,6 +330,7 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec return false; } try (InvocationPluginHelper helper = new InvocationPluginHelper(b, targetMethod)) { + ConstantNode arrayBaseOffset = ConstantNode.forLong(b.getMetaAccess().getArrayBaseOffset(kind), b.getGraph()); helper.emitReturnIf(b.add(new ObjectEqualsNode(arg1, arg2)), b.add(ConstantNode.forBoolean(true)), BranchProbabilityNode.SLOW_PATH_PROBABILITY); GuardingNode nonNullArg1guard = helper.emitReturnIf(IsNullNode.create(arg1), b.add(ConstantNode.forBoolean(false)), BranchProbabilityNode.SLOW_PATH_PROBABILITY); GuardingNode nonNullArg2guard = helper.emitReturnIf(IsNullNode.create(arg2), b.add(ConstantNode.forBoolean(false)), BranchProbabilityNode.SLOW_PATH_PROBABILITY); @@ -339,7 +341,7 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec ValueNode nonNullArg2 = b.add(new PiNode(arg2, stamp2, nonNullArg2guard.asNode())); ValueNode arg2Length = b.add(new ArrayLengthNode(nonNullArg2)); helper.emitReturnIfNot(IntegerEqualsNode.create(arg1Length, arg2Length, NodeView.DEFAULT), b.add(ConstantNode.forBoolean(false)), BranchProbabilityNode.FAST_PATH_PROBABILITY); - helper.emitFinalReturn(JavaKind.Boolean, b.append(new ArrayEqualsNode(nonNullArg1, nonNullArg2, arg1Length, kind))); + helper.emitFinalReturn(JavaKind.Boolean, b.append(new ArrayEqualsNode(nonNullArg1, arrayBaseOffset, nonNullArg2, arrayBaseOffset, arg1Length, kind))); } return true; } @@ -360,6 +362,7 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec try (InvocationPluginHelper helper = new InvocationPluginHelper(b, targetMethod)) { ConstantNode trueValue = ConstantNode.forBoolean(true); ConstantNode falseValue = ConstantNode.forBoolean(false); + ConstantNode arrayBaseOffset = ConstantNode.forLong(b.getMetaAccess().getArrayBaseOffset(JavaKind.Byte), b.getGraph()); // if (this == other) return true ValueNode thisString = receiver.get(); @@ -390,7 +393,7 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec helper.emitReturnIf(IntegerEqualsNode.create(thisLength, ConstantNode.forInt(0), NodeView.DEFAULT), trueValue, BranchProbabilityNode.SLOW_PATH_PROBABILITY); // compare the array bodies - helper.emitFinalReturn(JavaKind.Boolean, b.append(new ArrayEqualsNode(thisValue, thatValue, + helper.emitFinalReturn(JavaKind.Boolean, b.append(new ArrayEqualsNode(thisValue, arrayBaseOffset, thatValue, arrayBaseOffset, thisLength.isConstant() ? thisLength : thatLength, JavaKind.Byte))); } return true; @@ -1996,7 +1999,7 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec LogicNode condition = helper.createCompare(origFromIndex, CanonicalCondition.LT, zero); // fromIndex = max(fromIndex, 0) ValueNode fromIndex = ConditionalNode.create(condition, zero, origFromIndex, NodeView.DEFAULT); - helper.emitFinalReturn(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(JavaKind.Byte, JavaKind.Byte, nonNullValue, length, fromIndex, ch)); + helper.emitFinalReturn(JavaKind.Int, ArrayIndexOfNode.createIndexOfSingle(b, JavaKind.Byte, Stride.S1, nonNullValue, length, fromIndex, ch)); } return true; } diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringLatin1Snippets.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringLatin1Snippets.java index 312e0e87cbe2..5920a8d8a19e 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringLatin1Snippets.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringLatin1Snippets.java @@ -28,13 +28,13 @@ import static org.graalvm.compiler.api.directives.GraalDirectives.LIKELY_PROBABILITY; import static org.graalvm.compiler.api.directives.GraalDirectives.UNLIKELY_PROBABILITY; import static org.graalvm.compiler.api.directives.GraalDirectives.injectBranchProbability; -import static org.graalvm.compiler.core.common.StrideUtil.S1; import static org.graalvm.compiler.replacements.ReplacementsUtil.byteArrayBaseOffset; import static org.graalvm.compiler.replacements.ReplacementsUtil.byteArrayIndexScale; import static org.graalvm.compiler.replacements.StringHelperIntrinsics.getByte; import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; import org.graalvm.compiler.replacements.nodes.ArrayIndexOfNode; @@ -73,7 +73,7 @@ public static int indexOf(byte[] source, int sourceCount, byte[] target, int tar return -1; } if (injectBranchProbability(UNLIKELY_PROBABILITY, targetCount == 1)) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, source, 0, sourceCount, fromIndex, Byte.toUnsignedInt(getByte(target, 0))); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Byte, Stride.S1, false, false, source, byteArrayOffset(0), sourceCount, fromIndex, Byte.toUnsignedInt(getByte(target, 0))); } else { int haystackLength = sourceCount - (targetCount - 2); int offset = fromIndex; @@ -81,7 +81,8 @@ public static int indexOf(byte[] source, int sourceCount, byte[] target, int tar byte b1 = getByte(target, 0); byte b2 = getByte(target, 1); do { - int indexOfResult = ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, true, false, source, 0, haystackLength, offset, Byte.toUnsignedInt(b1), Byte.toUnsignedInt(b2)); + int indexOfResult = ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Byte, Stride.S1, true, false, source, byteArrayOffset(0), haystackLength, offset, Byte.toUnsignedInt(b1), + Byte.toUnsignedInt(b2)); if (injectBranchProbability(UNLIKELY_PROBABILITY, indexOfResult < 0)) { return -1; } @@ -90,7 +91,7 @@ public static int indexOf(byte[] source, int sourceCount, byte[] target, int tar return offset; } else { if (injectBranchProbability(UNLIKELY_PROBABILITY, - ArrayRegionEqualsNode.regionEquals(source, byteArrayOffset(offset), target, byteArrayOffset(0), targetCount, JavaKind.Byte))) { + ArrayRegionEqualsNode.regionEquals(source, byteArrayOffset(offset), target, byteArrayOffset(0), targetCount, JavaKind.Byte, Stride.S1, Stride.S1))) { return offset; } } diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringUTF16Snippets.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringUTF16Snippets.java index 4bcfe19e8725..783514fc563f 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringUTF16Snippets.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringUTF16Snippets.java @@ -28,14 +28,13 @@ import static org.graalvm.compiler.api.directives.GraalDirectives.LIKELY_PROBABILITY; import static org.graalvm.compiler.api.directives.GraalDirectives.UNLIKELY_PROBABILITY; import static org.graalvm.compiler.api.directives.GraalDirectives.injectBranchProbability; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; import static org.graalvm.compiler.replacements.ReplacementsUtil.byteArrayBaseOffset; import static org.graalvm.compiler.replacements.ReplacementsUtil.charArrayIndexScale; import static org.graalvm.compiler.replacements.StringHelperIntrinsics.getByte; import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.nodes.extended.JavaReadNode; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; import org.graalvm.compiler.options.OptionValues; @@ -85,12 +84,13 @@ public static int indexOfUnsafe(byte[] source, int sourceCount, byte[] target, i ReplacementsUtil.dynamicAssert(targetCount <= length(target), "StringUTF16.indexOfUnsafe invalid args: targetCount > length(target)"); ReplacementsUtil.dynamicAssert(sourceCount >= targetCount, "StringUTF16.indexOfUnsafe invalid args: sourceCount < targetCount"); if (targetCount == 1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, source, 0, sourceCount, fromIndex, getChar(target, 0)); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Byte, Stride.S2, false, false, source, byteArrayCharOffset(0), sourceCount, fromIndex, getChar(target, 0)); } else { int haystackLength = sourceCount - (targetCount - 2); int offset = fromIndex; while (injectBranchProbability(LIKELY_PROBABILITY, offset < haystackLength)) { - int indexOfResult = ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, true, false, source, 0, haystackLength, offset, getChar(target, 0), getChar(target, 1)); + int indexOfResult = ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Byte, Stride.S2, true, false, source, byteArrayCharOffset(0), haystackLength, offset, getChar(target, 0), + getChar(target, 1)); if (injectBranchProbability(UNLIKELY_PROBABILITY, indexOfResult < 0)) { return -1; } @@ -99,7 +99,7 @@ public static int indexOfUnsafe(byte[] source, int sourceCount, byte[] target, i return offset; } else { if (injectBranchProbability(UNLIKELY_PROBABILITY, - ArrayRegionEqualsNode.regionEquals(source, byteArrayCharOffset(offset), target, byteArrayCharOffset(0), targetCount, JavaKind.Char))) { + ArrayRegionEqualsNode.regionEquals(source, byteArrayCharOffset(offset), target, byteArrayCharOffset(0), targetCount, JavaKind.Byte, Stride.S2, Stride.S2))) { return offset; } } @@ -116,7 +116,8 @@ public static int indexOfLatin1Unsafe(byte[] source, int sourceCount, byte[] tar ReplacementsUtil.dynamicAssert(targetCount <= target.length, "StringUTF16.indexOfLatin1Unsafe invalid args: targetCount > length(target)"); ReplacementsUtil.dynamicAssert(sourceCount >= targetCount, "StringUTF16.indexOfLatin1Unsafe invalid args: sourceCount < targetCount"); if (targetCount == 1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, source, 0, sourceCount, fromIndex, (char) Byte.toUnsignedInt(getByte(target, 0))); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Byte, Stride.S2, false, false, source, byteArrayCharOffset(0), sourceCount, fromIndex, + (char) Byte.toUnsignedInt(getByte(target, 0))); } else { int haystackLength = sourceCount - (targetCount - 2); int offset = fromIndex; @@ -124,7 +125,7 @@ public static int indexOfLatin1Unsafe(byte[] source, int sourceCount, byte[] tar char c1 = (char) Byte.toUnsignedInt(getByte(target, 0)); char c2 = (char) Byte.toUnsignedInt(getByte(target, 1)); do { - int indexOfResult = ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, true, false, source, 0, haystackLength, offset, c1, c2); + int indexOfResult = ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Byte, Stride.S2, true, false, source, byteArrayCharOffset(0), haystackLength, offset, c1, c2); if (injectBranchProbability(UNLIKELY_PROBABILITY, indexOfResult < 0)) { return -1; } @@ -133,7 +134,7 @@ public static int indexOfLatin1Unsafe(byte[] source, int sourceCount, byte[] tar return offset; } else { if (injectBranchProbability(UNLIKELY_PROBABILITY, - ArrayRegionEqualsNode.regionEquals(source, byteArrayCharOffset(offset), target, byteArrayCharOffset(0), targetCount, JavaKind.Char, JavaKind.Byte))) { + ArrayRegionEqualsNode.regionEquals(source, byteArrayCharOffset(offset), target, byteArrayCharOffset(0), targetCount, JavaKind.Byte, Stride.S2, Stride.S1))) { return offset; } } diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToForeignCalls.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToForeignCalls.java index 97bda709493e..822a7ad4ebc6 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToForeignCalls.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToForeignCalls.java @@ -24,12 +24,11 @@ */ package org.graalvm.compiler.replacements.nodes; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.debug.GraalError; import org.graalvm.word.Pointer; -import jdk.vm.ci.meta.JavaKind; - public final class ArrayCompareToForeignCalls { private static final ForeignCallDescriptor STUB_BYTE_ARRAY_COMPARE_TO_BYTE_ARRAY = foreignCallDescriptor("byteArrayCompareToByteArray"); private static final ForeignCallDescriptor STUB_BYTE_ARRAY_COMPARE_TO_CHAR_ARRAY = foreignCallDescriptor("byteArrayCompareToCharArray"); @@ -43,22 +42,22 @@ public final class ArrayCompareToForeignCalls { STUB_CHAR_ARRAY_COMPARE_TO_CHAR_ARRAY}; private static ForeignCallDescriptor foreignCallDescriptor(String name) { - return ForeignCalls.pureFunctionForeignCallDescriptor(name, int.class, Pointer.class, Pointer.class, int.class, int.class); + return ForeignCalls.pureFunctionForeignCallDescriptor(name, int.class, Pointer.class, int.class, Pointer.class, int.class); } public static ForeignCallDescriptor getStub(ArrayCompareToNode arrayCompareToNode) { - JavaKind kind1 = arrayCompareToNode.getKind1(); - JavaKind kind2 = arrayCompareToNode.getKind2(); - if (kind1 == JavaKind.Byte) { - if (kind2 == JavaKind.Byte) { + Stride strideA = arrayCompareToNode.getStrideA(); + Stride strideB = arrayCompareToNode.getStrideB(); + if (strideA == Stride.S1) { + if (strideB == Stride.S1) { return STUB_BYTE_ARRAY_COMPARE_TO_BYTE_ARRAY; - } else if (kind2 == JavaKind.Char) { + } else if (strideB == Stride.S2) { return STUB_BYTE_ARRAY_COMPARE_TO_CHAR_ARRAY; } - } else if (kind1 == JavaKind.Char) { - if (kind2 == JavaKind.Byte) { + } else if (strideA == Stride.S2) { + if (strideB == Stride.S1) { return STUB_CHAR_ARRAY_COMPARE_TO_BYTE_ARRAY; - } else if (kind2 == JavaKind.Char) { + } else if (strideB == Stride.S2) { return STUB_CHAR_ARRAY_COMPARE_TO_CHAR_ARRAY; } } diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java index 7797656c6103..03e32213dfff 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java @@ -30,6 +30,7 @@ import java.util.EnumSet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.graph.Node; @@ -45,10 +46,8 @@ import org.graalvm.compiler.nodes.spi.Virtualizable; import org.graalvm.compiler.nodes.spi.VirtualizerTool; import org.graalvm.compiler.nodes.util.GraphUtil; -import org.graalvm.word.LocationIdentity; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.Value; // JaCoCo Exclude @@ -61,54 +60,54 @@ public class ArrayCompareToNode extends PureFunctionStubIntrinsicNode implements public static final NodeClass TYPE = NodeClass.create(ArrayCompareToNode.class); - /** {@link JavaKind} of one array to compare. */ - protected final JavaKind kind1; + /** {@link Stride} of one array to compare. */ + protected final Stride strideA; - /** {@link JavaKind} of the other array to compare. */ - protected final JavaKind kind2; + /** {@link Stride} of the other array to compare. */ + protected final Stride strideB; /** One array to be tested for equality. */ - @Input protected ValueNode array1; + @Input protected ValueNode arrayA; - /** The other array to be tested for equality. */ - @Input protected ValueNode array2; + /** Length of array A. */ + @Input protected ValueNode lengthA; - /** Length of one array. */ - @Input protected ValueNode length1; + /** The other array to be tested for equality. */ + @Input protected ValueNode arrayB; - /** Length of the other array. */ - @Input protected ValueNode length2; + /** Length of array B. */ + @Input protected ValueNode lengthB; - public ArrayCompareToNode(ValueNode array1, ValueNode array2, ValueNode length1, ValueNode length2, - @ConstantNodeParameter JavaKind kind1, - @ConstantNodeParameter JavaKind kind2) { - this(TYPE, array1, array2, length1, length2, kind1, kind2); + public ArrayCompareToNode(ValueNode arrayA, ValueNode lengthA, ValueNode arrayB, ValueNode lengthB, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB) { + this(TYPE, arrayA, lengthA, arrayB, lengthB, strideA, strideB); } - public ArrayCompareToNode(ValueNode array1, ValueNode array2, ValueNode length1, ValueNode length2, - @ConstantNodeParameter JavaKind kind1, - @ConstantNodeParameter JavaKind kind2, + public ArrayCompareToNode(ValueNode arrayA, ValueNode lengthA, ValueNode arrayB, ValueNode lengthB, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { - this(TYPE, array1, array2, length1, length2, kind1, kind2, runtimeCheckedCPUFeatures); + this(TYPE, arrayA, lengthA, arrayB, lengthB, strideA, strideB, runtimeCheckedCPUFeatures); } - protected ArrayCompareToNode(NodeClass c, ValueNode array1, ValueNode array2, ValueNode length1, ValueNode length2, - @ConstantNodeParameter JavaKind kind1, - @ConstantNodeParameter JavaKind kind2) { - this(c, array1, array2, length1, length2, kind1, kind2, null); + protected ArrayCompareToNode(NodeClass c, ValueNode arrayA, ValueNode lengthA, ValueNode arrayB, ValueNode lengthB, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB) { + this(c, arrayA, lengthA, arrayB, lengthB, strideA, strideB, null); } - protected ArrayCompareToNode(NodeClass c, ValueNode array1, ValueNode array2, ValueNode length1, ValueNode length2, - @ConstantNodeParameter JavaKind kind1, - @ConstantNodeParameter JavaKind kind2, + protected ArrayCompareToNode(NodeClass c, ValueNode arrayA, ValueNode lengthA, ValueNode arrayB, ValueNode lengthB, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { - super(c, StampFactory.forKind(JavaKind.Int), runtimeCheckedCPUFeatures, kind1 != kind2 ? LocationIdentity.ANY_LOCATION : NamedLocationIdentity.getArrayLocation(kind1)); - this.kind1 = kind1; - this.kind2 = kind2; - this.array1 = array1; - this.array2 = array2; - this.length1 = length1; - this.length2 = length2; + super(c, StampFactory.forKind(JavaKind.Int), runtimeCheckedCPUFeatures, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); + this.strideA = strideA; + this.strideB = strideB; + this.arrayA = arrayA; + this.lengthA = lengthA; + this.arrayB = arrayB; + this.lengthB = lengthB; } @Override @@ -116,8 +115,8 @@ public Node canonical(CanonicalizerTool tool) { if (tool.allUsagesAvailable() && hasNoUsages()) { return null; } - ValueNode a1 = GraphUtil.unproxify(array1); - ValueNode a2 = GraphUtil.unproxify(array2); + ValueNode a1 = GraphUtil.unproxify(arrayA); + ValueNode a2 = GraphUtil.unproxify(arrayB); if (a1 == a2) { return ConstantNode.forInt(0); } @@ -126,8 +125,8 @@ public Node canonical(CanonicalizerTool tool) { @Override public void virtualize(VirtualizerTool tool) { - ValueNode alias1 = tool.getAlias(array1); - ValueNode alias2 = tool.getAlias(array2); + ValueNode alias1 = tool.getAlias(arrayA); + ValueNode alias2 = tool.getAlias(arrayB); if (alias1 == alias2) { // the same virtual objects will always have the same contents tool.replaceWithValue(ConstantNode.forInt(0, graph())); @@ -135,38 +134,38 @@ public void virtualize(VirtualizerTool tool) { } @NodeIntrinsic - public static native int compareTo(Object array1, Object array2, int length1, int length2, - @ConstantNodeParameter JavaKind kind1, - @ConstantNodeParameter JavaKind kind2); + public static native int compareTo(Object arrayA, int lengthA, Object arrayB, int lengthB, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB); @NodeIntrinsic - public static native int compareTo(Object array1, Object array2, int length1, int length2, - @ConstantNodeParameter JavaKind kind1, - @ConstantNodeParameter JavaKind kind2, + public static native int compareTo(Object arrayA, int lengthA, Object arrayB, int lengthB, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures); - public JavaKind getKind1() { - return kind1; + public Stride getStrideA() { + return strideA; } - public JavaKind getKind2() { - return kind2; + public Stride getStrideB() { + return strideB; } - public ValueNode getArray1() { - return array1; + public ValueNode getArrayA() { + return arrayA; } - public ValueNode getArray2() { - return array2; + public ValueNode getLengthA() { + return lengthA; } - public ValueNode getLength1() { - return length1; + public ValueNode getArrayB() { + return arrayB; } - public ValueNode getLength2() { - return length2; + public ValueNode getLengthB() { + return lengthB; } @Override @@ -174,7 +173,8 @@ public void generate(NodeLIRBuilderTool gen) { if (UseGraalStubs.getValue(graph().getOptions())) { ForeignCallLinkage linkage = gen.lookupGraalStub(this); if (linkage != null) { - Value result = gen.getLIRGeneratorTool().emitForeignCall(linkage, null, gen.operand(array1), gen.operand(array2), gen.operand(length1), gen.operand(length2)); + Value result = gen.getLIRGeneratorTool().emitForeignCall(linkage, null, + gen.operand(arrayA), gen.operand(lengthA), gen.operand(arrayB), gen.operand(lengthB)); gen.setResult(this, result); return; } @@ -183,11 +183,8 @@ public void generate(NodeLIRBuilderTool gen) { } protected void generateArrayCompareTo(NodeLIRBuilderTool gen) { - MetaAccessProvider metaAccess = gen.getLIRGeneratorTool().getMetaAccess(); - int array1BaseOffset = metaAccess.getArrayBaseOffset(kind1); - int array2BaseOffset = metaAccess.getArrayBaseOffset(kind2); - Value result = gen.getLIRGeneratorTool().emitArrayCompareTo(kind1, kind2, array1BaseOffset, array2BaseOffset, getRuntimeCheckedCPUFeatures(), - gen.operand(array1), gen.operand(array2), gen.operand(length1), gen.operand(length2)); + Value result = gen.getLIRGeneratorTool().emitArrayCompareTo(strideA, strideB, getRuntimeCheckedCPUFeatures(), + gen.operand(arrayA), gen.operand(lengthA), gen.operand(arrayB), gen.operand(lengthB)); gen.setResult(this, result); } } diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCopyWithConversionsForeignCalls.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCopyWithConversionsForeignCalls.java index e3a519451626..402c55a58d6a 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCopyWithConversionsForeignCalls.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCopyWithConversionsForeignCalls.java @@ -26,18 +26,17 @@ import static org.graalvm.compiler.replacements.nodes.ArrayCopyWithConversionsNode.KILLED_LOCATIONS; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.replacements.NodeStrideUtil; -import jdk.vm.ci.meta.JavaKind; - public final class ArrayCopyWithConversionsForeignCalls { private static final ForeignCallDescriptor STUB_DYNAMIC_STRIDES = foreignCallDescriptor( "arrayCopyWithConversionsDynamicStrides", Object.class, long.class, Object.class, long.class, int.class, int.class); /** * CAUTION: the ordering here is important: entries 0-9 must match the indices generated by - * {@link NodeStrideUtil#getDirectStubCallIndex(ValueNode, JavaKind, JavaKind)}. + * {@link NodeStrideUtil#getDirectStubCallIndex(ValueNode, Stride, Stride)}. * * @see #getStub(ArrayCopyWithConversionsNode) */ diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCopyWithConversionsNode.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCopyWithConversionsNode.java index a1aebddaaced..cee5e929e028 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCopyWithConversionsNode.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCopyWithConversionsNode.java @@ -29,6 +29,7 @@ import java.util.EnumSet; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.graph.NodeClass; @@ -61,8 +62,8 @@ public class ArrayCopyWithConversionsNode extends AbstractMemoryCheckpoint imple public static final LocationIdentity[] KILLED_LOCATIONS = {NamedLocationIdentity.getArrayLocation(JavaKind.Byte), NamedLocationIdentity.OFF_HEAP_LOCATION}; - private final JavaKind strideSrc; - private final JavaKind strideDst; + private final Stride strideSrc; + private final Stride strideDst; protected final EnumSet runtimeCheckedCPUFeatures; @Input protected ValueNode arraySrc; @Input protected ValueNode offsetSrc; @@ -94,14 +95,14 @@ public class ArrayCopyWithConversionsNode extends AbstractMemoryCheckpoint imple * @param length length of the region to copy, scaled to strideDst. */ public ArrayCopyWithConversionsNode(ValueNode arraySrc, ValueNode offsetSrc, ValueNode arrayDst, ValueNode offsetDst, ValueNode length, - @ConstantNodeParameter JavaKind strideSrc, - @ConstantNodeParameter JavaKind strideDst) { + @ConstantNodeParameter Stride strideSrc, + @ConstantNodeParameter Stride strideDst) { this(TYPE, arraySrc, offsetSrc, arrayDst, offsetDst, length, null, strideSrc, strideDst, null); } public ArrayCopyWithConversionsNode(ValueNode arraySrc, ValueNode offsetSrc, ValueNode arrayDst, ValueNode offsetDst, ValueNode length, - @ConstantNodeParameter JavaKind strideSrc, - @ConstantNodeParameter JavaKind strideDst, + @ConstantNodeParameter Stride strideSrc, + @ConstantNodeParameter Stride strideDst, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { this(TYPE, arraySrc, offsetSrc, arrayDst, offsetDst, length, null, strideSrc, strideDst, runtimeCheckedCPUFeatures); } @@ -121,8 +122,8 @@ public ArrayCopyWithConversionsNode(ValueNode arraySrc, ValueNode offsetSrc, Val protected ArrayCopyWithConversionsNode(NodeClass c, ValueNode arraySrc, ValueNode offsetSrc, ValueNode arrayDst, ValueNode offsetDst, ValueNode length, ValueNode dynamicStrides, - @ConstantNodeParameter JavaKind strideSrc, - @ConstantNodeParameter JavaKind strideDst, + @ConstantNodeParameter Stride strideSrc, + @ConstantNodeParameter Stride strideDst, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { super(c, StampFactory.forKind(JavaKind.Void)); this.strideSrc = strideSrc; @@ -138,13 +139,13 @@ protected ArrayCopyWithConversionsNode(NodeClass runtimeCheckedCPUFeatures); @NodeIntrinsic diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayEqualsForeignCalls.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayEqualsForeignCalls.java index 34503bfae32c..2d6b0dbcc170 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayEqualsForeignCalls.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayEqualsForeignCalls.java @@ -24,6 +24,8 @@ */ package org.graalvm.compiler.replacements.nodes; +import org.graalvm.compiler.core.common.Stride; +import org.graalvm.compiler.core.common.StrideUtil; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.nodes.ValueNode; @@ -33,11 +35,6 @@ import jdk.vm.ci.meta.JavaKind; public final class ArrayEqualsForeignCalls { - private static final ForeignCallDescriptor STUB_BOOLEAN_ARRAY_EQUALS = foreignCallDescriptorPointers("booleanArraysEquals"); - private static final ForeignCallDescriptor STUB_BYTE_ARRAY_EQUALS = foreignCallDescriptorPointers("byteArraysEquals"); - private static final ForeignCallDescriptor STUB_CHAR_ARRAY_EQUALS = foreignCallDescriptorPointers("charArraysEquals"); - private static final ForeignCallDescriptor STUB_SHORT_ARRAY_EQUALS = foreignCallDescriptorPointers("shortArraysEquals"); - private static final ForeignCallDescriptor STUB_INT_ARRAY_EQUALS = foreignCallDescriptorPointers("intArraysEquals"); private static final ForeignCallDescriptor STUB_LONG_ARRAY_EQUALS = foreignCallDescriptorPointers("longArraysEquals"); private static final ForeignCallDescriptor STUB_FLOAT_ARRAY_EQUALS = foreignCallDescriptorPointers("floatArraysEquals"); private static final ForeignCallDescriptor STUB_DOUBLE_ARRAY_EQUALS = foreignCallDescriptorPointers("doubleArraysEquals"); @@ -45,7 +42,7 @@ public final class ArrayEqualsForeignCalls { "arrayRegionEqualsDynamicStrides", boolean.class, Object.class, long.class, Object.class, long.class, int.class, int.class); /** * CAUTION: the ordering here is important: entries 0-9 must match the indices generated by - * {@link NodeStrideUtil#getDirectStubCallIndex(ValueNode, JavaKind, JavaKind)}. + * {@link NodeStrideUtil#getDirectStubCallIndex(ValueNode, Stride, Stride)}. * * @see #getRegionEqualsStub(ArrayRegionEqualsNode) */ @@ -62,11 +59,6 @@ public final class ArrayEqualsForeignCalls { STUB_REGION_EQUALS_DYNAMIC_STRIDES, - STUB_BOOLEAN_ARRAY_EQUALS, - STUB_BYTE_ARRAY_EQUALS, - STUB_CHAR_ARRAY_EQUALS, - STUB_SHORT_ARRAY_EQUALS, - STUB_INT_ARRAY_EQUALS, STUB_LONG_ARRAY_EQUALS, STUB_FLOAT_ARRAY_EQUALS, STUB_DOUBLE_ARRAY_EQUALS, @@ -77,22 +69,18 @@ private static ForeignCallDescriptor foreignCallDescriptor(String name) { } private static ForeignCallDescriptor foreignCallDescriptorPointers(String name) { - return ForeignCalls.pureFunctionForeignCallDescriptor(name, boolean.class, Pointer.class, Pointer.class, int.class); + return ForeignCalls.pureFunctionForeignCallDescriptor(name, boolean.class, Pointer.class, long.class, Pointer.class, long.class, int.class); } public static ForeignCallDescriptor getArrayEqualsStub(ArrayEqualsNode arrayEqualsNode) { - JavaKind stride = arrayEqualsNode.getKind(); - switch (stride) { + JavaKind kind = arrayEqualsNode.getKind(); + switch (kind) { case Boolean: - return STUB_BOOLEAN_ARRAY_EQUALS; case Byte: - return STUB_BYTE_ARRAY_EQUALS; case Char: - return STUB_CHAR_ARRAY_EQUALS; case Short: - return STUB_SHORT_ARRAY_EQUALS; case Int: - return STUB_INT_ARRAY_EQUALS; + return STUBS[StrideUtil.getDirectStubCallIndex(Stride.fromJavaKind(kind).log2, Stride.fromJavaKind(kind).log2)]; case Long: return STUB_LONG_ARRAY_EQUALS; case Float: diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayEqualsNode.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayEqualsNode.java index 9fd1aa704c57..d31a0a0f5268 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayEqualsNode.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayEqualsNode.java @@ -31,6 +31,7 @@ import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; import org.graalvm.compiler.core.common.type.StampFactory; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.nodeinfo.NodeCycles; @@ -75,30 +76,38 @@ public class ArrayEqualsNode extends PureFunctionStubIntrinsicNode implements LI /** One array to be tested for equality. */ @Input protected ValueNode array1; + /** array base offset of array1. */ + @Input protected ValueNode offset1; + /** The other array to be tested for equality. */ @Input protected ValueNode array2; + /** array base offset of array2. */ + @Input protected ValueNode offset2; + /** Length of both arrays. */ @Input protected ValueNode length; - public ArrayEqualsNode(ValueNode array1, ValueNode array2, ValueNode length, + public ArrayEqualsNode(ValueNode array1, ValueNode offset1, ValueNode array2, ValueNode offset2, ValueNode length, @ConstantNodeParameter JavaKind kind) { - this(TYPE, array1, array2, length, kind, null); + this(TYPE, array1, offset1, array2, offset2, length, kind, null); } - public ArrayEqualsNode(ValueNode array1, ValueNode array2, ValueNode length, + public ArrayEqualsNode(ValueNode array1, ValueNode offset1, ValueNode array2, ValueNode offset2, ValueNode length, @ConstantNodeParameter JavaKind kind, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { - this(TYPE, array1, array2, length, kind, runtimeCheckedCPUFeatures); + this(TYPE, array1, offset1, array2, offset2, length, kind, runtimeCheckedCPUFeatures); } - protected ArrayEqualsNode(NodeClass c, ValueNode array1, ValueNode array2, ValueNode length, + protected ArrayEqualsNode(NodeClass c, ValueNode array1, ValueNode offset1, ValueNode array2, ValueNode offset2, ValueNode length, @ConstantNodeParameter JavaKind kind, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { super(c, StampFactory.forKind(JavaKind.Boolean), runtimeCheckedCPUFeatures, NamedLocationIdentity.getArrayLocation(kind)); this.kind = kind; this.array1 = array1; + this.offset1 = offset1; this.array2 = array2; + this.offset2 = offset2; this.length = length; } @@ -128,7 +137,9 @@ public Node canonical(CanonicalizerTool tool) { if (a1 == a2) { return ConstantNode.forBoolean(true); } - if (a1.isConstant() && a2.isConstant() && length.isConstant()) { + if (a1.isConstant() && offset1.isConstant() && a2.isConstant() && offset2.isConstant() && length.isConstant()) { + GraalError.guarantee(offset1.asJavaConstant().asLong() == tool.getMetaAccess().getArrayBaseOffset(kind), "offset must be exactly the array base offset"); + GraalError.guarantee(offset2.asJavaConstant().asLong() == tool.getMetaAccess().getArrayBaseOffset(kind), "offset must be exactly the array base offset"); ConstantNode c1 = (ConstantNode) a1; ConstantNode c2 = (ConstantNode) a2; if (c1.getStableDimension() >= 1 && c2.getStableDimension() >= 1) { @@ -199,11 +210,11 @@ public void virtualize(VirtualizerTool tool) { } @NodeIntrinsic - public static native boolean equals(Object array1, Object array2, int length, + public static native boolean equals(Object array1, long offset1, Object array2, long offset2, int length, @ConstantNodeParameter JavaKind kind); @NodeIntrinsic - public static native boolean equals(Object array1, Object array2, int length, + public static native boolean equals(Object array1, long offset1, Object array2, long offset2, int length, @ConstantNodeParameter JavaKind kind, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures); @@ -228,7 +239,7 @@ public void generate(NodeLIRBuilderTool gen) { if (UseGraalStubs.getValue(graph().getOptions())) { ForeignCallLinkage linkage = gen.lookupGraalStub(this); if (linkage != null) { - Value result = gen.getLIRGeneratorTool().emitForeignCall(linkage, null, gen.operand(array1), gen.operand(array2), gen.operand(length)); + Value result = gen.getLIRGeneratorTool().emitForeignCall(linkage, null, gen.operand(array1), gen.operand(offset1), gen.operand(array2), gen.operand(offset2), gen.operand(length)); gen.setResult(this, result); return; } @@ -237,9 +248,7 @@ public void generate(NodeLIRBuilderTool gen) { } protected void generateArrayEquals(NodeLIRBuilderTool gen) { - int array1BaseOffset = gen.getLIRGeneratorTool().getMetaAccess().getArrayBaseOffset(kind); - int array2BaseOffset = gen.getLIRGeneratorTool().getMetaAccess().getArrayBaseOffset(kind); - Value result = gen.getLIRGeneratorTool().emitArrayEquals(kind, array1BaseOffset, array2BaseOffset, getRuntimeCheckedCPUFeatures(), gen.operand(array1), gen.operand(array2), + Value result = gen.getLIRGeneratorTool().emitArrayEquals(kind, getRuntimeCheckedCPUFeatures(), gen.operand(array1), gen.operand(offset1), gen.operand(array2), gen.operand(offset2), gen.operand(length)); gen.setResult(this, result); } diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayIndexOfForeignCalls.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayIndexOfForeignCalls.java index 6eec882a4947..4df3144f3172 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayIndexOfForeignCalls.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayIndexOfForeignCalls.java @@ -24,24 +24,18 @@ */ package org.graalvm.compiler.replacements.nodes; -import static org.graalvm.compiler.core.common.StrideUtil.NONE; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - import java.util.Arrays; import java.util.stream.Stream; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.debug.GraalError; -import jdk.vm.ci.meta.JavaKind; - public class ArrayIndexOfForeignCalls { - private static ForeignCallDescriptor foreignCallDescriptor(String name, Class arrayArgType, int nValues) { + private static ForeignCallDescriptor foreignCallDescriptor(String name, int nValues) { Class[] argTypes = new Class[4 + nValues]; - argTypes[0] = arrayArgType; + argTypes[0] = Object.class; argTypes[1] = long.class; for (int i = 2; i < argTypes.length; i++) { argTypes[i] = int.class; @@ -49,71 +43,29 @@ private static ForeignCallDescriptor foreignCallDescriptor(String name, Class return ForeignCalls.pureFunctionForeignCallDescriptor(name, int.class, argTypes); } - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_B_S1 = foreignCallDescriptor("indexOfTwoConsecutiveBS1", byte[].class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_B_S2 = foreignCallDescriptor("indexOfTwoConsecutiveBS2", byte[].class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_C_S2 = foreignCallDescriptor("indexOfTwoConsecutiveCS2", char[].class, 2); - - public static final ForeignCallDescriptor STUB_INDEX_OF_B_1_S1 = foreignCallDescriptor("indexOfB1S1", byte[].class, 1); - public static final ForeignCallDescriptor STUB_INDEX_OF_B_2_S1 = foreignCallDescriptor("indexOfB2S1", byte[].class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_B_3_S1 = foreignCallDescriptor("indexOfB3S1", byte[].class, 3); - public static final ForeignCallDescriptor STUB_INDEX_OF_B_4_S1 = foreignCallDescriptor("indexOfB4S1", byte[].class, 4); - public static final ForeignCallDescriptor STUB_INDEX_OF_B_1_S2 = foreignCallDescriptor("indexOfB1S2", byte[].class, 1); - public static final ForeignCallDescriptor STUB_INDEX_OF_B_2_S2 = foreignCallDescriptor("indexOfB2S2", byte[].class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_B_3_S2 = foreignCallDescriptor("indexOfB3S2", byte[].class, 3); - public static final ForeignCallDescriptor STUB_INDEX_OF_B_4_S2 = foreignCallDescriptor("indexOfB4S2", byte[].class, 4); - - public static final ForeignCallDescriptor STUB_INDEX_OF_C_1_S2 = foreignCallDescriptor("indexOfC1S2", char[].class, 1); - public static final ForeignCallDescriptor STUB_INDEX_OF_C_2_S2 = foreignCallDescriptor("indexOfC2S2", char[].class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_C_3_S2 = foreignCallDescriptor("indexOfC3S2", char[].class, 3); - public static final ForeignCallDescriptor STUB_INDEX_OF_C_4_S2 = foreignCallDescriptor("indexOfC4S2", char[].class, 4); - - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_S1 = foreignCallDescriptor("indexOfTwoConsecutiveS1", Object.class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_S2 = foreignCallDescriptor("indexOfTwoConsecutiveS2", Object.class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_S4 = foreignCallDescriptor("indexOfTwoConsecutiveS4", Object.class, 2); - - public static final ForeignCallDescriptor STUB_INDEX_OF_1_S1 = foreignCallDescriptor("indexOf1S1", Object.class, 1); - public static final ForeignCallDescriptor STUB_INDEX_OF_2_S1 = foreignCallDescriptor("indexOf2S1", Object.class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_3_S1 = foreignCallDescriptor("indexOf3S1", Object.class, 3); - public static final ForeignCallDescriptor STUB_INDEX_OF_4_S1 = foreignCallDescriptor("indexOf4S1", Object.class, 4); - public static final ForeignCallDescriptor STUB_INDEX_OF_1_S2 = foreignCallDescriptor("indexOf1S2", Object.class, 1); - public static final ForeignCallDescriptor STUB_INDEX_OF_2_S2 = foreignCallDescriptor("indexOf2S2", Object.class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_3_S2 = foreignCallDescriptor("indexOf3S2", Object.class, 3); - public static final ForeignCallDescriptor STUB_INDEX_OF_4_S2 = foreignCallDescriptor("indexOf4S2", Object.class, 4); - public static final ForeignCallDescriptor STUB_INDEX_OF_1_S4 = foreignCallDescriptor("indexOf1S4", Object.class, 1); - public static final ForeignCallDescriptor STUB_INDEX_OF_2_S4 = foreignCallDescriptor("indexOf2S4", Object.class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_3_S4 = foreignCallDescriptor("indexOf3S4", Object.class, 3); - public static final ForeignCallDescriptor STUB_INDEX_OF_4_S4 = foreignCallDescriptor("indexOf4S4", Object.class, 4); - - public static final ForeignCallDescriptor STUB_INDEX_OF_WITH_MASK_B_S1 = foreignCallDescriptor("indexOfWithMaskBS1", byte[].class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_WITH_MASK_B_S2 = foreignCallDescriptor("indexOfWithMaskBS2", byte[].class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_WITH_MASK_C_S2 = foreignCallDescriptor("indexOfWithMaskCS2", char[].class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_B_S1 = foreignCallDescriptor("indexOfTwoConsecutiveWithMaskBS1", byte[].class, 4); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_B_S2 = foreignCallDescriptor("indexOfTwoConsecutiveWithMaskBS2", byte[].class, 4); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_C_S2 = foreignCallDescriptor("indexOfTwoConsecutiveWithMaskCS2", char[].class, 4); - public static final ForeignCallDescriptor STUB_INDEX_OF_WITH_MASK_S1 = foreignCallDescriptor("indexOfWithMaskS1", Object.class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_WITH_MASK_S2 = foreignCallDescriptor("indexOfWithMaskS2", Object.class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_WITH_MASK_S4 = foreignCallDescriptor("indexOfWithMaskS4", Object.class, 2); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S1 = foreignCallDescriptor("indexOfTwoConsecutiveWithMaskS1", Object.class, 4); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S2 = foreignCallDescriptor("indexOfTwoConsecutiveWithMaskS2", Object.class, 4); - public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S4 = foreignCallDescriptor("indexOfTwoConsecutiveWithMaskS4", Object.class, 4); - - private static final ForeignCallDescriptor[] STUBS_INDEX_OF_ANY_B = new ForeignCallDescriptor[]{ - STUB_INDEX_OF_B_1_S1, - STUB_INDEX_OF_B_2_S1, - STUB_INDEX_OF_B_3_S1, - STUB_INDEX_OF_B_4_S1, - STUB_INDEX_OF_B_1_S2, - STUB_INDEX_OF_B_2_S2, - STUB_INDEX_OF_B_3_S2, - STUB_INDEX_OF_B_4_S2, - }; - - private static final ForeignCallDescriptor[] STUBS_INDEX_OF_ANY_C = new ForeignCallDescriptor[]{ - STUB_INDEX_OF_C_1_S2, - STUB_INDEX_OF_C_2_S2, - STUB_INDEX_OF_C_3_S2, - STUB_INDEX_OF_C_4_S2, - }; + public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_S1 = foreignCallDescriptor("indexOfTwoConsecutiveS1", 2); + public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_S2 = foreignCallDescriptor("indexOfTwoConsecutiveS2", 2); + public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_S4 = foreignCallDescriptor("indexOfTwoConsecutiveS4", 2); + + public static final ForeignCallDescriptor STUB_INDEX_OF_1_S1 = foreignCallDescriptor("indexOf1S1", 1); + public static final ForeignCallDescriptor STUB_INDEX_OF_2_S1 = foreignCallDescriptor("indexOf2S1", 2); + public static final ForeignCallDescriptor STUB_INDEX_OF_3_S1 = foreignCallDescriptor("indexOf3S1", 3); + public static final ForeignCallDescriptor STUB_INDEX_OF_4_S1 = foreignCallDescriptor("indexOf4S1", 4); + public static final ForeignCallDescriptor STUB_INDEX_OF_1_S2 = foreignCallDescriptor("indexOf1S2", 1); + public static final ForeignCallDescriptor STUB_INDEX_OF_2_S2 = foreignCallDescriptor("indexOf2S2", 2); + public static final ForeignCallDescriptor STUB_INDEX_OF_3_S2 = foreignCallDescriptor("indexOf3S2", 3); + public static final ForeignCallDescriptor STUB_INDEX_OF_4_S2 = foreignCallDescriptor("indexOf4S2", 4); + public static final ForeignCallDescriptor STUB_INDEX_OF_1_S4 = foreignCallDescriptor("indexOf1S4", 1); + public static final ForeignCallDescriptor STUB_INDEX_OF_2_S4 = foreignCallDescriptor("indexOf2S4", 2); + public static final ForeignCallDescriptor STUB_INDEX_OF_3_S4 = foreignCallDescriptor("indexOf3S4", 3); + public static final ForeignCallDescriptor STUB_INDEX_OF_4_S4 = foreignCallDescriptor("indexOf4S4", 4); + + public static final ForeignCallDescriptor STUB_INDEX_OF_WITH_MASK_S1 = foreignCallDescriptor("indexOfWithMaskS1", 2); + public static final ForeignCallDescriptor STUB_INDEX_OF_WITH_MASK_S2 = foreignCallDescriptor("indexOfWithMaskS2", 2); + public static final ForeignCallDescriptor STUB_INDEX_OF_WITH_MASK_S4 = foreignCallDescriptor("indexOfWithMaskS4", 2); + public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S1 = foreignCallDescriptor("indexOfTwoConsecutiveWithMaskS1", 4); + public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S2 = foreignCallDescriptor("indexOfTwoConsecutiveWithMaskS2", 4); + public static final ForeignCallDescriptor STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S4 = foreignCallDescriptor("indexOfTwoConsecutiveWithMaskS4", 4); private static final ForeignCallDescriptor[] STUBS_INDEX_OF_ANY = new ForeignCallDescriptor[]{ STUB_INDEX_OF_1_S1, @@ -130,154 +82,75 @@ private static ForeignCallDescriptor foreignCallDescriptor(String name, Class STUB_INDEX_OF_4_S4, }; - public static final ForeignCallDescriptor[] STUBS_AMD64 = Stream.concat(Stream.concat(Stream.concat(Stream.of( - STUB_INDEX_OF_TWO_CONSECUTIVE_B_S1, - STUB_INDEX_OF_TWO_CONSECUTIVE_B_S2, - STUB_INDEX_OF_TWO_CONSECUTIVE_C_S2, + public static final ForeignCallDescriptor[] STUBS_AMD64 = Stream.concat(Stream.of( STUB_INDEX_OF_TWO_CONSECUTIVE_S1, STUB_INDEX_OF_TWO_CONSECUTIVE_S2, STUB_INDEX_OF_TWO_CONSECUTIVE_S4, - STUB_INDEX_OF_WITH_MASK_B_S1, - STUB_INDEX_OF_WITH_MASK_B_S2, - STUB_INDEX_OF_WITH_MASK_C_S2, - STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_B_S1, - STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_B_S2, - STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_C_S2, STUB_INDEX_OF_WITH_MASK_S1, STUB_INDEX_OF_WITH_MASK_S2, STUB_INDEX_OF_WITH_MASK_S4, STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S1, STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S2, STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S4), - Arrays.stream(STUBS_INDEX_OF_ANY_B)), - Arrays.stream(STUBS_INDEX_OF_ANY_C)), Arrays.stream(STUBS_INDEX_OF_ANY)).toArray(ForeignCallDescriptor[]::new); public static final ForeignCallDescriptor[] STUBS_AARCH64 = { - STUB_INDEX_OF_B_1_S1, - STUB_INDEX_OF_B_1_S2, - STUB_INDEX_OF_C_1_S2, STUB_INDEX_OF_1_S1, STUB_INDEX_OF_1_S2, STUB_INDEX_OF_1_S4, - STUB_INDEX_OF_TWO_CONSECUTIVE_B_S1, - STUB_INDEX_OF_TWO_CONSECUTIVE_B_S2, - STUB_INDEX_OF_TWO_CONSECUTIVE_C_S2, STUB_INDEX_OF_TWO_CONSECUTIVE_S1, STUB_INDEX_OF_TWO_CONSECUTIVE_S2, STUB_INDEX_OF_TWO_CONSECUTIVE_S4, }; public static ForeignCallDescriptor getStub(ArrayIndexOfNode indexOfNode) { - return getStub(indexOfNode.getArrayKind(), indexOfNode.getStride(), indexOfNode.getNumberOfValues(), indexOfNode.isFindTwoConsecutive(), indexOfNode.isWithMask()); + return getStub(indexOfNode.getStride(), indexOfNode.getNumberOfValues(), indexOfNode.isFindTwoConsecutive(), indexOfNode.isWithMask()); } - public static ForeignCallDescriptor getStub(JavaKind arrayKind, JavaKind stride, int valueCount, boolean findTwoConsecutive, boolean withMask) { - GraalError.guarantee(arrayKind == S1 || arrayKind == S2 || arrayKind == NONE, "unsupported arrayKind"); - GraalError.guarantee(stride == S1 || stride == S2 || stride == S4, "unsupported stride"); + public static ForeignCallDescriptor getStub(Stride stride, int valueCount, boolean findTwoConsecutive, boolean withMask) { + GraalError.guarantee(stride == Stride.S1 || stride == Stride.S2 || stride == Stride.S4, "unsupported stride"); GraalError.guarantee(valueCount >= 1 && valueCount <= 4, "unsupported valueCount"); if (withMask) { if (findTwoConsecutive) { GraalError.guarantee(valueCount == 4, "findTwoConsecutive with mask requires 4 values"); - switch (arrayKind) { - case Byte: - switch (stride) { - case Byte: - return STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_B_S1; - case Char: - return STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_B_S2; - default: - throw GraalError.shouldNotReachHere(); - } - case Char: - GraalError.guarantee(stride == JavaKind.Char, "unsupported stride for char array"); - return STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_C_S2; - case Void: - switch (stride) { - case Byte: - return STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S1; - case Char: - return STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S2; - case Int: - return STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S4; - default: - throw GraalError.shouldNotReachHere(); - } + switch (stride) { + case S1: + return STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S1; + case S2: + return STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S2; + case S4: + return STUB_INDEX_OF_TWO_CONSECUTIVE_WITH_MASK_S4; default: throw GraalError.shouldNotReachHere(); } } else { GraalError.guarantee(valueCount == 2, "indexOf with mask requires 2 values"); - switch (arrayKind) { - case Byte: - assert stride == S1 || stride == S2; - switch (stride) { - case Byte: - return STUB_INDEX_OF_WITH_MASK_B_S1; - case Char: - return STUB_INDEX_OF_WITH_MASK_B_S2; - default: - throw GraalError.shouldNotReachHere(); - } - case Char: - GraalError.guarantee(stride == JavaKind.Char, "unsupported stride for char array"); - return STUB_INDEX_OF_WITH_MASK_C_S2; - case Void: - switch (stride) { - case Byte: - return STUB_INDEX_OF_WITH_MASK_S1; - case Char: - return STUB_INDEX_OF_WITH_MASK_S2; - case Int: - return STUB_INDEX_OF_WITH_MASK_S4; - default: - throw GraalError.shouldNotReachHere(); - } + switch (stride) { + case S1: + return STUB_INDEX_OF_WITH_MASK_S1; + case S2: + return STUB_INDEX_OF_WITH_MASK_S2; + case S4: + return STUB_INDEX_OF_WITH_MASK_S4; default: throw GraalError.shouldNotReachHere(); } } } else if (findTwoConsecutive) { GraalError.guarantee(valueCount == 2, "findTwoConsecutive without mask requires 2 values"); - switch (arrayKind) { - case Byte: - switch (stride) { - case Byte: - return STUB_INDEX_OF_TWO_CONSECUTIVE_B_S1; - case Char: - return STUB_INDEX_OF_TWO_CONSECUTIVE_B_S2; - default: - throw GraalError.shouldNotReachHere(); - } - case Char: - GraalError.guarantee(stride == JavaKind.Char, "unsupported stride for char array"); - return STUB_INDEX_OF_TWO_CONSECUTIVE_C_S2; - case Void: - switch (stride) { - case Byte: - return STUB_INDEX_OF_TWO_CONSECUTIVE_S1; - case Char: - return STUB_INDEX_OF_TWO_CONSECUTIVE_S2; - case Int: - return STUB_INDEX_OF_TWO_CONSECUTIVE_S4; - default: - throw GraalError.shouldNotReachHere(); - } + switch (stride) { + case S1: + return STUB_INDEX_OF_TWO_CONSECUTIVE_S1; + case S2: + return STUB_INDEX_OF_TWO_CONSECUTIVE_S2; + case S4: + return STUB_INDEX_OF_TWO_CONSECUTIVE_S4; default: throw GraalError.shouldNotReachHere(); } } else { - int index = (4 * ForeignCalls.strideAsPowerOf2(stride)) + (valueCount - 1); - switch (arrayKind) { - case Byte: - return STUBS_INDEX_OF_ANY_B[index]; - case Char: - return STUBS_INDEX_OF_ANY_C[valueCount - 1]; - case Void: - return STUBS_INDEX_OF_ANY[index]; - default: - throw GraalError.shouldNotReachHere(); - } + int index = (4 * stride.log2) + (valueCount - 1); + return STUBS_INDEX_OF_ANY[index]; } } } diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayIndexOfNode.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayIndexOfNode.java index c33dcda00d39..226309a890f9 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayIndexOfNode.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayIndexOfNode.java @@ -25,15 +25,12 @@ package org.graalvm.compiler.replacements.nodes; import static org.graalvm.compiler.core.common.GraalOptions.UseGraalStubs; -import static org.graalvm.compiler.core.common.StrideUtil.NONE; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_16; import java.util.EnumSet; import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.debug.GraalError; @@ -46,6 +43,7 @@ import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; import org.graalvm.compiler.nodes.spi.Canonicalizable; import org.graalvm.compiler.nodes.spi.CanonicalizerTool; import org.graalvm.compiler.nodes.spi.LIRLowerable; @@ -65,8 +63,8 @@ * Parameters: *
    *
  • {@code arrayPointer}: pointer to a java array or native memory location.
    • - *
  • {@code arrayOffset}: byte offset to be added to the array pointer. If {@code arrayKind} is - * {@link JavaKind#Void}, this offset must include the array base offset!
    • + *
  • {@code arrayOffset}: byte offset to be added to the array pointer. This offset must include + * the array base offset!
    • *
  • {@code arrayLength}: array length respective to the element size given by * {@code stride}.
    • *
  • {@code fromIndex}: start index of the indexOf search, respective to the element size given by @@ -97,8 +95,7 @@ public class ArrayIndexOfNode extends PureFunctionStubIntrinsicNode implements C public static final NodeClass TYPE = NodeClass.create(ArrayIndexOfNode.class); - private final JavaKind arrayKind; - private final JavaKind stride; + private final Stride stride; private final boolean findTwoConsecutive; private final boolean withMask; @@ -110,51 +107,48 @@ public class ArrayIndexOfNode extends PureFunctionStubIntrinsicNode implements C public ArrayIndexOfNode( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind stride, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, ValueNode arrayPointer, ValueNode arrayOffset, ValueNode arrayLength, ValueNode fromIndex, ValueNode... searchValues) { - this(TYPE, arrayKind, stride, findTwoConsecutive, withMask, null, arrayKind == NONE ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(arrayKind), + this(TYPE, stride, findTwoConsecutive, withMask, null, defaultLocationIdentity(arrayKind), arrayPointer, arrayOffset, arrayLength, fromIndex, searchValues); } public ArrayIndexOfNode( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind stride, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures, ValueNode arrayPointer, ValueNode arrayOffset, ValueNode arrayLength, ValueNode fromIndex, ValueNode... searchValues) { - this(TYPE, arrayKind, stride, findTwoConsecutive, withMask, runtimeCheckedCPUFeatures, arrayKind == NONE ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(arrayKind), + this(TYPE, stride, findTwoConsecutive, withMask, runtimeCheckedCPUFeatures, defaultLocationIdentity(arrayKind), arrayPointer, arrayOffset, arrayLength, fromIndex, searchValues); } public ArrayIndexOfNode( - JavaKind arrayKind, - JavaKind stride, + Stride stride, boolean findTwoConsecutive, boolean withMask, EnumSet runtimeCheckedCPUFeatures, LocationIdentity locationIdentity, ValueNode arrayPointer, ValueNode arrayOffset, ValueNode arrayLength, ValueNode fromIndex, ValueNode... searchValues) { - this(TYPE, arrayKind, stride, findTwoConsecutive, withMask, runtimeCheckedCPUFeatures, locationIdentity, arrayPointer, arrayOffset, arrayLength, fromIndex, searchValues); + this(TYPE, stride, findTwoConsecutive, withMask, runtimeCheckedCPUFeatures, locationIdentity, arrayPointer, arrayOffset, arrayLength, fromIndex, searchValues); } public ArrayIndexOfNode( NodeClass c, - JavaKind arrayKind, - JavaKind stride, + Stride stride, boolean findTwoConsecutive, boolean withMask, EnumSet runtimeCheckedCPUFeatures, LocationIdentity locationIdentity, ValueNode arrayPointer, ValueNode arrayOffset, ValueNode arrayLength, ValueNode fromIndex, ValueNode... searchValues) { super(c, StampFactory.forKind(JavaKind.Int), runtimeCheckedCPUFeatures, locationIdentity); - GraalError.guarantee(arrayKind == S1 || arrayKind == S2 || arrayKind == S4 || arrayKind == NONE, "unsupported arrayKind"); + GraalError.guarantee(stride.value <= 4, "unsupported stride"); GraalError.guarantee(!(!withMask && findTwoConsecutive) || searchValues.length == 2, "findTwoConsecutive without mask requires exactly two search values"); GraalError.guarantee(!(withMask && findTwoConsecutive) || searchValues.length == 4, "findTwoConsecutive with mask requires exactly four search values"); GraalError.guarantee(!(withMask && !findTwoConsecutive) || searchValues.length == 2, "indexOf with mask requires exactly two search values"); - this.arrayKind = arrayKind; this.stride = stride; this.findTwoConsecutive = findTwoConsecutive; this.withMask = withMask; @@ -165,13 +159,14 @@ public ArrayIndexOfNode( this.searchValues = new NodeInputList<>(this, searchValues); } - public static ArrayIndexOfNode createIndexOfSingle(JavaKind arrayKind, JavaKind stride, ValueNode array, ValueNode arrayLength, ValueNode fromIndex, ValueNode searchValue) { - return new ArrayIndexOfNode(TYPE, arrayKind, stride, false, false, null, arrayKind == NONE ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(arrayKind), - array, ConstantNode.forLong(0), arrayLength, fromIndex, searchValue); + public static ArrayIndexOfNode createIndexOfSingle(GraphBuilderContext b, JavaKind arrayKind, Stride stride, ValueNode array, ValueNode arrayLength, ValueNode fromIndex, ValueNode searchValue) { + ValueNode baseOffset = ConstantNode.forLong(b.getMetaAccess().getArrayBaseOffset(arrayKind), b.getGraph()); + return new ArrayIndexOfNode(TYPE, stride, false, false, null, defaultLocationIdentity(arrayKind), + array, baseOffset, arrayLength, fromIndex, searchValue); } - public JavaKind getArrayKind() { - return arrayKind; + private static LocationIdentity defaultLocationIdentity(JavaKind arrayKind) { + return arrayKind == JavaKind.Void ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(arrayKind); } public boolean isFindTwoConsecutive() { @@ -206,7 +201,7 @@ public int getNumberOfValues() { return searchValues.size(); } - public JavaKind getStride() { + public Stride getStride() { return stride; } @@ -232,8 +227,7 @@ public void generate(NodeLIRBuilderTool gen) { } private void generateArrayIndexOf(NodeLIRBuilderTool gen) { - int arrayBaseOffset = arrayKind == NONE ? 0 : getArrayBaseOffset(gen.getLIRGeneratorTool().getMetaAccess(), arrayPointer, arrayKind); - Value result = gen.getLIRGeneratorTool().emitArrayIndexOf(arrayBaseOffset, stride, findTwoConsecutive, withMask, + Value result = gen.getLIRGeneratorTool().emitArrayIndexOf(stride, findTwoConsecutive, withMask, getRuntimeCheckedCPUFeatures(), gen.operand(arrayPointer), gen.operand(arrayOffset), gen.operand(arrayLength), gen.operand(fromIndex), searchValuesAsOperands(gen)); gen.setResult(this, result); } @@ -267,20 +261,18 @@ public Node canonical(CanonicalizerTool tool) { // arrayOffset is given in bytes, scale it to the stride. long arrayBaseOffsetBytesConstant = arrayOffset.asJavaConstant().asLong(); - if (arrayKind == NONE) { - arrayBaseOffsetBytesConstant -= getArrayBaseOffset(tool.getMetaAccess(), arrayPointer, constantArrayKind); - } - long arrayOffsetConstant = arrayBaseOffsetBytesConstant / stride.getByteCount(); + arrayBaseOffsetBytesConstant -= getArrayBaseOffset(tool.getMetaAccess(), arrayPointer, constantArrayKind); + long arrayOffsetConstant = arrayBaseOffsetBytesConstant / stride.value; int arrayLengthConstant = arrayLength.asJavaConstant().asInt(); - assert arrayLengthConstant * stride.getByteCount() <= actualArrayLength * constantArrayKind.getByteCount(); + assert arrayLengthConstant * stride.value <= actualArrayLength * constantArrayKind.getByteCount(); int fromIndexConstant = fromIndex.asJavaConstant().asInt(); int[] valuesConstant = new int[searchValues.size()]; for (int i = 0; i < searchValues.size(); i++) { valuesConstant[i] = searchValues.get(i).asJavaConstant().asInt(); } - if (arrayLengthConstant * stride.getByteCount() < GraalOptions.StringIndexOfConstantLimit.getValue(tool.getOptions())) { + if (arrayLengthConstant * stride.value < GraalOptions.StringIndexOfConstantLimit.getValue(tool.getOptions())) { if (findTwoConsecutive) { assert valuesConstant.length == (withMask ? 4 : 2); for (int i = fromIndexConstant; i < arrayLengthConstant - 1; i++) { @@ -331,7 +323,7 @@ private boolean searchValuesConstant() { @NodeIntrinsic public static native int optimizedArrayIndexOf( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind valueKind, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, Object array, long arrayOffset, int arrayLength, int fromIndex, int v1); @@ -339,7 +331,7 @@ public static native int optimizedArrayIndexOf( @NodeIntrinsic public static native int optimizedArrayIndexOf( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind valueKind, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures, @@ -348,7 +340,7 @@ public static native int optimizedArrayIndexOf( @NodeIntrinsic public static native int optimizedArrayIndexOf( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind valueKind, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, Object array, long arrayOffset, int arrayLength, int fromIndex, int v1, int v2); @@ -356,7 +348,7 @@ public static native int optimizedArrayIndexOf( @NodeIntrinsic public static native int optimizedArrayIndexOf( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind valueKind, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures, @@ -365,7 +357,7 @@ public static native int optimizedArrayIndexOf( @NodeIntrinsic public static native int optimizedArrayIndexOf( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind valueKind, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, Object array, long arrayOffset, int arrayLength, int fromIndex, int v1, int v2, int v3); @@ -373,7 +365,7 @@ public static native int optimizedArrayIndexOf( @NodeIntrinsic public static native int optimizedArrayIndexOf( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind valueKind, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures, @@ -382,7 +374,7 @@ public static native int optimizedArrayIndexOf( @NodeIntrinsic public static native int optimizedArrayIndexOf( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind valueKind, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, Object array, long arrayOffset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4); @@ -390,7 +382,7 @@ public static native int optimizedArrayIndexOf( @NodeIntrinsic public static native int optimizedArrayIndexOf( @ConstantNodeParameter JavaKind arrayKind, - @ConstantNodeParameter JavaKind valueKind, + @ConstantNodeParameter Stride stride, @ConstantNodeParameter boolean findTwoConsecutive, @ConstantNodeParameter boolean withMask, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures, diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionCompareToForeignCalls.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionCompareToForeignCalls.java index 42162165be75..74b043b5250f 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionCompareToForeignCalls.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionCompareToForeignCalls.java @@ -24,18 +24,17 @@ */ package org.graalvm.compiler.replacements.nodes; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.replacements.NodeStrideUtil; -import jdk.vm.ci.meta.JavaKind; - public final class ArrayRegionCompareToForeignCalls { private static final ForeignCallDescriptor STUB_DYNAMIC_STRIDES = ForeignCalls.pureFunctionForeignCallDescriptor("arrayRegionCompareToDynamicStrides", int.class, Object.class, long.class, Object.class, long.class, int.class, int.class); /** * CAUTION: the ordering here is important: entries 0-9 must match the indices generated by - * {@link NodeStrideUtil#getDirectStubCallIndex(ValueNode, JavaKind, JavaKind)}. + * {@link NodeStrideUtil#getDirectStubCallIndex(ValueNode, Stride, Stride)}. * * @see #getStub(ArrayRegionCompareToNode) */ diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionCompareToNode.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionCompareToNode.java index 7ece1ea68a92..7135dadbe168 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionCompareToNode.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionCompareToNode.java @@ -29,6 +29,7 @@ import java.util.EnumSet; import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.debug.GraalError; @@ -37,7 +38,6 @@ import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodeinfo.NodeSize; import org.graalvm.compiler.nodes.ConstantNode; -import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.spi.Canonicalizable; @@ -80,9 +80,14 @@ public class ArrayRegionCompareToNode extends PureFunctionStubIntrinsicNode impl public static final NodeClass TYPE = NodeClass.create(ArrayRegionCompareToNode.class); - /** {@link JavaKind} of the arrays to compare. */ - protected final JavaKind strideA; - protected final JavaKind strideB; + /** + * Element size of arrayA. + */ + protected final Stride strideA; + /** + * Element size of arrayB. + */ + protected final Stride strideB; /** * Pointer to the first array object. @@ -116,7 +121,7 @@ public class ArrayRegionCompareToNode extends PureFunctionStubIntrinsicNode impl */ @OptionalInput protected ValueNode dynamicStrides; - public ArrayRegionCompareToNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, JavaKind strideA, JavaKind strideB, LocationIdentity locationIdentity) { + public ArrayRegionCompareToNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, Stride strideA, Stride strideB, LocationIdentity locationIdentity) { this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, null, locationIdentity); } @@ -134,24 +139,22 @@ public ArrayRegionCompareToNode(ValueNode arrayA, ValueNode offsetA, ValueNode a } public ArrayRegionCompareToNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, - @ConstantNodeParameter JavaKind strideA, - @ConstantNodeParameter JavaKind strideB) { - this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, null, - strideA != strideB ? LocationIdentity.ANY_LOCATION : NamedLocationIdentity.getArrayLocation(strideA)); + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB) { + this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, null, LocationIdentity.ANY_LOCATION); } public ArrayRegionCompareToNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, - @ConstantNodeParameter JavaKind strideA, - @ConstantNodeParameter JavaKind strideB, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { - this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, runtimeCheckedCPUFeatures, - strideA != strideB ? LocationIdentity.ANY_LOCATION : NamedLocationIdentity.getArrayLocation(strideA)); + this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, runtimeCheckedCPUFeatures, LocationIdentity.ANY_LOCATION); } protected ArrayRegionCompareToNode(NodeClass c, ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, ValueNode dynamicStrides, - JavaKind strideA, - JavaKind strideB, + Stride strideA, + Stride strideB, EnumSet runtimeCheckedCPUFeatures, LocationIdentity locationIdentity) { super(c, StampFactory.forKind(JavaKind.Int), runtimeCheckedCPUFeatures, locationIdentity); @@ -163,23 +166,19 @@ protected ArrayRegionCompareToNode(NodeClass this.offsetB = offsetB; this.length = length; this.dynamicStrides = dynamicStrides; - GraalError.guarantee(strideA == null || allowedStrides().contains(strideA), "unsupported strideA"); - GraalError.guarantee(strideB == null || allowedStrides().contains(strideB), "unsupported strideB"); - } - - private static EnumSet allowedStrides() { - return EnumSet.of(JavaKind.Byte, JavaKind.Char, JavaKind.Int); + GraalError.guarantee(strideA == null || strideA.value <= 4, "unsupported strideA"); + GraalError.guarantee(strideB == null || strideB.value <= 4, "unsupported strideB"); } @NodeIntrinsic public static native int compare(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, - @ConstantNodeParameter JavaKind strideA, - @ConstantNodeParameter JavaKind strideB); + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB); @NodeIntrinsic public static native int compare(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, - @ConstantNodeParameter JavaKind strideA, - @ConstantNodeParameter JavaKind strideB, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures); @NodeIntrinsic @@ -205,11 +204,11 @@ public ValueNode getOffsetB() { return offsetB; } - public JavaKind getStrideA() { + public Stride getStrideA() { return strideA; } - public JavaKind getStrideB() { + public Stride getStrideB() { return strideB; } @@ -241,8 +240,8 @@ public void generate(NodeLIRBuilderTool gen) { } @Override - public int getArrayBaseOffset(MetaAccessProvider metaAccess, @SuppressWarnings("unused") ValueNode array, JavaKind elementKind) { - return metaAccess.getArrayBaseOffset(elementKind); + public int getArrayBaseOffset(MetaAccessProvider metaAccess, @SuppressWarnings("unused") ValueNode array, JavaKind arrayKind) { + return metaAccess.getArrayBaseOffset(arrayKind); } protected void generateArrayCompare(NodeLIRBuilderTool gen) { @@ -263,9 +262,9 @@ public ValueNode canonical(CanonicalizerTool tool) { } if ((dynamicStrides == null || dynamicStrides.isJavaConstant()) && length.isJavaConstant()) { int len = length.asJavaConstant().asInt(); - JavaKind constStrideA = NodeStrideUtil.getConstantStrideA(dynamicStrides, strideA); - JavaKind constStrideB = NodeStrideUtil.getConstantStrideB(dynamicStrides, strideB); - if (len * Math.max(constStrideA.getByteCount(), constStrideB.getByteCount()) < GraalOptions.ArrayRegionEqualsConstantLimit.getValue(tool.getOptions()) && + Stride constStrideA = NodeStrideUtil.getConstantStrideA(dynamicStrides, strideA); + Stride constStrideB = NodeStrideUtil.getConstantStrideB(dynamicStrides, strideB); + if (len * Math.max(constStrideA.value, constStrideB.value) < GraalOptions.ArrayRegionEqualsConstantLimit.getValue(tool.getOptions()) && ConstantReflectionUtil.canFoldReads(tool, arrayA, offsetA, constStrideA, len, this) && ConstantReflectionUtil.canFoldReads(tool, arrayB, offsetB, constStrideB, len, this)) { Integer startIndex1 = ConstantReflectionUtil.startIndex(tool, arrayA, offsetA.asJavaConstant(), constStrideA, this); @@ -276,7 +275,7 @@ public ValueNode canonical(CanonicalizerTool tool) { return this; } - private static int foldResult(CanonicalizerTool tool, JavaKind constStrideA, JavaKind constStrideB, ValueNode a, int startIndexA, ValueNode b, int startIndexB, int len) { + private static int foldResult(CanonicalizerTool tool, Stride constStrideA, Stride constStrideB, ValueNode a, int startIndexA, ValueNode b, int startIndexB, int len) { JavaKind arrayKindA = a.stamp(NodeView.DEFAULT).javaType(tool.getMetaAccess()).getComponentType().getJavaKind(); JavaKind arrayKindB = b.stamp(NodeView.DEFAULT).javaType(tool.getMetaAccess()).getComponentType().getJavaKind(); ConstantReflectionProvider constantReflection = tool.getConstantReflection(); diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java index de670d2d4444..71d3b30d0c59 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java @@ -29,7 +29,7 @@ import java.util.EnumSet; import org.graalvm.compiler.core.common.GraalOptions; -import org.graalvm.compiler.core.common.StrideUtil; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.graph.NodeClass; @@ -68,9 +68,8 @@ public class ArrayRegionEqualsNode extends PureFunctionStubIntrinsicNode impleme public static final NodeClass TYPE = NodeClass.create(ArrayRegionEqualsNode.class); - /** {@link JavaKind} of the arrays to compare. */ - protected final JavaKind strideA; - protected final JavaKind strideB; + private final Stride strideA; + private final Stride strideB; /** * Pointer to the first array object. @@ -103,27 +102,29 @@ public class ArrayRegionEqualsNode extends PureFunctionStubIntrinsicNode impleme */ @OptionalInput protected ValueNode dynamicStrides; - public ArrayRegionEqualsNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, ValueNode dynamicStrides, LocationIdentity locationIdentity) { - this(TYPE, arrayA, offsetA, arrayB, offsetB, length, dynamicStrides, null, null, null, locationIdentity); + public ArrayRegionEqualsNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, + @ConstantNodeParameter JavaKind arrayKind, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB) { + this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, null, NamedLocationIdentity.getArrayLocation(arrayKind)); } public ArrayRegionEqualsNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, - @ConstantNodeParameter JavaKind strideA, - @ConstantNodeParameter JavaKind strideB) { - this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, null, strideA != strideB ? LocationIdentity.ANY_LOCATION : NamedLocationIdentity.getArrayLocation(strideA)); + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB) { + this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, null, LocationIdentity.ANY_LOCATION); } public ArrayRegionEqualsNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, - @ConstantNodeParameter JavaKind strideA, - @ConstantNodeParameter JavaKind strideB, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { - this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, runtimeCheckedCPUFeatures, - strideA != strideB ? LocationIdentity.ANY_LOCATION : NamedLocationIdentity.getArrayLocation(strideA)); + this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, runtimeCheckedCPUFeatures, LocationIdentity.ANY_LOCATION); } public ArrayRegionEqualsNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, - @ConstantNodeParameter JavaKind strideA, - @ConstantNodeParameter JavaKind strideB, + Stride strideA, + Stride strideB, LocationIdentity locationIdentity) { this(TYPE, arrayA, offsetA, arrayB, offsetB, length, null, strideA, strideB, null, locationIdentity); } @@ -132,14 +133,18 @@ public ArrayRegionEqualsNode(ValueNode arrayA, ValueNode offsetA, ValueNode arra this(TYPE, arrayA, offsetA, arrayB, offsetB, length, dynamicStrides, null, null, null, LocationIdentity.ANY_LOCATION); } + public ArrayRegionEqualsNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, ValueNode dynamicStrides, LocationIdentity locationIdentity) { + this(TYPE, arrayA, offsetA, arrayB, offsetB, length, dynamicStrides, null, null, null, locationIdentity); + } + public ArrayRegionEqualsNode(ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, ValueNode dynamicStrides, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures) { this(TYPE, arrayA, offsetA, arrayB, offsetB, length, dynamicStrides, null, null, runtimeCheckedCPUFeatures, LocationIdentity.ANY_LOCATION); } protected ArrayRegionEqualsNode(NodeClass c, ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, ValueNode dynamicStrides, - JavaKind strideA, - JavaKind strideB, + Stride strideA, + Stride strideB, EnumSet runtimeCheckedCPUFeatures, LocationIdentity locationIdentity) { super(c, StampFactory.forKind(JavaKind.Boolean), runtimeCheckedCPUFeatures, locationIdentity); @@ -151,25 +156,30 @@ protected ArrayRegionEqualsNode(NodeClass c, Va this.offsetB = offsetB; this.length = length; this.dynamicStrides = dynamicStrides; - assert strideA == null || strideA.isPrimitive() && strideB.isPrimitive() : "expected primitive kinds, got: " + strideA + ", " + strideB; } - public static boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, @ConstantNodeParameter JavaKind kind) { - return regionEquals(arrayA, offsetA, arrayB, offsetB, length, kind, kind); - } + @NodeIntrinsic + public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB); @NodeIntrinsic - public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, @ConstantNodeParameter JavaKind kind1, @ConstantNodeParameter JavaKind kind2); + public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, + @ConstantNodeParameter JavaKind arrayKind, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB); @NodeIntrinsic - public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, @ConstantNodeParameter JavaKind kind1, @ConstantNodeParameter JavaKind kind2, + public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, + @ConstantNodeParameter Stride strideA, + @ConstantNodeParameter Stride strideB, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures); @NodeIntrinsic - public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, int stride); + public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, int dynamicStrides); @NodeIntrinsic - public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, int stride, + public static native boolean regionEquals(Object arrayA, long offsetA, Object arrayB, long offsetB, int length, int dynamicStrides, @ConstantNodeParameter EnumSet runtimeCheckedCPUFeatures); public ValueNode getArrayA() { @@ -188,11 +198,11 @@ public ValueNode getOffsetB() { return offsetB; } - public JavaKind getStrideA() { + public Stride getStrideA() { return strideA; } - public JavaKind getStrideB() { + public Stride getStrideB() { return strideB; } @@ -228,32 +238,23 @@ public void generate(NodeLIRBuilderTool gen) { } @Override - public int getArrayBaseOffset(MetaAccessProvider metaAccess, @SuppressWarnings("unused") ValueNode array, JavaKind elementKind) { - return metaAccess.getArrayBaseOffset(elementKind); + public int getArrayBaseOffset(MetaAccessProvider metaAccess, @SuppressWarnings("unused") ValueNode array, JavaKind arrayKind) { + return metaAccess.getArrayBaseOffset(arrayKind); } protected void generateArrayRegionEquals(NodeLIRBuilderTool gen) { final Value result; - if (strideA != null) { - if (strideA == strideB) { - result = gen.getLIRGeneratorTool().emitArrayEquals(strideA, - 0, 0, getRuntimeCheckedCPUFeatures(), gen.operand(arrayA), gen.operand(offsetA), gen.operand(arrayB), gen.operand(offsetB), gen.operand(length)); - } else { - result = gen.getLIRGeneratorTool().emitArrayEquals(strideA, strideB, - 0, 0, getRuntimeCheckedCPUFeatures(), gen.operand(arrayA), gen.operand(offsetA), gen.operand(arrayB), gen.operand(offsetB), gen.operand(length)); - } + int directStubCallIndex = getDirectStubCallIndex(); + if (directStubCallIndex < 0) { + result = gen.getLIRGeneratorTool().emitArrayEqualsDynamicStrides( + getRuntimeCheckedCPUFeatures(), + gen.operand(arrayA), gen.operand(offsetA), gen.operand(arrayB), gen.operand(offsetB), gen.operand(length), gen.operand(dynamicStrides)); } else { - int directStubCallIndex = getDirectStubCallIndex(); - if (directStubCallIndex < 0) { - result = gen.getLIRGeneratorTool().emitArrayEquals( - 0, 0, getRuntimeCheckedCPUFeatures(), gen.operand(arrayA), gen.operand(offsetA), gen.operand(arrayB), gen.operand(offsetB), gen.operand(length), - gen.operand(dynamicStrides)); - } else { - result = gen.getLIRGeneratorTool().emitArrayEquals( - StrideUtil.getConstantStrideA(directStubCallIndex), - StrideUtil.getConstantStrideB(directStubCallIndex), - 0, 0, getRuntimeCheckedCPUFeatures(), gen.operand(arrayA), gen.operand(offsetA), gen.operand(arrayB), gen.operand(offsetB), gen.operand(length)); - } + result = gen.getLIRGeneratorTool().emitArrayEquals( + NodeStrideUtil.getConstantStrideA(dynamicStrides, strideA), + NodeStrideUtil.getConstantStrideB(dynamicStrides, strideB), + getRuntimeCheckedCPUFeatures(), + gen.operand(arrayA), gen.operand(offsetA), gen.operand(arrayB), gen.operand(offsetB), gen.operand(length)); } gen.setResult(this, result); } @@ -265,9 +266,9 @@ public ValueNode canonical(CanonicalizerTool tool) { } if ((dynamicStrides == null || dynamicStrides.isJavaConstant()) && length.isJavaConstant()) { int len = length.asJavaConstant().asInt(); - JavaKind constStrideA = NodeStrideUtil.getConstantStrideA(dynamicStrides, strideA); - JavaKind constStrideB = NodeStrideUtil.getConstantStrideB(dynamicStrides, strideB); - if (len * Math.max(constStrideA.getByteCount(), constStrideB.getByteCount()) < GraalOptions.ArrayRegionEqualsConstantLimit.getValue(tool.getOptions()) && + Stride constStrideA = NodeStrideUtil.getConstantStrideA(dynamicStrides, strideA); + Stride constStrideB = NodeStrideUtil.getConstantStrideB(dynamicStrides, strideB); + if (len * Math.max(constStrideA.value, constStrideB.value) < GraalOptions.ArrayRegionEqualsConstantLimit.getValue(tool.getOptions()) && ConstantReflectionUtil.canFoldReads(tool, arrayA, offsetA, constStrideA, len, this) && ConstantReflectionUtil.canFoldReads(tool, arrayB, offsetB, constStrideB, len, this)) { Integer startIndex1 = ConstantReflectionUtil.startIndex(tool, arrayA, offsetA.asJavaConstant(), constStrideA, this); @@ -278,7 +279,7 @@ public ValueNode canonical(CanonicalizerTool tool) { return this; } - private static boolean arrayRegionEquals(CanonicalizerTool tool, ValueNode a, JavaKind constStrideA, int startIndexA, ValueNode b, JavaKind constStrideB, int startIndexB, int len) { + private static boolean arrayRegionEquals(CanonicalizerTool tool, ValueNode a, Stride constStrideA, int startIndexA, ValueNode b, Stride constStrideB, int startIndexB, int len) { JavaKind arrayKindA = a.stamp(NodeView.DEFAULT).javaType(tool.getMetaAccess()).getComponentType().getJavaKind(); JavaKind arrayKindB = b.stamp(NodeView.DEFAULT).javaType(tool.getMetaAccess()).getComponentType().getJavaKind(); ConstantReflectionProvider constantReflection = tool.getConstantReflection(); diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ForeignCalls.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ForeignCalls.java index 9869b6f037ac..e3043afd0032 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ForeignCalls.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ForeignCalls.java @@ -24,7 +24,6 @@ */ package org.graalvm.compiler.replacements.nodes; -import jdk.vm.ci.meta.JavaKind; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.word.LocationIdentity; @@ -35,8 +34,4 @@ public final class ForeignCalls { public static ForeignCallDescriptor pureFunctionForeignCallDescriptor(String name, Class resultType, Class... argTypes) { return new ForeignCallDescriptor(name, resultType, argTypes, true, NO_LOCATIONS, false, false); } - - public static int strideAsPowerOf2(JavaKind stride) { - return Integer.numberOfTrailingZeros(stride.getByteCount()); - } } diff --git a/compiler/src/org.graalvm.compiler.truffle.compiler.amd64/src/org/graalvm/compiler/truffle/compiler/amd64/substitutions/AMD64TruffleInvocationPlugins.java b/compiler/src/org.graalvm.compiler.truffle.compiler.amd64/src/org/graalvm/compiler/truffle/compiler/amd64/substitutions/AMD64TruffleInvocationPlugins.java index 22d14d85b64d..cb6847a22c54 100644 --- a/compiler/src/org.graalvm.compiler.truffle.compiler.amd64/src/org/graalvm/compiler/truffle/compiler/amd64/substitutions/AMD64TruffleInvocationPlugins.java +++ b/compiler/src/org.graalvm.compiler.truffle.compiler.amd64/src/org/graalvm/compiler/truffle/compiler/amd64/substitutions/AMD64TruffleInvocationPlugins.java @@ -24,10 +24,9 @@ */ package org.graalvm.compiler.truffle.compiler.amd64.substitutions; -import static org.graalvm.compiler.core.common.StrideUtil.NONE; import static org.graalvm.compiler.nodes.NamedLocationIdentity.getArrayLocation; -import static org.graalvm.compiler.replacements.nodes.ForeignCalls.strideAsPowerOf2; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.StrideUtil; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.amd64.AMD64CalcStringAttributesOp; @@ -54,7 +53,6 @@ import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.code.Architecture; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaMethod; public class AMD64TruffleInvocationPlugins { @@ -69,30 +67,29 @@ public static void register(Architecture architecture, InvocationPlugins plugins private static void registerArrayUtilsPlugins(InvocationPlugins plugins, Replacements replacements) { plugins.registerIntrinsificationPredicate(t -> t.getName().equals("Lcom/oracle/truffle/api/ArrayUtils;")); InvocationPlugins.Registration r = new InvocationPlugins.Registration(plugins, "com.oracle.truffle.api.ArrayUtils", replacements); - for (JavaKind stride : new JavaKind[]{JavaKind.Byte, JavaKind.Char}) { - String strideStr = stride == JavaKind.Byte ? "1" : "2"; - r.register(new InlineOnlyInvocationPlugin("stubIndexOfB1S" + strideStr, byte[].class, long.class, int.class, int.class) { + for (Stride stride : new Stride[]{Stride.S1, Stride.S2}) { + r.register(new InlineOnlyInvocationPlugin("stubIndexOfB1" + stride.name(), byte[].class, long.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0) { return arrayUtilsIndexOfAny(b, JavaKind.Byte, stride, array, fromIndex, maxIndex, v0); } }); - r.register(new InlineOnlyInvocationPlugin("stubIndexOfB2S" + strideStr, byte[].class, long.class, int.class, int.class, int.class) { + r.register(new InlineOnlyInvocationPlugin("stubIndexOfB2" + stride.name(), byte[].class, long.class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0, ValueNode v1) { return arrayUtilsIndexOfAny(b, JavaKind.Byte, stride, array, fromIndex, maxIndex, v0, v1); } }); - r.register(new InlineOnlyInvocationPlugin("stubIndexOfB3S" + strideStr, byte[].class, long.class, int.class, int.class, int.class, int.class) { + r.register(new InlineOnlyInvocationPlugin("stubIndexOfB3" + stride.name(), byte[].class, long.class, int.class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0, ValueNode v1, ValueNode v2) { return arrayUtilsIndexOfAny(b, JavaKind.Byte, stride, array, fromIndex, maxIndex, v0, v1, v2); } }); - r.register(new InlineOnlyInvocationPlugin("stubIndexOfB4S" + strideStr, byte[].class, long.class, int.class, int.class, int.class, int.class, int.class) { + r.register(new InlineOnlyInvocationPlugin("stubIndexOfB4" + stride.name(), byte[].class, long.class, int.class, int.class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0, ValueNode v1, ValueNode v2, ValueNode v3) { @@ -104,49 +101,49 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0) { - return arrayUtilsIndexOfAny(b, JavaKind.Char, JavaKind.Char, array, fromIndex, maxIndex, v0); + return arrayUtilsIndexOfAny(b, JavaKind.Char, Stride.S2, array, fromIndex, maxIndex, v0); } }); r.register(new InlineOnlyInvocationPlugin("stubIndexOfC2S2", char[].class, long.class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0, ValueNode v1) { - return arrayUtilsIndexOfAny(b, JavaKind.Char, JavaKind.Char, array, fromIndex, maxIndex, v0, v1); + return arrayUtilsIndexOfAny(b, JavaKind.Char, Stride.S2, array, fromIndex, maxIndex, v0, v1); } }); r.register(new InlineOnlyInvocationPlugin("stubIndexOfC3S2", char[].class, long.class, int.class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0, ValueNode v1, ValueNode v2) { - return arrayUtilsIndexOfAny(b, JavaKind.Char, JavaKind.Char, array, fromIndex, maxIndex, v0, v1, v2); + return arrayUtilsIndexOfAny(b, JavaKind.Char, Stride.S2, array, fromIndex, maxIndex, v0, v1, v2); } }); r.register(new InlineOnlyInvocationPlugin("stubIndexOfC4S2", char[].class, long.class, int.class, int.class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0, ValueNode v1, ValueNode v2, ValueNode v3) { - return arrayUtilsIndexOfAny(b, JavaKind.Char, JavaKind.Char, array, fromIndex, maxIndex, v0, v1, v2, v3); + return arrayUtilsIndexOfAny(b, JavaKind.Char, Stride.S2, array, fromIndex, maxIndex, v0, v1, v2, v3); } }); r.register(new InlineOnlyInvocationPlugin("stubIndexOf2ConsecutiveS1", byte[].class, long.class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0, ValueNode v1) { - return arrayUtilsIndexOf(b, JavaKind.Byte, JavaKind.Byte, true, false, array, fromIndex, maxIndex, v0, v1); + return arrayUtilsIndexOf(b, JavaKind.Byte, Stride.S1, true, false, array, fromIndex, maxIndex, v0, v1); } }); r.register(new InlineOnlyInvocationPlugin("stubIndexOf2ConsecutiveS2", byte[].class, long.class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0, ValueNode v1) { - return arrayUtilsIndexOf(b, JavaKind.Byte, JavaKind.Char, true, false, array, fromIndex, maxIndex, v0, v1); + return arrayUtilsIndexOf(b, JavaKind.Byte, Stride.S2, true, false, array, fromIndex, maxIndex, v0, v1); } }); r.register(new InlineOnlyInvocationPlugin("stubIndexOf2ConsecutiveS2", char[].class, long.class, int.class, int.class, int.class) { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode v0, ValueNode v1) { - return arrayUtilsIndexOf(b, JavaKind.Char, JavaKind.Char, true, false, array, fromIndex, maxIndex, v0, v1); + return arrayUtilsIndexOf(b, JavaKind.Char, Stride.S2, true, false, array, fromIndex, maxIndex, v0, v1); } }); @@ -154,55 +151,55 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec @Override public boolean apply(GraphBuilderContext graph, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length) { - return arrayUtilsRegionEquals(graph.getMetaAccess(), graph, arrayA, offsetA, arrayB, offsetB, length, JavaKind.Byte, JavaKind.Byte, JavaKind.Byte); + return arrayUtilsRegionEquals(graph, arrayA, offsetA, arrayB, offsetB, length, JavaKind.Byte, Stride.S1, Stride.S1); } }); r.register(new InlineOnlyInvocationPlugin("stubRegionEqualsS2S1", byte[].class, long.class, byte[].class, long.class, int.class) { @Override public boolean apply(GraphBuilderContext graph, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length) { - return arrayUtilsRegionEquals(graph.getMetaAccess(), graph, arrayA, offsetA, arrayB, offsetB, length, JavaKind.Byte, JavaKind.Char, JavaKind.Byte); + return arrayUtilsRegionEquals(graph, arrayA, offsetA, arrayB, offsetB, length, JavaKind.Byte, Stride.S2, Stride.S1); } }); r.register(new InlineOnlyInvocationPlugin("stubRegionEqualsS2", char[].class, long.class, char[].class, long.class, int.class) { @Override public boolean apply(GraphBuilderContext graph, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length) { - return arrayUtilsRegionEquals(graph.getMetaAccess(), graph, arrayA, offsetA, arrayB, offsetB, length, JavaKind.Char, JavaKind.Char, JavaKind.Char); + return arrayUtilsRegionEquals(graph, arrayA, offsetA, arrayB, offsetB, length, JavaKind.Char, Stride.S2, Stride.S2); } }); } - private static boolean arrayUtilsIndexOfAny(GraphBuilderContext b, JavaKind arrayKind, JavaKind stride, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode... values) { + private static boolean arrayUtilsIndexOfAny(GraphBuilderContext b, JavaKind arrayKind, Stride stride, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode... values) { return arrayUtilsIndexOf(b, arrayKind, stride, false, false, array, fromIndex, maxIndex, values); } - public static boolean arrayUtilsIndexOf(GraphBuilderContext b, JavaKind arrayKind, JavaKind stride, boolean findTwoConsecutive, boolean withMask, ValueNode array, ValueNode fromIndex, + public static boolean arrayUtilsIndexOf(GraphBuilderContext b, JavaKind arrayKind, Stride stride, boolean findTwoConsecutive, boolean withMask, ValueNode array, ValueNode fromIndex, ValueNode maxIndex, ValueNode... values) { - ConstantNode zero = ConstantNode.forInt(0, b.getGraph()); - b.addPush(JavaKind.Int, new ArrayIndexOfNode(arrayKind, stride, findTwoConsecutive, withMask, null, getArrayLocation(arrayKind), array, zero, maxIndex, fromIndex, values)); + ValueNode baseOffset = ConstantNode.forLong(b.getMetaAccess().getArrayBaseOffset(arrayKind), b.getGraph()); + b.addPush(JavaKind.Int, new ArrayIndexOfNode(stride, findTwoConsecutive, withMask, null, getArrayLocation(arrayKind), array, baseOffset, maxIndex, fromIndex, values)); return true; } - private static boolean arrayUtilsRegionEquals(MetaAccessProvider metaAccess, GraphBuilderContext graph, ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, - JavaKind arrayKind, JavaKind strideA, JavaKind strideB) { - ValueNode byteOffsetA = toByteOffset(metaAccess, graph, offsetA, strideA); - ValueNode byteOffsetB = toByteOffset(metaAccess, graph, offsetB, strideB); + private static boolean arrayUtilsRegionEquals(GraphBuilderContext graph, ValueNode arrayA, ValueNode offsetA, ValueNode arrayB, ValueNode offsetB, ValueNode length, + JavaKind arrayKind, Stride strideA, Stride strideB) { + ValueNode byteOffsetA = toByteOffset(graph, arrayKind, strideA, offsetA); + ValueNode byteOffsetB = toByteOffset(graph, arrayKind, strideB, offsetB); graph.addPush(JavaKind.Boolean, new ArrayRegionEqualsNode(arrayA, byteOffsetA, arrayB, byteOffsetB, length, strideA, strideB, getArrayLocation(arrayKind))); return true; } - public static ValueNode toByteOffset(MetaAccessProvider metaAccess, GraphBuilderContext graph, ValueNode offset, JavaKind stride) { - ValueNode shifted = stride == JavaKind.Byte ? offset : graph.add(LeftShiftNode.create(offset, ConstantNode.forInt(strideAsPowerOf2(stride), graph.getGraph()), NodeView.DEFAULT)); - return graph.add(AddNode.create(shifted, ConstantNode.forLong(metaAccess.getArrayBaseOffset(stride), graph.getGraph()), NodeView.DEFAULT)); + public static ValueNode toByteOffset(GraphBuilderContext graph, JavaKind arrayKind, Stride stride, ValueNode offset) { + ValueNode shifted = stride == Stride.S1 ? offset : graph.add(LeftShiftNode.create(offset, ConstantNode.forInt(stride.log2, graph.getGraph()), NodeView.DEFAULT)); + return graph.add(AddNode.create(shifted, ConstantNode.forLong(graph.getMetaAccess().getArrayBaseOffset(arrayKind), graph.getGraph()), NodeView.DEFAULT)); } - public static JavaKind constantStrideParam(ValueNode param) { + public static Stride constantStrideParam(ValueNode param) { if (!param.isJavaConstant()) { throw GraalError.shouldNotReachHere(); } // TruffleString stores strides in log2 - return StrideUtil.log2ToStride(param.asJavaConstant().asInt()); + return Stride.fromLog2(param.asJavaConstant().asInt()); } private static boolean asBoolean(ValueNode param) { @@ -419,9 +416,9 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec public static boolean applyIndexOf(GraphBuilderContext b, boolean findTwoConsecutive, boolean withMask, ValueNode array, ValueNode offset, ValueNode length, ValueNode stride, ValueNode isNative, ValueNode fromIndex, ValueNode... values) { - JavaKind constStride = constantStrideParam(stride); + Stride constStride = constantStrideParam(stride); LocationIdentity locationIdentity = inferLocationIdentity(isNative); - b.addPush(JavaKind.Int, new ArrayIndexOfNode(NONE, constStride, findTwoConsecutive, withMask, null, locationIdentity, array, offset, length, fromIndex, values)); + b.addPush(JavaKind.Int, new ArrayIndexOfNode(constStride, findTwoConsecutive, withMask, null, locationIdentity, array, offset, length, fromIndex, values)); return true; } } diff --git a/compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotPartialEvaluator.java b/compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotPartialEvaluator.java index 8bcfc99a9dfb..3ad050712df3 100644 --- a/compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotPartialEvaluator.java +++ b/compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotPartialEvaluator.java @@ -35,6 +35,7 @@ import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; import org.graalvm.compiler.truffle.compiler.PartialEvaluator; import org.graalvm.compiler.truffle.compiler.TruffleCompilerConfiguration; +import org.graalvm.compiler.truffle.compiler.substitutions.KnownTruffleTypes; import org.graalvm.options.OptionValues; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -47,8 +48,8 @@ public final class HotSpotPartialEvaluator extends PartialEvaluator { private boolean disableEncodedGraphCachePurges; - public HotSpotPartialEvaluator(TruffleCompilerConfiguration config, GraphBuilderConfiguration configForRoot) { - super(config, configForRoot, new HotSpotKnownTruffleTypes(config.lastTier().providers().getMetaAccess())); + public HotSpotPartialEvaluator(TruffleCompilerConfiguration config, GraphBuilderConfiguration configForRoot, KnownTruffleTypes knownTruffleTypes) { + super(config, configForRoot, knownTruffleTypes); this.graphCacheRef = new AtomicReference<>(); this.disableEncodedGraphCachePurges = false; } diff --git a/compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotTruffleCompilerImpl.java b/compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotTruffleCompilerImpl.java index 7c8b141dffa9..8b8971ecd0f6 100644 --- a/compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotTruffleCompilerImpl.java +++ b/compiler/src/org.graalvm.compiler.truffle.compiler.hotspot/src/org/graalvm/compiler/truffle/compiler/hotspot/HotSpotTruffleCompilerImpl.java @@ -90,6 +90,7 @@ import org.graalvm.compiler.truffle.compiler.TruffleCompilerConfiguration; import org.graalvm.compiler.truffle.compiler.TruffleCompilerImpl; import org.graalvm.compiler.truffle.compiler.TruffleTierConfiguration; +import org.graalvm.compiler.truffle.options.PolyglotCompilerOptions; import jdk.vm.ci.code.BailoutException; import jdk.vm.ci.code.CodeCacheProvider; @@ -103,7 +104,6 @@ import jdk.vm.ci.meta.Assumptions; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.runtime.JVMCICompiler; -import org.graalvm.compiler.truffle.options.PolyglotCompilerOptions; public final class HotSpotTruffleCompilerImpl extends TruffleCompilerImpl implements HotSpotTruffleCompiler { @@ -127,8 +127,9 @@ public static HotSpotTruffleCompilerImpl create(final TruffleCompilerRuntime run HotSpotBackend backend = hotspotGraalRuntime.getHostBackend(); GraphBuilderPhase phase = (GraphBuilderPhase) backend.getSuites().getDefaultGraphBuilderSuite().findPhase(GraphBuilderPhase.class).previous(); Plugins plugins = phase.getGraphBuilderConfig().getPlugins(); + HotSpotKnownTruffleTypes knownTruffleTypes = new HotSpotKnownTruffleTypes(backend.getProviders().getMetaAccess()); final PartialEvaluatorConfiguration lastTierPe = createPartialEvaluatorConfiguration(hotspotGraalRuntime.getCompilerConfigurationName()); - final TruffleTierConfiguration lastTierSetup = new TruffleTierConfiguration(lastTierPe, backend, options); + final TruffleTierConfiguration lastTierSetup = new TruffleTierConfiguration(lastTierPe, backend, options, knownTruffleTypes); CompilerConfigurationFactory lowTierCompilerConfigurationFactory = new EconomyCompilerConfigurationFactory(); CompilerConfiguration compilerConfiguration = lowTierCompilerConfigurationFactory.createCompilerConfiguration(); @@ -139,8 +140,8 @@ public static HotSpotTruffleCompilerImpl create(final TruffleCompilerRuntime run Providers firstTierProviders = firstTierBackend.getProviders(); PartialEvaluatorConfiguration firstTierPe = new EconomyPartialEvaluatorConfiguration(); firstTierBackend.completeInitialization(HotSpotJVMCIRuntime.runtime(), options); - TruffleTierConfiguration firstTierSetup = new TruffleTierConfiguration(firstTierPe, firstTierBackend, firstTierProviders, firstTierSuites, firstTierLirSuites); - final TruffleCompilerConfiguration compilerConfig = new TruffleCompilerConfiguration(runtime, plugins, snippetReflection, firstTierSetup, lastTierSetup); + TruffleTierConfiguration firstTierSetup = new TruffleTierConfiguration(firstTierPe, firstTierBackend, firstTierProviders, firstTierSuites, firstTierLirSuites, knownTruffleTypes); + final TruffleCompilerConfiguration compilerConfig = new TruffleCompilerConfiguration(runtime, plugins, snippetReflection, firstTierSetup, lastTierSetup, knownTruffleTypes); return new HotSpotTruffleCompilerImpl(hotspotGraalRuntime, compilerConfig); } @@ -200,7 +201,7 @@ protected DebugContext createDebugContext(OptionValues options, CompilationIdent @Override protected HotSpotPartialEvaluator createPartialEvaluator(TruffleCompilerConfiguration configuration) { - return new HotSpotPartialEvaluator(configuration, builderConfig); + return new HotSpotPartialEvaluator(configuration, builderConfig, configuration.getKnownTruffleTypes()); } @Override diff --git a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleCompilerConfiguration.java b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleCompilerConfiguration.java index 7b95068ac255..0047bec1a6a6 100644 --- a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleCompilerConfiguration.java +++ b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleCompilerConfiguration.java @@ -31,6 +31,7 @@ import org.graalvm.compiler.core.target.Backend; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; import org.graalvm.compiler.truffle.common.TruffleCompilerRuntime; +import org.graalvm.compiler.truffle.compiler.substitutions.KnownTruffleTypes; import jdk.vm.ci.code.Architecture; @@ -40,14 +41,21 @@ public final class TruffleCompilerConfiguration { private final SnippetReflectionProvider provider; private final TruffleTierConfiguration firstTier; private final TruffleTierConfiguration lastTier; + private final KnownTruffleTypes knownTruffleTypes; - public TruffleCompilerConfiguration(TruffleCompilerRuntime runtime, GraphBuilderConfiguration.Plugins plugins, SnippetReflectionProvider provider, TruffleTierConfiguration firstTier, - TruffleTierConfiguration lastTier) { + public TruffleCompilerConfiguration( + TruffleCompilerRuntime runtime, + GraphBuilderConfiguration.Plugins plugins, + SnippetReflectionProvider provider, + TruffleTierConfiguration firstTier, + TruffleTierConfiguration lastTier, + KnownTruffleTypes knownTruffleTypes) { this.runtime = runtime; this.plugins = plugins; this.provider = provider; this.firstTier = firstTier; this.lastTier = lastTier; + this.knownTruffleTypes = knownTruffleTypes; } public TruffleCompilerRuntime runtime() { @@ -70,8 +78,12 @@ public TruffleTierConfiguration lastTier() { return lastTier; } + public KnownTruffleTypes getKnownTruffleTypes() { + return knownTruffleTypes; + } + public TruffleCompilerConfiguration withFirstTier(TruffleTierConfiguration tier) { - return new TruffleCompilerConfiguration(runtime, plugins, provider, tier, lastTier); + return new TruffleCompilerConfiguration(runtime, plugins, provider, tier, lastTier, knownTruffleTypes); } public List backends() { diff --git a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleConstantFieldProvider.java b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleConstantFieldProvider.java index 3330c5663886..0d2e370afc11 100644 --- a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleConstantFieldProvider.java +++ b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleConstantFieldProvider.java @@ -35,20 +35,12 @@ import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaField; -import jdk.vm.ci.meta.ResolvedJavaType; -public class TruffleConstantFieldProvider implements ConstantFieldProvider { - private final ConstantFieldProvider graalConstantFieldProvider; - private final MetaAccessProvider metaAccess; - private final KnownTruffleTypes types; +public class TruffleConstantFieldProvider extends TruffleStringConstantFieldProvider { private final ConcurrentHashMap cachedConstantFieldInfo; - private final ResolvedJavaType byteArrayType; public TruffleConstantFieldProvider(ConstantFieldProvider graalConstantFieldProvider, MetaAccessProvider metaAccess, KnownTruffleTypes types) { - this.graalConstantFieldProvider = graalConstantFieldProvider; - this.metaAccess = metaAccess; - this.types = types; - this.byteArrayType = metaAccess.lookupJavaType(byte[].class); + super(graalConstantFieldProvider, metaAccess, types); this.cachedConstantFieldInfo = new ConcurrentHashMap<>(); } @@ -59,27 +51,9 @@ public T readConstantField(ResolvedJavaField field, ConstantFieldTool too // can't be optimized return null; } - - // well-known internal fields of AbstractTruffleString - if (types.truffleStringDataField.equals(field) || types.truffleStringHashCodeField.equals(field)) { - // only applies to the immutable subclass TruffleString, not MutableTruffleString - if (types.truffleStringType.isAssignableFrom(metaAccess.lookupJavaType(tool.getReceiver()))) { - JavaConstant value = tool.readValue(); - if (value != null) { - if (types.truffleStringDataField.equals(field)) { - // the "data" field is implicitly stable if it contains a byte array - if (byteArrayType.isAssignableFrom(metaAccess.lookupJavaType(value))) { - return tool.foldStableArray(value, 1, true); - } - } else { - assert types.truffleStringHashCodeField.equals(field); - // the "hashCode" field is stable if its value is not zero - if (!value.isDefaultForKind()) { - return tool.foldConstant(value); - } - } - } - } + T wellKnownField = readWellKnownConstantTruffleField(field, tool); + if (wellKnownField != null) { + return wellKnownField; } boolean isArrayField = field.getType().isArray(); @@ -119,11 +93,6 @@ public T readConstantField(ResolvedJavaField field, ConstantFieldTool too return null; } - @Override - public boolean maybeFinal(ResolvedJavaField field) { - return types.truffleStringDataField.equals(field) || types.truffleStringHashCodeField.equals(field) || graalConstantFieldProvider.maybeFinal(field); - } - private ConstantFieldInfo getConstantFieldInfo(ResolvedJavaField field) { return cachedConstantFieldInfo.computeIfAbsent(field, f -> TruffleCompilerRuntime.getRuntime().getConstantFieldInfo(f)); } diff --git a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleStringConstantFieldProvider.java b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleStringConstantFieldProvider.java new file mode 100644 index 000000000000..364777022c9a --- /dev/null +++ b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleStringConstantFieldProvider.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.truffle.compiler; + +import org.graalvm.compiler.core.common.spi.ConstantFieldProvider; +import org.graalvm.compiler.truffle.compiler.substitutions.KnownTruffleTypes; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaType; + +public class TruffleStringConstantFieldProvider implements ConstantFieldProvider { + protected final ConstantFieldProvider graalConstantFieldProvider; + protected final MetaAccessProvider metaAccess; + private final KnownTruffleTypes types; + private final ResolvedJavaType byteArrayType; + + public TruffleStringConstantFieldProvider(ConstantFieldProvider graalConstantFieldProvider, MetaAccessProvider metaAccess, KnownTruffleTypes types) { + this.graalConstantFieldProvider = graalConstantFieldProvider; + this.metaAccess = metaAccess; + this.types = types; + this.byteArrayType = metaAccess.lookupJavaType(byte[].class); + } + + @Override + public T readConstantField(ResolvedJavaField field, ConstantFieldTool tool) { + boolean isStaticField = field.isStatic(); + if (!isStaticField && tool.getReceiver().isNull()) { + // can't be optimized + return null; + } + T wellKnownField = readWellKnownConstantTruffleField(field, tool); + if (wellKnownField != null) { + return wellKnownField; + } + return graalConstantFieldProvider.readConstantField(field, tool); + + } + + protected T readWellKnownConstantTruffleField(ResolvedJavaField field, ConstantFieldTool tool) { + // well-known internal fields of AbstractTruffleString + if (types.truffleStringDataField.equals(field) || types.truffleStringHashCodeField.equals(field)) { + // only applies to the immutable subclass TruffleString, not MutableTruffleString + if (types.truffleStringType.isAssignableFrom(metaAccess.lookupJavaType(tool.getReceiver()))) { + JavaConstant value = tool.readValue(); + if (value != null) { + if (types.truffleStringDataField.equals(field)) { + // the "data" field is implicitly stable if it contains a byte array + if (byteArrayType.isAssignableFrom(metaAccess.lookupJavaType(value))) { + return tool.foldStableArray(value, 1, true); + } + } else { + assert types.truffleStringHashCodeField.equals(field); + // the "hashCode" field is stable if its value is not zero + if (!value.isDefaultForKind()) { + return tool.foldConstant(value); + } + } + } + } + } + return null; + } + + @Override + public boolean maybeFinal(ResolvedJavaField field) { + return types.truffleStringDataField.equals(field) || types.truffleStringHashCodeField.equals(field) || graalConstantFieldProvider.maybeFinal(field); + } +} diff --git a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleTierConfiguration.java b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleTierConfiguration.java index 5bcfd82dcec3..1c5c433b7def 100644 --- a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleTierConfiguration.java +++ b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/TruffleTierConfiguration.java @@ -30,6 +30,7 @@ import org.graalvm.compiler.phases.tiers.Suites; import org.graalvm.compiler.phases.util.Providers; import org.graalvm.compiler.truffle.compiler.phases.TruffleCompilerPhases; +import org.graalvm.compiler.truffle.compiler.substitutions.KnownTruffleTypes; public final class TruffleTierConfiguration { private final PartialEvaluatorConfiguration configuration; @@ -38,14 +39,14 @@ public final class TruffleTierConfiguration { private final Suites suites; private final LIRSuites lirSuites; - public TruffleTierConfiguration(PartialEvaluatorConfiguration configuration, Backend backend, OptionValues options) { - this(configuration, backend, backend.getProviders(), backend.getSuites().getDefaultSuites(options), backend.getSuites().getDefaultLIRSuites(options)); + public TruffleTierConfiguration(PartialEvaluatorConfiguration configuration, Backend backend, OptionValues options, KnownTruffleTypes knownTruffleTypes) { + this(configuration, backend, backend.getProviders(), backend.getSuites().getDefaultSuites(options), backend.getSuites().getDefaultLIRSuites(options), knownTruffleTypes); } - public TruffleTierConfiguration(PartialEvaluatorConfiguration configuration, Backend backend, Providers providers, Suites suites, LIRSuites lirSuites) { + public TruffleTierConfiguration(PartialEvaluatorConfiguration configuration, Backend backend, Providers providers, Suites suites, LIRSuites lirSuites, KnownTruffleTypes knownTruffleTypes) { this.configuration = configuration; this.backend = backend; - this.providers = providers; + this.providers = providers.copyWith(new TruffleStringConstantFieldProvider(providers.getConstantFieldProvider(), providers.getMetaAccess(), knownTruffleTypes)); this.suites = suites; this.lirSuites = lirSuites; TruffleCompilerPhases.register(providers, suites); diff --git a/compiler/src/org.graalvm.compiler.truffle.test/src/org/graalvm/compiler/truffle/test/strings/TStringOpsRegionEqualsConstantLengthTest.java b/compiler/src/org.graalvm.compiler.truffle.test/src/org/graalvm/compiler/truffle/test/strings/TStringOpsRegionEqualsConstantLengthTest.java index d9c26ab3c39c..55a82c830f20 100644 --- a/compiler/src/org.graalvm.compiler.truffle.test/src/org/graalvm/compiler/truffle/test/strings/TStringOpsRegionEqualsConstantLengthTest.java +++ b/compiler/src/org.graalvm.compiler.truffle.test/src/org/graalvm/compiler/truffle/test/strings/TStringOpsRegionEqualsConstantLengthTest.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.stream.Collectors; -import org.graalvm.compiler.core.common.StrideUtil; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.lir.amd64.AMD64ArrayEqualsOp; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; @@ -45,6 +45,7 @@ import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.code.InstalledCode; +import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.ResolvedJavaMethod; @RunWith(Parameterized.class) @@ -87,7 +88,7 @@ protected InstalledCode getCode(final ResolvedJavaMethod installedCodeOwner, Str @Test public void testRegionEquals() { Assume.assumeTrue(getTarget().arch instanceof AMD64); - Assume.assumeTrue(AMD64ArrayEqualsOp.canGenerateConstantLengthCompare(getTarget(), null, StrideUtil.log2ToStride(strideA), StrideUtil.log2ToStride(strideB), lengthCMP, getMaxVectorSize())); + Assume.assumeTrue(AMD64ArrayEqualsOp.canGenerateConstantLengthCompare(getTarget(), null, JavaKind.Byte, Stride.fromLog2(strideA), Stride.fromLog2(strideB), lengthCMP, getMaxVectorSize())); constantArgs[4] = strideA; constantArgs[9] = strideB; constantArgs[12] = lengthCMP; diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/EspressoBindings.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/EspressoBindings.java index ccfb0bf549d9..d35baaafd865 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/EspressoBindings.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/EspressoBindings.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -67,6 +67,10 @@ public EspressoBindings(@JavaType(ClassLoader.class) StaticObject loader, boolea this.loader = loader; } + public StaticObject getBindingsLoader() { + return loader; + } + @ExportMessage @SuppressWarnings("static-method") Object getMembers(@SuppressWarnings("unused") boolean includeInternal) { diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/EspressoOptions.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/EspressoOptions.java index 319372d0b52e..256dddcda4b6 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/EspressoOptions.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/EspressoOptions.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -49,6 +49,7 @@ public final class EspressoOptions { private static final Path EMPTY = Paths.get(""); private static final String PATH_SEPARATOR_INSERT = "\" + java.io.File.pathSeparator + \""; + private static final String SEMI_COLON = ";"; /** * File.pathSeparator-delimited list of paths. @@ -86,6 +87,17 @@ public Path apply(String path) { } }); + private static final OptionType> STRINGS_OPTION_TYPE_SEPARATED_BY_SEMI_COLON = new OptionType<>("Strings", new Function>() { + @Override + public List apply(String strings) { + try { + return Collections.unmodifiableList(splitBySemiColon(strings)); + } catch (InvalidPathException e) { + throw new IllegalArgumentException(e); + } + } + }); + @Option(help = "User-defined system properties.", // category = OptionCategory.USER, // stability = OptionStability.STABLE, // @@ -188,6 +200,12 @@ public Path apply(String path) { ) // public static final OptionKey> ExtDirs = new OptionKey<>(Collections.emptyList(), PATHS_OPTION_TYPE); + @Option(help = "A '" + SEMI_COLON + "' separated list of fully qualified interface names that enables interface type mapping in polyglot usage.", // + category = OptionCategory.USER, // + stability = OptionStability.EXPERIMENTAL, // + usageSyntax = "my.first.MyInterface;my.second.MySecondInterface;...") // + public static final OptionKey> PolyglotInterfaceMappings = new OptionKey<>(Collections.emptyList(), STRINGS_OPTION_TYPE_SEPARATED_BY_SEMI_COLON); + @Option(help = "Enable assertions.", // category = OptionCategory.USER, // stability = OptionStability.STABLE, // @@ -212,6 +230,10 @@ private static List splitByFileSeparator(String strings) { return new ArrayList<>(Arrays.asList(strings.split(File.pathSeparator))); } + private static List splitBySemiColon(String strings) { + return new ArrayList<>(Arrays.asList(strings.split(EspressoOptions.SEMI_COLON))); + } + public enum SpecComplianceMode { STRICT, HOTSPOT diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/ObjectKlass.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/ObjectKlass.java index de41d75c4eb5..8073d4b0116a 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/ObjectKlass.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/ObjectKlass.java @@ -697,7 +697,7 @@ private void verifyImpl() { try { MethodVerifier.verify(m); } catch (MethodVerifier.VerifierError e) { - String message = String.format("Verification for class `%s` failed for method `%s`", getExternalName(), m.getNameAsString()); + String message = String.format("Verification for class `%s` failed for method `%s` with message `%s`", getExternalName(), m.getNameAsString(), e.getMessage()); switch (e.kind()) { case Verify: throw meta.throwExceptionWithMessage(meta.java_lang_VerifyError, message); diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/meta/JavaKind.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/meta/JavaKind.java index e4c16fac5723..936766fb7b28 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/meta/JavaKind.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/meta/JavaKind.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -96,6 +96,9 @@ public enum JavaKind { private final int basicType; private final Symbol type; private final Symbol name; + private final String unwrapMethodName; + private final String unwrapMethodDesc; + private final String wrapperValueOfDesc; JavaKind(char typeChar, int basicType, String javaName, int slotCount, boolean isStackInt, Class primitiveJavaClass, Class boxedJavaClass) { this.typeChar = typeChar; @@ -107,6 +110,9 @@ public enum JavaKind { this.basicType = basicType; this.type = (primitiveJavaClass != null) ? StaticSymbols.putType("" + typeChar) : null; this.name = StaticSymbols.putName(javaName); + this.unwrapMethodName = (primitiveJavaClass != null) ? javaName + "Value" : null; + this.unwrapMethodDesc = (primitiveJavaClass != null) ? "()" + typeChar : null; + this.wrapperValueOfDesc = (primitiveJavaClass != null) ? "(" + typeChar + ")Ljava/lang/" + boxedJavaClass.getSimpleName() + ";" : null; assert primitiveJavaClass == null || javaName.equals(primitiveJavaClass.getName()); } @@ -442,6 +448,18 @@ public Symbol getPrimitiveBinaryName() { return name; } + public String getUnwrapMethodName() { + return unwrapMethodName; + } + + public String getUnwrapMethodDesc() { + return unwrapMethodDesc; + } + + public String getWrapperValueOfDesc() { + return wrapperValueOfDesc; + } + /** * Marker interface for types that should be {@linkplain JavaKind#format(Object) formatted} with * their {@link Object#toString()} value. Calling {@link Object#toString()} on other objects diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/BytecodeNode.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/BytecodeNode.java index 4fe2e8e83495..23badb5c1e57 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/BytecodeNode.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/BytecodeNode.java @@ -1718,15 +1718,57 @@ private static boolean takeBranchPrimitive2(int operand1, int operand2, int opco // @formatter:on } - private static boolean takeBranchRef2(StaticObject operand1, StaticObject operand2, int opcode) { + private boolean takeBranchRef2(StaticObject operand1, StaticObject operand2, int opcode) { assert IF_ACMPEQ <= opcode && opcode <= IF_ACMPNE; // @formatter:off - switch (opcode) { - case IF_ACMPEQ : return operand1 == operand2; - case IF_ACMPNE : return operand1 != operand2; - default : - CompilerDirectives.transferToInterpreterAndInvalidate(); - throw EspressoError.shouldNotReachHere("expecting IF_ACMPEQ,IF_ACMPNE"); + if (noForeignObjects.isValid()) { + switch (opcode) { + case IF_ACMPEQ: + return operand1 == operand2; + case IF_ACMPNE: + return operand1 != operand2; + default: + CompilerDirectives.transferToInterpreterAndInvalidate(); + throw EspressoError.shouldNotReachHere("expecting IF_ACMPEQ,IF_ACMPNE"); + } + } else { + switch (opcode) { + case IF_ACMPEQ: { + if (operand1 == operand2) { + return true; + } + // Espresso null == foreign null + if (StaticObject.isNull(operand1) && StaticObject.isNull(operand2)) { + return true; + } + // an Espresso object can never be identical to a foreign object + if (operand1.isForeignObject() && operand2.isForeignObject()) { + InteropLibrary operand1Lib = InteropLibrary.getUncached(operand1); + InteropLibrary operand2Lib = InteropLibrary.getUncached(operand2); + return operand1Lib.isIdentical(operand1, operand2, operand2Lib); + } + return false; + } + case IF_ACMPNE: { + if (operand1 == operand2) { + return false; + } + // Espresso null == foreign null + if (StaticObject.isNull(operand1) && StaticObject.isNull(operand2)) { + return false; + } + // an Espresso object can never be identical to a foreign object + if (operand1.isForeignObject() && operand2.isForeignObject()) { + InteropLibrary operand1Lib = InteropLibrary.getUncached(operand1); + InteropLibrary operand2Lib = InteropLibrary.getUncached(operand2); + return !operand1Lib.isIdentical(operand1, operand2, operand2Lib); + } + return operand1 != operand2; + } + default: + CompilerDirectives.transferToInterpreterAndInvalidate(); + throw EspressoError.shouldNotReachHere("expecting IF_ACMPEQ,IF_ACMPNE"); + } } // @formatter:on } diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/EspressoForeignProxyGenerator.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/EspressoForeignProxyGenerator.java new file mode 100644 index 000000000000..7498ec2b53f6 --- /dev/null +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/EspressoForeignProxyGenerator.java @@ -0,0 +1,814 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.truffle.espresso.nodes.interop; + +import static com.oracle.truffle.api.impl.asm.Opcodes.AASTORE; +import static com.oracle.truffle.api.impl.asm.Opcodes.ACC_FINAL; +import static com.oracle.truffle.api.impl.asm.Opcodes.ACC_PUBLIC; +import static com.oracle.truffle.api.impl.asm.Opcodes.ACC_SUPER; +import static com.oracle.truffle.api.impl.asm.Opcodes.ACC_VARARGS; +import static com.oracle.truffle.api.impl.asm.Opcodes.ALOAD; +import static com.oracle.truffle.api.impl.asm.Opcodes.ANEWARRAY; +import static com.oracle.truffle.api.impl.asm.Opcodes.ARETURN; +import static com.oracle.truffle.api.impl.asm.Opcodes.ASTORE; +import static com.oracle.truffle.api.impl.asm.Opcodes.ATHROW; +import static com.oracle.truffle.api.impl.asm.Opcodes.BIPUSH; +import static com.oracle.truffle.api.impl.asm.Opcodes.CHECKCAST; +import static com.oracle.truffle.api.impl.asm.Opcodes.DLOAD; +import static com.oracle.truffle.api.impl.asm.Opcodes.DRETURN; +import static com.oracle.truffle.api.impl.asm.Opcodes.DUP; +import static com.oracle.truffle.api.impl.asm.Opcodes.FLOAD; +import static com.oracle.truffle.api.impl.asm.Opcodes.FRETURN; +import static com.oracle.truffle.api.impl.asm.Opcodes.ICONST_0; +import static com.oracle.truffle.api.impl.asm.Opcodes.ILOAD; +import static com.oracle.truffle.api.impl.asm.Opcodes.INVOKESPECIAL; +import static com.oracle.truffle.api.impl.asm.Opcodes.INVOKESTATIC; +import static com.oracle.truffle.api.impl.asm.Opcodes.INVOKEVIRTUAL; +import static com.oracle.truffle.api.impl.asm.Opcodes.IRETURN; +import static com.oracle.truffle.api.impl.asm.Opcodes.LLOAD; +import static com.oracle.truffle.api.impl.asm.Opcodes.LRETURN; +import static com.oracle.truffle.api.impl.asm.Opcodes.NEW; +import static com.oracle.truffle.api.impl.asm.Opcodes.POP; +import static com.oracle.truffle.api.impl.asm.Opcodes.RETURN; +import static com.oracle.truffle.api.impl.asm.Opcodes.SIPUSH; +import static com.oracle.truffle.api.impl.asm.Opcodes.V1_8; + +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.impl.asm.ClassWriter; +import com.oracle.truffle.api.impl.asm.Label; +import com.oracle.truffle.api.impl.asm.MethodVisitor; +import com.oracle.truffle.espresso.impl.Klass; +import com.oracle.truffle.espresso.impl.Method; +import com.oracle.truffle.espresso.impl.ObjectKlass; +import com.oracle.truffle.espresso.meta.EspressoError; +import com.oracle.truffle.espresso.meta.JavaKind; +import com.oracle.truffle.espresso.meta.Meta; +import com.oracle.truffle.espresso.runtime.EspressoContext; + +/** + * EspressoForeignProxyGenerator contains the code to generate a dynamic Espresso proxy class for + * foreign objects. + * + * Large parts of this class is copied from java.lang.reflect.ProxyGenerator. + */ +public final class EspressoForeignProxyGenerator extends ClassWriter { + + private static final String JL_OBJECT = "java/lang/Object"; + private static final String JL_THROWABLE = "java/lang/Throwable"; + private static final String JLR_UNDECLARED_THROWABLE_EX = "java/lang/reflect/UndeclaredThrowableException"; + private static final int VARARGS = 0x00000080; + + /* name of the superclass of proxy classes */ + private static final String superclassName = "java/lang/Object"; + + private Meta meta; + + /* name of proxy class */ + private String className; + + /* proxy interfaces */ + private ObjectKlass[] interfaces; + + /* proxy class access flags */ + private int accessFlags; + + /* + * Maps method signature string to list of ProxyMethod objects for proxy methods with that + * signature. + */ + private Map> proxyMethods = new HashMap<>(); + + // next number to use for generation of unique proxy class names + private static final AtomicLong nextUniqueNumber = new AtomicLong(); + + private static final String proxyNamePrefix = "com.oracle.truffle.espresso.polyglot.Foreign$Proxy$"; + + /** + * Construct a ProxyGenerator to generate a proxy class with the specified name and for the + * given interfaces. + * + * A ProxyGenerator object contains the state for the ongoing generation of a particular proxy + * class. + */ + private EspressoForeignProxyGenerator(Meta meta, ObjectKlass[] interfaces) { + super(ClassWriter.COMPUTE_FRAMES); + this.meta = meta; + this.className = nextClassName(); + this.interfaces = interfaces; + this.accessFlags = ACC_PUBLIC | ACC_FINAL | ACC_SUPER; + } + + public static class GeneratedProxyBytes { + public final byte[] bytes; + public final String name; + + GeneratedProxyBytes(byte[] bytes, String name) { + this.bytes = bytes; + this.name = name; + } + } + + @TruffleBoundary + public static GeneratedProxyBytes getProxyKlassBytes(String metaName, ObjectKlass[] interfaces, EspressoContext context) { + synchronized (context) { + GeneratedProxyBytes generatedProxyBytes = context.getProxyBytesOrNull(metaName); + if (generatedProxyBytes == null) { + EspressoForeignProxyGenerator generator = new EspressoForeignProxyGenerator(context.getMeta(), interfaces); + generatedProxyBytes = new GeneratedProxyBytes(generator.generateClassFile(), generator.className); + context.registerProxyBytes(metaName, generatedProxyBytes); + } + return generatedProxyBytes; + } + } + + private static String nextClassName() { + return proxyNamePrefix + nextUniqueNumber.getAndIncrement(); + } + + /** + * Generate a class file for the proxy class. This method drives the class file generation + * process. + */ + private byte[] generateClassFile() { + visit(V1_8, accessFlags, dotToSlash(className), null, + superclassName, typeNames(interfaces)); + + // toString is implemented by interop protocol by means of + // toDisplayString and asString + generateToStringMethod(); + + /* + * Accumulate all of the methods from the proxy interfaces. + */ + for (ObjectKlass intf : interfaces) { + for (Method m : intf.getDeclaredMethods()) { + if (!Modifier.isStatic(m.getModifiers()) && !Modifier.isPrivate(m.getModifiers())) { + addProxyMethod(m); + } + } + } + + /* + * For each set of proxy methods with the same signature, verify that the methods' return + * types are compatible. + */ + for (List sigmethods : proxyMethods.values()) { + checkReturnTypes(sigmethods); + } + + generateConstructor(); + + for (List sigmethods : proxyMethods.values()) { + for (ProxyMethod pm : sigmethods) { + // Generate code for proxy method + pm.generateMethod(this); + } + } + + return toByteArray(); + } + + private void generateToStringMethod() { + MethodVisitor mv = visitMethod(ACC_PUBLIC | ACC_FINAL, + "toString", "()Ljava/lang/String;", null, + null); + + mv.visitCode(); + Label startBlock = new Label(); + Label endBlock = new Label(); + Label runtimeHandler = new Label(); + Label throwableHandler = new Label(); + + mv.visitLabel(startBlock); + + mv.visitVarInsn(ALOAD, 0); + + mv.visitMethodInsn(INVOKESTATIC, "com/oracle/truffle/espresso/polyglot/Interop", + "toDisplayString", + "(Ljava/lang/Object;)Ljava/lang/Object;", + false); + + mv.visitMethodInsn(INVOKESTATIC, "com/oracle/truffle/espresso/polyglot/Interop", + "asString", + "(Ljava/lang/Object;)Ljava/lang/String;", + false); + + mv.visitInsn(ARETURN); + + mv.visitLabel(endBlock); + + // Generate exception handler + mv.visitLabel(runtimeHandler); + mv.visitInsn(ATHROW); // just rethrow the exception + + mv.visitLabel(throwableHandler); + mv.visitVarInsn(ASTORE, 1); + mv.visitTypeInsn(NEW, JLR_UNDECLARED_THROWABLE_EX); + mv.visitInsn(DUP); + mv.visitVarInsn(ALOAD, 1); + mv.visitMethodInsn(INVOKESPECIAL, JLR_UNDECLARED_THROWABLE_EX, + "", "(Ljava/lang/Throwable;)V", false); + mv.visitInsn(ATHROW); + // Maxs computed by ClassWriter.COMPUTE_FRAMES, these arguments ignored + mv.visitMaxs(-1, -1); + mv.visitEnd(); + } + + /** + * Return an array of the class and interface names from an array of Classes. + * + * @param classes an array of classes or interfaces + * @return the array of class and interface names; or null if classes is null or empty + */ + private static String[] typeNames(Klass[] classes) { + if (classes == null || classes.length == 0) { + return null; + } + int size = classes.length; + String[] ifaces = new String[size]; + for (int i = 0; i < size; i++) { + ifaces[i] = dotToSlash(classes[i].getNameAsString()); + } + return ifaces; + } + + /** + * Generate the constructor method for the proxy class. + */ + private void generateConstructor() { + MethodVisitor ctor = visitMethod(Modifier.PUBLIC, "", + "()V", null, null); + ctor.visitParameter(null, 0); + ctor.visitCode(); + ctor.visitVarInsn(ALOAD, 0); + ctor.visitMethodInsn(INVOKESPECIAL, superclassName, "", + "()V", false); + ctor.visitInsn(RETURN); + + // Maxs computed by ClassWriter.COMPUTE_FRAMES, these arguments ignored + ctor.visitMaxs(-1, -1); + ctor.visitEnd(); + } + + /** + * Add another method to be proxied, either by creating a new ProxyMethod object or augmenting + * an old one for a duplicate method. + * + * "fromClass" indicates the proxy interface that the method was found through, which may be + * different from (a subinterface of) the method's "declaring class". Note that the first Method + * object passed for a given name and descriptor identifies the Method object (and thus the + * declaring class) that will be passed to the invocation handler's "invoke" method for a given + * set of duplicate methods. + */ + private void addProxyMethod(Method m) { + String name = m.getNameAsString(); + Klass[] parameterTypes = m.resolveParameterKlasses(); + Klass returnType = m.resolveReturnKlass(); + ObjectKlass[] exceptionTypes = m.getCheckedExceptions(); + + String sig = name + getParameterDescriptors(parameterTypes); + List sigmethods = proxyMethods.get(sig); + if (sigmethods != null) { + for (ProxyMethod pm : sigmethods) { + if (returnType == pm.returnType) { + /* + * Found a match: reduce exception types to the greatest set of exceptions that + * can thrown compatibly with the throws clauses of both overridden methods. + */ + List legalExceptions = new ArrayList<>(); + collectCompatibleTypes( + exceptionTypes, pm.exceptionTypes, legalExceptions); + collectCompatibleTypes( + pm.exceptionTypes, exceptionTypes, legalExceptions); + pm.exceptionTypes = new Klass[legalExceptions.size()]; + pm.exceptionTypes = legalExceptions.toArray(pm.exceptionTypes); + return; + } + } + } else { + sigmethods = new ArrayList<>(3); + proxyMethods.put(sig, sigmethods); + } + sigmethods.add(new ProxyMethod(name, parameterTypes, returnType, + exceptionTypes, isVarArgs(m.getModifiers()))); + } + + private static boolean isVarArgs(int modifiers) { + return (modifiers & VARARGS) != 0; + } + + /** + * For a given set of proxy methods with the same signature, check that their return types are + * compatible according to the Proxy specification. + * + * Specifically, if there is more than one such method, then all of the return types must be + * reference types, and there must be one return type that is assignable to each of the rest of + * them. + */ + private static void checkReturnTypes(List methods) { + /* + * If there is only one method with a given signature, there cannot be a conflict. This is + * the only case in which a primitive (or void) return type is allowed. + */ + if (methods.size() < 2) { + return; + } + + /* + * List of return types that are not yet known to be assignable from ("covered" by) any of + * the others. + */ + LinkedList uncoveredReturnTypes = new LinkedList<>(); + + nextNewReturnType: for (ProxyMethod pm : methods) { + Klass newReturnType = pm.returnType; + if (newReturnType.isPrimitive()) { + throw new IllegalArgumentException( + "methods with same signature " + + getFriendlyMethodSignature(pm.methodName, + pm.parameterTypes) + + " but incompatible return types: " + + newReturnType.getName() + " and others"); + } + boolean added = false; + + /* + * Compare the new return type to the existing uncovered return types. + */ + ListIterator liter = uncoveredReturnTypes.listIterator(); + while (liter.hasNext()) { + Klass uncoveredReturnType = liter.next(); + + /* + * If an existing uncovered return type is assignable to this new one, then we can + * forget the new one. + */ + if (newReturnType.isAssignableFrom(uncoveredReturnType)) { + assert !added; + continue nextNewReturnType; + } + + /* + * If the new return type is assignable to an existing uncovered one, then should + * replace the existing one with the new one (or just forget the existing one, if + * the new one has already be put in the list). + */ + if (uncoveredReturnType.isAssignableFrom(newReturnType)) { + // (we can assume that each return type is unique) + if (!added) { + liter.set(newReturnType); + added = true; + } else { + liter.remove(); + } + } + } + + /* + * If we got through the list of existing uncovered return types without an + * assignability relationship, then add the new return type to the list of uncovered + * ones. + */ + if (!added) { + uncoveredReturnTypes.add(newReturnType); + } + } + + /* + * We shouldn't end up with more than one return type that is not assignable from any of the + * others. + */ + if (uncoveredReturnTypes.size() > 1) { + ProxyMethod pm = methods.get(0); + throw new IllegalArgumentException( + "methods with same signature " + + getFriendlyMethodSignature(pm.methodName, pm.parameterTypes) + + " but incompatible return types: " + uncoveredReturnTypes); + } + } + + /** + * A ProxyMethod object represents a proxy method in the proxy class being generated: a method + * whose implementation will encode and dispatch invocations to the proxy instance's invocation + * handler. + */ + private final class ProxyMethod { + + public String methodName; + public Klass[] parameterTypes; + public Klass returnType; + public Klass[] exceptionTypes; + boolean isVarArgs; + + private ProxyMethod(String methodName, Klass[] parameterTypes, + Klass returnType, Klass[] exceptionTypes, boolean isVarArgs) { + this.methodName = methodName; + this.parameterTypes = parameterTypes; + this.returnType = returnType; + this.exceptionTypes = exceptionTypes; + this.isVarArgs = isVarArgs; + } + + private void generateMethod(ClassWriter cw) { + String desc = getMethodDescriptor(parameterTypes, returnType); + int methodAccess = ACC_PUBLIC | ACC_FINAL; + + if (isVarArgs) { + methodAccess |= ACC_VARARGS; + } + + MethodVisitor mv = cw.visitMethod(methodAccess, + methodName, desc, null, + typeNames(exceptionTypes)); + + int[] parameterSlot = new int[parameterTypes.length]; + int nextSlot = 1; + for (int i = 0; i < parameterSlot.length; i++) { + parameterSlot[i] = nextSlot; + nextSlot += getWordsPerType(parameterTypes[i]); + } + + mv.visitCode(); + Label startBlock = new Label(); + Label endBlock = new Label(); + Label runtimeHandler = new Label(); + Label throwableHandler = new Label(); + + List catchList = computeUniqueCatchList(exceptionTypes); + if (catchList.size() > 0) { + for (Klass ex : catchList) { + mv.visitTryCatchBlock(startBlock, endBlock, runtimeHandler, + dotToSlash(ex.getNameAsString())); + } + + mv.visitTryCatchBlock(startBlock, endBlock, throwableHandler, + JL_THROWABLE); + } + mv.visitLabel(startBlock); + + mv.visitVarInsn(ALOAD, 0); + mv.visitLdcInsn(methodName); + + if (parameterTypes.length > 0) { + // Create an array and fill with the parameters converting primitives to wrappers + emitIconstInsn(mv, parameterTypes.length); + mv.visitTypeInsn(ANEWARRAY, JL_OBJECT); + for (int i = 0; i < parameterTypes.length; i++) { + mv.visitInsn(DUP); + emitIconstInsn(mv, i); + codeWrapArgument(mv, parameterTypes[i], parameterSlot[i]); + mv.visitInsn(AASTORE); + } + } else { + mv.visitInsn(ICONST_0); + mv.visitTypeInsn(ANEWARRAY, JL_OBJECT); + } + + mv.visitMethodInsn(INVOKESTATIC, "com/oracle/truffle/espresso/polyglot/Interop", + "invokeMember", + "(Ljava/lang/Object;Ljava/lang/String;" + + "[Ljava/lang/Object;)Ljava/lang/Object;", + false); + + if (returnType == meta._void) { + mv.visitInsn(POP); + mv.visitInsn(RETURN); + } else { + codeUnwrapReturnValue(mv, returnType); + } + + mv.visitLabel(endBlock); + + // Generate exception handler + mv.visitLabel(runtimeHandler); + mv.visitInsn(ATHROW); // just rethrow the exception + + mv.visitLabel(throwableHandler); + mv.visitVarInsn(ASTORE, 1); + mv.visitTypeInsn(NEW, JLR_UNDECLARED_THROWABLE_EX); + mv.visitInsn(DUP); + mv.visitVarInsn(ALOAD, 1); + mv.visitMethodInsn(INVOKESPECIAL, JLR_UNDECLARED_THROWABLE_EX, + "", "(Ljava/lang/Throwable;)V", false); + mv.visitInsn(ATHROW); + // Maxs computed by ClassWriter.COMPUTE_FRAMES, these arguments ignored + mv.visitMaxs(-1, -1); + mv.visitEnd(); + } + + /** + * Generate code for wrapping an argument of the given type whose value can be found at the + * specified local variable index, in order for it to be passed (as an Object) to the + * invocation handler's "invoke" method. + */ + private void codeWrapArgument(MethodVisitor mv, Klass type, int slot) { + if (type.isPrimitive()) { + JavaKind kind = JavaKind.fromTypeString(type.getTypeAsString()); + + switch (kind) { + case Int: + case Boolean: + case Char: + case Byte: + case Short: + mv.visitVarInsn(ILOAD, slot); + break; + case Long: + mv.visitVarInsn(LLOAD, slot); + break; + case Float: + mv.visitVarInsn(FLOAD, slot); + break; + case Double: + mv.visitVarInsn(DLOAD, slot); + break; + default: + throw EspressoError.shouldNotReachHere(); + } + + mv.visitMethodInsn(INVOKESTATIC, dotToSlash(kind.toBoxedJavaClass().getName()), "valueOf", + kind.getWrapperValueOfDesc(), false); + } else { + mv.visitVarInsn(ALOAD, slot); + } + } + + /** + * Generate code for unwrapping a return value of the given type from the invocation + * handler's "invoke" method (as type Object) to its correct type. + */ + private void codeUnwrapReturnValue(MethodVisitor mv, Klass type) { + if (type.isPrimitive()) { + JavaKind kind = JavaKind.fromTypeString(type.getTypeAsString()); + String wrapperClassName = dotToSlash(kind.toBoxedJavaClass().getName()); + mv.visitTypeInsn(CHECKCAST, wrapperClassName); + mv.visitMethodInsn(INVOKEVIRTUAL, + wrapperClassName, + kind.getUnwrapMethodName(), kind.getUnwrapMethodDesc(), false); + + switch (kind) { + case Int: + case Boolean: + case Byte: + case Short: + case Char: + mv.visitInsn(IRETURN); + break; + case Long: + mv.visitInsn(LRETURN); + break; + case Float: + mv.visitInsn(FRETURN); + break; + case Double: + mv.visitInsn(DRETURN); + break; + default: + throw new AssertionError(); + } + } else { + mv.visitTypeInsn(CHECKCAST, dotToSlash(type.getNameAsString())); + mv.visitInsn(ARETURN); + } + } + + /* + * =============== Code Generation Utility Methods =============== + */ + + /** + * Visit a bytecode for a constant. + * + * @param mv The MethodVisitor + * @param cst The constant value + */ + private void emitIconstInsn(MethodVisitor mv, final int cst) { + if (cst >= -1 && cst <= 5) { + mv.visitInsn(ICONST_0 + cst); + } else if (cst >= Byte.MIN_VALUE && cst <= Byte.MAX_VALUE) { + mv.visitIntInsn(BIPUSH, cst); + } else if (cst >= Short.MIN_VALUE && cst <= Short.MAX_VALUE) { + mv.visitIntInsn(SIPUSH, cst); + } else { + mv.visitLdcInsn(cst); + } + } + } + + /* + * ==================== General Utility Methods ==================== + */ + + /** + * Convert a fully qualified class name that uses '.' as the package separator, the external + * representation used by the Java language and APIs, to a fully qualified class name that uses + * '/' as the package separator, the representation used in the class file format (see JVMS + * section 4.2). + */ + private static String dotToSlash(String name) { + return name.replace('.', '/'); + } + + /** + * Return the "method descriptor" string for a method with the given parameter types and return + * type. See JVMS section 4.3.3. + */ + private String getMethodDescriptor(Klass[] parameterTypes, + Klass returnType) { + return getParameterDescriptors(parameterTypes) + + ((returnType == meta._void) ? "V" : getFieldType(returnType)); + } + + /** + * Return the list of "parameter descriptor" strings enclosed in parentheses corresponding to + * the given parameter types (in other words, a method descriptor without a return descriptor). + * This string is useful for constructing string keys for methods without regard to their return + * type. + */ + private static String getParameterDescriptors(Klass[] parameterTypes) { + StringBuilder desc = new StringBuilder("("); + for (int i = 0; i < parameterTypes.length; i++) { + desc.append(getFieldType(parameterTypes[i])); + } + desc.append(')'); + return desc.toString(); + } + + /** + * Return the "field type" string for the given type, appropriate for a field descriptor, a + * parameter descriptor, or a return descriptor other than "void". See JVMS section 4.3.2. + */ + private static String getFieldType(Klass type) { + if (type.isPrimitive()) { + return String.valueOf(JavaKind.fromTypeString(type.getTypeAsString()).getTypeChar()); + } else if (type.isArray()) { + /* + * According to JLS 20.3.2, the getName() method on Class does return the VM type + * descriptor format for array classes (only); using that should be quicker than the + * otherwise obvious code: + * + * return "[" + getTypeDescriptor(type.getComponentType()); + */ + return type.getNameAsString().replace('.', '/'); + } else { + return "L" + dotToSlash(type.getNameAsString()) + ";"; + } + } + + /** + * Returns a human-readable string representing the signature of a method with the given name + * and parameter types. + */ + private static String getFriendlyMethodSignature(String name, + Klass[] parameterTypes) { + StringBuilder sig = new StringBuilder(name); + sig.append('('); + for (int i = 0; i < parameterTypes.length; i++) { + if (i > 0) { + sig.append(','); + } + Klass parameterType = parameterTypes[i]; + int dimensions = 0; + while (parameterType.isArray()) { + parameterType = parameterType.getElementalType(); + dimensions++; + } + sig.append(parameterType.getName()); + while (dimensions-- > 0) { + sig.append("[]"); + } + } + sig.append(')'); + return sig.toString(); + } + + /** + * Return the number of abstract "words", or consecutive local variable indexes, required to + * contain a value of the given type. See JVMS section 3.6.1. + * + * Note that the original version of the JVMS contained a definition of this abstract notion of + * a "word" in section 3.4, but that definition was removed for the second edition. + */ + private int getWordsPerType(Klass type) { + if (type == meta._long || type == meta._double) { + return 2; + } else { + return 1; + } + } + + /** + * Add to the given list all of the types in the "from" array that are not already contained in + * the list and are assignable to at least one of the types in the "with" array. + * + * This method is useful for computing the greatest common set of declared exceptions from + * duplicate methods inherited from different interfaces. + */ + private static void collectCompatibleTypes(Klass[] from, + Klass[] with, + List list) { + for (Klass fc : from) { + if (!list.contains(fc)) { + for (Klass wc : with) { + if (wc.isAssignableFrom(fc)) { + list.add(fc); + break; + } + } + } + } + } + + /** + * Given the exceptions declared in the throws clause of a proxy method, compute the exceptions + * that need to be caught from the invocation handler's invoke method and rethrown intact in the + * method's implementation before catching other Throwables and wrapping them in + * UndeclaredThrowableExceptions. + * + * The exceptions to be caught are returned in a List object. Each exception in the returned + * list is guaranteed to not be a subclass of any of the other exceptions in the list, so the + * catch blocks for these exceptions may be generated in any order relative to each other. + * + * Error and RuntimeException are each always contained by the returned list (if none of their + * superclasses are contained), since those unchecked exceptions should always be rethrown + * intact, and thus their subclasses will never appear in the returned list. + * + * The returned List will be empty if java.lang.Throwable is in the given list of declared + * exceptions, indicating that no exceptions need to be caught. + */ + private List computeUniqueCatchList(Klass[] exceptions) { + List uniqueList = new ArrayList<>(); + // unique exceptions to catch + + uniqueList.add(meta.java_lang_Error); // always catch/rethrow these + uniqueList.add(meta.java_lang_RuntimeException); + + nextException: for (Klass ex : exceptions) { + if (ex.isAssignableFrom(meta.java_lang_Throwable)) { + /* + * If Throwable is declared to be thrown by the proxy method, then no catch blocks + * are necessary, because the invoke can, at most, throw Throwable anyway. + */ + uniqueList.clear(); + break; + } else if (!meta.java_lang_Throwable.isAssignableFrom(ex)) { + /* + * Ignore types that cannot be thrown by the invoke method. + */ + continue; + } + /* + * Compare this exception against the current list of exceptions that need to be caught: + */ + for (int j = 0; j < uniqueList.size();) { + Klass ex2 = uniqueList.get(j); + if (ex2.isAssignableFrom(ex)) { + /* + * if a superclass of this exception is already on the list to catch, then + * ignore this one and continue; + */ + continue nextException; + } else if (ex.isAssignableFrom(ex2)) { + /* + * if a subclass of this exception is on the list to catch, then remove it; + */ + uniqueList.remove(j); + } else { + j++; // else continue comparing. + } + } + // This exception is unique (so far): add it to the list to catch. + uniqueList.add(ex); + } + return uniqueList; + } +} diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/LookupProxyKlassNode.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/LookupProxyKlassNode.java new file mode 100644 index 000000000000..d20525715bc8 --- /dev/null +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/LookupProxyKlassNode.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.truffle.espresso.nodes.interop; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.GenerateUncached; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.InvalidArrayIndexException; +import com.oracle.truffle.api.interop.UnsupportedMessageException; +import com.oracle.truffle.api.library.CachedLibrary; +import com.oracle.truffle.espresso.descriptors.Symbol; +import com.oracle.truffle.espresso.impl.ClassRegistry; +import com.oracle.truffle.espresso.impl.Klass; +import com.oracle.truffle.espresso.impl.ObjectKlass; +import com.oracle.truffle.espresso.meta.EspressoError; +import com.oracle.truffle.espresso.nodes.EspressoNode; +import com.oracle.truffle.espresso.runtime.EspressoContext; +import com.oracle.truffle.espresso.runtime.PolyglotInterfaceMappings; + +import java.util.HashSet; +import java.util.Set; + +@GenerateUncached +public abstract class LookupProxyKlassNode extends EspressoNode { + static final int LIMIT = 3; + + LookupProxyKlassNode() { + } + + public abstract ObjectKlass execute(Object metaObject, Klass targetType) throws ClassCastException; + + static String getMetaName(Object metaObject, InteropLibrary interop) { + assert interop.isMetaObject(metaObject); + try { + return interop.asString(interop.getMetaQualifiedName(metaObject)); + } catch (UnsupportedMessageException e) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + throw EspressoError.shouldNotReachHere(); + } + } + + static boolean isSame(Object metaObject, String metaQualifiedName, InteropLibrary interop) { + assert interop.isMetaObject(metaObject); + return getMetaName(metaObject, interop).equals(metaQualifiedName); + } + + @SuppressWarnings("unused") + @Specialization(guards = {"isSame(metaObject, metaName, interop)", "targetType == cachedTargetType"}, limit = "LIMIT") + ObjectKlass doCached(Object metaObject, Klass targetType, + @Cached("targetType") Klass cachedTargetType, + @CachedLibrary(limit = "LIMIT") InteropLibrary interop, + @Cached("getMetaName(metaObject, interop)") String metaName, + @Cached("doUncached(metaObject, targetType, interop)") ObjectKlass cachedProxyKlass) throws ClassCastException { + assert cachedProxyKlass == doUncached(metaObject, targetType, interop); + return cachedProxyKlass; + } + + @TruffleBoundary + @Specialization(replaces = "doCached") + ObjectKlass doUncached(Object metaObject, Klass targetType, + @CachedLibrary(limit = "LIMIT") InteropLibrary interop) throws ClassCastException { + + assert interop.isMetaObject(metaObject); + String metaName; + try { + metaName = interop.asString(interop.getMetaQualifiedName(metaObject)); + } catch (UnsupportedMessageException e) { + throw EspressoError.shouldNotReachHere(); + } + EspressoForeignProxyGenerator.GeneratedProxyBytes proxyBytes = getContext().getProxyBytesOrNull(metaName); + if (proxyBytes == null) { + // cache miss + Set parentInterfaces = new HashSet<>(); + fillParentInterfaces(metaObject, interop, getContext().getPolyglotInterfaceMappings(), parentInterfaces); + if (parentInterfaces.isEmpty()) { + getContext().registerProxyBytes(metaName, null); + return null; + } + proxyBytes = EspressoForeignProxyGenerator.getProxyKlassBytes(metaName, parentInterfaces.toArray(new ObjectKlass[parentInterfaces.size()]), getContext()); + } + + Klass proxyKlass = lookupOrDefineInBindingsLoader(proxyBytes, getContext()); + + if (!targetType.isAssignableFrom(proxyKlass)) { + throw new ClassCastException("proxy object is not instance of expected type: " + targetType.getName()); + } + return (ObjectKlass) proxyKlass; + } + + private static Klass lookupOrDefineInBindingsLoader(EspressoForeignProxyGenerator.GeneratedProxyBytes proxyBytes, EspressoContext context) { + ClassRegistry registry = context.getRegistries().getClassRegistry(context.getBindings().getBindingsLoader()); + + Symbol proxyName = context.getTypes().fromClassGetName(proxyBytes.name); + Klass proxyKlass = registry.findLoadedKlass(proxyName); + if (proxyKlass == null) { + proxyKlass = registry.defineKlass(proxyName, proxyBytes.bytes); + } + return proxyKlass; + } + + private static void fillParentInterfaces(Object metaObject, InteropLibrary interop, PolyglotInterfaceMappings mappings, Set parents) throws ClassCastException { + try { + if (interop.hasMetaParents(metaObject)) { + Object metaParents = interop.getMetaParents(metaObject); + + long arraySize = interop.getArraySize(metaParents); + for (long i = 0; i < arraySize; i++) { + Object parent = interop.readArrayElement(metaParents, i); + ObjectKlass mappedKlass = mappings.mapName(interop.asString(interop.getMetaQualifiedName(parent))); + if (mappedKlass != null) { + parents.add(mappedKlass); + } + fillParentInterfaces(parent, interop, mappings, parents); + } + } + } catch (InvalidArrayIndexException | UnsupportedMessageException e) { + throw new ClassCastException(); + } + } +} diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/ToEspressoNode.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/ToEspressoNode.java index 5315cf997735..9afebe7989fc 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/ToEspressoNode.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/nodes/interop/ToEspressoNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -47,6 +47,7 @@ import com.oracle.truffle.espresso.nodes.EspressoNode; import com.oracle.truffle.espresso.nodes.bytecodes.InitCheck; import com.oracle.truffle.espresso.nodes.bytecodes.InstanceOf; +import com.oracle.truffle.espresso.runtime.EspressoContext; import com.oracle.truffle.espresso.runtime.EspressoException; import com.oracle.truffle.espresso.runtime.StaticObject; @@ -93,6 +94,55 @@ static boolean isEspressoException(Object object) { return object instanceof EspressoException; } + static boolean isBoxedPrimitive(Object obj) { + return obj instanceof Number || obj instanceof Character || obj instanceof Boolean; + } + + static boolean isForeignException(Klass klass) { + Meta meta = klass.getMeta(); + return meta.polyglot != null /* polyglot enabled */ && meta.polyglot.ForeignException.equals(klass); + } + + static boolean isIntegerCompatible(Klass klass) { + return klass.isAssignableFrom(klass.getMeta().java_lang_Integer); + } + + static boolean isBooleanCompatible(Klass klass) { + return klass.isAssignableFrom(klass.getMeta().java_lang_Boolean); + } + + static boolean isByteCompatible(Klass klass) { + return klass.isAssignableFrom(klass.getMeta().java_lang_Byte); + } + + static boolean isShortCompatible(Klass klass) { + return klass.isAssignableFrom(klass.getMeta().java_lang_Short); + } + + static boolean isCharacterCompatible(Klass klass) { + return klass.isAssignableFrom(klass.getMeta().java_lang_Character); + } + + static boolean isLongCompatible(Klass klass) { + return klass.isAssignableFrom(klass.getMeta().java_lang_Long); + } + + static boolean isFloatCompatible(Klass klass) { + return klass.isAssignableFrom(klass.getMeta().java_lang_Float); + } + + static boolean isDoubleCompatible(Klass klass) { + return klass.isAssignableFrom(klass.getMeta().java_lang_Double); + } + + static boolean isTypeMappingEnabled(Klass klass) { + return klass == klass.getMeta().java_lang_Object && klass.getContext().explicitTypeMappingsEnabled(); + } + + static boolean isHostObject(EspressoContext context, Object value) { + return context.getEnv().isHostObject(value); + } + // endregion Specialization predicates public abstract Object execute(Object value, Klass targetType) throws UnsupportedTypeException; @@ -225,6 +275,46 @@ Object doForeignBuffer(Object value, ArrayKlass klass, return StaticObject.createForeign(EspressoLanguage.get(this), klass, value, interop); } + @Specialization(guards = {"isIntegerCompatible(klass)"}) + Object doHostInteger(Integer value, @SuppressWarnings("unused") ObjectKlass klass) { + return getMeta().boxInteger(value); + } + + @Specialization(guards = {"isBooleanCompatible(klass)"}) + Object doHostBoolean(Boolean value, @SuppressWarnings("unused") ObjectKlass klass) { + return getMeta().boxBoolean(value); + } + + @Specialization(guards = {"isByteCompatible(klass)"}) + Object doHostByte(Byte value, @SuppressWarnings("unused") ObjectKlass klass) { + return getMeta().boxByte(value); + } + + @Specialization(guards = {"isCharacterCompatible(klass)"}) + Object doHostChar(Character value, @SuppressWarnings("unused") ObjectKlass klass) { + return getMeta().boxCharacter(value); + } + + @Specialization(guards = {"isShortCompatible(klass)"}) + Object doHostShort(Short value, @SuppressWarnings("unused") ObjectKlass klass) { + return getMeta().boxShort(value); + } + + @Specialization(guards = {"isLongCompatible(klass)"}) + Object doHostLong(Long value, @SuppressWarnings("unused") ObjectKlass klass) { + return getMeta().boxLong(value); + } + + @Specialization(guards = {"isFloatCompatible(klass)"}) + Object doHostFloat(Float value, @SuppressWarnings("unused") ObjectKlass klass) { + return getMeta().boxFloat(value); + } + + @Specialization(guards = {"isDoubleCompatible(klass)"}) + Object doHostDouble(Double value, @SuppressWarnings("unused") ObjectKlass klass) { + return getMeta().boxDouble(value); + } + @Specialization(guards = { "!isStaticObject(value)", "!interop.isNull(value)", @@ -232,23 +322,85 @@ Object doForeignBuffer(Object value, ArrayKlass klass, "!isEspressoException(value)", "!isForeignException(meta, klass)", "!klass.isAbstract()", - "!isString(meta, klass)" + "!isString(meta, klass)", + "!isTypeMappingEnabled(klass)" }) Object doForeignConcreteClassWrapper(Object value, ObjectKlass klass, - @Shared("value") @CachedLibrary(limit = "LIMIT") InteropLibrary interop, + @CachedLibrary(limit = "LIMIT") InteropLibrary interop, @Cached BranchProfile errorProfile, @Cached InitCheck initCheck, - @SuppressWarnings("unused") @Bind("getMeta()") Meta meta) throws UnsupportedTypeException { + @Bind("getMeta()") Meta meta) throws UnsupportedTypeException { try { checkHasAllFieldsOrThrow(value, klass, interop, meta); } catch (ClassCastException e) { errorProfile.enter(); - throw UnsupportedTypeException.create(new Object[]{value}, EspressoError.format("Could not cast foreign object to %s: ", klass.getNameAsString(), e.getMessage())); + throw UnsupportedTypeException.create(new Object[]{value}, EspressoError.format("Could not cast foreign object to %s: due to: %s", klass.getNameAsString(), e.getMessage())); } initCheck.execute(klass); return StaticObject.createForeign(getLanguage(), klass, value, interop); } + @Specialization(guards = { + "isTypeMappingEnabled(klass)", + "!isStaticObject(value)", + "!interop.isNull(value)", + "!isString(meta, klass)", + "!isForeignException(klass)", + "!klass.isAbstract()", + "!isBoxedPrimitive(value)", + "isHostObject(getContext(), value)" + }) + Object doForeignClassProxy(Object value, ObjectKlass klass, + @CachedLibrary(limit = "LIMIT") InteropLibrary interop, + @Cached LookupProxyKlassNode lookupProxyKlassNode, + @Cached BranchProfile errorProfile, + @Bind("getMeta()") Meta meta) throws UnsupportedTypeException { + try { + checkHasAllFieldsOrThrow(value, klass, interop, meta); + ObjectKlass proxyKlass = lookupProxyKlassNode.execute(getMetaObjectOrThrow(value, interop), klass); + if (proxyKlass != null) { + return StaticObject.createForeign(getLanguage(), proxyKlass, value, interop); + } else { + return StaticObject.createForeign(getLanguage(), klass, value, interop); + } + } catch (ClassCastException e) { + errorProfile.enter(); + throw UnsupportedTypeException.create(new Object[]{value}, EspressoError.format("Could not cast foreign object to %s: due to: %s", klass.getNameAsString(), e.getMessage())); + } + } + + @Specialization(guards = {"!isStaticObject(value)", "!interop.isNull(value)", "klass.isInterface()", "isHostObject(getContext(), value)"}) + Object doForeignInterface(Object value, ObjectKlass klass, + @SuppressWarnings("unused") @CachedLibrary(limit = "LIMIT") InteropLibrary interop, + @Cached InitCheck initCheck, + @Cached LookupProxyKlassNode lookupProxyKlassNode, + @Cached BranchProfile errorProfile) throws UnsupportedTypeException { + try { + if (getContext().explicitTypeMappingsEnabled()) { + ObjectKlass proxyKlass = lookupProxyKlassNode.execute(getMetaObjectOrThrow(value, interop), klass); + if (proxyKlass != null) { + initCheck.execute(klass); + return StaticObject.createForeign(getLanguage(), proxyKlass, value, interop); + } + } + throw new ClassCastException(); + } catch (ClassCastException e) { + errorProfile.enter(); + throw UnsupportedTypeException.create(new Object[]{value}, EspressoError.format("Could not cast foreign object to %s: ", klass.getNameAsString(), e.getMessage())); + } + } + + private static Object getMetaObjectOrThrow(Object value, InteropLibrary interop) throws ClassCastException { + if (interop.hasMetaObject(value)) { + try { + return interop.getMetaObject(value); + } catch (UnsupportedMessageException e) { + throw new ClassCastException("Could not lookup meta object"); + } + } + throw new ClassCastException("Unable to lookup meta object for foreign object: " + value.getClass()); + } + @Fallback Object doUnsupportedType(Object value, Klass klass) throws UnsupportedTypeException { throw UnsupportedTypeException.create(new Object[]{value}, klass.getTypeAsString()); diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/runtime/EspressoContext.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/runtime/EspressoContext.java index 00ac3eeb3dc3..b8b691e14e32 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/runtime/EspressoContext.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/runtime/EspressoContext.java @@ -95,6 +95,7 @@ import com.oracle.truffle.espresso.jni.JniEnv; import com.oracle.truffle.espresso.meta.EspressoError; import com.oracle.truffle.espresso.meta.Meta; +import com.oracle.truffle.espresso.nodes.interop.EspressoForeignProxyGenerator; import com.oracle.truffle.espresso.perf.DebugCloseable; import com.oracle.truffle.espresso.perf.DebugTimer; import com.oracle.truffle.espresso.perf.TimerCollection; @@ -204,6 +205,8 @@ public final class EspressoContext { public final int TrivialMethodSize; public final boolean UseHostFinalReference; public final EspressoOptions.JImageMode jimageMode; + private final PolyglotInterfaceMappings polyglotInterfaceMappings; + private final HashMap proxyCache; // Debug option public final com.oracle.truffle.espresso.jdwp.api.JDWPOptions JDWPOptions; @@ -227,7 +230,7 @@ public final class EspressoContext { @CompilationFinal private EspressoException stackOverflow; @CompilationFinal private EspressoException outOfMemory; - @CompilationFinal private TruffleObject topBindings; + @CompilationFinal private EspressoBindings topBindings; private final WeakHashMap hostSignalHandlers = new WeakHashMap<>(); public TruffleLogger getLogger() { @@ -311,6 +314,8 @@ public EspressoContext(TruffleLanguage.Env env, EspressoLanguage language) { this.NativeAccessAllowed = env.isNativeAccessAllowed(); this.Polyglot = env.getOptions().get(EspressoOptions.Polyglot); this.HotSwapAPI = env.getOptions().get(EspressoOptions.HotSwapAPI); + this.polyglotInterfaceMappings = new PolyglotInterfaceMappings(env.getOptions().get(EspressoOptions.PolyglotInterfaceMappings)); + this.proxyCache = polyglotInterfaceMappings.hasMappings() ? new HashMap<>() : null; EspressoOptions.JImageMode requestedJImageMode = env.getOptions().get(EspressoOptions.JImage); if (!NativeAccessAllowed && requestedJImageMode == EspressoOptions.JImageMode.NATIVE) { @@ -463,6 +468,7 @@ public void initializeContext() { if (JDWPOptions != null) { jdwpContext.jdwpInit(env, getMainThread(), eventListener); } + polyglotInterfaceMappings.resolve(this); referenceDrainer.startReferenceDrain(); } @@ -1028,7 +1034,7 @@ public TimerCollection getTimers() { return timers; } - public TruffleObject getBindings() { + public EspressoBindings getBindings() { return topBindings; } @@ -1135,4 +1141,22 @@ public boolean isFinalized() { public void setFinalized() { isFinalized = true; } + + public boolean explicitTypeMappingsEnabled() { + return polyglotInterfaceMappings.hasMappings(); + } + + public PolyglotInterfaceMappings getPolyglotInterfaceMappings() { + return polyglotInterfaceMappings; + } + + public EspressoForeignProxyGenerator.GeneratedProxyBytes getProxyBytesOrNull(String name) { + assert proxyCache != null; + return proxyCache.get(name); + } + + public void registerProxyBytes(String name, EspressoForeignProxyGenerator.GeneratedProxyBytes generatedProxyBytes) { + assert proxyCache != null; + proxyCache.put(name, generatedProxyBytes); + } } diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/runtime/PolyglotInterfaceMappings.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/runtime/PolyglotInterfaceMappings.java new file mode 100644 index 000000000000..1ef740981952 --- /dev/null +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/runtime/PolyglotInterfaceMappings.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.truffle.espresso.runtime; + +import java.util.List; + +import org.graalvm.collections.EconomicMap; +import org.graalvm.collections.UnmodifiableEconomicMap; + +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.espresso.impl.Klass; +import com.oracle.truffle.espresso.impl.ObjectKlass; + +public class PolyglotInterfaceMappings { + + private final boolean hasMappings; + private final List mappings; + private UnmodifiableEconomicMap resolvedKlasses; + + PolyglotInterfaceMappings(List mappings) { + this.mappings = mappings; + this.hasMappings = !mappings.isEmpty(); + } + + @TruffleBoundary + void resolve(EspressoContext context) { + assert mappings != null; + if (!hasMappings) { + return; + } + EconomicMap temp = EconomicMap.create(mappings.size()); + StaticObject bindingsLoader = context.getBindings().getBindingsLoader(); + + for (String mapping : mappings) { + Klass parent = context.getRegistries().loadKlass(context.getTypes().fromClassGetName(mapping), bindingsLoader, StaticObject.NULL); + if (parent.isInterface()) { + temp.put(mapping, (ObjectKlass) parent); + } else { + throw new IllegalStateException("invalid interface type mapping specified: " + mapping); + } + } + resolvedKlasses = EconomicMap.create(temp); + } + + @TruffleBoundary + public ObjectKlass mapName(String name) { + assert resolvedKlasses != null; + return resolvedKlasses.get(name); + } + + public boolean hasMappings() { + return hasMappings; + } +} diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/substitutions/Target_com_oracle_truffle_espresso_polyglot_Interop.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/substitutions/Target_com_oracle_truffle_espresso_polyglot_Interop.java index 74147d4b97ed..f62aaa240eab 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/substitutions/Target_com_oracle_truffle_espresso_polyglot_Interop.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/substitutions/Target_com_oracle_truffle_espresso_polyglot_Interop.java @@ -51,6 +51,7 @@ import com.oracle.truffle.espresso.EspressoLanguage; import com.oracle.truffle.espresso.meta.EspressoError; import com.oracle.truffle.espresso.meta.Meta; +import com.oracle.truffle.espresso.nodes.interop.ToEspressoNode; import com.oracle.truffle.espresso.runtime.EspressoContext; import com.oracle.truffle.espresso.runtime.EspressoException; import com.oracle.truffle.espresso.runtime.StaticObject; @@ -1950,7 +1951,7 @@ StaticObject doCached( @JavaType(Object[].class) StaticObject arguments, @CachedLibrary(limit = "LIMIT") InteropLibrary interop, @CachedLibrary(limit = "LIMIT") InteropLibrary memberInterop, - @CachedLibrary(limit = "LIMIT") InteropLibrary resultInterop, + @Cached ToEspressoNode toEspressoNode, @Cached ThrowInteropExceptionAsGuest throwInteropExceptionAsGuest, @Cached ToHostArguments toHostArguments, @Cached BranchProfile exceptionProfile) { @@ -1959,7 +1960,7 @@ StaticObject doCached( try { Object[] hostArguments = toHostArguments.execute(receiver.isForeignObject(), arguments); Object result = interop.invokeMember(InteropUtils.unwrap(getLanguage(), receiver), hostMember, hostArguments); - return InteropUtils.maybeWrapAsObject(result, resultInterop, getContext()); + return (StaticObject) toEspressoNode.execute(result, getMeta().java_lang_Object); } catch (InteropException e) { exceptionProfile.enter(); throw throwInteropExceptionAsGuest.execute(e); diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/vm/VM.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/vm/VM.java index 9360c64fde75..2a3b1a28ce2c 100644 --- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/vm/VM.java +++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/vm/VM.java @@ -500,6 +500,17 @@ public static int JVM_IHashCode(@JavaType(Object.class) StaticObject object) { * On SVM + Windows, the System.identityHashCode substitution calls methods blocked for PE * (System.currentTimeMillis?). */ + if (object.isForeignObject()) { + InteropLibrary library = InteropLibrary.getUncached(object); + if (library.hasIdentity(object.rawForeignObject(object.getKlass().getLanguage()))) { + try { + return library.identityHashCode(object); + } catch (UnsupportedMessageException e) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + throw EspressoError.shouldNotReachHere(); + } + } + } return System.identityHashCode(MetaUtil.maybeUnwrapNull(object)); } diff --git a/substratevm/ci.jsonnet b/substratevm/ci.jsonnet index 196583c485cc..e1232b7ae535 100644 --- a/substratevm/ci.jsonnet +++ b/substratevm/ci.jsonnet @@ -39,6 +39,12 @@ }, }, + local jsonschema = { + packages+: { + "pip:jsonschema": "==4.6.1", + }, + }, + local musl_toolchain = { downloads+: { "MUSL_TOOLCHAIN": { @@ -73,7 +79,7 @@ darwin_jdk17 + gate("js-quickbuild", "build,js_quickbuild") + clone_js_benchmarks, linux_amd64_jdk11 + gate("build-ce", "build,checkstubs,helloworld,test,nativeimagehelp,muslcbuild,debuginfotest") + maven + svm_unittest + t("35:00") + musl_toolchain + gdb("10.2"), linux_amd64_jdk11 + gate("modules-basic", "build,hellomodule,test") + maven + svm_unittest + t("30:00"), - linux_amd64_jdk17 + gate("style-fullbuild", "style,fullbuild,helloworld,test,svmjunit,debuginfotest") + common.eclipse + common.jdt + maven + svm_unittest + t("50:00") + mx_build_exploded + gdb("10.2"), + linux_amd64_jdk17 + gate("style-fullbuild", "style,fullbuild,helloworld,test,svmjunit,debuginfotest") + common.eclipse + common.jdt + maven + jsonschema + svm_unittest + t("50:00") + mx_build_exploded + gdb("10.2"), linux_amd64_jdk19 + gate("build-ce", "build") + { run: [ # cannot yet use mx gate --tag build due to compile errors in /compiler diff --git a/substratevm/mx.substratevm/mx_substratevm.py b/substratevm/mx.substratevm/mx_substratevm.py index 4b268d3a6904..4341c17ff7f5 100644 --- a/substratevm/mx.substratevm/mx_substratevm.py +++ b/substratevm/mx.substratevm/mx_substratevm.py @@ -475,6 +475,29 @@ def help_stdout_check(output): if t: hellomodule([]) + with Task('Validate JSON build output', tasks, tags=[mx_gate.Tags.style]) as t: + if t: + import json + try: + from jsonschema import validate as json_validate + from jsonschema.exceptions import ValidationError, SchemaError + except ImportError: + mx.abort('Unable to import jsonschema') + with open(join(suite.dir, '..', 'docs', 'reference-manual', 'native-image', 'assets', 'build-output-schema-v0.9.0.json')) as f: + json_schema = json.load(f) + with tempfile.NamedTemporaryFile(prefix='build_json') as json_file: + helloworld(['--output-path', svmbuild_dir(), f'-H:BuildOutputJSONFile={json_file.name}']) + try: + with open(json_file.name) as f: + json_output = json.load(f) + json_validate(json_output, json_schema) + except IOError as e: + mx.abort(f'Unable to load JSON build output: {e}') + except ValidationError as e: + mx.abort(f'Unable to validate JSON build output against the schema: {e}') + except SchemaError as e: + mx.abort(f'JSON schema not valid: {e}') + def native_unittests_task(extra_build_args=None): if mx.is_windows(): diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisType.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisType.java index 4803c99ef040..99742d5b0006 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisType.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/meta/AnalysisType.java @@ -24,6 +24,7 @@ */ package com.oracle.graal.pointsto.meta; +import java.lang.reflect.AnnotatedElement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1092,6 +1093,11 @@ public AnalysisField[] getStaticFields() { return convertFields(wrapped.getStaticFields(), new ArrayList<>(), false); } + @Override + public AnnotatedElement getAnnotationRoot() { + return wrapped; + } + @Override public String getSourceFileName() { // getSourceFileName is not implemented for primitive types diff --git a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/AMD64CalleeSavedRegisters.java b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/AMD64CalleeSavedRegisters.java index ccca90e0a89c..b1c6ed3d4b7f 100644 --- a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/AMD64CalleeSavedRegisters.java +++ b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/AMD64CalleeSavedRegisters.java @@ -40,6 +40,7 @@ import org.graalvm.compiler.asm.amd64.AMD64Assembler; import org.graalvm.compiler.asm.amd64.AMD64BaseAssembler; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import org.graalvm.nativeimage.ImageSingletons; @@ -449,7 +450,7 @@ private AMD64Address getFeatureMapAddress() { GraalError.guarantee(ConfigurationValues.getTarget().inlineObjects, "Dynamic feature check for callee saved registers requires inlined objects"); Register heapBase = ReservedRegisters.singleton().getHeapBaseRegister(); GraalError.guarantee(heapBase != null, "Heap base register must not be null"); - return new AMD64Address(heapBase, Register.None, AMD64Address.Scale.Times1, displacement(object, (SharedConstantReflectionProvider) crb.providers.getConstantReflection()) + fieldOffset, + return new AMD64Address(heapBase, Register.None, Stride.S1, displacement(object, (SharedConstantReflectionProvider) crb.providers.getConstantReflection()) + fieldOffset, displacementAnnotation(object)); } diff --git a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64AddressLowering.java b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64AddressLowering.java index 661dffa1125e..19a2e6c0c7e9 100644 --- a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64AddressLowering.java +++ b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64AddressLowering.java @@ -24,7 +24,8 @@ */ package com.oracle.svm.core.graal.amd64; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.amd64.AMD64AddressNode; import org.graalvm.compiler.core.amd64.AMD64CompressAddressLowering; import org.graalvm.compiler.core.common.CompressEncoding; @@ -50,7 +51,7 @@ protected final boolean improveUncompression(AMD64AddressNode addr, CompressionN assert SubstrateOptions.SpawnIsolates.getValue(); CompressEncoding encoding = compression.getEncoding(); - if (!Scale.isScaleShiftSupported(encoding.getShift())) { + if (!AMD64Address.isScaleShiftSupported(encoding.getShift())) { return false; } @@ -67,9 +68,9 @@ protected final boolean improveUncompression(AMD64AddressNode addr, CompressionN } } - Scale scale = Scale.fromShift(encoding.getShift()); + Stride stride = Stride.fromLog2(encoding.getShift()); addr.setBase(base); - addr.setScale(scale); + addr.setScale(stride); addr.setIndex(compression.getValue()); return true; } diff --git a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java index a45f7f2bc299..e6e7395aaba5 100644 --- a/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java +++ b/substratevm/src/com.oracle.svm.core.graal.amd64/src/com/oracle/svm/core/graal/amd64/SubstrateAMD64Backend.java @@ -58,6 +58,7 @@ import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.CompressEncoding; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; import org.graalvm.compiler.core.common.memory.MemoryExtendKind; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; @@ -356,7 +357,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { * an optional shift that is known to be a valid addressing mode. */ memoryAddress = new AMD64Address(ReservedRegisters.singleton().getHeapBaseRegister(), - computeRegister, AMD64Address.Scale.fromShift(compressEncoding.getShift()), + computeRegister, Stride.fromLog2(compressEncoding.getShift()), field.getOffset()); } else { memoryAddress = new AMD64Address(computeRegister, field.getOffset()); @@ -375,7 +376,7 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { * the heap base is a constant. */ memoryAddress = new AMD64Address(ReservedRegisters.singleton().getHeapBaseRegister(), - Register.None, AMD64Address.Scale.Times1, + Register.None, Stride.S1, field.getOffset() + addressDisplacement(object, constantReflection), addressDisplacementAnnotation(object)); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreads.java index 4129ff0f3b69..02d8e48d0e55 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreads.java @@ -43,6 +43,7 @@ import com.oracle.svm.core.jdk.Target_jdk_internal_misc_VM; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.jfr.events.ThreadSleepEvent; +import com.oracle.svm.util.ReflectionUtil; /** * Implements operations on {@linkplain Target_java_lang_Thread Java threads}, which are on a higher @@ -99,11 +100,17 @@ static Thread.State getThreadState(Thread thread) { return Target_jdk_internal_misc_VM.toThreadState(getThreadStatus(thread)); } - @SuppressWarnings("deprecation") - @AlwaysInline("handle Thread.getId deprecation") + /** + * Returns the unique identifier of this thread. This method is necessary because + * {@link Thread#getId()} is a non-final method that can be overridden. + */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getThreadId(Thread thread) { - return thread.getId(); + if (SubstrateUtil.HOSTED) { + return ReflectionUtil.readField(Thread.class, "tid", thread); + } else { + return toTarget(thread).tid; + } } /** diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreadsFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreadsFeature.java index b713a589483b..e80338cfff75 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreadsFeature.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/JavaThreadsFeature.java @@ -131,7 +131,10 @@ public void afterAnalysis(AfterAnalysisAccess access) { } static long threadId(Thread thread) { - return thread == PlatformThreads.singleton().mainThread ? 1 : JavaThreads.getThreadId(thread); + if (thread == PlatformThreads.singleton().mainThread) { + return 1; + } + return JavaThreads.getThreadId(thread); } private static final String AUTONUMBER_PREFIX = "Thread-"; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_Thread.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_Thread.java index 633193862042..cfcce8485714 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_Thread.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_Thread.java @@ -125,7 +125,7 @@ public final class Target_java_lang_Thread { /* Thread ID */ @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Custom, declClass = ThreadIdRecomputation.class) // - long tid; + public long tid; /** We have our own atomic number in {@link JavaThreads#threadSeqNumber}. */ @Delete @TargetElement(onlyWith = JDK17OrEarlier.class)// diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMAMD64ArrayEqualsWithMaskForeignCalls.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMAMD64ArrayEqualsWithMaskForeignCalls.java index 376e9c62cfb7..2515af355be2 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMAMD64ArrayEqualsWithMaskForeignCalls.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMAMD64ArrayEqualsWithMaskForeignCalls.java @@ -24,14 +24,11 @@ */ package com.oracle.svm.graal.stubs; -import static org.graalvm.compiler.core.common.StrideUtil.NONE; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.replacements.amd64.AMD64ArrayRegionEqualsWithMaskNode; import org.graalvm.nativeimage.Platform.AMD64; import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.Pointer; import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.cpufeature.Stubs; @@ -49,167 +46,82 @@ class SVMAMD64ArrayEqualsWithMaskForeignCalls { @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS1S1S1(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S1, S1, S1); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS1S2S1(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S1, S2, S1); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS1S2S2(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S1, S2, S2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS2S1S1(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S2, S1, S1); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS2S2S1(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S2, S2, S1); + private static boolean arrayRegionEqualsS1S2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S2, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS2S2S2(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S2, S2, S2); + private static boolean arrayRegionEqualsS2S2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S2, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsC(char[] arrayA, long offsetA, char[] arrayB, long offsetB, char[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S2, S2, S2, S2); + private static boolean arrayRegionEqualsS1S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S1, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS1S1(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S1, S1, S1); + private static boolean arrayRegionEqualsS1S2(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S2, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS1S2(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S1, S2, S2); + private static boolean arrayRegionEqualsS1S4(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S4, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS1S4(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S1, S4, S4); + private static boolean arrayRegionEqualsS2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S1, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S2, S1, S1); + private static boolean arrayRegionEqualsS2S2(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S2, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS2S2(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S2, S2, S2); + private static boolean arrayRegionEqualsS2S4(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S4, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS2S4(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S2, S4, S4); + private static boolean arrayRegionEqualsS4S1(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S4, Stride.S1, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS4S1(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S4, S1, S1); + private static boolean arrayRegionEqualsS4S2(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S4, Stride.S2, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS4S2(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S4, S2, S2); + private static boolean arrayRegionEqualsS4S4(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S4, Stride.S4, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS4S4(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S4, S4, S4); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsDynamicStrides(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length, int dynamicStrides) { + private static boolean arrayRegionEqualsDynamicStrides(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length, int dynamicStrides) { return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, dynamicStrides); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS1S1S1RTC(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S1, S1, S1, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS1S2S1RTC(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S1, S2, S1, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS1S2S2RTC(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S1, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS2S1S1RTC(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S2, S1, S1, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS2S2S1RTC(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S2, S2, S1, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsBS2S2S2RTC(byte[] arrayA, long offsetA, byte[] arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS1S2S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S1, S2, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S2, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -217,10 +129,10 @@ private static boolean arrayRegionEqualsBS2S2S2RTC(byte[] arrayA, long offsetA, @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsCRTC(char[] arrayA, long offsetA, char[] arrayB, long offsetB, char[] mask, int length) { + private static boolean arrayRegionEqualsS2S2S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, S2, S2, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S2, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -228,10 +140,10 @@ private static boolean arrayRegionEqualsCRTC(char[] arrayA, long offsetA, char[] @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS1S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS1S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S1, S1, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S1, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -239,10 +151,10 @@ private static boolean arrayRegionEqualsS1S1RTC(Object arrayA, long offsetA, Obj @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS1S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS1S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S1, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S2, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -250,10 +162,10 @@ private static boolean arrayRegionEqualsS1S2RTC(Object arrayA, long offsetA, Obj @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS1S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS1S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S1, S4, S4, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S1, Stride.S4, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -261,10 +173,10 @@ private static boolean arrayRegionEqualsS1S4RTC(Object arrayA, long offsetA, Obj @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS2S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS2S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S2, S1, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S1, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -272,10 +184,10 @@ private static boolean arrayRegionEqualsS2S1RTC(Object arrayA, long offsetA, Obj @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS2S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS2S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S2, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S2, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -283,10 +195,10 @@ private static boolean arrayRegionEqualsS2S2RTC(Object arrayA, long offsetA, Obj @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS2S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS2S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S2, S4, S4, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S2, Stride.S4, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -294,10 +206,10 @@ private static boolean arrayRegionEqualsS2S4RTC(Object arrayA, long offsetA, Obj @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS4S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS4S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S4, S1, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S4, Stride.S1, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -305,10 +217,10 @@ private static boolean arrayRegionEqualsS4S1RTC(Object arrayA, long offsetA, Obj @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS4S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS4S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S4, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S4, Stride.S2, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -316,10 +228,10 @@ private static boolean arrayRegionEqualsS4S2RTC(Object arrayA, long offsetA, Obj @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsS4S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length) { + private static boolean arrayRegionEqualsS4S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, NONE, S4, S4, S4, Stubs.getRuntimeCheckedCPUFeatures()); + return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, Stride.S4, Stride.S4, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -327,7 +239,7 @@ private static boolean arrayRegionEqualsS4S4RTC(Object arrayA, long offsetA, Obj @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean arrayRegionEqualsDynamicStridesRTC(Object arrayA, long offsetA, Object arrayB, long offsetB, byte[] mask, int length, int dynamicStrides) { + private static boolean arrayRegionEqualsDynamicStridesRTC(Object arrayA, long offsetA, Object arrayB, long offsetB, Pointer mask, int length, int dynamicStrides) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { return AMD64ArrayRegionEqualsWithMaskNode.regionEquals(arrayA, offsetA, arrayB, offsetB, mask, length, dynamicStrides, Stubs.getRuntimeCheckedCPUFeatures()); diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayCompareToForeignCalls.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayCompareToForeignCalls.java index 26bb6d12493b..be8aba97c967 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayCompareToForeignCalls.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayCompareToForeignCalls.java @@ -24,9 +24,7 @@ */ package com.oracle.svm.graal.stubs; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; - +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.replacements.nodes.ArrayCompareToNode; import org.graalvm.nativeimage.Platform.AMD64; import org.graalvm.nativeimage.Platforms; @@ -48,34 +46,34 @@ class SVMArrayCompareToForeignCalls { @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int byteArrayCompareToByteArray(Pointer array1, Pointer array2, int length1, int length2) { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S1, S1); + private static int byteArrayCompareToByteArray(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S1, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int byteArrayCompareToCharArray(Pointer array1, Pointer array2, int length1, int length2) { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S1, S2); + private static int byteArrayCompareToCharArray(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S1, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int charArrayCompareToByteArray(Pointer array1, Pointer array2, int length1, int length2) { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S2, S1); + private static int charArrayCompareToByteArray(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S2, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int charArrayCompareToCharArray(Pointer array1, Pointer array2, int length1, int length2) { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S2, S2); + private static int charArrayCompareToCharArray(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S2, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int byteArrayCompareToByteArrayRTC(Pointer array1, Pointer array2, int length1, int length2) { + private static int byteArrayCompareToByteArrayRTC(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S1, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S1, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -83,10 +81,10 @@ private static int byteArrayCompareToByteArrayRTC(Pointer array1, Pointer array2 @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int byteArrayCompareToCharArrayRTC(Pointer array1, Pointer array2, int length1, int length2) { + private static int byteArrayCompareToCharArrayRTC(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S1, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S1, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -94,10 +92,10 @@ private static int byteArrayCompareToCharArrayRTC(Pointer array1, Pointer array2 @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int charArrayCompareToByteArrayRTC(Pointer array1, Pointer array2, int length1, int length2) { + private static int charArrayCompareToByteArrayRTC(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S2, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S2, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -105,10 +103,10 @@ private static int charArrayCompareToByteArrayRTC(Pointer array1, Pointer array2 @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int charArrayCompareToCharArrayRTC(Pointer array1, Pointer array2, int length1, int length2) { + private static int charArrayCompareToCharArrayRTC(Pointer arrayA, int lengthA, Pointer arrayB, int lengthB) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayCompareToNode.compareTo(array1, array2, length1, length2, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayCompareToNode.compareTo(arrayA, lengthA, arrayB, lengthB, Stride.S2, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayCopyWithConversionsForeignCalls.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayCopyWithConversionsForeignCalls.java index 7a3d2b77a84c..2033108c841d 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayCopyWithConversionsForeignCalls.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayCopyWithConversionsForeignCalls.java @@ -24,10 +24,7 @@ */ package com.oracle.svm.graal.stubs; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.replacements.nodes.ArrayCopyWithConversionsNode; import org.graalvm.nativeimage.Platform.AMD64; import org.graalvm.nativeimage.Platforms; @@ -49,55 +46,55 @@ class SVMArrayCopyWithConversionsForeignCalls { @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static void arrayCopyWithConversionsS1S1(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S1, S1); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S1, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static void arrayCopyWithConversionsS1S2(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S1, S2); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S1, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static void arrayCopyWithConversionsS1S4(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S1, S4); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S1, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static void arrayCopyWithConversionsS2S1(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S2, S1); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S2, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static void arrayCopyWithConversionsS2S2(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S2, S2); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S2, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static void arrayCopyWithConversionsS2S4(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S2, S4); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S2, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static void arrayCopyWithConversionsS4S1(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S4, S1); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S4, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static void arrayCopyWithConversionsS4S2(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S4, S2); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S4, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static void arrayCopyWithConversionsS4S4(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S4, S4); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S4, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @@ -111,7 +108,7 @@ private static void arrayCopyWithConversionsDynamicStrides(Object arraySrc, long private static void arrayCopyWithConversionsS1S1RTC(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S1, S1, Stubs.getRuntimeCheckedCPUFeatures()); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S1, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -122,7 +119,7 @@ private static void arrayCopyWithConversionsS1S1RTC(Object arraySrc, long offset private static void arrayCopyWithConversionsS1S2RTC(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S1, S2, Stubs.getRuntimeCheckedCPUFeatures()); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S1, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -133,7 +130,7 @@ private static void arrayCopyWithConversionsS1S2RTC(Object arraySrc, long offset private static void arrayCopyWithConversionsS1S4RTC(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S1, S4, Stubs.getRuntimeCheckedCPUFeatures()); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S1, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -144,7 +141,7 @@ private static void arrayCopyWithConversionsS1S4RTC(Object arraySrc, long offset private static void arrayCopyWithConversionsS2S1RTC(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S2, S1, Stubs.getRuntimeCheckedCPUFeatures()); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S2, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -155,7 +152,7 @@ private static void arrayCopyWithConversionsS2S1RTC(Object arraySrc, long offset private static void arrayCopyWithConversionsS2S2RTC(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S2, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -166,7 +163,7 @@ private static void arrayCopyWithConversionsS2S2RTC(Object arraySrc, long offset private static void arrayCopyWithConversionsS2S4RTC(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S2, S4, Stubs.getRuntimeCheckedCPUFeatures()); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S2, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -177,7 +174,7 @@ private static void arrayCopyWithConversionsS2S4RTC(Object arraySrc, long offset private static void arrayCopyWithConversionsS4S1RTC(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S4, S1, Stubs.getRuntimeCheckedCPUFeatures()); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S4, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -188,7 +185,7 @@ private static void arrayCopyWithConversionsS4S1RTC(Object arraySrc, long offset private static void arrayCopyWithConversionsS4S2RTC(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S4, S2, Stubs.getRuntimeCheckedCPUFeatures()); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S4, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -199,7 +196,7 @@ private static void arrayCopyWithConversionsS4S2RTC(Object arraySrc, long offset private static void arrayCopyWithConversionsS4S4RTC(Object arraySrc, long offsetSrc, Object arrayDst, long offsetDst, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, S4, S4, Stubs.getRuntimeCheckedCPUFeatures()); + ArrayCopyWithConversionsNode.arrayCopy(arraySrc, offsetSrc, arrayDst, offsetDst, length, Stride.S4, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayEqualsForeignCalls.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayEqualsForeignCalls.java index 3c1cb23b4c85..9609b31a1317 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayEqualsForeignCalls.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayEqualsForeignCalls.java @@ -24,10 +24,7 @@ */ package com.oracle.svm.graal.stubs; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.replacements.nodes.ArrayEqualsNode; import org.graalvm.compiler.replacements.nodes.ArrayRegionEqualsNode; import org.graalvm.nativeimage.Platform.AMD64; @@ -52,50 +49,20 @@ class SVMArrayEqualsForeignCalls { @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean booleanArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Boolean); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean byteArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Byte); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean charArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Char); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean shortArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Short); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean intArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Int); + private static boolean longArraysEquals(Pointer array1, long offset1, Pointer array2, long offset2, int length) { + return ArrayEqualsNode.equals(array1, offset1, array2, offset2, length, JavaKind.Long); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean longArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Long); + private static boolean floatArraysEquals(Pointer array1, long offset1, Pointer array2, long offset2, int length) { + return ArrayEqualsNode.equals(array1, offset1, array2, offset2, length, JavaKind.Float); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean floatArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Float); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean doubleArraysEquals(Pointer array1, Pointer array2, int length) { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Double); + private static boolean doubleArraysEquals(Pointer array1, long offset1, Pointer array2, long offset2, int length) { + return ArrayEqualsNode.equals(array1, offset1, array2, offset2, length, JavaKind.Double); } @Uninterruptible(reason = "Must not do a safepoint check.") @@ -107,118 +74,63 @@ private static boolean arrayRegionEqualsDynamicStrides(Object arrayA, long offse @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static boolean arrayRegionEqualsS1S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S1, S1); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static boolean arrayRegionEqualsS1S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S1, S2); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static boolean arrayRegionEqualsS1S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S1, S4); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static boolean arrayRegionEqualsS2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S2, S1); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static boolean arrayRegionEqualsS2S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S2, S2); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static boolean arrayRegionEqualsS2S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S2, S4); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static boolean arrayRegionEqualsS4S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S4, S1); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static boolean arrayRegionEqualsS4S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S4, S2); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static boolean arrayRegionEqualsS4S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S4, S4); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean booleanArraysEqualsRTC(Pointer array1, Pointer array2, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Boolean, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean byteArraysEqualsRTC(Pointer array1, Pointer array2, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Byte, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean charArraysEqualsRTC(Pointer array1, Pointer array2, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Char, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean shortArraysEqualsRTC(Pointer array1, Pointer array2, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Short, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean intArraysEqualsRTC(Pointer array1, Pointer array2, int length) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Int, Stubs.getRuntimeCheckedCPUFeatures()); - } finally { - region.leave(); - } + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean longArraysEqualsRTC(Pointer array1, Pointer array2, int length) { + private static boolean longArraysEqualsRTC(Pointer array1, long offset1, Pointer array2, long offset2, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Long, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayEqualsNode.equals(array1, offset1, array2, offset2, length, JavaKind.Long, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -226,10 +138,10 @@ private static boolean longArraysEqualsRTC(Pointer array1, Pointer array2, int l @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean floatArraysEqualsRTC(Pointer array1, Pointer array2, int length) { + private static boolean floatArraysEqualsRTC(Pointer array1, long offset1, Pointer array2, long offset2, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Float, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayEqualsNode.equals(array1, offset1, array2, offset2, length, JavaKind.Float, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -237,10 +149,10 @@ private static boolean floatArraysEqualsRTC(Pointer array1, Pointer array2, int @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static boolean doubleArraysEqualsRTC(Pointer array1, Pointer array2, int length) { + private static boolean doubleArraysEqualsRTC(Pointer array1, long offset1, Pointer array2, long offset2, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayEqualsNode.equals(array1, array2, length, JavaKind.Double, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayEqualsNode.equals(array1, offset1, array2, offset2, length, JavaKind.Double, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -262,7 +174,7 @@ private static boolean arrayRegionEqualsDynamicStridesRTC(Object arrayA, long of private static boolean arrayRegionEqualsS1S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S1, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -273,7 +185,7 @@ private static boolean arrayRegionEqualsS1S1RTC(Object arrayA, long offsetA, Obj private static boolean arrayRegionEqualsS1S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S1, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -284,7 +196,7 @@ private static boolean arrayRegionEqualsS1S2RTC(Object arrayA, long offsetA, Obj private static boolean arrayRegionEqualsS1S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S1, S4, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -295,7 +207,7 @@ private static boolean arrayRegionEqualsS1S4RTC(Object arrayA, long offsetA, Obj private static boolean arrayRegionEqualsS2S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S2, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -306,7 +218,7 @@ private static boolean arrayRegionEqualsS2S1RTC(Object arrayA, long offsetA, Obj private static boolean arrayRegionEqualsS2S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -317,7 +229,7 @@ private static boolean arrayRegionEqualsS2S2RTC(Object arrayA, long offsetA, Obj private static boolean arrayRegionEqualsS2S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S2, S4, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -328,7 +240,7 @@ private static boolean arrayRegionEqualsS2S4RTC(Object arrayA, long offsetA, Obj private static boolean arrayRegionEqualsS4S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S4, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -339,7 +251,7 @@ private static boolean arrayRegionEqualsS4S1RTC(Object arrayA, long offsetA, Obj private static boolean arrayRegionEqualsS4S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S4, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -350,7 +262,7 @@ private static boolean arrayRegionEqualsS4S2RTC(Object arrayA, long offsetA, Obj private static boolean arrayRegionEqualsS4S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, S4, S4, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionEqualsNode.regionEquals(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayIndexOfForeignCalls.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayIndexOfForeignCalls.java index 47d8013e440d..02e841000ece 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayIndexOfForeignCalls.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayIndexOfForeignCalls.java @@ -24,11 +24,8 @@ */ package com.oracle.svm.graal.stubs; -import static org.graalvm.compiler.core.common.StrideUtil.NONE; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.replacements.nodes.ArrayIndexOfNode; import org.graalvm.nativeimage.Platform.AARCH64; import org.graalvm.nativeimage.Platform.AMD64; @@ -48,421 +45,130 @@ class SVMArrayIndexOfForeignCalls { // compiler/src/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/ArrayIndexOfStub.java // BY: "mx svm-sync-graal-stubs" - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveBS1(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, true, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveBS2(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, true, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveCS2(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, true, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB1S1(byte[] array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, array, offset, arrayLength, fromIndex, v1); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB2S1(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB3S1(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB4S1(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB1S2(byte[] array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, array, offset, arrayLength, fromIndex, v1); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB2S2(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB3S2(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB4S2(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfC1S2(char[] array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, array, offset, arrayLength, fromIndex, v1); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfC2S2(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfC3S2(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfC4S2(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } - @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOfTwoConsecutiveS1(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, true, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, true, false, array, offset, arrayLength, fromIndex, v1, v2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOfTwoConsecutiveS2(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, true, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, true, false, array, offset, arrayLength, fromIndex, v1, v2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOfTwoConsecutiveS4(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, true, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, true, false, array, offset, arrayLength, fromIndex, v1, v2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf1S1(Object array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, array, offset, arrayLength, fromIndex, v1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, array, offset, arrayLength, fromIndex, v1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf2S1(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, array, offset, arrayLength, fromIndex, v1, v2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf3S1(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf4S1(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf1S2(Object array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, array, offset, arrayLength, fromIndex, v1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, array, offset, arrayLength, fromIndex, v1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf2S2(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, array, offset, arrayLength, fromIndex, v1, v2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf3S2(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf4S2(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf1S4(Object array, long offset, int arrayLength, int fromIndex, int v1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, array, offset, arrayLength, fromIndex, v1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, array, offset, arrayLength, fromIndex, v1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf2S4(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, array, offset, arrayLength, fromIndex, v1, v2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf3S4(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOf4S4(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfWithMaskBS1(byte[] array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, true, array, offset, length, fromIndex, v1, mask1); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfWithMaskBS2(byte[] array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, true, array, offset, length, fromIndex, v1, mask1); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveWithMaskBS1(byte[] array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveWithMaskBS2(byte[] array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfWithMaskCS2(char[] array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, true, array, offset, length, fromIndex, v1, mask1); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveWithMaskCS2(char[] array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, array, offset, arrayLength, fromIndex, v1, v2, v3, v4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOfWithMaskS1(Object array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, true, array, offset, length, fromIndex, v1, mask1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, true, array, offset, length, fromIndex, v1, mask1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOfWithMaskS2(Object array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, true, array, offset, length, fromIndex, v1, mask1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, true, array, offset, length, fromIndex, v1, mask1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOfWithMaskS4(Object array, long offset, int length, int fromIndex, int v1, int mask1) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, true, array, offset, length, fromIndex, v1, mask1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, true, array, offset, length, fromIndex, v1, mask1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOfTwoConsecutiveWithMaskS1(Object array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOfTwoConsecutiveWithMaskS2(Object array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int indexOfTwoConsecutiveWithMaskS4(Object array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveBS1RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, true, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveBS2RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, true, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveCS2RTC(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, true, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB1S1RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB2S1RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB3S1RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB4S1RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB1S2RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB2S2RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB3S2RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfB4S2RTC(byte[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfC1S2RTC(char[] array, long offset, int arrayLength, int fromIndex, int v1) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfC2S2RTC(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfC3S2RTC(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfC4S2RTC(char[] array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } finally { - region.leave(); - } + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, true, true, array, offset, length, fromIndex, v1, v2, mask1, mask2); } @Uninterruptible(reason = "Must not do a safepoint check.") @@ -470,7 +176,7 @@ private static int indexOfC4S2RTC(char[] array, long offset, int arrayLength, in private static int indexOfTwoConsecutiveS1RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, true, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, true, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); } finally { region.leave(); } @@ -481,7 +187,7 @@ private static int indexOfTwoConsecutiveS1RTC(Object array, long offset, int arr private static int indexOfTwoConsecutiveS2RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, true, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, true, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); } finally { region.leave(); } @@ -492,7 +198,7 @@ private static int indexOfTwoConsecutiveS2RTC(Object array, long offset, int arr private static int indexOfTwoConsecutiveS4RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, true, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, true, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); } finally { region.leave(); } @@ -503,7 +209,7 @@ private static int indexOfTwoConsecutiveS4RTC(Object array, long offset, int arr private static int indexOf1S1RTC(Object array, long offset, int arrayLength, int fromIndex, int v1) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1); } finally { region.leave(); } @@ -514,7 +220,7 @@ private static int indexOf1S1RTC(Object array, long offset, int arrayLength, int private static int indexOf2S1RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); } finally { region.leave(); } @@ -525,7 +231,7 @@ private static int indexOf2S1RTC(Object array, long offset, int arrayLength, int private static int indexOf3S1RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3); } finally { region.leave(); } @@ -536,7 +242,7 @@ private static int indexOf3S1RTC(Object array, long offset, int arrayLength, int private static int indexOf4S1RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3, v4); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3, v4); } finally { region.leave(); } @@ -547,7 +253,7 @@ private static int indexOf4S1RTC(Object array, long offset, int arrayLength, int private static int indexOf1S2RTC(Object array, long offset, int arrayLength, int fromIndex, int v1) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1); } finally { region.leave(); } @@ -558,7 +264,7 @@ private static int indexOf1S2RTC(Object array, long offset, int arrayLength, int private static int indexOf2S2RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); } finally { region.leave(); } @@ -569,7 +275,7 @@ private static int indexOf2S2RTC(Object array, long offset, int arrayLength, int private static int indexOf3S2RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3); } finally { region.leave(); } @@ -580,7 +286,7 @@ private static int indexOf3S2RTC(Object array, long offset, int arrayLength, int private static int indexOf4S2RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3, v4); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3, v4); } finally { region.leave(); } @@ -591,7 +297,7 @@ private static int indexOf4S2RTC(Object array, long offset, int arrayLength, int private static int indexOf1S4RTC(Object array, long offset, int arrayLength, int fromIndex, int v1) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1); } finally { region.leave(); } @@ -602,7 +308,7 @@ private static int indexOf1S4RTC(Object array, long offset, int arrayLength, int private static int indexOf2S4RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2); } finally { region.leave(); } @@ -613,7 +319,7 @@ private static int indexOf2S4RTC(Object array, long offset, int arrayLength, int private static int indexOf3S4RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3); } finally { region.leave(); } @@ -624,73 +330,7 @@ private static int indexOf3S4RTC(Object array, long offset, int arrayLength, int private static int indexOf4S4RTC(Object array, long offset, int arrayLength, int fromIndex, int v1, int v2, int v3, int v4) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3, v4); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfWithMaskBS1RTC(byte[] array, long offset, int length, int fromIndex, int v1, int mask1) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, false, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, mask1); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfWithMaskBS2RTC(byte[] array, long offset, int length, int fromIndex, int v1, int mask1) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, false, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, mask1); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveWithMaskBS1RTC(byte[] array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S1, true, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, v2, mask1, mask2); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveWithMaskBS2RTC(byte[] array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S1, S2, true, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, v2, mask1, mask2); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfWithMaskCS2RTC(char[] array, long offset, int length, int fromIndex, int v1, int mask1) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, false, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, mask1); - } finally { - region.leave(); - } - } - - @Uninterruptible(reason = "Must not do a safepoint check.") - @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) - private static int indexOfTwoConsecutiveWithMaskCS2RTC(char[] array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { - RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); - try { - return ArrayIndexOfNode.optimizedArrayIndexOf(S2, S2, true, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, false, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, arrayLength, fromIndex, v1, v2, v3, v4); } finally { region.leave(); } @@ -701,7 +341,7 @@ private static int indexOfTwoConsecutiveWithMaskCS2RTC(char[] array, long offset private static int indexOfWithMaskS1RTC(Object array, long offset, int length, int fromIndex, int v1, int mask1) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, false, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, mask1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, false, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, mask1); } finally { region.leave(); } @@ -712,7 +352,7 @@ private static int indexOfWithMaskS1RTC(Object array, long offset, int length, i private static int indexOfWithMaskS2RTC(Object array, long offset, int length, int fromIndex, int v1, int mask1) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, false, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, mask1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, false, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, mask1); } finally { region.leave(); } @@ -723,7 +363,7 @@ private static int indexOfWithMaskS2RTC(Object array, long offset, int length, i private static int indexOfWithMaskS4RTC(Object array, long offset, int length, int fromIndex, int v1, int mask1) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, false, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, mask1); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, false, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, mask1); } finally { region.leave(); } @@ -734,7 +374,7 @@ private static int indexOfWithMaskS4RTC(Object array, long offset, int length, i private static int indexOfTwoConsecutiveWithMaskS1RTC(Object array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S1, true, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S1, true, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, v2, mask1, mask2); } finally { region.leave(); } @@ -745,7 +385,7 @@ private static int indexOfTwoConsecutiveWithMaskS1RTC(Object array, long offset, private static int indexOfTwoConsecutiveWithMaskS2RTC(Object array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S2, true, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S2, true, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, v2, mask1, mask2); } finally { region.leave(); } @@ -756,7 +396,7 @@ private static int indexOfTwoConsecutiveWithMaskS2RTC(Object array, long offset, private static int indexOfTwoConsecutiveWithMaskS4RTC(Object array, long offset, int length, int fromIndex, int v1, int v2, int mask1, int mask2) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayIndexOfNode.optimizedArrayIndexOf(NONE, S4, true, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, v2, mask1, mask2); + return ArrayIndexOfNode.optimizedArrayIndexOf(JavaKind.Void, Stride.S4, true, true, Stubs.getRuntimeCheckedCPUFeatures(), array, offset, length, fromIndex, v1, v2, mask1, mask2); } finally { region.leave(); } diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayRegionCompareToForeignCalls.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayRegionCompareToForeignCalls.java index 0e2c0f7df2cb..3e1175d9c35b 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayRegionCompareToForeignCalls.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/stubs/SVMArrayRegionCompareToForeignCalls.java @@ -24,10 +24,7 @@ */ package com.oracle.svm.graal.stubs; -import static org.graalvm.compiler.core.common.StrideUtil.S1; -import static org.graalvm.compiler.core.common.StrideUtil.S2; -import static org.graalvm.compiler.core.common.StrideUtil.S4; - +import org.graalvm.compiler.core.common.Stride; import org.graalvm.compiler.replacements.nodes.ArrayRegionCompareToNode; import org.graalvm.nativeimage.Platform.AMD64; import org.graalvm.nativeimage.Platforms; @@ -49,55 +46,55 @@ class SVMArrayRegionCompareToForeignCalls { @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int arrayRegionCompareToS1S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S1, S1); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int arrayRegionCompareToS1S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S1, S2); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int arrayRegionCompareToS1S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S1, S4); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int arrayRegionCompareToS2S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S2, S1); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int arrayRegionCompareToS2S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S2, S2); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int arrayRegionCompareToS2S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S2, S4); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int arrayRegionCompareToS4S1(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S4, S1); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S1); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int arrayRegionCompareToS4S2(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S4, S2); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S2); } @Uninterruptible(reason = "Must not do a safepoint check.") @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true) private static int arrayRegionCompareToS4S4(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S4, S4); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S4); } @Uninterruptible(reason = "Must not do a safepoint check.") @@ -111,7 +108,7 @@ private static int arrayRegionCompareToDynamicStrides(Object arrayA, long offset private static int arrayRegionCompareToS1S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S1, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -122,7 +119,7 @@ private static int arrayRegionCompareToS1S1RTC(Object arrayA, long offsetA, Obje private static int arrayRegionCompareToS1S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S1, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -133,7 +130,7 @@ private static int arrayRegionCompareToS1S2RTC(Object arrayA, long offsetA, Obje private static int arrayRegionCompareToS1S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S1, S4, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S1, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -144,7 +141,7 @@ private static int arrayRegionCompareToS1S4RTC(Object arrayA, long offsetA, Obje private static int arrayRegionCompareToS2S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S2, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -155,7 +152,7 @@ private static int arrayRegionCompareToS2S1RTC(Object arrayA, long offsetA, Obje private static int arrayRegionCompareToS2S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S2, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -166,7 +163,7 @@ private static int arrayRegionCompareToS2S2RTC(Object arrayA, long offsetA, Obje private static int arrayRegionCompareToS2S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S2, S4, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S2, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -177,7 +174,7 @@ private static int arrayRegionCompareToS2S4RTC(Object arrayA, long offsetA, Obje private static int arrayRegionCompareToS4S1RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S4, S1, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S1, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -188,7 +185,7 @@ private static int arrayRegionCompareToS4S1RTC(Object arrayA, long offsetA, Obje private static int arrayRegionCompareToS4S2RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S4, S2, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S2, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } @@ -199,7 +196,7 @@ private static int arrayRegionCompareToS4S2RTC(Object arrayA, long offsetA, Obje private static int arrayRegionCompareToS4S4RTC(Object arrayA, long offsetA, Object arrayB, long offsetB, int length) { RuntimeCPUFeatureRegion region = RuntimeCPUFeatureRegion.enterSet(Stubs.getRuntimeCheckedCPUFeatures()); try { - return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, S4, S4, Stubs.getRuntimeCheckedCPUFeatures()); + return ArrayRegionCompareToNode.compare(arrayA, offsetA, arrayB, offsetB, length, Stride.S4, Stride.S4, Stubs.getRuntimeCheckedCPUFeatures()); } finally { region.leave(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 7b99816c112d..c99619020c6e 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -124,7 +124,7 @@ public class ProgressReporter { private long lastGCCheckTimeMillis = System.currentTimeMillis(); private GCStats lastGCStats = GCStats.getCurrent(); private long numRuntimeCompiledMethods = -1; - private long graphEncodingByteLength = 0; + private long graphEncodingByteLength = -1; private int numJNIClasses = -1; private int numJNIFields = -1; private int numJNIMethods = -1; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/annotation/SubstrateAnnotationExtracter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/annotation/SubstrateAnnotationExtracter.java index 72536131ad08..92a04c730ab9 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/annotation/SubstrateAnnotationExtracter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/annotation/SubstrateAnnotationExtracter.java @@ -62,7 +62,7 @@ * avoid initializing classes that should be initialized at run-time, since looking up annotations * through the JDK's {@link AnnotationParser} initializes the class of every annotation on the * queried element. - * + * * When queried, the extracter looks for the root of the provided element, which can be a * {@link Field}, {@link Method}, {@link Constructor} or {@link Class} object, as well as a record * component on JDK 17. It then looks into the byte arrays representing annotations in the root @@ -70,7 +70,7 @@ * annotation on demand in an {@link AnnotationValue} or {@link TypeAnnotationValue} object or any * subclass of {@link AnnotationMemberValue}. The actual annotation can then be created using the * {@link AnnotationMemberValue#get(Class)} method. - * + * * The {@link SubstrateAnnotationExtracter} is tightly coupled with {@link GuardedAnnotationAccess}, * which provides implementations of {@link AnnotatedElement#isAnnotationPresent(Class)} and * {@link AnnotatedElement#getAnnotation(Class)}. {@link AnnotatedElement#getAnnotations()} should @@ -262,7 +262,8 @@ private AnnotationValue[] getDeclaredAnnotationDataFromRoot(AnnotatedElement roo } public AnnotationValue[][] getParameterAnnotationData(AnnotatedElement element) { - return getParameterAnnotationDataFromRoot((Executable) getRoot(element)); + AnnotatedElement root = getRoot(element); + return root != null ? getParameterAnnotationDataFromRoot((Executable) root) : NO_PARAMETER_ANNOTATIONS; } private AnnotationValue[][] getParameterAnnotationDataFromRoot(Executable rootElement) { @@ -290,7 +291,8 @@ private AnnotationValue[][] getParameterAnnotationDataFromRoot(Executable rootEl } public TypeAnnotationValue[] getTypeAnnotationData(AnnotatedElement element) { - return getTypeAnnotationDataFromRoot(getRoot(element)); + AnnotatedElement root = getRoot(element); + return root != null ? getTypeAnnotationDataFromRoot(root) : NO_TYPE_ANNOTATIONS; } private TypeAnnotationValue[] getTypeAnnotationDataFromRoot(AnnotatedElement rootElement) { @@ -310,7 +312,8 @@ private TypeAnnotationValue[] getTypeAnnotationDataFromRoot(AnnotatedElement roo } public AnnotationMemberValue getAnnotationDefaultData(AnnotatedElement element) { - return getAnnotationDefaultDataFromRoot((Method) getRoot(element)); + AnnotatedElement root = getRoot(element); + return root != null ? getAnnotationDefaultDataFromRoot((Method) root) : null; } private AnnotationMemberValue getAnnotationDefaultDataFromRoot(Method accessorMethod) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/SubstitutionField.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/SubstitutionField.java index 54c2d365a718..51e1f17d8ab9 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/SubstitutionField.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/substitute/SubstitutionField.java @@ -24,12 +24,13 @@ */ package com.oracle.svm.hosted.substitute; -import java.lang.annotation.Annotation; +import java.lang.reflect.AnnotatedElement; import java.lang.reflect.Field; import com.oracle.graal.pointsto.infrastructure.OriginalFieldProvider; import com.oracle.graal.pointsto.util.GraalAccess; import com.oracle.svm.core.meta.ReadableJavaField; +import com.oracle.svm.util.AnnotationWrapper; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaType; @@ -37,7 +38,7 @@ import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaType; -public class SubstitutionField implements ReadableJavaField, OriginalFieldProvider { +public class SubstitutionField implements ReadableJavaField, OriginalFieldProvider, AnnotationWrapper { private final ResolvedJavaField original; private final ResolvedJavaField annotated; @@ -126,18 +127,8 @@ public ResolvedJavaType getDeclaringClass() { } @Override - public T getAnnotation(Class annotationClass) { - return annotated.getAnnotation(annotationClass); - } - - @Override - public Annotation[] getAnnotations() { - return annotated.getAnnotations(); - } - - @Override - public Annotation[] getDeclaredAnnotations() { - return annotated.getDeclaredAnnotations(); + public AnnotatedElement getAnnotationRoot() { + return annotated; } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/JavaxXmlClassAndResourcesLoaderFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/JavaxXmlClassAndResourcesLoaderFeature.java index 82fc9babff13..12abd35dd1b8 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/JavaxXmlClassAndResourcesLoaderFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/JavaxXmlClassAndResourcesLoaderFeature.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,6 +33,7 @@ import static com.oracle.svm.hosted.xml.XMLParsersRegistration.DOMParserClasses; import static com.oracle.svm.hosted.xml.XMLParsersRegistration.DatatypeFactoryClasses; import static com.oracle.svm.hosted.xml.XMLParsersRegistration.SAXParserClasses; +import static com.oracle.svm.hosted.xml.XMLParsersRegistration.SchemaFactoryClasses; import static com.oracle.svm.hosted.xml.XMLParsersRegistration.StAXParserClasses; import static com.oracle.svm.hosted.xml.XMLParsersRegistration.TransformerClassesAndResources; @@ -58,5 +59,8 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { access.registerReachabilityHandler(new DatatypeFactoryClasses()::registerConfigs, method(access, "javax.xml.datatype.DatatypeFactory", "newInstance")); + + access.registerReachabilityHandler(new SchemaFactoryClasses()::registerConfigs, + method(access, "javax.xml.validation.SchemaFactory", "newSchema")); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/XMLParsersRegistration.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/XMLParsersRegistration.java index 8f5ce06e88b9..1a8473521182 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/XMLParsersRegistration.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/xml/XMLParsersRegistration.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -96,6 +96,16 @@ List xmlParserClasses() { } } + static class SchemaFactoryClasses extends XMLParsersRegistration { + + @Override + List xmlParserClasses() { + return Arrays.asList( + "com.sun.org.apache.xerces.internal.impl.dv.xs.ExtendedSchemaDVFactoryImpl", + "com.sun.org.apache.xerces.internal.impl.dv.xs.SchemaDVFactoryImpl"); + } + } + static class StAXParserClasses extends XMLParsersRegistration { @Override diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleSupport.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleSupport.java index 1704e49fd455..8eb1e61c079d 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleSupport.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleSupport.java @@ -28,9 +28,6 @@ import java.util.function.Consumer; import java.util.function.Supplier; -import com.oracle.svm.core.graal.code.SubstrateBackend; -import com.oracle.svm.truffle.api.SubstrateOptimizedCallTargetInstalledCode; -import com.oracle.svm.util.ReflectionUtil; import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; import org.graalvm.compiler.truffle.common.CompilableTruffleAST; @@ -50,16 +47,20 @@ import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.graal.code.SubstrateBackend; import com.oracle.svm.core.meta.SubstrateObjectConstant; import com.oracle.svm.graal.GraalSupport; import com.oracle.svm.graal.hosted.GraalFeature; +import com.oracle.svm.truffle.api.SubstrateKnownTruffleTypes; import com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget; +import com.oracle.svm.truffle.api.SubstrateOptimizedCallTargetInstalledCode; import com.oracle.svm.truffle.api.SubstratePartialEvaluator; import com.oracle.svm.truffle.api.SubstrateTruffleCompiler; import com.oracle.svm.truffle.api.SubstrateTruffleCompilerImpl; import com.oracle.svm.truffle.api.SubstrateTruffleRuntime; import com.oracle.svm.truffle.isolated.IsolateAwareTruffleCompiler; import com.oracle.svm.truffle.isolated.IsolatedTruffleRuntimeSupport; +import com.oracle.svm.util.ReflectionUtil; import com.oracle.truffle.api.nodes.RootNode; import com.oracle.truffle.api.utilities.TriState; @@ -87,7 +88,7 @@ private static Method getOptimizedCallTargetInvokeMethod() { } public SubstratePartialEvaluator createPartialEvaluator(TruffleCompilerConfiguration config, GraphBuilderConfiguration graphBuilderConfigForRoot) { - return new SubstratePartialEvaluator(config, graphBuilderConfigForRoot); + return new SubstratePartialEvaluator(config, graphBuilderConfigForRoot, config.getKnownTruffleTypes()); } @SuppressWarnings("unused") @@ -104,17 +105,18 @@ public SubstrateTruffleCompiler createTruffleCompiler(SubstrateTruffleRuntime ru protected static TruffleCompilerConfiguration createSubstrateTruffleCompilerConfig(SubstrateTruffleRuntime runtime, String compilerConfigurationName, Method optimizedCallTargetMethod) { GraalFeature graalFeature = ImageSingletons.lookup(GraalFeature.class); + SubstrateKnownTruffleTypes knownTruffleTypes = new SubstrateKnownTruffleTypes(GraalSupport.getRuntimeConfig().getProviders().getMetaAccess()); SnippetReflectionProvider snippetReflectionProvider = graalFeature.getHostedProviders().getSnippetReflection(); final GraphBuilderConfiguration.Plugins graphBuilderPlugins = graalFeature.getHostedProviders().getGraphBuilderPlugins(); SubstrateBackend substrateBackend = GraalSupport.getRuntimeConfig().getBackendForNormalMethod(); substrateBackend.setRuntimeToRuntimeInvokeMethod(optimizedCallTargetMethod); final TruffleTierConfiguration firstTier = new TruffleTierConfiguration(new EconomyPartialEvaluatorConfiguration(), substrateBackend, - GraalSupport.getFirstTierProviders(), GraalSupport.getFirstTierSuites(), GraalSupport.getFirstTierLirSuites()); + GraalSupport.getFirstTierProviders(), GraalSupport.getFirstTierSuites(), GraalSupport.getFirstTierLirSuites(), knownTruffleTypes); PartialEvaluatorConfiguration peConfig = TruffleCompilerImpl.createPartialEvaluatorConfiguration(compilerConfigurationName); final TruffleTierConfiguration lastTier = new TruffleTierConfiguration(peConfig, substrateBackend, - GraalSupport.getRuntimeConfig().getProviders(), GraalSupport.getSuites(), GraalSupport.getLIRSuites()); - return new TruffleCompilerConfiguration(runtime, graphBuilderPlugins, snippetReflectionProvider, firstTier, lastTier); + GraalSupport.getRuntimeConfig().getProviders(), GraalSupport.getSuites(), GraalSupport.getLIRSuites(), knownTruffleTypes); + return new TruffleCompilerConfiguration(runtime, graphBuilderPlugins, snippetReflectionProvider, firstTier, lastTier, knownTruffleTypes); } public static boolean isIsolatedCompilation() { diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstratePartialEvaluator.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstratePartialEvaluator.java index 3263b6235f40..18e082e40687 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstratePartialEvaluator.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstratePartialEvaluator.java @@ -44,6 +44,7 @@ import org.graalvm.compiler.truffle.compiler.PartialEvaluatorConfiguration; import org.graalvm.compiler.truffle.compiler.TruffleCompilerConfiguration; import org.graalvm.compiler.truffle.compiler.TruffleTierContext; +import org.graalvm.compiler.truffle.compiler.substitutions.KnownTruffleTypes; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; @@ -55,8 +56,8 @@ public class SubstratePartialEvaluator extends PartialEvaluator { private final ConcurrentHashMap specialCallTargetCache; @Platforms(Platform.HOSTED_ONLY.class) - public SubstratePartialEvaluator(TruffleCompilerConfiguration config, GraphBuilderConfiguration graphBuilderConfigForRoot) { - super(config, graphBuilderConfigForRoot, new SubstrateKnownTruffleTypes(config.lastTier().providers().getMetaAccess())); + public SubstratePartialEvaluator(TruffleCompilerConfiguration config, GraphBuilderConfiguration graphBuilderConfigForRoot, KnownTruffleTypes knownTruffleTypes) { + super(config, graphBuilderConfigForRoot, knownTruffleTypes); this.invocationPluginsCache = new ConcurrentHashMap<>(); this.specialCallTargetCache = new ConcurrentHashMap<>(); } diff --git a/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/nodes/intrinsics/llvm/x86_win/LLVMX86_64_WinVaListStorage.java b/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/nodes/intrinsics/llvm/x86_win/LLVMX86_64_WinVaListStorage.java index 41c7376cb001..bf47f58d19b1 100644 --- a/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/nodes/intrinsics/llvm/x86_win/LLVMX86_64_WinVaListStorage.java +++ b/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/nodes/intrinsics/llvm/x86_win/LLVMX86_64_WinVaListStorage.java @@ -35,7 +35,6 @@ import com.oracle.truffle.api.Truffle; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.Cached.Exclusive; -import com.oracle.truffle.api.dsl.Cached.Shared; import com.oracle.truffle.api.dsl.GenerateUncached; import com.oracle.truffle.api.dsl.ImportStatic; import com.oracle.truffle.api.dsl.Specialization; @@ -67,7 +66,7 @@ import com.oracle.truffle.llvm.runtime.pointer.LLVMPointer; import com.oracle.truffle.llvm.runtime.types.Type; -// @ExportLibrary(value = LLVMManagedWriteLibrary.class, useForAOT = true, useForAOTPriority = 2) +@ExportLibrary(value = LLVMVaListLibrary.class, useForAOT = true, useForAOTPriority = 2) @ExportLibrary(value = LLVMManagedReadLibrary.class, useForAOT = true, useForAOTPriority = 1) @ExportLibrary(InteropLibrary.class) public final class LLVMX86_64_WinVaListStorage implements TruffleObject { @@ -256,15 +255,33 @@ void toNative( initNative(realArguments, numberOfExplicitArguments, nativeAreaPointer, storeI64Node, storePointerNode); } + @ExportMessage public void initialize(Object[] realArgs, int numOfExpArgs, Frame frame, - @Shared("stackAllocationNode") @Cached StackAllocationNode stackAllocationNode) { + @Cached StackAllocationNode stackAllocationNode) { realArguments = realArgs; numberOfExplicitArguments = numOfExpArgs; argsArea = new WinArgsArea(realArgs, numOfExpArgs); assert numOfExpArgs <= realArgs.length; nativeAreaPointer = stackAllocationNode.executeWithTarget((realArgs.length - numOfExpArgs) * Long.BYTES, frame); + } + + @SuppressWarnings("static-method") + @ExportMessage + void cleanup(@SuppressWarnings("unused") Frame frame) { + throw CompilerDirectives.shouldNotReachHere("should only be called on LLVMMaybeVaPointer"); + } + @SuppressWarnings("static-method") + @ExportMessage + void copy(@SuppressWarnings("unused") Object dest, @SuppressWarnings("unused") Frame frame) { + throw CompilerDirectives.shouldNotReachHere("should never be called directly"); + } + + @SuppressWarnings("static-method") + @ExportMessage + Object shift(@SuppressWarnings("unused") Type type, @SuppressWarnings("unused") Frame frame) { + throw CompilerDirectives.shouldNotReachHere("should never be called directly"); } @ExportLibrary(value = LLVMVaListLibrary.class, useForAOT = true, useForAOTPriority = 0) diff --git a/vm/ci_common/benchmark-forks.json b/vm/ci_common/benchmark-forks.json index 5a41e7dc974b..9f68d731bea5 100644 --- a/vm/ci_common/benchmark-forks.json +++ b/vm/ci_common/benchmark-forks.json @@ -1,12 +1,12 @@ { - "warmup/specjbb2005.js": 3, - "warmup/acorn-polybench.js": 10, - "warmup/typescript-polybench.js": 20, - "warmup/asciidoctor-convert.rb": 20, - "warmup/asciidoctor-load-file.rb": 20, - "warmup/pyflate-fast.py": 3, - "warmup/raytracer.py": 3, - "interpreter/jsinit.js": 5, - "interpreter/rbinit.rb": 5, - "interpreter/pyinit.py": 5 + "polybench:warmup/specjbb2005.js": 3, + "polybench:warmup/acorn-polybench.js": 10, + "polybench:warmup/typescript-polybench.js": 20, + "polybench:warmup/asciidoctor-convert.rb": 20, + "polybench:warmup/asciidoctor-load-file.rb": 20, + "polybench:warmup/pyflate-fast.py": 3, + "polybench:warmup/raytracer.py": 3, + "polybench:interpreter/jsinit.js": 5, + "polybench:interpreter/rbinit.rb": 5, + "polybench:interpreter/pyinit.py": 5 }