From da93881b6bfc4a5041e29e462524654d62b49b31 Mon Sep 17 00:00:00 2001 From: Fred Morcos Date: Wed, 26 May 2021 17:37:35 +0200 Subject: [PATCH 1/2] Sulong: Intrinsify implicitly typed XADD instructions --- sulong/CHANGELOG.md | 4 +++ .../truffle/llvm/asm/amd64/AsmFactory.java | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/sulong/CHANGELOG.md b/sulong/CHANGELOG.md index 1e273c4a45a1..50323f0a6627 100644 --- a/sulong/CHANGELOG.md +++ b/sulong/CHANGELOG.md @@ -4,6 +4,10 @@ Fixes: * Fix LLVM toolchain not working correctly for C++ on MacOS 11.3. +New features: + +* Support implicitly typed XADD instructions in inline assembly. + # Version 21.1.0 New features: diff --git a/sulong/projects/com.oracle.truffle.llvm.asm.amd64/src/com/oracle/truffle/llvm/asm/amd64/AsmFactory.java b/sulong/projects/com.oracle.truffle.llvm.asm.amd64/src/com/oracle/truffle/llvm/asm/amd64/AsmFactory.java index 8830a557345e..7ce4a5836b02 100644 --- a/sulong/projects/com.oracle.truffle.llvm.asm.amd64/src/com/oracle/truffle/llvm/asm/amd64/AsmFactory.java +++ b/sulong/projects/com.oracle.truffle.llvm.asm.amd64/src/com/oracle/truffle/llvm/asm/amd64/AsmFactory.java @@ -1218,6 +1218,41 @@ void createBinaryOperationImplicitSize(String operation, AsmOperand a, AsmOperan LLVMX86_ConversionNode.LLVMX86_Pmovmskb128 pmovmskb128 = LLVMX86_ConversionNodeFactory.LLVMX86_Pmovmskb128NodeGen.create(srcA); out = pmovmskb128; break; + case "xadd": + srcA = getOperandLoad(dstType, a); + srcB = getOperandLoad(dstType, b); + switch (getPrimitiveType(dstType)) { + case I8: { + LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I8, a); + LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); + LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); + statements.add(LLVMAMD64XaddbNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + return; + } + case I16: { + LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I16, a); + LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); + LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); + statements.add(LLVMAMD64XaddwNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + return; + } + case I32: { + LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I32, a); + LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); + LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); + statements.add(LLVMAMD64XaddlNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + return; + } + case I64: { + LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I64, a); + LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); + LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); + statements.add(LLVMAMD64XaddqNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + return; + } + default: + throw invalidOperandType(dstType); + } default: statements.add(LLVMUnsupportedInstructionNode.create(UnsupportedReason.INLINE_ASSEMBLER, operation)); return; From 1662979489701142c205044f8b78ce51a1980516 Mon Sep 17 00:00:00 2001 From: Fred Morcos Date: Fri, 28 May 2021 08:09:47 +0200 Subject: [PATCH 2/2] Sulong: Support atomic inline assembly instructions --- sulong/CHANGELOG.md | 4 + .../truffle/llvm/asm/amd64/AsmFactory.java | 84 ++++++++++++++----- .../truffle/llvm/runtime/LLVMContext.java | 1 + .../LLVMAMD64LockedInstructionNode.java | 66 +++++++++++++++ 4 files changed, 136 insertions(+), 19 deletions(-) create mode 100644 sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/nodes/asm/support/LLVMAMD64LockedInstructionNode.java diff --git a/sulong/CHANGELOG.md b/sulong/CHANGELOG.md index 50323f0a6627..af280f00af02 100644 --- a/sulong/CHANGELOG.md +++ b/sulong/CHANGELOG.md @@ -8,6 +8,10 @@ New features: * Support implicitly typed XADD instructions in inline assembly. +* Support some atomic variants of certain inline assembly instructions. Currently + supported instructions are: xchg, cmpxchg and xadd along with their explicitly typed + variants, and various unary instructions including incb/w/l/q and decb/w/l/q. + # Version 21.1.0 New features: diff --git a/sulong/projects/com.oracle.truffle.llvm.asm.amd64/src/com/oracle/truffle/llvm/asm/amd64/AsmFactory.java b/sulong/projects/com.oracle.truffle.llvm.asm.amd64/src/com/oracle/truffle/llvm/asm/amd64/AsmFactory.java index 7ce4a5836b02..e84ce9c4ca5c 100644 --- a/sulong/projects/com.oracle.truffle.llvm.asm.amd64/src/com/oracle/truffle/llvm/asm/amd64/AsmFactory.java +++ b/sulong/projects/com.oracle.truffle.llvm.asm.amd64/src/com/oracle/truffle/llvm/asm/amd64/AsmFactory.java @@ -197,6 +197,7 @@ import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64AddressComputationNodeFactory.LLVMAMD64AddressSegmentComputationNodeGen; import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64Flags; import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64GetTlsNodeGen; +import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64LockedInstructionNodeGen; import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64ReadAddressNodeGen; import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64ReadRegisterNodeGen; import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64Target; @@ -951,7 +952,7 @@ void createUnaryOperation(String operation, AsmOperand operand) { statements.add(LLVMUnsupportedInstructionNode.create(UnsupportedReason.INLINE_ASSEMBLER, operation)); return; } - statements.add(getOperandStore(dstType, dst, out)); + statements.add(maybeMakeAtomic(getOperandStore(dstType, dst, out), dst)); } private static boolean isShiftOperation(String operation) { @@ -1035,6 +1036,41 @@ private Type getType(AsmOperand dst, AsmOperand src) { return type; } + /** + * Ensures that at least the instruction's operand passed in is a memory operand, and that the + * instruction has a lock prefix, in which case it returns the instruction ("the statement") + * wrapped in an + * {@link com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64LockedInstructionNode}. + * Otherwise returns the bare instruction. + */ + private LLVMStatementNode maybeMakeAtomic(LLVMStatementNode statement, AsmOperand maybeMemoryOperand) { + if ("lock".equals(currentPrefix)) { + return makeAtomic(statement, maybeMemoryOperand); + } + + return statement; + } + + /** + * Ensures that at least the instruction's operand passed in is a memory operand, in which case + * it returns the instruction ("the statement") wrapped in an + * {@link com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64LockedInstructionNode}. + * Otherwise returns the bare instruction. + */ + private LLVMStatementNode makeAtomic(LLVMStatementNode statement, AsmOperand maybeMemoryOperand) { + if (maybeMemoryOperand instanceof AsmMemoryOperand) { + return LLVMAMD64LockedInstructionNodeGen.create(statement, getOperandAddress(maybeMemoryOperand)); + } else if (maybeMemoryOperand instanceof AsmArgumentOperand) { + AsmArgumentOperand op = (AsmArgumentOperand) maybeMemoryOperand; + Argument info = argInfo.get(op.getIndex()); + if (info.isMemory()) { + return LLVMAMD64LockedInstructionNodeGen.create(statement, getOperandAddress(maybeMemoryOperand)); + } + } + + return statement; + } + void createBinaryOperationImplicitSize(String operation, AsmOperand a, AsmOperand b) { AsmOperand dst = b; AsmOperand src = a; @@ -1130,7 +1166,17 @@ void createBinaryOperationImplicitSize(String operation, AsmOperand a, AsmOperan default: throw invalidOperandType(dstType); } - statements.add(res); + /** + * Make xchg unconditionally atomic. Some code (e.g. musl libc) using xchg relies on + * its (and any other instruction's) non-atomic behavior in concurrency with other + * locked/atomic instructions to be atomic. + * + * The issue is that we should avoid globally locking memory operations, and so + * instead we hand-pick instructions that are used in code related to + * synchronization, and xchg is one of those (e.g. in the musl libc). This is not + * entirely correct behavior, but should cover most practical use-cases. + */ + statements.add(makeAtomic(res, b)); return; } case "cmpxchg": { @@ -1170,7 +1216,7 @@ void createBinaryOperationImplicitSize(String operation, AsmOperand a, AsmOperan throw invalidOperandType(dstType); } } - statements.add(res); + statements.add(maybeMakeAtomic(res, b)); return; } case "and": @@ -1226,28 +1272,28 @@ void createBinaryOperationImplicitSize(String operation, AsmOperand a, AsmOperan LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I8, a); LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); - statements.add(LLVMAMD64XaddbNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64XaddbNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b)); return; } case I16: { LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I16, a); LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); - statements.add(LLVMAMD64XaddwNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64XaddwNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b)); return; } case I32: { LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I32, a); LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); - statements.add(LLVMAMD64XaddlNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64XaddlNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b)); return; } case I64: { LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I64, a); LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); - statements.add(LLVMAMD64XaddqNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64XaddqNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b)); return; } default: @@ -1511,22 +1557,22 @@ void createBinaryOperation(String operation, AsmOperand a, AsmOperand b) { break; case "xchgb": { XchgOperands operands = new XchgOperands(a, b, dstType); - statements.add(LLVMAMD64XchgbNodeGen.create(operands.dst, operands.srcA, operands.srcB)); + statements.add(makeAtomic(LLVMAMD64XchgbNodeGen.create(operands.dst, operands.srcA, operands.srcB), b)); return; } case "xchgw": { XchgOperands operands = new XchgOperands(a, b, dstType); - statements.add(LLVMAMD64XchgwNodeGen.create(operands.dst, operands.srcA, operands.srcB)); + statements.add(makeAtomic(LLVMAMD64XchgwNodeGen.create(operands.dst, operands.srcA, operands.srcB), b)); return; } case "xchgl": { XchgOperands operands = new XchgOperands(a, b, dstType); - statements.add(LLVMAMD64XchglNodeGen.create(operands.dst, operands.srcA, operands.srcB)); + statements.add(makeAtomic(LLVMAMD64XchglNodeGen.create(operands.dst, operands.srcA, operands.srcB), b)); return; } case "xchgq": { XchgOperands operands = new XchgOperands(a, b, dstType); - statements.add(LLVMAMD64XchgqNodeGen.create(operands.dst, operands.srcA, operands.srcB)); + statements.add(makeAtomic(LLVMAMD64XchgqNodeGen.create(operands.dst, operands.srcA, operands.srcB), b)); return; } case "cmpb": @@ -1545,56 +1591,56 @@ void createBinaryOperation(String operation, AsmOperand a, AsmOperand b) { LLVMAMD64WriteValueNode dst1 = getStore(dstType, b); LLVMAMD64WriteValueNode dst2 = getRegisterStore("al"); LLVMExpressionNode accumulator = getOperandLoad(PrimitiveType.I8, new AsmRegisterOperand("al")); - statements.add(LLVMAMD64CmpXchgbNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64CmpXchgbNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB), b)); return; } case "cmpxchgw": { LLVMAMD64WriteValueNode dst1 = getStore(dstType, b); LLVMAMD64WriteValueNode dst2 = getRegisterStore("ax"); LLVMExpressionNode accumulator = getOperandLoad(PrimitiveType.I16, new AsmRegisterOperand("ax")); - statements.add(LLVMAMD64CmpXchgwNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64CmpXchgwNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB), b)); return; } case "cmpxchgl": { LLVMAMD64WriteValueNode dst1 = getStore(dstType, b); LLVMAMD64WriteValueNode dst2 = getRegisterStore("eax"); LLVMExpressionNode accumulator = getOperandLoad(PrimitiveType.I32, new AsmRegisterOperand("eax")); - statements.add(LLVMAMD64CmpXchglNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64CmpXchglNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB), b)); return; } case "cmpxchgq": { LLVMAMD64WriteValueNode dst1 = getStore(dstType, b); LLVMAMD64WriteValueNode dst2 = getRegisterStore("rax"); LLVMExpressionNode accumulator = getOperandLoad(PrimitiveType.I64, new AsmRegisterOperand("rax")); - statements.add(LLVMAMD64CmpXchgqNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64CmpXchgqNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB), b)); return; } case "xaddb": { LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I8, a); LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); - statements.add(LLVMAMD64XaddbNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64XaddbNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b)); return; } case "xaddw": { LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I16, a); LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); - statements.add(LLVMAMD64XaddwNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64XaddwNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b)); return; } case "xaddl": { LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I32, a); LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); - statements.add(LLVMAMD64XaddlNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64XaddlNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b)); return; } case "xaddq": { LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I64, a); LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst); LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2); - statements.add(LLVMAMD64XaddqNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB)); + statements.add(maybeMakeAtomic(LLVMAMD64XaddqNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b)); return; } case "xorb": diff --git a/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/LLVMContext.java b/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/LLVMContext.java index 0358e666440d..1aee736bc1c2 100644 --- a/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/LLVMContext.java +++ b/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/LLVMContext.java @@ -100,6 +100,7 @@ public final class LLVMContext { protected final ArrayList globalsNonPointerStore = new ArrayList<>(); protected final EconomicMap globalsReadOnlyStore = EconomicMap.create(); private final Object globalsStoreLock = new Object(); + public final Object atomicInstructionsLock = new Object(); private final List runningThreads = new ArrayList<>(); @CompilationFinal private LLVMThreadingStack threadingStack; diff --git a/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/nodes/asm/support/LLVMAMD64LockedInstructionNode.java b/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/nodes/asm/support/LLVMAMD64LockedInstructionNode.java new file mode 100644 index 000000000000..7059a546429a --- /dev/null +++ b/sulong/projects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/nodes/asm/support/LLVMAMD64LockedInstructionNode.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.oracle.truffle.llvm.runtime.nodes.asm.support; + +import com.oracle.truffle.api.dsl.CachedContext; +import com.oracle.truffle.api.dsl.NodeChild; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.llvm.runtime.LLVMContext; +import com.oracle.truffle.llvm.runtime.LLVMLanguage; +import com.oracle.truffle.llvm.runtime.nodes.api.LLVMExpressionNode; +import com.oracle.truffle.llvm.runtime.nodes.api.LLVMStatementNode; +import com.oracle.truffle.llvm.runtime.pointer.LLVMManagedPointer; +import com.oracle.truffle.llvm.runtime.pointer.LLVMNativePointer; + +@NodeChild(value = "address", type = LLVMExpressionNode.class) +public abstract class LLVMAMD64LockedInstructionNode extends LLVMStatementNode { + @Child LLVMStatementNode statement; + + public LLVMAMD64LockedInstructionNode(LLVMStatementNode statement) { + this.statement = statement; + } + + @Specialization + public void doNative(VirtualFrame frame, + @SuppressWarnings("unused") LLVMNativePointer address, + @CachedContext(LLVMLanguage.class) LLVMContext context) { + synchronized (context.atomicInstructionsLock) { + statement.execute(frame); + } + } + + @Specialization + public void doManaged(VirtualFrame frame, LLVMManagedPointer address) { + synchronized (address.getObject()) { + statement.execute(frame); + } + } +}