Skip to content

Commit

Permalink
[GR-31706] [GR-31549] Support atomic inline asm instructions.
Browse files Browse the repository at this point in the history 
PullRequest: graal/9025
  • Loading branch information
@fmorcos
fmorcos committed May 29, 2021
2 parents 4befcba + 1662979 commit 5fa3162
Show file tree
Hide file tree
Showing 4 changed files with 171 additions and 15 deletions.
8 changes: 8 additions & 0 deletions sulong/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,14 @@ Fixes:

* Fix LLVM toolchain not working correctly for C++ on MacOS 11.3.

New features:

* Support implicitly typed XADD instructions in inline assembly.

* Support some atomic variants of certain inline assembly instructions. Currently
supported instructions are: xchg, cmpxchg and xadd along with their explicitly typed
variants, and various unary instructions including incb/w/l/q and decb/w/l/q.

# Version 21.1.0

New features:
Expand Down
111 changes: 96 additions & 15 deletions ...s/com.oracle.truffle.llvm.asm.amd64/src/com/oracle/truffle/llvm/asm/amd64/AsmFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@
import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64AddressComputationNodeFactory.LLVMAMD64AddressSegmentComputationNodeGen;
import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64Flags;
import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64GetTlsNodeGen;
import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64LockedInstructionNodeGen;
import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64ReadAddressNodeGen;
import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64ReadRegisterNodeGen;
import com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64Target;
Expand Down Expand Up @@ -951,7 +952,7 @@ void createUnaryOperation(String operation, AsmOperand operand) {
statements.add(LLVMUnsupportedInstructionNode.create(UnsupportedReason.INLINE_ASSEMBLER, operation));
return;
}
statements.add(getOperandStore(dstType, dst, out));
statements.add(maybeMakeAtomic(getOperandStore(dstType, dst, out), dst));
}

private static boolean isShiftOperation(String operation) {
Expand Down Expand Up @@ -1035,6 +1036,41 @@ private Type getType(AsmOperand dst, AsmOperand src) {
return type;
}

/**
* Ensures that at least the instruction's operand passed in is a memory operand, and that the
* instruction has a lock prefix, in which case it returns the instruction ("the statement")
* wrapped in an
* {@link com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64LockedInstructionNode}.
* Otherwise returns the bare instruction.
*/
private LLVMStatementNode maybeMakeAtomic(LLVMStatementNode statement, AsmOperand maybeMemoryOperand) {
if ("lock".equals(currentPrefix)) {
return makeAtomic(statement, maybeMemoryOperand);
}

return statement;
}

/**
* Ensures that at least the instruction's operand passed in is a memory operand, in which case
* it returns the instruction ("the statement") wrapped in an
* {@link com.oracle.truffle.llvm.runtime.nodes.asm.support.LLVMAMD64LockedInstructionNode}.
* Otherwise returns the bare instruction.
*/
private LLVMStatementNode makeAtomic(LLVMStatementNode statement, AsmOperand maybeMemoryOperand) {
if (maybeMemoryOperand instanceof AsmMemoryOperand) {
return LLVMAMD64LockedInstructionNodeGen.create(statement, getOperandAddress(maybeMemoryOperand));
} else if (maybeMemoryOperand instanceof AsmArgumentOperand) {
AsmArgumentOperand op = (AsmArgumentOperand) maybeMemoryOperand;
Argument info = argInfo.get(op.getIndex());
if (info.isMemory()) {
return LLVMAMD64LockedInstructionNodeGen.create(statement, getOperandAddress(maybeMemoryOperand));
}
}

return statement;
}

void createBinaryOperationImplicitSize(String operation, AsmOperand a, AsmOperand b) {
AsmOperand dst = b;
AsmOperand src = a;
Expand Down Expand Up @@ -1130,7 +1166,17 @@ void createBinaryOperationImplicitSize(String operation, AsmOperand a, AsmOperan
default:
throw invalidOperandType(dstType);
}
statements.add(res);
/**
* Make xchg unconditionally atomic. Some code (e.g. musl libc) using xchg relies on
* its (and any other instruction's) non-atomic behavior in concurrency with other
* locked/atomic instructions to be atomic.
*
* The issue is that we should avoid globally locking memory operations, and so
* instead we hand-pick instructions that are used in code related to
* synchronization, and xchg is one of those (e.g. in the musl libc). This is not
* entirely correct behavior, but should cover most practical use-cases.
*/
statements.add(makeAtomic(res, b));
return;
}
case "cmpxchg": {
Expand Down Expand Up @@ -1170,7 +1216,7 @@ void createBinaryOperationImplicitSize(String operation, AsmOperand a, AsmOperan
throw invalidOperandType(dstType);
}
}
statements.add(res);
statements.add(maybeMakeAtomic(res, b));
return;
}
case "and":
Expand Down Expand Up @@ -1218,6 +1264,41 @@ void createBinaryOperationImplicitSize(String operation, AsmOperand a, AsmOperan
LLVMX86_ConversionNode.LLVMX86_Pmovmskb128 pmovmskb128 = LLVMX86_ConversionNodeFactory.LLVMX86_Pmovmskb128NodeGen.create(srcA);
out = pmovmskb128;
break;
case "xadd":
srcA = getOperandLoad(dstType, a);
srcB = getOperandLoad(dstType, b);
switch (getPrimitiveType(dstType)) {
case I8: {
LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I8, a);
LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst);
LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2);
statements.add(maybeMakeAtomic(LLVMAMD64XaddbNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b));
return;
}
case I16: {
LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I16, a);
LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst);
LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2);
statements.add(maybeMakeAtomic(LLVMAMD64XaddwNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b));
return;
}
case I32: {
LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I32, a);
LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst);
LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2);
statements.add(maybeMakeAtomic(LLVMAMD64XaddlNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b));
return;
}
case I64: {
LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I64, a);
LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst);
LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2);
statements.add(maybeMakeAtomic(LLVMAMD64XaddqNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b));
return;
}
default:
throw invalidOperandType(dstType);
}
default:
statements.add(LLVMUnsupportedInstructionNode.create(UnsupportedReason.INLINE_ASSEMBLER, operation));
return;
Expand Down Expand Up @@ -1476,22 +1557,22 @@ void createBinaryOperation(String operation, AsmOperand a, AsmOperand b) {
break;
case "xchgb": {
XchgOperands operands = new XchgOperands(a, b, dstType);
statements.add(LLVMAMD64XchgbNodeGen.create(operands.dst, operands.srcA, operands.srcB));
statements.add(makeAtomic(LLVMAMD64XchgbNodeGen.create(operands.dst, operands.srcA, operands.srcB), b));
return;
}
case "xchgw": {
XchgOperands operands = new XchgOperands(a, b, dstType);
statements.add(LLVMAMD64XchgwNodeGen.create(operands.dst, operands.srcA, operands.srcB));
statements.add(makeAtomic(LLVMAMD64XchgwNodeGen.create(operands.dst, operands.srcA, operands.srcB), b));
return;
}
case "xchgl": {
XchgOperands operands = new XchgOperands(a, b, dstType);
statements.add(LLVMAMD64XchglNodeGen.create(operands.dst, operands.srcA, operands.srcB));
statements.add(makeAtomic(LLVMAMD64XchglNodeGen.create(operands.dst, operands.srcA, operands.srcB), b));
return;
}
case "xchgq": {
XchgOperands operands = new XchgOperands(a, b, dstType);
statements.add(LLVMAMD64XchgqNodeGen.create(operands.dst, operands.srcA, operands.srcB));
statements.add(makeAtomic(LLVMAMD64XchgqNodeGen.create(operands.dst, operands.srcA, operands.srcB), b));
return;
}
case "cmpb":
Expand All @@ -1510,56 +1591,56 @@ void createBinaryOperation(String operation, AsmOperand a, AsmOperand b) {
LLVMAMD64WriteValueNode dst1 = getStore(dstType, b);
LLVMAMD64WriteValueNode dst2 = getRegisterStore("al");
LLVMExpressionNode accumulator = getOperandLoad(PrimitiveType.I8, new AsmRegisterOperand("al"));
statements.add(LLVMAMD64CmpXchgbNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB));
statements.add(maybeMakeAtomic(LLVMAMD64CmpXchgbNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB), b));
return;
}
case "cmpxchgw": {
LLVMAMD64WriteValueNode dst1 = getStore(dstType, b);
LLVMAMD64WriteValueNode dst2 = getRegisterStore("ax");
LLVMExpressionNode accumulator = getOperandLoad(PrimitiveType.I16, new AsmRegisterOperand("ax"));
statements.add(LLVMAMD64CmpXchgwNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB));
statements.add(maybeMakeAtomic(LLVMAMD64CmpXchgwNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB), b));
return;
}
case "cmpxchgl": {
LLVMAMD64WriteValueNode dst1 = getStore(dstType, b);
LLVMAMD64WriteValueNode dst2 = getRegisterStore("eax");
LLVMExpressionNode accumulator = getOperandLoad(PrimitiveType.I32, new AsmRegisterOperand("eax"));
statements.add(LLVMAMD64CmpXchglNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB));
statements.add(maybeMakeAtomic(LLVMAMD64CmpXchglNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB), b));
return;
}
case "cmpxchgq": {
LLVMAMD64WriteValueNode dst1 = getStore(dstType, b);
LLVMAMD64WriteValueNode dst2 = getRegisterStore("rax");
LLVMExpressionNode accumulator = getOperandLoad(PrimitiveType.I64, new AsmRegisterOperand("rax"));
statements.add(LLVMAMD64CmpXchgqNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB));
statements.add(maybeMakeAtomic(LLVMAMD64CmpXchgqNodeGen.create(getUpdateCPAZSOFlagsNode(), dst1, dst2, accumulator, srcA, srcB), b));
return;
}
case "xaddb": {
LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I8, a);
LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst);
LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2);
statements.add(LLVMAMD64XaddbNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB));
statements.add(maybeMakeAtomic(LLVMAMD64XaddbNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b));
return;
}
case "xaddw": {
LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I16, a);
LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst);
LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2);
statements.add(LLVMAMD64XaddwNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB));
statements.add(maybeMakeAtomic(LLVMAMD64XaddwNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b));
return;
}
case "xaddl": {
LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I32, a);
LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst);
LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2);
statements.add(LLVMAMD64XaddlNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB));
statements.add(maybeMakeAtomic(LLVMAMD64XaddlNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b));
return;
}
case "xaddq": {
LLVMAMD64WriteValueNode dst1 = getRegisterStore(PrimitiveType.I64, a);
LLVMAMD64WriteValueNode dst2 = getStore(dstType, dst);
LLVMAMD64WriteTupelNode res = LLVMAMD64WriteTupelNodeGen.create(dst1, dst2);
statements.add(LLVMAMD64XaddqNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB));
statements.add(maybeMakeAtomic(LLVMAMD64XaddqNodeGen.create(getUpdateCPZSOFlagsNode(), res, srcA, srcB), b));
return;
}
case "xorb":
Expand Down
1 change: 1 addition & 0 deletions ...ects/com.oracle.truffle.llvm.runtime/src/com/oracle/truffle/llvm/runtime/LLVMContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ public final class LLVMContext {
protected final ArrayList<LLVMPointer> globalsNonPointerStore = new ArrayList<>();
protected final EconomicMap<Integer, LLVMPointer> globalsReadOnlyStore = EconomicMap.create();
private final Object globalsStoreLock = new Object();
public final Object atomicInstructionsLock = new Object();

private final List<LLVMThread> runningThreads = new ArrayList<>();
@CompilationFinal private LLVMThreadingStack threadingStack;
Expand Down
66 changes: 66 additions & 0 deletions ...src/com/oracle/truffle/llvm/runtime/nodes/asm/support/LLVMAMD64LockedInstructionNode.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates.
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are
* permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.oracle.truffle.llvm.runtime.nodes.asm.support;

import com.oracle.truffle.api.dsl.CachedContext;
import com.oracle.truffle.api.dsl.NodeChild;
import com.oracle.truffle.api.dsl.Specialization;
import com.oracle.truffle.api.frame.VirtualFrame;
import com.oracle.truffle.llvm.runtime.LLVMContext;
import com.oracle.truffle.llvm.runtime.LLVMLanguage;
import com.oracle.truffle.llvm.runtime.nodes.api.LLVMExpressionNode;
import com.oracle.truffle.llvm.runtime.nodes.api.LLVMStatementNode;
import com.oracle.truffle.llvm.runtime.pointer.LLVMManagedPointer;
import com.oracle.truffle.llvm.runtime.pointer.LLVMNativePointer;

@NodeChild(value = "address", type = LLVMExpressionNode.class)
public abstract class LLVMAMD64LockedInstructionNode extends LLVMStatementNode {
@Child LLVMStatementNode statement;

public LLVMAMD64LockedInstructionNode(LLVMStatementNode statement) {
this.statement = statement;
}

@Specialization
public void doNative(VirtualFrame frame,
@SuppressWarnings("unused") LLVMNativePointer address,
@CachedContext(LLVMLanguage.class) LLVMContext context) {
synchronized (context.atomicInstructionsLock) {
statement.execute(frame);
}
}

@Specialization
public void doManaged(VirtualFrame frame, LLVMManagedPointer address) {
synchronized (address.getObject()) {
statement.execute(frame);
}
}
}

0 comments on commit 5fa3162

Please sign in to comment.