Skip to content

Commit

Permalink
[AMDGPU] Allow potentially negative flat scratch offsets on GFX12 (ll…
Browse files Browse the repository at this point in the history 
…vm#78193)

llvm#70634 has disabled use
of potentially negative scratch offsets, but we can use it on GFX12.

---------

Co-authored-by: Stanislav Mekhanoshin <[email protected]>
  • Loading branch information
@jayfoad @rampitec
jayfoad and rampitec authored Jan 18, 2024
1 parent 172dbdf commit c111dc7
Show file tree
Hide file tree
Showing 9 changed files with 185 additions and 190 deletions.
12 changes: 11 additions & 1 deletion llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1159,7 +1159,7 @@ bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegal(SDValue Addr) const {

// Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
// values.
if (AMDGPU::isGFX12Plus(*Subtarget))
if (Subtarget->hasSignedScratchOffsets())
return true;

auto LHS = Addr.getOperand(0);
Expand All @@ -1184,6 +1184,11 @@ bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSV(SDValue Addr) const {
if (isNoUnsignedWrap(Addr))
return true;

// Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
// values.
if (Subtarget->hasSignedScratchOffsets())
return true;

auto LHS = Addr.getOperand(0);
auto RHS = Addr.getOperand(1);
return CurDAG->SignBitIsZero(RHS) && CurDAG->SignBitIsZero(LHS);
Expand All @@ -1192,6 +1197,11 @@ bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSV(SDValue Addr) const {
// Check address value in SGPR/VGPR are legal for flat scratch in the form
// of: SGPR + VGPR + Imm.
bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSVImm(SDValue Addr) const {
// Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
// values.
if (AMDGPU::isGFX12Plus(*Subtarget))
return true;

auto Base = Addr.getOperand(0);
auto *RHSImm = cast<ConstantSDNode>(Addr.getOperand(1));
// If the immediate offset is negative and within certain range, the base
Expand Down
12 changes: 11 additions & 1 deletion llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4557,7 +4557,7 @@ bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(Register Addr) const {

// Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
// values.
if (AMDGPU::isGFX12Plus(STI))
if (STI.hasSignedScratchOffsets())
return true;

Register LHS = AddrMI->getOperand(1).getReg();
Expand Down Expand Up @@ -4586,6 +4586,11 @@ bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const {
if (isNoUnsignedWrap(AddrMI))
return true;

// Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
// values.
if (STI.hasSignedScratchOffsets())
return true;

Register LHS = AddrMI->getOperand(1).getReg();
Register RHS = AddrMI->getOperand(2).getReg();
return KB->signBitIsZero(RHS) && KB->signBitIsZero(LHS);
Expand All @@ -4595,6 +4600,11 @@ bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const {
// of: SGPR + VGPR + Imm.
bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
Register Addr) const {
// Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
// values.
if (STI.hasSignedScratchOffsets())
return true;

MachineInstr *AddrMI = getDefIgnoringCopies(Addr, *MRI);
Register Base = AddrMI->getOperand(1).getReg();
std::optional<DefinitionAndSourceRegister> BaseDef =
Expand Down
4 changes: 4 additions & 0 deletions llvm/lib/Target/AMDGPU/GCNSubtarget.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1274,6 +1274,10 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
// \returns true if the target has WG_RR_MODE kernel descriptor mode bit
bool hasRrWGMode() const { return getGeneration() >= GFX12; }

/// \returns true if VADDR and SADDR fields in VSCRATCH can use negative
/// values.
bool hasSignedScratchOffsets() const { return getGeneration() >= GFX12; }

/// \returns SGPR allocation granularity supported by the subtarget.
unsigned getSGPRAllocGranule() const {
return AMDGPU::IsaInfo::getSGPRAllocGranule(this);
Expand Down
105 changes: 51 additions & 54 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,16 +79,17 @@ define amdgpu_kernel void @store_load_sindex_kernel(i32 %idx) {
; GFX12-LABEL: store_load_sindex_kernel:
; GFX12: ; %bb.0: ; %bb
; GFX12-NEXT: s_load_b32 s0, s[0:1], 0x24
; GFX12-NEXT: v_mov_b32_e32 v1, 15
; GFX12-NEXT: s_waitcnt lgkmcnt(0)
; GFX12-NEXT: s_and_b32 s1, s0, 15
; GFX12-NEXT: s_lshl_b32 s1, s0, 2
; GFX12-NEXT: s_and_b32 s0, s0, 15
; GFX12-NEXT: v_mov_b32_e32 v0, s1
; GFX12-NEXT: s_lshl_b32 s0, s0, 2
; GFX12-NEXT: s_lshl_b32 s1, s1, 2
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v0, 15 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: s_add_co_i32 s0, s0, 4
; GFX12-NEXT: scratch_store_b32 off, v0, s0 th:TH_STORE_NT_RT
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: scratch_store_b32 v0, v1, off offset:4 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: scratch_load_b32 v0, v1, off offset:4 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v2, off offset:4 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_endpgm
bb:
Expand Down Expand Up @@ -170,8 +171,7 @@ define amdgpu_kernel void @store_load_vindex_kernel() {
; GFX12-NEXT: v_dual_mov_b32 v2, 15 :: v_dual_lshlrev_b32 v1, 2, v1
; GFX12-NEXT: scratch_store_b32 v0, v2, off offset:4 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: v_add_nc_u32_e32 v1, 4, v1
; GFX12-NEXT: scratch_load_b32 v0, v1, off offset:124 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v1, off offset:128 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_endpgm
bb:
Expand Down Expand Up @@ -248,14 +248,13 @@ define void @store_load_vindex_foo(i32 %idx) {
; GFX12-LABEL: store_load_vindex_foo:
; GFX12: ; %bb.0: ; %bb
; GFX12-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX12-NEXT: v_dual_mov_b32 v2, 15 :: v_dual_lshlrev_b32 v1, 2, v0
; GFX12-NEXT: v_and_b32_e32 v0, 15, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_add_nc_u32_e32 v1, s32, v1
; GFX12-NEXT: v_dual_mov_b32 v2, 15 :: v_dual_and_b32 v1, 15, v0
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT: scratch_store_b32 v1, v2, off th:TH_STORE_NT_RT
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_lshlrev_b32_e32 v1, 2, v1
; GFX12-NEXT: scratch_store_b32 v0, v2, s32 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: scratch_load_b32 v0, v0, s32 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v1, s32 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
bb:
Expand Down Expand Up @@ -391,17 +390,19 @@ define amdgpu_kernel void @store_load_sindex_small_offset_kernel(i32 %idx) {
; GFX12-LABEL: store_load_sindex_small_offset_kernel:
; GFX12: ; %bb.0: ; %bb
; GFX12-NEXT: s_load_b32 s0, s[0:1], 0x24
; GFX12-NEXT: scratch_load_b32 v2, off, off offset:4 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX12-NEXT: s_and_b32 s1, s0, 15
; GFX12-NEXT: scratch_load_b32 v3, off, off offset:4 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: v_mov_b32_e32 v1, 15
; GFX12-NEXT: s_waitcnt lgkmcnt(0)
; GFX12-NEXT: s_lshl_b32 s1, s0, 2
; GFX12-NEXT: s_and_b32 s0, s0, 15
; GFX12-NEXT: v_mov_b32_e32 v0, s1
; GFX12-NEXT: s_lshl_b32 s0, s0, 2
; GFX12-NEXT: s_lshl_b32 s1, s1, 2
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v0, 15 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: s_addk_co_i32 s0, 0x104
; GFX12-NEXT: scratch_store_b32 off, v0, s0 th:TH_STORE_NT_RT
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: scratch_store_b32 v0, v1, off offset:260 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: scratch_load_b32 v0, v1, off offset:260 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v2, off offset:260 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_endpgm
bb:
Expand Down Expand Up @@ -490,13 +491,13 @@ define amdgpu_kernel void @store_load_vindex_small_offset_kernel() {
; GFX12: ; %bb.0: ; %bb
; GFX12-NEXT: v_sub_nc_u32_e32 v1, 0, v0
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT: v_mov_b32_e32 v2, 15
; GFX12-NEXT: scratch_load_b32 v3, off, off offset:4 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: v_dual_mov_b32 v2, 15 :: v_dual_lshlrev_b32 v1, 2, v1
; GFX12-NEXT: v_lshlrev_b32_e32 v1, 2, v1
; GFX12-NEXT: scratch_store_b32 v0, v2, off offset:260 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: v_add_nc_u32_e32 v1, 0x104, v1
; GFX12-NEXT: scratch_load_b32 v0, v1, off offset:124 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v1, off offset:384 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_endpgm
bb:
Expand Down Expand Up @@ -589,16 +590,14 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
; GFX12-LABEL: store_load_vindex_small_offset_foo:
; GFX12: ; %bb.0: ; %bb
; GFX12-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX12-NEXT: v_dual_mov_b32 v2, 15 :: v_dual_lshlrev_b32 v1, 2, v0
; GFX12-NEXT: v_and_b32_e32 v0, 15, v0
; GFX12-NEXT: s_add_co_i32 s0, s32, 0x100
; GFX12-NEXT: v_dual_mov_b32 v2, 15 :: v_dual_and_b32 v1, 15, v0
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT: scratch_load_b32 v3, off, s32 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: v_add_nc_u32_e32 v1, s0, v1
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT: scratch_store_b32 v1, v2, off th:TH_STORE_NT_RT
; GFX12-NEXT: v_lshlrev_b32_e32 v1, 2, v1
; GFX12-NEXT: scratch_store_b32 v0, v2, s32 offset:256 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: scratch_load_b32 v0, v0, s32 offset:256 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v1, s32 offset:256 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
bb:
Expand Down Expand Up @@ -697,17 +696,19 @@ define amdgpu_kernel void @store_load_sindex_large_offset_kernel(i32 %idx) {
; GFX12-LABEL: store_load_sindex_large_offset_kernel:
; GFX12: ; %bb.0: ; %bb
; GFX12-NEXT: s_load_b32 s0, s[0:1], 0x24
; GFX12-NEXT: scratch_load_b32 v2, off, off offset:4 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX12-NEXT: s_and_b32 s1, s0, 15
; GFX12-NEXT: scratch_load_b32 v3, off, off offset:4 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: v_mov_b32_e32 v1, 15
; GFX12-NEXT: s_waitcnt lgkmcnt(0)
; GFX12-NEXT: s_lshl_b32 s1, s0, 2
; GFX12-NEXT: s_and_b32 s0, s0, 15
; GFX12-NEXT: v_mov_b32_e32 v0, s1
; GFX12-NEXT: s_lshl_b32 s0, s0, 2
; GFX12-NEXT: s_lshl_b32 s1, s1, 2
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v0, 15 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: s_addk_co_i32 s0, 0x4004
; GFX12-NEXT: scratch_store_b32 off, v0, s0 th:TH_STORE_NT_RT
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: scratch_store_b32 v0, v1, off offset:16388 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: scratch_load_b32 v0, v1, off offset:16388 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v2, off offset:16388 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_endpgm
bb:
Expand Down Expand Up @@ -798,13 +799,13 @@ define amdgpu_kernel void @store_load_vindex_large_offset_kernel() {
; GFX12: ; %bb.0: ; %bb
; GFX12-NEXT: v_sub_nc_u32_e32 v1, 0, v0
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT: v_mov_b32_e32 v2, 15
; GFX12-NEXT: scratch_load_b32 v3, off, off offset:4 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: v_dual_mov_b32 v2, 15 :: v_dual_lshlrev_b32 v1, 2, v1
; GFX12-NEXT: v_lshlrev_b32_e32 v1, 2, v1
; GFX12-NEXT: scratch_store_b32 v0, v2, off offset:16388 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: v_add_nc_u32_e32 v1, 0x4004, v1
; GFX12-NEXT: scratch_load_b32 v0, v1, off offset:124 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v1, off offset:16512 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_endpgm
bb:
Expand Down Expand Up @@ -899,16 +900,14 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
; GFX12-LABEL: store_load_vindex_large_offset_foo:
; GFX12: ; %bb.0: ; %bb
; GFX12-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX12-NEXT: v_dual_mov_b32 v2, 15 :: v_dual_lshlrev_b32 v1, 2, v0
; GFX12-NEXT: v_and_b32_e32 v0, 15, v0
; GFX12-NEXT: s_add_co_i32 s0, s32, 0x4000
; GFX12-NEXT: v_dual_mov_b32 v2, 15 :: v_dual_and_b32 v1, 15, v0
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT: scratch_load_b32 v3, off, s32 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: v_add_nc_u32_e32 v1, s0, v1
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT: scratch_store_b32 v1, v2, off th:TH_STORE_NT_RT
; GFX12-NEXT: v_lshlrev_b32_e32 v1, 2, v1
; GFX12-NEXT: scratch_store_b32 v0, v2, s32 offset:16384 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: scratch_load_b32 v0, v0, s32 offset:16384 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v1, s32 offset:16384 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_setpc_b64 s[30:31]
bb:
Expand Down Expand Up @@ -1154,11 +1153,9 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
; GFX12-NEXT: v_mov_b32_e32 v1, 15
; GFX12-NEXT: s_waitcnt lgkmcnt(0)
; GFX12-NEXT: v_add_lshl_u32 v0, s0, v0, 2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_add_nc_u32_e32 v0, 4, v0
; GFX12-NEXT: scratch_store_b32 v0, v1, off offset:1024 th:TH_STORE_NT_RT
; GFX12-NEXT: scratch_store_b32 v0, v1, off offset:1028 th:TH_STORE_NT_RT
; GFX12-NEXT: s_waitcnt_vscnt null, 0x0
; GFX12-NEXT: scratch_load_b32 v0, v0, off offset:1024 th:TH_LOAD_RT_NT
; GFX12-NEXT: scratch_load_b32 v0, v0, off offset:1028 th:TH_LOAD_RT_NT
; GFX12-NEXT: s_waitcnt vmcnt(0)
; GFX12-NEXT: s_endpgm
bb:
Expand Down
Loading

0 comments on commit c111dc7

Please sign in to comment.