diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 3261b26e74cd71..1729d606f6cc5a 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -845,7 +845,8 @@ class CombinerHelper { bool matchExtractVectorElement(MachineInstr &MI, BuildFnTy &MatchInfo); /// Combine extract vector element with a build vector on the vector register. - bool matchExtractVectorElementWithBuildVector(const MachineOperand &MO, + bool matchExtractVectorElementWithBuildVector(const MachineInstr &MI, + const MachineInstr &MI2, BuildFnTy &MatchInfo); /// Combine extract vector element with a build vector trunc on the vector @@ -855,7 +856,8 @@ class CombinerHelper { /// Combine extract vector element with a shuffle vector on the vector /// register. - bool matchExtractVectorElementWithShuffleVector(const MachineOperand &MO, + bool matchExtractVectorElementWithShuffleVector(const MachineInstr &MI, + const MachineInstr &MI2, BuildFnTy &MatchInfo); /// Combine extract vector element with a insert vector element on the vector diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index e75cf0b7d4afc1..be33c77adffb51 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -1373,110 +1373,23 @@ def extract_vector_element_different_indices : GICombineRule< [{ return Helper.matchExtractVectorElementWithDifferentIndices(${root}, ${matchinfo}); }]), (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; -def extract_vector_element_build_vector2 : GICombineRule< +def extract_vector_element_build_vector : GICombineRule< (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector3 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector4 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector5 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector6 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector7 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector8 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector9 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector10 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector11 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector12 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector13 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + (match (G_CONSTANT $idx, $imm), + (G_BUILD_VECTOR $src, GIVariadic<>:$unused):$Build, + (G_EXTRACT_VECTOR_ELT $root, $src, $idx):$Extract, + [{ return Helper.matchExtractVectorElementWithBuildVector(*${Extract}, *${Build}, + ${matchinfo}); }]), + (apply [{ Helper.applyBuildFn(*${Extract}, ${matchinfo}); }])>; -def extract_vector_element_build_vector14 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector15 : GICombineRule< +def extract_vector_element_shuffle_vector : GICombineRule< (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - -def extract_vector_element_build_vector16 : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_BUILD_VECTOR $src, $x, $y, $z, $a, $b, $c, $d, $e, $f, $g, $h, $i, $j, $k, $l, $m), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithBuildVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; + (match (G_CONSTANT $idx, $imm), + (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask):$Shuffle, + (G_EXTRACT_VECTOR_ELT $root, $src, $idx):$Extract, + [{ return Helper.matchExtractVectorElementWithShuffleVector(*${Extract}, *${Shuffle}, + ${matchinfo}); }]), + (apply [{ Helper.applyBuildFn(*${Extract}, ${matchinfo}); }])>; def extract_vector_element_build_vector_trunc2 : GICombineRule< (defs root:$root, build_fn_matchinfo:$matchinfo), @@ -1547,13 +1460,6 @@ def nneg_zext : GICombineRule< [{ return Helper.matchNonNegZext(${root}, ${matchinfo}); }]), (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; -def extract_vector_element_shuffle_vector : GICombineRule< - (defs root:$root, build_fn_matchinfo:$matchinfo), - (match (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask), - (G_EXTRACT_VECTOR_ELT $root, $src, $idx), - [{ return Helper.matchExtractVectorElementWithShuffleVector(${root}, ${matchinfo}); }]), - (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>; - // Combines concat operations def concat_matchinfo : GIDefMatchData<"SmallVector">; def combine_concat_vector : GICombineRule< @@ -1647,20 +1553,7 @@ match_extract_of_element, insert_vector_elt_oob, extract_vector_element_not_const, extract_vector_element_different_indices, -extract_vector_element_build_vector2, -extract_vector_element_build_vector3, -extract_vector_element_build_vector4, -extract_vector_element_build_vector5, -extract_vector_element_build_vector7, -extract_vector_element_build_vector8, -extract_vector_element_build_vector9, -extract_vector_element_build_vector10, -extract_vector_element_build_vector11, -extract_vector_element_build_vector12, -extract_vector_element_build_vector13, -extract_vector_element_build_vector14, -extract_vector_element_build_vector15, -extract_vector_element_build_vector16, +extract_vector_element_build_vector, extract_vector_element_build_vector_trunc2, extract_vector_element_build_vector_trunc3, extract_vector_element_build_vector_trunc4, diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp index 66b1c5f8ca82c7..84fb3b59658956 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp @@ -146,9 +146,9 @@ bool CombinerHelper::matchExtractVectorElementWithDifferentIndices( } bool CombinerHelper::matchExtractVectorElementWithBuildVector( - const MachineOperand &MO, BuildFnTy &MatchInfo) { - MachineInstr *Root = getDefIgnoringCopies(MO.getReg(), MRI); - GExtractVectorElement *Extract = cast(Root); + const MachineInstr &MI, const MachineInstr &MI2, BuildFnTy &MatchInfo) { + const GExtractVectorElement *Extract = cast(&MI); + const GBuildVector *Build = cast(&MI2); // // %zero:_(s64) = G_CONSTANT i64 0 @@ -160,23 +160,8 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVector( // %extract:_(32) = COPY %arg1(s32) // // - // - // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) - // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) - // - // --> - // - // %bv:_(<2 x s32>) = G_BUILD_VECTOR %arg1(s32), %arg2(s32) - // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %bv(<2 x s32>), %opaque(s64) - // Register Vector = Extract->getVectorReg(); - - // We expect a buildVector on the Vector register. - GBuildVector *Build = getOpcodeDef(Vector, MRI); - if (!Build) - return false; - LLT VectorTy = MRI.getType(Vector); // There is a one-use check. There are more combines on build vectors. @@ -185,14 +170,7 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVector( !getTargetLowering().aggressivelyPreferBuildVectorSources(Ty)) return false; - Register Index = Extract->getIndexReg(); - - // If the Index is constant, then we can extract the element from the given - // offset. - std::optional MaybeIndex = - getIConstantVRegValWithLookThrough(Index, MRI); - if (!MaybeIndex) - return false; + APInt Index = getIConstantFromReg(Extract->getIndexReg(), MRI); // We now know that there is a buildVector def'd on the Vector register and // the index is const. The combine will succeed. @@ -200,7 +178,7 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVector( Register Dst = Extract->getReg(0); MatchInfo = [=](MachineIRBuilder &B) { - B.buildCopy(Dst, Build->getSourceReg(MaybeIndex->Value.getZExtValue())); + B.buildCopy(Dst, Build->getSourceReg(Index.getZExtValue())); }; return true; @@ -274,9 +252,9 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc( } bool CombinerHelper::matchExtractVectorElementWithShuffleVector( - const MachineOperand &MO, BuildFnTy &MatchInfo) { - GExtractVectorElement *Extract = - cast(getDefIgnoringCopies(MO.getReg(), MRI)); + const MachineInstr &MI, const MachineInstr &MI2, BuildFnTy &MatchInfo) { + const GExtractVectorElement *Extract = cast(&MI); + const GShuffleVector *Shuffle = cast(&MI2); // // %zero:_(s64) = G_CONSTANT i64 0 @@ -302,32 +280,12 @@ bool CombinerHelper::matchExtractVectorElementWithShuffleVector( // %extract:_(s32) = G_IMPLICIT_DEF // // - // - // - // - // %sv:_(<4 x s32>) = G_SHUFFLE_SHUFFLE %arg1(<4 x s32>), %arg2(<4 x s32>), - // shufflemask(0, 0, 0, -1) - // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %opaque(s64) - // - // --> - // - // %sv:_(<4 x s32>) = G_SHUFFLE_SHUFFLE %arg1(<4 x s32>), %arg2(<4 x s32>), - // shufflemask(0, 0, 0, -1) - // %extract:_(s32) = G_EXTRACT_VECTOR_ELT %sv(<4 x s32>), %opaque(s64) - // - - // We try to get the value of the Index register. - std::optional MaybeIndex = - getIConstantVRegValWithLookThrough(Extract->getIndexReg(), MRI); - if (!MaybeIndex) - return false; - GShuffleVector *Shuffle = - cast(getDefIgnoringCopies(Extract->getVectorReg(), MRI)); + APInt Index = getIConstantFromReg(Extract->getIndexReg(), MRI); ArrayRef Mask = Shuffle->getMask(); - unsigned Offset = MaybeIndex->Value.getZExtValue(); + unsigned Offset = Index.getZExtValue(); int SrcIdx = Mask[Offset]; LLT Src1Type = MRI.getType(Shuffle->getSrc1Reg()); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir index c98dcf6ccb7966..83b5c388520eb3 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-extract-vec-elt.mir @@ -634,3 +634,30 @@ body: | RET_ReallyLR implicit $x0 ... --- +name: extract_from_build_vector_const_huge +alignment: 4 +liveins: + - { reg: '$x0' } + - { reg: '$x1' } +frameInfo: + maxAlignment: 1 +body: | + bb.1: + liveins: $x0, $x1 + ; CHECK-LABEL: name: extract_from_build_vector_const_huge + ; CHECK: liveins: $x0, $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %arg1:_(s64) = COPY $x0 + ; CHECK-NEXT: $x0 = COPY %arg1(s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %vec:_(<2 x s64>) = COPY $q0 + %idx:_(s64) = G_CONSTANT i64 0 + %arg1:_(s64) = COPY $x0 + %arg2:_(s64) = COPY $x1 + %bv:_(<18 x s64>) = G_BUILD_VECTOR %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64), %arg1(s64), %arg2(s64) + %extract:_(s64) = G_EXTRACT_VECTOR_ELT %bv(<18 x s64>), %idx(s64) + $x0 = COPY %extract(s64) + RET_ReallyLR implicit $x0 + +... +---