From f0bd2db68cf226b8fb73157df192da445fc4c12e Mon Sep 17 00:00:00 2001 From: nella Date: Tue, 8 Sep 2026 21:47:32 +0200 Subject: [PATCH 1/3] Split long pack functions. --- src/backend/pass/opt/slp/slp_pack.h | 26 ++++- src/backend/pass/opt/slp/slp_packer.cpp | 143 ++++++++++++++---------- src/backend/pass/opt/slp/slp_reduce.cpp | 63 ++++++----- 3 files changed, 138 insertions(+), 94 deletions(-) diff --git a/src/backend/pass/opt/slp/slp_pack.h b/src/backend/pass/opt/slp/slp_pack.h index 5a6ff2d..d37beeb 100644 --- a/src/backend/pass/opt/slp/slp_pack.h +++ b/src/backend/pass/opt/slp/slp_pack.h @@ -82,6 +82,16 @@ namespace rat { I64 minC = 0, maxC = 0; }; + // how a load tuple is addressed + struct LoadShape { + LoadNode* first = nullptr; + RefinedAddr k0; // address of lane 0 + U32 esz = 0; + B32 sharedState = false; // every lane reads one memory state + B32 adjacent = false; // lane i is at k0 + i*esz + B32 equal = false; // every lane hits k0 + }; + Packer(Slp& drv, Node* memIn, const RefinedAddr* windowKey); void collectRun(const Segment& seg, U32 begin, U32 count); @@ -92,6 +102,10 @@ namespace rat { static String tupleKey(const List& lanes); Node* packTuple(const List& lanes, Type* elemTy, U32 depth); Node* packTupleUncached(const List& lanes, Type* elemTy, Type* vecTy, U32 depth); + B32 matchLoadShape(const List& lanes, Type* elemTy, LoadShape& out) const; + B32 straddlesWindow(const LoadShape& sh, U32 w) const; + B32 innerStatesHoistable(const List& lanes, const LoadShape& sh); + void guardAgainstWindow(const LoadShape& sh, U32 w); Node* packLoads(const List& lanes, Type* elemTy, Type* vecTy); Node* packBinaryLanes(const List& lanes, Type* elemTy, Type* vecTy, U32 depth); Node* packWideOrSplat(Node* mem, @@ -108,10 +122,12 @@ namespace rat { // window context, null for reductions Node* memIn; const RefinedAddr* windowKey; - Set runStores; // the scalar stores being fused - Map> interWritten; // inner store -> lane offsets stored before it - Set observers; // loads reading an inner store's state - B32 dead = false; // a matched strategy could not be built, the window is rejected + Set runStores; // the scalar stores being fused + // inner store -> lane offsets already written in the state that store produces, + // its own offset included + Map> interWritten; + Set observers; // loads reading an inner store's state + B32 dead = false; // a matched strategy could not be built, the window is rejected List guardGroups; // splat reloads kept for post-commit coalescing into wide loads List> splatLoads; @@ -167,6 +183,8 @@ namespace rat { // steer every other user of the pre-branch control to the right arm void rerouteBlock(Node* startCtrl, Node* iff, Node* region, Node* elseP, const Packer& packer); + void sortReductionTerms(List& terms, U32 esz); + Node* emitHsum(Node* acc, Opcode addOp, Type* vecTy, U32 w); U32 packReduction(BinaryNode* root); U32 packReductions(); U32 run(); diff --git a/src/backend/pass/opt/slp/slp_packer.cpp b/src/backend/pass/opt/slp/slp_packer.cpp index 27f11aa..cca3e60 100644 --- a/src/backend/pass/opt/slp/slp_packer.cpp +++ b/src/backend/pass/opt/slp/slp_packer.cpp @@ -187,84 +187,103 @@ namespace rat { return fn.create(op, vecTy, lv, rv); } - Node* slp::Packer::packLoads(const List& lanes, Type* elemTy, Type* vecTy) { + B32 slp::Packer::matchLoadShape(const List& lanes, Type* elemTy, LoadShape& out) const { U32 w = (U32)lanes.size(); - U32 esz = elemTy->byteSize(drv.ptrBytes); + out.esz = elemTy->byteSize(drv.ptrBytes); for(Node* n : lanes) if(!isa(n) || n->getType() != elemTy) - return nullptr; - LoadNode* first = cast(lanes[0]); - RefinedAddr k0 = refineAddr(first->getPointer(), esz); - if(!k0.valid()) - return nullptr; - B32 sharedState = true; - B32 adjacent = true, equal = true; + return false; + out.first = cast(lanes[0]); + out.k0 = refineAddr(out.first->getPointer(), out.esz); + if(!out.k0.valid()) + return false; + + out.sharedState = true; + out.adjacent = true; + out.equal = true; for(U32 i = 1; i < w; ++i) { LoadNode* l = cast(lanes[i]); - if(l->getControl() != first->getControl()) - return nullptr; - sharedState &= l->getMemory() == first->getMemory(); - RefinedAddr k = refineAddr(l->getPointer(), esz); - if(!k.valid() || !k.sameGroup(k0)) - return nullptr; - adjacent &= k.constant == k0.constant + (I64)(i * esz); - equal &= k.constant == k0.constant; - } - if(!adjacent && !equal) - return nullptr; - - // a same-group load range that straddles the store window without matching - // it exactly is a store-forward trap - if(windowKey && !equal && k0.sameGroup(*windowKey)) { - I64 sLo = windowKey->constant, sHi = sLo + (I64)(w * esz); - I64 lLo = k0.constant, lHi = lLo + (I64)(w * esz); - if(lLo != sLo && lLo < sHi && sLo < lHi) { - ++drv.stats.rejectedOverlap; - dead = true; - return nullptr; - } + if(l->getControl() != out.first->getControl()) + return false; + out.sharedState &= l->getMemory() == out.first->getMemory(); + RefinedAddr k = refineAddr(l->getPointer(), out.esz); + if(!k.valid() || !k.sameGroup(out.k0)) + return false; + out.adjacent &= k.constant == out.k0.constant + (I64)(i * out.esz); + out.equal &= k.constant == out.k0.constant; } + return out.adjacent || out.equal; + } - // all lanes read one pre-window state: a single wide load (or splat) - if(sharedState && !interWritten.count(first->getMemory())) - return packWideOrSplat(first->getMemory(), first, k0, elemTy, vecTy, w, equal); + // a same-group load range that overlaps the store window without starting at it + // reads bytes the window itself writes + B32 slp::Packer::straddlesWindow(const LoadShape& sh, U32 w) const { + if(!windowKey || sh.equal || !sh.k0.sameGroup(*windowKey)) + return false; + I64 sLo = windowKey->constant, sHi = sLo + (I64)(w * sh.esz); + I64 lLo = sh.k0.constant, lHi = lLo + (I64)(w * sh.esz); + return lLo != sLo && lLo < sHi && sLo < lHi; + } - // lanes read inner window states: hoist them to memIn unless one reads a - // lane stored earlier in the window (never reached by reductions, their - // interWritten is empty) - for(U32 i = 0; i < w; ++i) { - LoadNode* l = cast(lanes[i]); - Node* m = l->getMemory(); + // lanes read states inside the window + B32 slp::Packer::innerStatesHoistable(const List& lanes, const LoadShape& sh) { + for(U32 i = 0; i < (U32)lanes.size(); ++i) { + Node* m = cast(lanes[i])->getMemory(); if(m == memIn) continue; auto it = interWritten.find(m); if(it == interWritten.end()) - return nullptr; // some other state, not this window's business - if(k0.sameGroup(*windowKey)) { - I64 laneOff = 0; - if(adjacent) - laneOff = (I64)(i * esz); - // this lane reads [c, c + esz), a lane stored before it covers [written, written + ssz). - // any overlap, not just an exact hit, means the scalar sees a freshly stored byte - I64 c = k0.constant + laneOff; - I64 ssz = (I64)windowKey->size; - for(I64 written : it->second) - if(written < c + (I64)esz && c < written + ssz) { - dead = true; // scalar reads a freshly stored lane - return nullptr; - } - } + return false; // some other state, not this window's business + if(!sh.k0.sameGroup(*windowKey)) + continue; + I64 laneOff = 0; + if(sh.adjacent) + laneOff = (I64)(i * sh.esz); + // this lane reads [c, c + esz), a lane stored before it covers [written, written + ssz). + // any overlap, not just an exact hit, means the scalar sees a freshly stored byte + I64 c = sh.k0.constant + laneOff; + I64 ssz = (I64)windowKey->size; + for(I64 written : it->second) + if(written < c + (I64)sh.esz && c < written + ssz) { + dead = true; // scalar reads a freshly stored lane + return false; + } } + return true; + } + + // a load outside the window's own group needs a runtime disjointness check, + // unless the two ranges provably miss each other + void slp::Packer::guardAgainstWindow(const LoadShape& sh, U32 w) { + if(sh.k0.sameGroup(*windowKey)) + return; + U32 guardBytes = w * sh.esz; + if(sh.equal) + guardBytes = sh.esz; + if(!provablyDisjoint(sh.k0, guardBytes, *windowKey, w * windowKey->size)) + addGuard(sh.k0, sh.first->getPointer(), guardBytes); + } - if(!k0.sameGroup(*windowKey)) { - U32 guardBytes = w * esz; - if(equal) - guardBytes = esz; - if(!provablyDisjoint(k0, guardBytes, *windowKey, w * windowKey->size)) - addGuard(k0, first->getPointer(), guardBytes); + Node* slp::Packer::packLoads(const List& lanes, Type* elemTy, Type* vecTy) { + U32 w = (U32)lanes.size(); + LoadShape sh; + if(!matchLoadShape(lanes, elemTy, sh)) + return nullptr; + + if(straddlesWindow(sh, w)) { + ++drv.stats.rejectedOverlap; + dead = true; + return nullptr; } - return packWideOrSplat(memIn, first, k0, elemTy, vecTy, w, equal); + // all lanes read one state outside the window + if(sh.sharedState && !interWritten.count(sh.first->getMemory())) + return packWideOrSplat(sh.first->getMemory(), sh.first, sh.k0, elemTy, vecTy, w, sh.equal); + if(!innerStatesHoistable(lanes, sh)) + return nullptr; + assert(windowKey && "inner window states imply a window"); + guardAgainstWindow(sh, w); + return packWideOrSplat(memIn, sh.first, sh.k0, elemTy, vecTy, w, sh.equal); } // materialize a whole-vector load, or a splat when every lane hits one address diff --git a/src/backend/pass/opt/slp/slp_reduce.cpp b/src/backend/pass/opt/slp/slp_reduce.cpp index b5f6b53..76bcbd5 100644 --- a/src/backend/pass/opt/slp/slp_reduce.cpp +++ b/src/backend/pass/opt/slp/slp_reduce.cpp @@ -19,22 +19,8 @@ namespace rat { out = refineAddr(l->getPointer(), esz); return out.valid(); } - } // namespace detail - U32 slp::Slp::packReduction(BinaryNode* root) { - Type* t = root->getType(); - U32 esz = t->byteSize(ptrBytes); - U32 w = laneCountFor(esz); - Opcode addOp = root->getOpcode(); - - List terms = flattenAddChain(root, addOp, t); - U32 n = (U32)terms.size(); - if(n < 2 * w || n % w != 0) - return 0; - ++stats.windowsSeen; - - // canonical term order: sort by the first leaf load's refined address so - // grouping is robust against source-level reassociation + // a term tagged with its leaf address, ordered by group then offset struct Keyed { B32 operator<(const Keyed& o) const { if(sig != o.sig) @@ -46,18 +32,45 @@ namespace rat { I64 c; Node* term; }; - List keyed; + } // namespace detail + + void slp::Slp::sortReductionTerms(List& terms, U32 esz) { + List keyed; for(Node* term : terms) { RefinedAddr k; if(!detail::leafKey(term, esz, k)) - break; + return; keyed.push_back({groupSig(k), k.constant, term}); } - if(keyed.size() == terms.size()) { - std::stable_sort(keyed.begin(), keyed.end()); - for(U32 i = 0; i < n; ++i) - terms[i] = keyed[i].term; + std::stable_sort(keyed.begin(), keyed.end()); + for(U32 i = 0; i < (U32)terms.size(); ++i) + terms[i] = keyed[i].term; + } + + // log2 shuffle+add finish + Node* slp::Slp::emitHsum(Node* acc, Opcode addOp, Type* vecTy, U32 w) { + Node* s1 = fn.create(vecTy, acc, (U8)0x4e); // swap 64-bit halves + acc = fn.create(addOp, vecTy, acc, s1); + if(w == 4) { + Node* s2 = fn.create(vecTy, acc, (U8)0xb1); // swap 32-bit pairs + acc = fn.create(addOp, vecTy, acc, s2); } + return acc; + } + + U32 slp::Slp::packReduction(BinaryNode* root) { + Type* t = root->getType(); + U32 esz = t->byteSize(ptrBytes); + U32 w = laneCountFor(esz); + Opcode addOp = root->getOpcode(); + + List terms = flattenAddChain(root, addOp, t); + U32 n = (U32)terms.size(); + if(n < 2 * w || n % w != 0) + return 0; + ++stats.windowsSeen; + + sortReductionTerms(terms, esz); // no window - every packed load must read one shared pre-state Packer packer(*this, nullptr, nullptr); @@ -88,13 +101,7 @@ namespace rat { Node* acc = vecs[0]; for(U32 g = 1; g < k; ++g) acc = fn.create(addOp, vecTy, acc, vecs[g]); - // log2 shuffle+add finish, result in every lane - Node* s1 = fn.create(vecTy, acc, (U8)0x4e); // swap 64-bit halves - acc = fn.create(addOp, vecTy, acc, s1); - if(w == 4) { - Node* s2 = fn.create(vecTy, acc, (U8)0xb1); // swap 32-bit pairs - acc = fn.create(addOp, vecTy, acc, s2); - } + acc = emitHsum(acc, addOp, vecTy, w); Node* res = fn.create(t, acc, 0); root->replaceAllUsesWith(res); ++stats.packedReduction; From 358b32c5d6bed2349e9c905e56e7210fc671ff2d Mon Sep 17 00:00:00 2001 From: nella Date: Mon, 14 Sep 2026 06:41:19 +0200 Subject: [PATCH 2/3] Guard speculated loads. --- src/backend/pass/opt/slp/slp_commit.cpp | 3 +- src/backend/pass/opt/slp/slp_pack.h | 1 + src/backend/pass/opt/slp/slp_packer.cpp | 7 ++++ .../custom/slp_guard_pre_state_load.c | 34 +++++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 src/compiler/test/correctness/custom/slp_guard_pre_state_load.c diff --git a/src/backend/pass/opt/slp/slp_commit.cpp b/src/backend/pass/opt/slp/slp_commit.cpp index 81213c2..b2de356 100644 --- a/src/backend/pass/opt/slp/slp_commit.cpp +++ b/src/backend/pass/opt/slp/slp_commit.cpp @@ -94,6 +94,7 @@ namespace rat { B32 profitable = packer.profit - guardCost >= (I32)(kMinProfit * n) && packer.interior >= n; B32 withinBudget = packer.guardGroups.size() <= kMaxGuards; B32 accept = profitable && withinBudget && !packer.coneTouchesObserver(wPtr); + accept = accept && packer.madeLoadsReadMemIn(); for(const Packer::GuardGroup& g : packer.guardGroups) accept = accept && !packer.coneTouchesObserver(g.ptr); // every observer must feed only the scalar stores of this run @@ -258,7 +259,7 @@ namespace rat { dest = elseP; else if(m && !preSet.count(m)) dest = region; // post-run - else if(m && isa(u) && coneEndsInStores(u, packer.runStores, 64)) + else if(m == packer.memIn && isa(u) && coneEndsInStores(u, packer.runStores, 64)) dest = elseP; // pre-run load only the scalar arm reads } if(dest) diff --git a/src/backend/pass/opt/slp/slp_pack.h b/src/backend/pass/opt/slp/slp_pack.h index d37beeb..c27f3a9 100644 --- a/src/backend/pass/opt/slp/slp_pack.h +++ b/src/backend/pass/opt/slp/slp_pack.h @@ -99,6 +99,7 @@ namespace rat { Node* anchorPtr(Node* ptr, const RefinedAddr& k); void coalesceSplats(); B32 coneTouchesObserver(const Node* n) const; + B32 madeLoadsReadMemIn() const; static String tupleKey(const List& lanes); Node* packTuple(const List& lanes, Type* elemTy, U32 depth); Node* packTupleUncached(const List& lanes, Type* elemTy, Type* vecTy, U32 depth); diff --git a/src/backend/pass/opt/slp/slp_packer.cpp b/src/backend/pass/opt/slp/slp_packer.cpp index cca3e60..b1d497f 100644 --- a/src/backend/pass/opt/slp/slp_packer.cpp +++ b/src/backend/pass/opt/slp/slp_packer.cpp @@ -73,6 +73,13 @@ namespace rat { return false; } + B32 slp::Packer::madeLoadsReadMemIn() const { + for(const Node* n : madeLoads) + if(cast(n)->getMemory() != memIn) + return false; + return true; + } + String slp::Packer::tupleKey(const List& lanes) { String k; k.reserve(lanes.size() * 10); diff --git a/src/compiler/test/correctness/custom/slp_guard_pre_state_load.c b/src/compiler/test/correctness/custom/slp_guard_pre_state_load.c new file mode 100644 index 0000000..ec6636f --- /dev/null +++ b/src/compiler/test/correctness/custom/slp_guard_pre_state_load.c @@ -0,0 +1,34 @@ +// expect: 0 +// a guarded pack may only speculate loads that read the run's incoming memory +// state +unsigned C[32], B[32]; + +__attribute__((noinline)) void k(unsigned *c, unsigned *a, unsigned *b) { + unsigned t0 = a[0], t1 = a[1], t2 = a[2], t3 = a[3]; + unsigned t4 = a[4], t5 = a[5], t6 = a[6], t7 = a[7]; + c[8] = 7; // clobbers a[0] when a == c + 8, and is not part of the run + c[0] = t0 + b[0]; + c[1] = t1 + b[1]; + c[2] = t2 + b[2]; + c[3] = t3 + b[3]; + c[4] = t4 + b[4]; + c[5] = t5 + b[5]; + c[6] = t6 + b[6]; + c[7] = t7 + b[7]; +} + +int main(void) { + for(int i = 0; i < 32; i++) { + C[i] = 100 + i; + B[i] = 1000 + i; + } + k(C, C + 8, B); // guard passes: vector arm + if(C[0] != 1108) + return 1; + for(int i = 0; i < 32; i++) + C[i] = 100 + i; + k(C, C + 8, C); // self-aliased: scalar arm + if(C[0] != 208) + return 2; + return 0; +} From c5e5f4935d0f09d8b78775fdbb05fa79251ed391 Mon Sep 17 00:00:00 2001 From: nella Date: Mon, 14 Sep 2026 07:53:04 +0200 Subject: [PATCH 3/3] Split vector packs per lane. --- src/backend/pass/emit/x86/x86_emit.cpp | 17 +++++- src/backend/pass/emit/x86/x86_encode.cpp | 4 ++ src/backend/pass/emit/x86/x86_encode.h | 2 + src/backend/pass/emit/x86/x86_encode_ops.cpp | 35 ++++++------ src/backend/pass/emit/x86/x86_lower.h | 3 +- src/backend/pass/emit/x86/x86_lower_ops.cpp | 10 ++-- src/backend/pass/emit/x86/x86_op.cpp | 6 ++- src/backend/pass/emit/x86/x86_op.h | 8 +-- .../custom/slp_vpack_mem_scratch.c | 53 +++++++++++++++++++ .../correctness/custom/slp_vpack_scratch.c | 38 +++++++++++++ 10 files changed, 147 insertions(+), 29 deletions(-) create mode 100644 src/compiler/test/correctness/custom/slp_vpack_mem_scratch.c create mode 100644 src/compiler/test/correctness/custom/slp_vpack_scratch.c diff --git a/src/backend/pass/emit/x86/x86_emit.cpp b/src/backend/pass/emit/x86/x86_emit.cpp index 259565f..29abb3b 100644 --- a/src/backend/pass/emit/x86/x86_emit.cpp +++ b/src/backend/pass/emit/x86/x86_emit.cpp @@ -263,8 +263,21 @@ namespace rat { m.regClass = isInt ? detail::kGp : detail::kFp; } - void X86LowerPass::vpack(X86Op op, VReg d, List lanes, U32 esz, B32 isInt) { - put(op, {detail::vr(d, 16)}, std::move(lanes), (I64)esz, isInt ? 1 : 0); + void X86LowerPass::vpackMem(VReg d, const List& lanes, U32 esz, B32 isInt) { + for(U32 i = 0; i < (U32)lanes.size(); ++i) { + I64 desc = ((I64)i << 1) | (isInt ? 1 : 0); + put(X86Op::VPackLane, {}, {lanes[i]}, (I64)esz, desc).regClass = + isInt ? detail::kGp : detail::kFp; + } + put(X86Op::VPack, {detail::vr(d, 16)}, {}, (I64)esz, isInt ? 1 : 0); + } + + void X86LowerPass::vpackReg(VReg d, const List& lanes, U32 esz) { + put(X86Op::VPackReg, {detail::vr(d, 16)}, {lanes[0]}, (I64)esz, 1); + for(U32 i = 1; i < (U32)lanes.size(); ++i) { + List uses = {detail::vr(d, 16), lanes[i]}; + put(X86Op::VInsertReg, {detail::vr(d, 16)}, std::move(uses), (I64)esz, (I64)i); + } } void X86LowerPass::vshuf(VReg d, VReg s, U8 sel) { diff --git a/src/backend/pass/emit/x86/x86_encode.cpp b/src/backend/pass/emit/x86/x86_encode.cpp index 3650f36..b1e052e 100644 --- a/src/backend/pass/emit/x86/x86_encode.cpp +++ b/src/backend/pass/emit/x86/x86_encode.cpp @@ -367,8 +367,12 @@ namespace rat { return emitVExtract(in); case X86Op::VPack: return emitVPack(in); + case X86Op::VPackLane: + return emitVPackLane(in); case X86Op::VPackReg: return emitVPackReg(in); + case X86Op::VInsertReg: + return emitVInsertReg(in); case X86Op::VShuf: return a->pshufd(xmmOf(in.defs[0]), xmmOf(in.uses[0]), (U8)in.imm); case X86Op::X87LoadMem: diff --git a/src/backend/pass/emit/x86/x86_encode.h b/src/backend/pass/emit/x86/x86_encode.h index 0a95f61..76806e3 100644 --- a/src/backend/pass/emit/x86/x86_encode.h +++ b/src/backend/pass/emit/x86/x86_encode.h @@ -64,7 +64,9 @@ namespace rat { void emitVSplat(const MachineInstr& in); void emitVExtract(const MachineInstr& in); void emitVPack(const MachineInstr& in); + void emitVPackLane(const MachineInstr& in); void emitVPackReg(const MachineInstr& in); + void emitVInsertReg(const MachineInstr& in); void fldSlot(I32 slot); void fstpSlot(I32 slot); void emitX87LoadMem(const MachineInstr& in); diff --git a/src/backend/pass/emit/x86/x86_encode_ops.cpp b/src/backend/pass/emit/x86/x86_encode_ops.cpp index 20f1d42..99cde88 100644 --- a/src/backend/pass/emit/x86/x86_encode_ops.cpp +++ b/src/backend/pass/emit/x86/x86_encode_ops.cpp @@ -293,27 +293,30 @@ namespace rat { a->loadExt(d, RBP, fl->vecScratch + (I32)(lane * 4), 4, true); } - // gather the lanes through the 16-byte vec scratch slot (float, or int without sse4.1) - void X86EncodePass::emitVPack(const MachineInstr& in) { + // the lanes are gathered through the 16-byte vec scratch slot (float, or int without + // sse4.1), one lane per instruction + void X86EncodePass::emitVPackLane(const MachineInstr& in) { U32 esz = (U32)in.imm; - B32 isInt = in.imm2 != 0; - for(U32 i = 0; i < (U32)in.uses.size(); ++i) { - I32 disp = fl->vecScratch + (I32)(i * esz); - if(isInt) - a->storeMem(RBP, disp, gpOf(in.uses[i]), esz); - else - a->storeXmm(xmmOf(in.uses[i]), RBP, disp, esz); - } + I32 disp = fl->vecScratch + (I32)(((U32)in.imm2 >> 1) * esz); + if(in.imm2 & 1) + a->storeMem(RBP, disp, gpOf(in.uses[0]), esz); + else + a->storeXmm(xmmOf(in.uses[0]), RBP, disp, esz); + } + + // the lanes are already in the slot, just pick the whole vector up + void X86EncodePass::emitVPack(const MachineInstr& in) { a->loadXmm(xmmOf(in.defs[0]), RBP, fl->vecScratch, 16); } - // build the vector in-register: movd/movq lane 0, then pinsrd/pinsrq the rest (sse4.1, int lanes) + // build the vector in-register (sse4.1, int lanes): movd/movq lane 0, then one + // pinsrd/pinsrq per remaining lane void X86EncodePass::emitVPackReg(const MachineInstr& in) { - U32 d = xmmOf(in.defs[0]); - B32 wide = (U32)in.imm == 8; - a->movdXmmGp(d, gpOf(in.uses[0]), wide); - for(U32 i = 1; i < (U32)in.uses.size(); ++i) - a->pinsr(d, gpOf(in.uses[i]), (U8)i, wide); + a->movdXmmGp(xmmOf(in.defs[0]), gpOf(in.uses[0]), (U32)in.imm == 8); + } + + void X86EncodePass::emitVInsertReg(const MachineInstr& in) { + a->pinsr(xmmOf(in.defs[0]), gpOf(in.uses[1]), (U8)in.imm2, (U32)in.imm == 8); } void X86EncodePass::emitFNeg(const MachineInstr& in) { diff --git a/src/backend/pass/emit/x86/x86_lower.h b/src/backend/pass/emit/x86/x86_lower.h index 2115a11..0df24fd 100644 --- a/src/backend/pass/emit/x86/x86_lower.h +++ b/src/backend/pass/emit/x86/x86_lower.h @@ -142,7 +142,8 @@ namespace rat { void varith(VReg d, VReg a, VReg b, U8 pfx, U8 opc, B32 esc38); void vsplat(VReg d, VReg s, U32 esz, B32 isInt); void vextract(VReg d, VReg s, U32 lane, U32 esz, B32 isInt); - void vpack(X86Op op, VReg d, List lanes, U32 esz, B32 isInt); + void vpackMem(VReg d, const List& lanes, U32 esz, B32 isInt); + void vpackReg(VReg d, const List& lanes, U32 esz); void vshuf(VReg d, VReg s, U8 sel); // x87 void fld(Slot d, VReg addr); diff --git a/src/backend/pass/emit/x86/x86_lower_ops.cpp b/src/backend/pass/emit/x86/x86_lower_ops.cpp index 99929a3..757520f 100644 --- a/src/backend/pass/emit/x86/x86_lower_ops.cpp +++ b/src/backend/pass/emit/x86/x86_lower_ops.cpp @@ -380,10 +380,7 @@ namespace rat { // int lanes with sse4.1 build the vector in-register (VPackReg); float and pre-sse4.1 // fall back to gathering through the vec scratch slot (VPack) B32 useReg = isInt && sse41; - X86Op op = X86Op::VPack; - if(useReg) - op = X86Op::VPackReg; - else + if(!useReg) needVecScratch(); List lanes; for(U32 i = 0; i < w; ++i) { @@ -393,7 +390,10 @@ namespace rat { else lanes.push_back(MachineOperand::vr(sseValue(lane), esz)); } - vpack(op, vregFor(n), std::move(lanes), esz, isInt); + if(useReg) + vpackReg(vregFor(n), lanes, esz); + else + vpackMem(vregFor(n), lanes, esz, isInt); } void X86LowerPass::emitX87Binary(BinaryNode* n, U32 idx) { diff --git a/src/backend/pass/emit/x86/x86_op.cpp b/src/backend/pass/emit/x86/x86_op.cpp index 1d1f471..67ad8c2 100644 --- a/src/backend/pass/emit/x86/x86_op.cpp +++ b/src/backend/pass/emit/x86/x86_op.cpp @@ -65,8 +65,10 @@ namespace rat { {"varith", kFp, 1, 2, 0, ImmKind::Other, ImmKind::None}, {"vsplat", kFp, 1, 1, 0, ImmKind::Lane, ImmKind::Other}, {"vextract", kFp, 1, 1, 0, ImmKind::Lane, ImmKind::Other}, - {"vpack", kFp, 1, -1, 0, ImmKind::Lane, ImmKind::Other}, - {"vpackreg", kFp, 1, -1, 0, ImmKind::Lane, ImmKind::Other}, + {"vpacklane", kFp, 0, 1, 0, ImmKind::Lane, ImmKind::Other}, + {"vpack", kFp, 1, 0, 0, ImmKind::Lane, ImmKind::Other}, + {"vpackreg", kFp, 1, 1, 0, ImmKind::Lane, ImmKind::Other}, + {"vinsertreg", kFp, 1, 2, 0, ImmKind::Lane, ImmKind::Other}, {"vshuf", kFp, 1, 1, 0, ImmKind::Lane, ImmKind::None}, // x87 // imm is the memory width, or -1 / -2 to store-and-pop or discard st(0) diff --git a/src/backend/pass/emit/x86/x86_op.h b/src/backend/pass/emit/x86/x86_op.h index 0d2801f..f5e1f33 100644 --- a/src/backend/pass/emit/x86/x86_op.h +++ b/src/backend/pass/emit/x86/x86_op.h @@ -92,9 +92,11 @@ namespace rat { VArith, // dst = dst OP use[1] packed; imm = (0f38 escape << 16) | (prefix << 8) | opcode byte VSplat, // dst = broadcast use[0] to all lanes; imm = elem bytes, imm2 = int? VExtract, // dst = lane imm of use[0]; imm2 = (elem bytes << 1) | int? - VPack, // dst = int/fp lanes use[0..k-1] gathered through the vec scratch slot; imm = elem bytes - VPackReg, // dst = int lanes use[0..k-1] built in-register via movd/movq + pinsr (sse4.1); imm = elem bytes - VShuf, // dst = pshufd(use[0], imm) + VPackLane, // [vec scratch + imm2>>1 * imm] = use[0] + VPack, // dst = the vec scratch slot + VPackReg, // dst = movd/movq of int lane use[0] + VInsertReg, // dst = use[0] + VShuf, // dst = pshufd(use[0], imm) // x87 ops X87LoadMem, // def(slot) = fld [use0 addr]; imm = mem width (4/8/80) X87StoreMem, // [use0 addr] = fstp use1(slot); imm = mem width (4/8/80) diff --git a/src/compiler/test/correctness/custom/slp_vpack_mem_scratch.c b/src/compiler/test/correctness/custom/slp_vpack_mem_scratch.c new file mode 100644 index 0000000..16d7cfd --- /dev/null +++ b/src/compiler/test/correctness/custom/slp_vpack_mem_scratch.c @@ -0,0 +1,53 @@ +// expect: 0 +// a float vector pack gathers its lanes through the frame scratch slot +typedef float T; +T A[128], B[128], C[128]; + +void k(void) { + A[8 + 7] = B[8 + 7] - C[8 + 7]; + A[8 + 6] = B[8 + 6] - C[8 + 6]; + A[8 + 5] = B[8 + 5] - C[8 + 5]; + A[8 + 3] = B[8 + 3] - C[8 + 3]; + { + T L[16]; + L[0] = B[8 + 0] * C[8 + 0]; + L[1] = B[8 + 1] * C[8 + 1]; + L[2] = B[8 + 2] * C[8 + 2]; + L[3] = B[8 + 3] * C[8 + 3]; + L[4] = B[8 + 4] + C[8 + 4]; + L[5] = B[8 + 5] * C[8 + 5]; + L[6] = B[8 + 6] + C[8 + 6]; + L[7] = B[8 + 7] - C[8 + 7]; + A[0] = L[0] + 8.5f; + } + { + T s0 = B[0], s1 = B[1], s2 = B[2], s3 = B[3]; + A[4 + 2] = s1 * C[4 + 2]; + A[4 + 3] = s1 * C[4 + 3]; + A[8 + 0] = s2 * C[8 + 0]; + A[8 + 1] = s2 * C[8 + 1]; + A[12 + 5] = s3 * C[12 + 5]; + A[2] = s0 * C[2]; + } + { + T L[16]; + L[0] = B[8 + 0] - C[8 + 0]; + L[1] = B[8 + 1] + C[8 + 1]; + A[0] = L[0] - 13.5f; + A[1] = L[1] - 28.5f; + } +} + +int main(void) { + for(int i = 0; i < 128; i++) { + A[i] = (T)(i % 17); + B[i] = (T)((i * 3) % 23); + C[i] = (T)((i * 5) % 11); + } + k(); + // sum in a fixed-point integer so the check is bit-exact + unsigned long total = 0; + for(int i = 0; i < 32; i++) + total = total * 3u + (unsigned long)(long)(A[i] * 16.0f); + return total != 18178825671880576560ul; +} diff --git a/src/compiler/test/correctness/custom/slp_vpack_scratch.c b/src/compiler/test/correctness/custom/slp_vpack_scratch.c new file mode 100644 index 0000000..573c1ed --- /dev/null +++ b/src/compiler/test/correctness/custom/slp_vpack_scratch.c @@ -0,0 +1,38 @@ +// expect: 0 +typedef unsigned T; +T A[128], B[128], C[128]; + +void k(void) { + A[8 + 0] = B[8 + 0] + C[8 + 0]; + A[8 + 1] = B[8 + 1] + C[8 + 1]; + A[8 + 2] = B[8 + 2] + C[8 + 2]; + A[8 + 3] = B[8 + 3] + C[8 + 3]; + { + T s0 = B[0], s1 = B[1], s2 = B[2], s3 = B[3]; + A[0] = s0 & C[0]; + A[1] = s0 ^ C[1]; + A[2] = s0 * C[2]; + A[3] = s0 & C[3]; + A[4 + 0] = s1 ^ C[4 + 0]; + A[4 + 1] = s1 - C[4 + 1]; + A[4 + 2] = s1 | C[4 + 2]; + A[8 + 0] = s2 + C[8 + 0]; + A[12 + 3] = s3 ^ C[12 + 3]; + } + A[16 + 0] = B[16 + 0] ^ C[16 + 0]; + A[16 + 1] = B[16 + 1] ^ C[16 + 1]; + A[16 + 2] = B[16 + 2] ^ C[16 + 2]; +} + +int main(void) { + for(int i = 0; i < 128; i++) { + A[i] = (T)(i % 17); + B[i] = (T)((i * 3) % 23); + C[i] = (T)((i * 5) % 11); + } + k(); + unsigned h = 0; + for(int i = 0; i < 32; i++) + h = h * 31u + A[i]; + return h != 2059802793u; +}