Nella/slp correctness

src/backend/pass/emit/x86/x86_emit.cpp

@@ -263,8 +263,21 @@ namespace rat {
 		m.regClass = isInt ? detail::kGp : detail::kFp;
 	}
 
-	void X86LowerPass::vpack(X86Op op, VReg d, List<MachineOperand> lanes, U32 esz, B32 isInt) {
-		put(op, {detail::vr(d, 16)}, std::move(lanes), (I64)esz, isInt ? 1 : 0);
+	void X86LowerPass::vpackMem(VReg d, const List<MachineOperand>& lanes, U32 esz, B32 isInt) {
+		for(U32 i = 0; i < (U32)lanes.size(); ++i) {
+			I64 desc = ((I64)i << 1) | (isInt ? 1 : 0);
+			put(X86Op::VPackLane, {}, {lanes[i]}, (I64)esz, desc).regClass =
+					isInt ? detail::kGp : detail::kFp;
+		}
+		put(X86Op::VPack, {detail::vr(d, 16)}, {}, (I64)esz, isInt ? 1 : 0);
+	}
+
+	void X86LowerPass::vpackReg(VReg d, const List<MachineOperand>& lanes, U32 esz) {
+		put(X86Op::VPackReg, {detail::vr(d, 16)}, {lanes[0]}, (I64)esz, 1);
+		for(U32 i = 1; i < (U32)lanes.size(); ++i) {
+			List<MachineOperand> uses = {detail::vr(d, 16), lanes[i]};
+			put(X86Op::VInsertReg, {detail::vr(d, 16)}, std::move(uses), (I64)esz, (I64)i);
+		}
 	}
 
 	void X86LowerPass::vshuf(VReg d, VReg s, U8 sel) {

src/backend/pass/emit/x86/x86_encode.cpp

@@ -367,8 +367,12 @@ namespace rat {
 			return emitVExtract(in);
 		case X86Op::VPack:
 			return emitVPack(in);
+		case X86Op::VPackLane:
+			return emitVPackLane(in);
 		case X86Op::VPackReg:
 			return emitVPackReg(in);
+		case X86Op::VInsertReg:
+			return emitVInsertReg(in);
 		case X86Op::VShuf:
 			return a->pshufd(xmmOf(in.defs[0]), xmmOf(in.uses[0]), (U8)in.imm);
 		case X86Op::X87LoadMem:

src/backend/pass/emit/x86/x86_encode.h

@@ -64,7 +64,9 @@ namespace rat {
 		void emitVSplat(const MachineInstr& in);
 		void emitVExtract(const MachineInstr& in);
 		void emitVPack(const MachineInstr& in);
+		void emitVPackLane(const MachineInstr& in);
 		void emitVPackReg(const MachineInstr& in);
+		void emitVInsertReg(const MachineInstr& in);
 		void fldSlot(I32 slot);
 		void fstpSlot(I32 slot);
 		void emitX87LoadMem(const MachineInstr& in);

src/backend/pass/emit/x86/x86_encode_ops.cpp

@@ -293,27 +293,30 @@ namespace rat {
 			a->loadExt(d, RBP, fl->vecScratch + (I32)(lane * 4), 4, true);
 	}
 
-	// gather the lanes through the 16-byte vec scratch slot (float, or int without sse4.1)
-	void X86EncodePass::emitVPack(const MachineInstr& in) {
+	// the lanes are gathered through the 16-byte vec scratch slot (float, or int without
+	// sse4.1), one lane per instruction
+	void X86EncodePass::emitVPackLane(const MachineInstr& in) {
 		U32 esz = (U32)in.imm;
-		B32 isInt = in.imm2 != 0;
-		for(U32 i = 0; i < (U32)in.uses.size(); ++i) {
-			I32 disp = fl->vecScratch + (I32)(i * esz);
-			if(isInt)
-				a->storeMem(RBP, disp, gpOf(in.uses[i]), esz);
-			else
-				a->storeXmm(xmmOf(in.uses[i]), RBP, disp, esz);
-		}
+		I32 disp = fl->vecScratch + (I32)(((U32)in.imm2 >> 1) * esz);
+		if(in.imm2 & 1)
+			a->storeMem(RBP, disp, gpOf(in.uses[0]), esz);
+		else
+			a->storeXmm(xmmOf(in.uses[0]), RBP, disp, esz);
+	}
+
+	// the lanes are already in the slot, just pick the whole vector up
+	void X86EncodePass::emitVPack(const MachineInstr& in) {
 		a->loadXmm(xmmOf(in.defs[0]), RBP, fl->vecScratch, 16);
 	}
 
-	// build the vector in-register: movd/movq lane 0, then pinsrd/pinsrq the rest (sse4.1, int lanes)
+	// build the vector in-register (sse4.1, int lanes): movd/movq lane 0, then one
+	// pinsrd/pinsrq per remaining lane
 	void X86EncodePass::emitVPackReg(const MachineInstr& in) {
-		U32 d = xmmOf(in.defs[0]);
-		B32 wide = (U32)in.imm == 8;
-		a->movdXmmGp(d, gpOf(in.uses[0]), wide);
-		for(U32 i = 1; i < (U32)in.uses.size(); ++i)
-			a->pinsr(d, gpOf(in.uses[i]), (U8)i, wide);
+		a->movdXmmGp(xmmOf(in.defs[0]), gpOf(in.uses[0]), (U32)in.imm == 8);
+	}
+
+	void X86EncodePass::emitVInsertReg(const MachineInstr& in) {
+		a->pinsr(xmmOf(in.defs[0]), gpOf(in.uses[1]), (U8)in.imm2, (U32)in.imm == 8);
 	}
 
 	void X86EncodePass::emitFNeg(const MachineInstr& in) {

src/backend/pass/emit/x86/x86_lower.h

@@ -142,7 +142,8 @@ namespace rat {
 		void varith(VReg d, VReg a, VReg b, U8 pfx, U8 opc, B32 esc38);
 		void vsplat(VReg d, VReg s, U32 esz, B32 isInt);
 		void vextract(VReg d, VReg s, U32 lane, U32 esz, B32 isInt);
-		void vpack(X86Op op, VReg d, List<MachineOperand> lanes, U32 esz, B32 isInt);
+		void vpackMem(VReg d, const List<MachineOperand>& lanes, U32 esz, B32 isInt);
+		void vpackReg(VReg d, const List<MachineOperand>& lanes, U32 esz);
 		void vshuf(VReg d, VReg s, U8 sel);
 		// x87
 		void fld(Slot d, VReg addr);

src/backend/pass/emit/x86/x86_lower_ops.cpp

@@ -380,10 +380,7 @@ namespace rat {
 		// int lanes with sse4.1 build the vector in-register (VPackReg); float and pre-sse4.1
 		// fall back to gathering through the vec scratch slot (VPack)
 		B32 useReg = isInt && sse41;
-		X86Op op = X86Op::VPack;
-		if(useReg)
-			op = X86Op::VPackReg;
-		else
+		if(!useReg)
 			needVecScratch();
 		List<MachineOperand> lanes;
 		for(U32 i = 0; i < w; ++i) {
@@ -393,7 +390,10 @@ namespace rat {
 			else
 				lanes.push_back(MachineOperand::vr(sseValue(lane), esz));
 		}
-		vpack(op, vregFor(n), std::move(lanes), esz, isInt);
+		if(useReg)
+			vpackReg(vregFor(n), lanes, esz);
+		else
+			vpackMem(vregFor(n), lanes, esz, isInt);
 	}
 
 	void X86LowerPass::emitX87Binary(BinaryNode* n, U32 idx) {

src/backend/pass/emit/x86/x86_op.cpp

@@ -65,8 +65,10 @@ namespace rat {
 		{"varith",       kFp,    1,   2, 0,        ImmKind::Other, ImmKind::None},
 		{"vsplat",       kFp,    1,   1, 0,        ImmKind::Lane,  ImmKind::Other},
 		{"vextract",     kFp,    1,   1, 0,        ImmKind::Lane,  ImmKind::Other},
-		{"vpack",        kFp,    1,  -1, 0,        ImmKind::Lane,  ImmKind::Other},
-		{"vpackreg",     kFp,    1,  -1, 0,        ImmKind::Lane,  ImmKind::Other},
+		{"vpacklane",    kFp,    0,   1, 0,        ImmKind::Lane,  ImmKind::Other},
+		{"vpack",        kFp,    1,   0, 0,        ImmKind::Lane,  ImmKind::Other},
+		{"vpackreg",     kFp,    1,   1, 0,        ImmKind::Lane,  ImmKind::Other},
+		{"vinsertreg",   kFp,    1,   2, 0,        ImmKind::Lane,  ImmKind::Other},
 		{"vshuf",        kFp,    1,   1, 0,        ImmKind::Lane,  ImmKind::None},
 		// x87
 		// imm is the memory width, or -1 / -2 to store-and-pop or discard st(0)

src/backend/pass/emit/x86/x86_op.h

@@ -92,9 +92,11 @@ namespace rat {
 		VArith,		// dst = dst OP use[1] packed; imm = (0f38 escape << 16) | (prefix << 8) | opcode byte
 		VSplat,		// dst = broadcast use[0] to all lanes; imm = elem bytes, imm2 = int?
 		VExtract, // dst = lane imm of use[0]; imm2 = (elem bytes << 1) | int?
-		VPack, // dst = int/fp lanes use[0..k-1] gathered through the vec scratch slot; imm = elem bytes
-		VPackReg, // dst = int lanes use[0..k-1] built in-register via movd/movq + pinsr (sse4.1); imm = elem bytes
-		VShuf, // dst = pshufd(use[0], imm)
+		VPackLane,	// [vec scratch + imm2>>1 * imm] = use[0]
+		VPack,			// dst = the vec scratch slot
+		VPackReg,		// dst = movd/movq of int lane use[0]
+		VInsertReg, // dst = use[0]
+		VShuf,			// dst = pshufd(use[0], imm)
 		// x87 ops
 		X87LoadMem,	 // def(slot) = fld [use0 addr]; imm = mem width (4/8/80)
 		X87StoreMem, // [use0 addr] = fstp use1(slot); imm = mem width (4/8/80)

src/backend/pass/opt/slp/slp_commit.cpp

@@ -94,6 +94,7 @@ namespace rat {
 		B32 profitable = packer.profit - guardCost >= (I32)(kMinProfit * n) && packer.interior >= n;
 		B32 withinBudget = packer.guardGroups.size() <= kMaxGuards;
 		B32 accept = profitable && withinBudget && !packer.coneTouchesObserver(wPtr);
+		accept = accept && packer.madeLoadsReadMemIn();
 		for(const Packer::GuardGroup& g : packer.guardGroups)
 			accept = accept && !packer.coneTouchesObserver(g.ptr);
 		// every observer must feed only the scalar stores of this run
@@ -258,7 +259,7 @@ namespace rat {
 					dest = elseP;
 				else if(m && !preSet.count(m))
 					dest = region; // post-run
-				else if(m && isa<LoadNode>(u) && coneEndsInStores(u, packer.runStores, 64))
+				else if(m == packer.memIn && isa<LoadNode>(u) && coneEndsInStores(u, packer.runStores, 64))
 					dest = elseP; // pre-run load only the scalar arm reads
 			}
 			if(dest)

src/backend/pass/opt/slp/slp_pack.h

@@ -82,16 +82,31 @@ namespace rat {
 				I64 minC = 0, maxC = 0;
 			};
 
+			// how a load tuple is addressed
+			struct LoadShape {
+				LoadNode* first = nullptr;
+				RefinedAddr k0; // address of lane 0
+				U32 esz = 0;
+				B32 sharedState = false; // every lane reads one memory state
+				B32 adjacent = false;		 // lane i is at k0 + i*esz
+				B32 equal = false;			 // every lane hits k0
+			};
+
 			Packer(Slp& drv, Node* memIn, const RefinedAddr* windowKey);
 
 			void collectRun(const Segment& seg, U32 begin, U32 count);
 			void addGuard(const RefinedAddr& k, Node* lane0Ptr, U32 bytes);
 			Node* anchorPtr(Node* ptr, const RefinedAddr& k);
 			void coalesceSplats();
 			B32 coneTouchesObserver(const Node* n) const;
+			B32 madeLoadsReadMemIn() const;
 			static String tupleKey(const List<Node*>& lanes);
 			Node* packTuple(const List<Node*>& lanes, Type* elemTy, U32 depth);
 			Node* packTupleUncached(const List<Node*>& lanes, Type* elemTy, Type* vecTy, U32 depth);
+			B32 matchLoadShape(const List<Node*>& lanes, Type* elemTy, LoadShape& out) const;
+			B32 straddlesWindow(const LoadShape& sh, U32 w) const;
+			B32 innerStatesHoistable(const List<Node*>& lanes, const LoadShape& sh);
+			void guardAgainstWindow(const LoadShape& sh, U32 w);
 			Node* packLoads(const List<Node*>& lanes, Type* elemTy, Type* vecTy);
 			Node* packBinaryLanes(const List<Node*>& lanes, Type* elemTy, Type* vecTy, U32 depth);
 			Node* packWideOrSplat(Node* mem,
@@ -108,10 +123,12 @@ namespace rat {
 			// window context, null for reductions
 			Node* memIn;
 			const RefinedAddr* windowKey;
-			Set<const Node*> runStores;								// the scalar stores being fused
-			Map<const Node*, List<I64>> interWritten; // inner store -> lane offsets stored before it
-			Set<const Node*> observers;								// loads reading an inner store's state
-			B32 dead = false; // a matched strategy could not be built, the window is rejected
+			Set<const Node*> runStores; // the scalar stores being fused
+			// inner store -> lane offsets already written in the state that store produces,
+			// its own offset included
+			Map<const Node*, List<I64>> interWritten;
+			Set<const Node*> observers; // loads reading an inner store's state
+			B32 dead = false;						// a matched strategy could not be built, the window is rejected
 			List<GuardGroup> guardGroups;
 			// splat reloads kept for post-commit coalescing into wide loads
 			List<Pair<Node*, LoadNode*>> splatLoads;
@@ -167,6 +184,8 @@ namespace rat {
 			// steer every other user of the pre-branch control to the right arm
 			void
 			rerouteBlock(Node* startCtrl, Node* iff, Node* region, Node* elseP, const Packer& packer);
+			void sortReductionTerms(List<Node*>& terms, U32 esz);
+			Node* emitHsum(Node* acc, Opcode addOp, Type* vecTy, U32 w);
 			U32 packReduction(BinaryNode* root);
 			U32 packReductions();
 			U32 run();