Add fallback implementation for popcnt

Move popcnt fallback up into the macroassembler.

Share code between 32-bit and 64-bit popcnt

Add Popcnt to winch differential fuzzing

fuzz/fuzz_targets/differential.rs

@@ -350,6 +350,8 @@ fn winch_supports_module(module: &[u8]) -> bool {
                         | I64Clz { .. }
                         | I32Ctz { .. }
                         | I64Ctz { .. }
+                        | I32Popcnt { .. }
+                        | I64Popcnt { .. }
                         | LocalGet { .. }
                         | LocalSet { .. }
                         | Call { .. }

winch/codegen/src/isa/aarch64/masm.rs

@@ -210,7 +210,7 @@ impl Masm for MacroAssembler {
         self.asm.load_constant(0, reg);
     }
 
-    fn popcnt(&mut self, _reg: Reg, _size: OperandSize) {
+    fn popcnt(&mut self, _src: Reg, _dst: Reg, _size: OperandSize) {
         todo!()
     }
 

winch/codegen/src/isa/x64/asm.rs

@@ -689,16 +689,13 @@ impl Assembler {
         });
     }
 
-    pub fn popcnt(&mut self, reg: Reg, size: OperandSize) {
-        assert!(
-            self.isa_flags.has_popcnt(),
-            "has_popcnt isa flag required for winch"
-        );
+    pub fn popcnt(&mut self, src: Reg, dst: Reg, size: OperandSize) {
+        assert!(self.isa_flags.has_popcnt(), "Requires has_popcnt flag");
         self.emit(Inst::UnaryRmR {
             size: size.into(),
             op: args::UnaryRmROpcode::Popcnt,
-            src: Gpr::new(reg.into()).unwrap().into(),
-            dst: Writable::from_reg(Gpr::new(reg.into()).unwrap()),
+            src: Gpr::new(src.into()).unwrap().into(),
+            dst: Writable::from_reg(Gpr::new(dst.into()).unwrap()),
         });
     }
 

winch/codegen/src/isa/x64/masm.rs

@@ -419,8 +419,55 @@ impl Masm for MacroAssembler {
         self.asm.jmp(target);
     }
 
-    fn popcnt(&mut self, reg: Reg, size: OperandSize) {
-        self.asm.popcnt(reg, size)
+    fn popcnt(&mut self, src: Reg, dst: Reg, size: OperandSize) {
+        if self.flags.has_popcnt() {
+            self.asm.popcnt(src, dst, size)
+        } else {
+            let (masks, shift_amt) = match size {
+                OperandSize::S64 => (
+                    [
+                        0x5555555555555555, // m1
+                        0x3333333333333333, // m2
+                        0x0f0f0f0f0f0f0f0f, // m4
+                        0x0101010101010101, // h01
+                    ],
+                    56u8,
+                ),
+                // 32-bit popcount is the same, except the masks are half as
+                // wide and we shift by 24 at the end rather than 56
+                OperandSize::S32 => (
+                    [0x55555555i64, 0x33333333i64, 0x0f0f0f0fi64, 0x01010101i64],
+                    24u8,
+                ),
+            };
+            let tmp = regs::scratch();
+            self.asm.mov_rr(src, tmp, size);
+            if src != dst {
+                self.asm.mov_rr(src, dst, size);
+            }
+
+            // x -= (x >> 1) & m1;
+            self.asm.shift_ir(1u8, dst, ShiftKind::ShrU, size);
+            self.asm.and(RegImm::imm(masks[0]).into(), dst.into(), size);
+            self.asm.sub(dst.into(), tmp.into(), size);
+
+            // x = (x & m2) + ((x >> 2) & m2);
+            self.asm.mov(tmp.into(), dst.into(), size);
+            self.asm.and(RegImm::imm(masks[1]).into(), dst.into(), size);
+            self.asm.shift_ir(2u8, tmp, ShiftKind::ShrU, size);
+            self.asm.and(RegImm::imm(masks[1]).into(), tmp.into(), size);
+            self.asm.add(dst.into(), tmp.into(), size);
+
+            // x = (x + (x >> 4)) & m4;
+            self.asm.mov(tmp.into(), dst.into(), size);
+            self.asm.shift_ir(4u8, dst, ShiftKind::ShrU, size);
+            self.asm.add_rr(tmp, dst, size);
+            self.asm.and(RegImm::imm(masks[2]).into(), dst.into(), size);
+
+            // (x * h01) >> shift_amt
+            self.asm.mul(RegImm::imm(masks[3]).into(), dst.into(), size);
+            self.asm.shift_ir(shift_amt, dst, ShiftKind::ShrU, size);
+        }
     }
 }
 

winch/codegen/src/masm.rs

@@ -266,7 +266,9 @@ pub(crate) trait MacroAssembler {
     /// Zero a particular register.
     fn zero(&mut self, reg: Reg);
 
-    fn popcnt(&mut self, reg: Reg, size: OperandSize);
+    /// Count the number of 1 bits in src and put the result in dst. In x64,
+    /// this will emit multiple instructions if the `has_popcnt` flag is false.
+    fn popcnt(&mut self, src: Reg, dst: Reg, size: OperandSize);
 
     /// Zero a given memory range.
     ///

winch/codegen/src/visitor.rs

@@ -456,15 +456,15 @@ where
         use OperandSize::*;
 
         self.context.unop(self.masm, S32, &mut |masm, reg, size| {
-            masm.popcnt(reg, size);
+            masm.popcnt(reg, reg, size);
         })
     }
 
     fn visit_i64_popcnt(&mut self) {
         use OperandSize::*;
 
         self.context.unop(self.masm, S64, &mut |masm, reg, size| {
-            masm.popcnt(reg, size);
+            masm.popcnt(reg, reg, size);
         })
     }