cmd/compile: use OpMove instead of memmove more on arm64

OpMove is faster for small moves of fixed size.

For safety, we have to rewrite the Move rewrite rules a bit so that
all the loads are done before any stores happen.

Also use an 8-byte move instead of a 16-byte move if the tail is
at most 8 bytes.

Change-Id: I7f6c7496ac6d5eb2e0706fd59ca4b5d797c51101
Reviewed-on: https://go-review.googlesource.com/c/go/+/672997
Reviewed-by: Keith Randall <khr@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: David Chase <drchase@google.com>

Author: randall77

src/cmd/compile/internal/ssa/_gen/ARM64.rules

@@ -472,26 +472,39 @@
 		(MOVDstore dst (MOVDload src mem) mem))
 (Move [16] dst src mem) =>
 	(STP dst (Select0 <typ.UInt64> (LDP src mem)) (Select1 <typ.UInt64> (LDP src mem)) mem)
-(Move [32] dst src mem) =>
-	(STP [16] dst (Select0 <typ.UInt64> (LDP [16] src mem)) (Select1 <typ.UInt64> (LDP [16] src mem))
+
+(Move [s] dst src mem) && s > 16 && s <= 24 =>
+	(MOVDstore [int32(s-8)] dst (MOVDload [int32(s-8)] src mem)
+		(STP dst (Select0 <typ.UInt64> (LDP src mem)) (Select1 <typ.UInt64> (LDP src mem)) mem))
+(Move [s] dst src mem) && s > 24 && s <= 32 =>
+	(STP [int32(s-16)] dst (Select0 <typ.UInt64> (LDP [int32(s-16)] src mem)) (Select1 <typ.UInt64> (LDP [int32(s-16)] src mem))
 		(STP dst (Select0 <typ.UInt64> (LDP src mem)) (Select1 <typ.UInt64> (LDP src mem)) mem))
-(Move [48] dst src mem) =>
-	(STP [32] dst (Select0 <typ.UInt64> (LDP [32] src mem)) (Select1 <typ.UInt64> (LDP [32] src mem))
+(Move [s] dst src mem) && s > 32 && s <= 40 =>
+	(MOVDstore [int32(s-8)] dst (MOVDload [int32(s-8)] src mem)
+		(STP [16] dst (Select0 <typ.UInt64> (LDP [16] src mem)) (Select1 <typ.UInt64> (LDP [16] src mem))
+			(STP dst (Select0 <typ.UInt64> (LDP src mem)) (Select1 <typ.UInt64> (LDP src mem)) mem)))
+(Move [s] dst src mem) && s > 40 && s <= 48 =>
+	(STP [int32(s-16)] dst (Select0 <typ.UInt64> (LDP [int32(s-16)] src mem)) (Select1 <typ.UInt64> (LDP [int32(s-16)] src mem))
 		(STP [16] dst (Select0 <typ.UInt64> (LDP [16] src mem)) (Select1 <typ.UInt64> (LDP [16] src mem))
 			(STP dst (Select0 <typ.UInt64> (LDP src mem)) (Select1 <typ.UInt64> (LDP src mem)) mem)))
-(Move [64] dst src mem) =>
-	(STP [48] dst (Select0 <typ.UInt64> (LDP [48] src mem)) (Select1 <typ.UInt64> (LDP [48] src mem))
+(Move [s] dst src mem) && s > 48 && s <= 56 =>
+	(MOVDstore [int32(s-8)] dst (MOVDload [int32(s-8)] src mem)
+		(STP [32] dst (Select0 <typ.UInt64> (LDP [32] src mem)) (Select1 <typ.UInt64> (LDP [32] src mem))
+			(STP [16] dst (Select0 <typ.UInt64> (LDP [16] src mem)) (Select1 <typ.UInt64> (LDP [16] src mem))
+				(STP dst (Select0 <typ.UInt64> (LDP src mem)) (Select1 <typ.UInt64> (LDP src mem)) mem))))
+(Move [s] dst src mem) && s > 56 && s <= 64 =>
+	(STP [int32(s-16)] dst (Select0 <typ.UInt64> (LDP [int32(s-16)] src mem)) (Select1 <typ.UInt64> (LDP [int32(s-16)] src mem))
 		(STP [32] dst (Select0 <typ.UInt64> (LDP [32] src mem)) (Select1 <typ.UInt64> (LDP [32] src mem))
 			(STP [16] dst (Select0 <typ.UInt64> (LDP [16] src mem)) (Select1 <typ.UInt64> (LDP [16] src mem))
 				(STP dst (Select0 <typ.UInt64> (LDP src mem)) (Select1 <typ.UInt64> (LDP src mem)) mem))))
 
 // strip off fractional word move
-(Move [s] dst src mem) && s%16 != 0 && s%16 <= 8 && s > 16 =>
+(Move [s] dst src mem) && s%16 != 0 && s%16 <= 8 && s > 64 =>
 	(Move [8]
 		(OffPtr <dst.Type> dst [s-8])
 		(OffPtr <src.Type> src [s-8])
 		(Move [s-s%16] dst src mem))
-(Move [s] dst src mem) && s%16 != 0 && s%16 > 8 && s > 16 =>
+(Move [s] dst src mem) && s%16 != 0 && s%16 > 8 && s > 64 =>
 	(Move [16]
 		(OffPtr <dst.Type> dst [s-16])
 		(OffPtr <src.Type> src [s-16])

src/cmd/compile/internal/ssa/rewrite.go

@@ -1490,7 +1490,9 @@ func isInlinableMemmove(dst, src *Value, sz int64, c *Config) bool {
 	switch c.arch {
 	case "amd64":
 		return sz <= 16 || (sz < 1024 && disjoint(dst, sz, src, sz))
-	case "386", "arm64":
+	case "arm64":
+		return sz <= 64 || (sz <= 1024 && disjoint(dst, sz, src, sz))
+	case "386":
 		return sz <= 8
 	case "s390x", "ppc64", "ppc64le":
 		return sz <= 8 || disjoint(dst, sz, src, sz)