diff --git a/go.mod b/go.mod index 1d105f246..0039d504b 100644 --- a/go.mod +++ b/go.mod @@ -120,7 +120,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect - github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/compress v1.20.2 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mattn/go-colorable v0.1.16 // indirect diff --git a/go.sum b/go.sum index 7a7ac7f4a..18f71846a 100644 --- a/go.sum +++ b/go.sum @@ -299,8 +299,8 @@ github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXw github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= diff --git a/vendor/github.com/klauspost/compress/s2/encode_amd64.go b/vendor/github.com/klauspost/compress/s2/encode_asm.go similarity index 98% rename from vendor/github.com/klauspost/compress/s2/encode_amd64.go rename to vendor/github.com/klauspost/compress/s2/encode_asm.go index 68d72a41d..eee24e721 100644 --- a/vendor/github.com/klauspost/compress/s2/encode_amd64.go +++ b/vendor/github.com/klauspost/compress/s2/encode_asm.go @@ -1,4 +1,4 @@ -//go:build !appengine && !noasm && gc +//go:build (amd64 || arm64) && !appengine && !noasm && gc package s2 @@ -8,7 +8,7 @@ import ( "github.com/klauspost/compress/internal/race" ) -const hasAmd64Asm = true +const hasAsm = true var encPools [4]sync.Pool diff --git a/vendor/github.com/klauspost/compress/s2/encode_better.go b/vendor/github.com/klauspost/compress/s2/encode_better.go index adbc57c4d..2509d1c99 100644 --- a/vendor/github.com/klauspost/compress/s2/encode_better.go +++ b/vendor/github.com/klauspost/compress/s2/encode_better.go @@ -86,12 +86,18 @@ func encodeBlockBetterGo(dst, src []byte) (d int) { // We initialize repeat to 0, so we never match on first attempt repeat := 0 + // Cap the adaptive skip, as encodeBetterBlockAsm and the snappy Go encoder + // below already do. Without a cap, a long stretch with no matches lets the + // skip grow geometrically until it strides past the start of the next + // compressible region and loses the matches there. + const maxSkip = 100 + for { candidateL := 0 nextS := 0 for { // Next src position to check - nextS = s + (s-nextEmit)>>7 + 1 + nextS = s + min((s-nextEmit)>>7+1, maxSkip) if nextS > sLimit { goto emitRemainder } diff --git a/vendor/github.com/klauspost/compress/s2/encode_go.go b/vendor/github.com/klauspost/compress/s2/encode_go.go index 5597f3ef2..0270d33bc 100644 --- a/vendor/github.com/klauspost/compress/s2/encode_go.go +++ b/vendor/github.com/klauspost/compress/s2/encode_go.go @@ -1,4 +1,4 @@ -//go:build !amd64 || appengine || !gc || noasm +//go:build (!amd64 && !arm64) || appengine || !gc || noasm package s2 @@ -7,7 +7,7 @@ import ( "math/bits" ) -const hasAmd64Asm = false +const hasAsm = false // encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It // assumes that the varint-encoded length of the decompressed bytes has already diff --git a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s b/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s index df9be687b..fc8c0f4b6 100644 --- a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s +++ b/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s @@ -1,6 +1,6 @@ -// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_asm.go -pkg=s2. DO NOT EDIT. -//go:build !appengine && !noasm && gc && !noasm +//go:build (amd64 || arm64) && !appengine && !noasm && gc && !noasm #include "textflag.h" @@ -274,16 +274,10 @@ matchlen_loopback_16_repeat_extend_encodeBlockAsm: JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm matchlen_bsf_16repeat_extend_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm matchlen_match8_repeat_extend_encodeBlockAsm: CMPL R9, $0x08 @@ -296,16 +290,10 @@ matchlen_match8_repeat_extend_encodeBlockAsm: JMP matchlen_match4_repeat_extend_encodeBlockAsm matchlen_bsf_8_repeat_extend_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm matchlen_match4_repeat_extend_encodeBlockAsm: CMPL R9, $0x04 @@ -898,16 +886,10 @@ matchlen_loopback_16_match_nolit_encodeBlockAsm: JMP matchlen_loopback_16_match_nolit_encodeBlockAsm matchlen_bsf_16match_nolit_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm matchlen_match8_match_nolit_encodeBlockAsm: CMPL DI, $0x08 @@ -920,16 +902,10 @@ matchlen_match8_match_nolit_encodeBlockAsm: JMP matchlen_match4_match_nolit_encodeBlockAsm matchlen_bsf_8_match_nolit_encodeBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm matchlen_match4_match_nolit_encodeBlockAsm: CMPL DI, $0x04 @@ -1689,16 +1665,10 @@ matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB: JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB matchlen_bsf_16repeat_extend_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm4MB + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm4MB matchlen_match8_repeat_extend_encodeBlockAsm4MB: CMPL R9, $0x08 @@ -1711,16 +1681,10 @@ matchlen_match8_repeat_extend_encodeBlockAsm4MB: JMP matchlen_match4_repeat_extend_encodeBlockAsm4MB matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm4MB + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm4MB matchlen_match4_repeat_extend_encodeBlockAsm4MB: CMPL R9, $0x04 @@ -2263,16 +2227,10 @@ matchlen_loopback_16_match_nolit_encodeBlockAsm4MB: JMP matchlen_loopback_16_match_nolit_encodeBlockAsm4MB matchlen_bsf_16match_nolit_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm4MB + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm4MB matchlen_match8_match_nolit_encodeBlockAsm4MB: CMPL DI, $0x08 @@ -2285,16 +2243,10 @@ matchlen_match8_match_nolit_encodeBlockAsm4MB: JMP matchlen_match4_match_nolit_encodeBlockAsm4MB matchlen_bsf_8_match_nolit_encodeBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm4MB + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm4MB matchlen_match4_match_nolit_encodeBlockAsm4MB: CMPL DI, $0x04 @@ -3006,16 +2958,10 @@ matchlen_loopback_16_repeat_extend_encodeBlockAsm12B: JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm12B matchlen_bsf_16repeat_extend_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm12B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm12B matchlen_match8_repeat_extend_encodeBlockAsm12B: CMPL R9, $0x08 @@ -3028,16 +2974,10 @@ matchlen_match8_repeat_extend_encodeBlockAsm12B: JMP matchlen_match4_repeat_extend_encodeBlockAsm12B matchlen_bsf_8_repeat_extend_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm12B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm12B matchlen_match4_repeat_extend_encodeBlockAsm12B: CMPL R9, $0x04 @@ -3458,16 +3398,10 @@ matchlen_loopback_16_match_nolit_encodeBlockAsm12B: JMP matchlen_loopback_16_match_nolit_encodeBlockAsm12B matchlen_bsf_16match_nolit_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm12B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm12B matchlen_match8_match_nolit_encodeBlockAsm12B: CMPL DI, $0x08 @@ -3480,16 +3414,10 @@ matchlen_match8_match_nolit_encodeBlockAsm12B: JMP matchlen_match4_match_nolit_encodeBlockAsm12B matchlen_bsf_8_match_nolit_encodeBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm12B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm12B matchlen_match4_match_nolit_encodeBlockAsm12B: CMPL DI, $0x04 @@ -4091,16 +4019,10 @@ matchlen_loopback_16_repeat_extend_encodeBlockAsm10B: JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm10B matchlen_bsf_16repeat_extend_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm10B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm10B matchlen_match8_repeat_extend_encodeBlockAsm10B: CMPL R9, $0x08 @@ -4113,16 +4035,10 @@ matchlen_match8_repeat_extend_encodeBlockAsm10B: JMP matchlen_match4_repeat_extend_encodeBlockAsm10B matchlen_bsf_8_repeat_extend_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm10B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm10B matchlen_match4_repeat_extend_encodeBlockAsm10B: CMPL R9, $0x04 @@ -4543,16 +4459,10 @@ matchlen_loopback_16_match_nolit_encodeBlockAsm10B: JMP matchlen_loopback_16_match_nolit_encodeBlockAsm10B matchlen_bsf_16match_nolit_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm10B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm10B matchlen_match8_match_nolit_encodeBlockAsm10B: CMPL DI, $0x08 @@ -4565,16 +4475,10 @@ matchlen_match8_match_nolit_encodeBlockAsm10B: JMP matchlen_match4_match_nolit_encodeBlockAsm10B matchlen_bsf_8_match_nolit_encodeBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm10B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm10B matchlen_match4_match_nolit_encodeBlockAsm10B: CMPL DI, $0x04 @@ -5176,16 +5080,10 @@ matchlen_loopback_16_repeat_extend_encodeBlockAsm8B: JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm8B matchlen_bsf_16repeat_extend_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm8B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm8B matchlen_match8_repeat_extend_encodeBlockAsm8B: CMPL R9, $0x08 @@ -5198,16 +5096,10 @@ matchlen_match8_repeat_extend_encodeBlockAsm8B: JMP matchlen_match4_repeat_extend_encodeBlockAsm8B matchlen_bsf_8_repeat_extend_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP repeat_extend_forward_end_encodeBlockAsm8B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP repeat_extend_forward_end_encodeBlockAsm8B matchlen_match4_repeat_extend_encodeBlockAsm8B: CMPL R9, $0x04 @@ -5614,16 +5506,10 @@ matchlen_loopback_16_match_nolit_encodeBlockAsm8B: JMP matchlen_loopback_16_match_nolit_encodeBlockAsm8B matchlen_bsf_16match_nolit_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeBlockAsm8B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeBlockAsm8B matchlen_match8_match_nolit_encodeBlockAsm8B: CMPL DI, $0x08 @@ -5636,16 +5522,10 @@ matchlen_match8_match_nolit_encodeBlockAsm8B: JMP matchlen_match4_match_nolit_encodeBlockAsm8B matchlen_bsf_8_match_nolit_encodeBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeBlockAsm8B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeBlockAsm8B matchlen_match4_match_nolit_encodeBlockAsm8B: CMPL DI, $0x04 @@ -6140,16 +6020,10 @@ matchlen_loopback_16_match_nolit_encodeBetterBlockAsm: JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm matchlen_bsf_16match_nolit_encodeBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm matchlen_match8_match_nolit_encodeBetterBlockAsm: CMPL R8, $0x08 @@ -6162,16 +6036,10 @@ matchlen_match8_match_nolit_encodeBetterBlockAsm: JMP matchlen_match4_match_nolit_encodeBetterBlockAsm matchlen_bsf_8_match_nolit_encodeBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm matchlen_match4_match_nolit_encodeBetterBlockAsm: CMPL R8, $0x04 @@ -7235,16 +7103,10 @@ matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB: JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm4MB + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm4MB matchlen_match8_match_nolit_encodeBetterBlockAsm4MB: CMPL R8, $0x08 @@ -7257,16 +7119,10 @@ matchlen_match8_match_nolit_encodeBetterBlockAsm4MB: JMP matchlen_match4_match_nolit_encodeBetterBlockAsm4MB matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm4MB + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm4MB matchlen_match4_match_nolit_encodeBetterBlockAsm4MB: CMPL R8, $0x04 @@ -8256,16 +8112,10 @@ matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B: JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm12B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm12B matchlen_match8_match_nolit_encodeBetterBlockAsm12B: CMPL R8, $0x08 @@ -8278,16 +8128,10 @@ matchlen_match8_match_nolit_encodeBetterBlockAsm12B: JMP matchlen_match4_match_nolit_encodeBetterBlockAsm12B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm12B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm12B matchlen_match4_match_nolit_encodeBetterBlockAsm12B: CMPL R8, $0x04 @@ -9130,16 +8974,10 @@ matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B: JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm10B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm10B matchlen_match8_match_nolit_encodeBetterBlockAsm10B: CMPL R8, $0x08 @@ -9152,16 +8990,10 @@ matchlen_match8_match_nolit_encodeBetterBlockAsm10B: JMP matchlen_match4_match_nolit_encodeBetterBlockAsm10B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm10B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm10B matchlen_match4_match_nolit_encodeBetterBlockAsm10B: CMPL R8, $0x04 @@ -10004,16 +9836,10 @@ matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B: JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm8B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm8B matchlen_match8_match_nolit_encodeBetterBlockAsm8B: CMPL R8, $0x08 @@ -10026,16 +9852,10 @@ matchlen_match8_match_nolit_encodeBetterBlockAsm8B: JMP matchlen_match4_match_nolit_encodeBetterBlockAsm8B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeBetterBlockAsm8B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeBetterBlockAsm8B matchlen_match4_match_nolit_encodeBetterBlockAsm8B: CMPL R8, $0x04 @@ -10985,16 +10805,10 @@ matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm: JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm matchlen_match8_repeat_extend_encodeSnappyBlockAsm: CMPL R8, $0x08 @@ -11007,16 +10821,10 @@ matchlen_match8_repeat_extend_encodeSnappyBlockAsm: JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm matchlen_match4_repeat_extend_encodeSnappyBlockAsm: CMPL R8, $0x04 @@ -11329,16 +11137,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm: JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm matchlen_bsf_16match_nolit_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm matchlen_match8_match_nolit_encodeSnappyBlockAsm: CMPL DI, $0x08 @@ -11351,16 +11153,10 @@ matchlen_match8_match_nolit_encodeSnappyBlockAsm: JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm matchlen_match4_match_nolit_encodeSnappyBlockAsm: CMPL DI, $0x04 @@ -11901,16 +11697,10 @@ matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K: JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K: CMPL R8, $0x08 @@ -11923,16 +11713,10 @@ matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K: JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K: CMPL R8, $0x04 @@ -12205,16 +11989,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K: JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm64K + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm64K matchlen_match8_match_nolit_encodeSnappyBlockAsm64K: CMPL DI, $0x08 @@ -12227,16 +12005,10 @@ matchlen_match8_match_nolit_encodeSnappyBlockAsm64K: JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm64K matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm64K + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm64K matchlen_match4_match_nolit_encodeSnappyBlockAsm64K: CMPL DI, $0x04 @@ -12737,16 +12509,10 @@ matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B: JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B: CMPL R8, $0x08 @@ -12759,16 +12525,10 @@ matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B: JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B: CMPL R8, $0x04 @@ -13041,16 +12801,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B: JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm12B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm12B matchlen_match8_match_nolit_encodeSnappyBlockAsm12B: CMPL DI, $0x08 @@ -13063,16 +12817,10 @@ matchlen_match8_match_nolit_encodeSnappyBlockAsm12B: JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm12B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm12B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm12B matchlen_match4_match_nolit_encodeSnappyBlockAsm12B: CMPL DI, $0x04 @@ -13573,16 +13321,10 @@ matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B: JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B: CMPL R8, $0x08 @@ -13595,16 +13337,10 @@ matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B: JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B: CMPL R8, $0x04 @@ -13877,16 +13613,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B: JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm10B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm10B matchlen_match8_match_nolit_encodeSnappyBlockAsm10B: CMPL DI, $0x08 @@ -13899,16 +13629,10 @@ matchlen_match8_match_nolit_encodeSnappyBlockAsm10B: JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm10B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm10B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm10B matchlen_match4_match_nolit_encodeSnappyBlockAsm10B: CMPL DI, $0x04 @@ -14409,16 +14133,10 @@ matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B: JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B: CMPL R8, $0x08 @@ -14431,16 +14149,10 @@ matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B: JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B: CMPL R8, $0x04 @@ -14711,16 +14423,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B: JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm8B + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm8B matchlen_match8_match_nolit_encodeSnappyBlockAsm8B: CMPL DI, $0x08 @@ -14733,16 +14439,10 @@ matchlen_match8_match_nolit_encodeSnappyBlockAsm8B: JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm8B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_encodeSnappyBlockAsm8B + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_encodeSnappyBlockAsm8B matchlen_match4_match_nolit_encodeSnappyBlockAsm8B: CMPL DI, $0x04 @@ -15144,16 +14844,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm: JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm: CMPL R8, $0x08 @@ -15166,16 +14860,10 @@ matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm: JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm: CMPL R8, $0x04 @@ -15790,16 +15478,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K: JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm64K + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm64K matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K: CMPL R8, $0x08 @@ -15812,16 +15494,10 @@ matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K: JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm64K + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm64K matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K: CMPL R8, $0x04 @@ -16371,16 +16047,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B: JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm12B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm12B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B: CMPL R8, $0x08 @@ -16393,16 +16063,10 @@ matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B: JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm12B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm12B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B: CMPL R8, $0x04 @@ -16952,16 +16616,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B: JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm10B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm10B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B: CMPL R8, $0x08 @@ -16974,16 +16632,10 @@ matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B: JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm10B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm10B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B: CMPL R8, $0x04 @@ -17533,16 +17185,10 @@ matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B: JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R13, R13 - -#else - BSFQ R13, R13 - -#endif - SARQ $0x03, R13 - LEAL 8(R12)(R13*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm8B + SARQ $0x03, R13 + LEAL 8(R12)(R13*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm8B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B: CMPL R8, $0x08 @@ -17555,16 +17201,10 @@ matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B: JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL (R12)(R11*1), R12 - JMP match_nolit_end_encodeSnappyBetterBlockAsm8B + SARQ $0x03, R11 + LEAL (R12)(R11*1), R12 + JMP match_nolit_end_encodeSnappyBetterBlockAsm8B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B: CMPL R8, $0x04 @@ -18138,16 +17778,10 @@ matchlen_loopback_16_repeat_extend_calcBlockSize: JMP matchlen_loopback_16_repeat_extend_calcBlockSize matchlen_bsf_16repeat_extend_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_calcBlockSize + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_calcBlockSize matchlen_match8_repeat_extend_calcBlockSize: CMPL R8, $0x08 @@ -18160,16 +17794,10 @@ matchlen_match8_repeat_extend_calcBlockSize: JMP matchlen_match4_repeat_extend_calcBlockSize matchlen_bsf_8_repeat_extend_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_calcBlockSize + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_calcBlockSize matchlen_match4_repeat_extend_calcBlockSize: CMPL R8, $0x04 @@ -18371,16 +17999,10 @@ matchlen_loopback_16_match_nolit_calcBlockSize: JMP matchlen_loopback_16_match_nolit_calcBlockSize matchlen_bsf_16match_nolit_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_calcBlockSize + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_calcBlockSize matchlen_match8_match_nolit_calcBlockSize: CMPL DI, $0x08 @@ -18393,16 +18015,10 @@ matchlen_match8_match_nolit_calcBlockSize: JMP matchlen_match4_match_nolit_calcBlockSize matchlen_bsf_8_match_nolit_calcBlockSize: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_calcBlockSize + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_calcBlockSize matchlen_match4_match_nolit_calcBlockSize: CMPL DI, $0x04 @@ -18723,16 +18339,10 @@ matchlen_loopback_16_repeat_extend_calcBlockSizeSmall: JMP matchlen_loopback_16_repeat_extend_calcBlockSizeSmall matchlen_bsf_16repeat_extend_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R12, R12 - -#else - BSFQ R12, R12 - -#endif - SARQ $0x03, R12 - LEAL 8(R11)(R12*1), R11 - JMP repeat_extend_forward_end_calcBlockSizeSmall + SARQ $0x03, R12 + LEAL 8(R11)(R12*1), R11 + JMP repeat_extend_forward_end_calcBlockSizeSmall matchlen_match8_repeat_extend_calcBlockSizeSmall: CMPL R8, $0x08 @@ -18745,16 +18355,10 @@ matchlen_match8_repeat_extend_calcBlockSizeSmall: JMP matchlen_match4_repeat_extend_calcBlockSizeSmall matchlen_bsf_8_repeat_extend_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R10, R10 - -#else - BSFQ R10, R10 - -#endif - SARQ $0x03, R10 - LEAL (R11)(R10*1), R11 - JMP repeat_extend_forward_end_calcBlockSizeSmall + SARQ $0x03, R10 + LEAL (R11)(R10*1), R11 + JMP repeat_extend_forward_end_calcBlockSizeSmall matchlen_match4_repeat_extend_calcBlockSizeSmall: CMPL R8, $0x04 @@ -18926,16 +18530,10 @@ matchlen_loopback_16_match_nolit_calcBlockSizeSmall: JMP matchlen_loopback_16_match_nolit_calcBlockSizeSmall matchlen_bsf_16match_nolit_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R11, R11 - -#else - BSFQ R11, R11 - -#endif - SARQ $0x03, R11 - LEAL 8(R10)(R11*1), R10 - JMP match_nolit_end_calcBlockSizeSmall + SARQ $0x03, R11 + LEAL 8(R10)(R11*1), R10 + JMP match_nolit_end_calcBlockSizeSmall matchlen_match8_match_nolit_calcBlockSizeSmall: CMPL DI, $0x08 @@ -18948,16 +18546,10 @@ matchlen_match8_match_nolit_calcBlockSizeSmall: JMP matchlen_match4_match_nolit_calcBlockSizeSmall matchlen_bsf_8_match_nolit_calcBlockSizeSmall: -#ifdef GOAMD64_v3 TZCNTQ R9, R9 - -#else - BSFQ R9, R9 - -#endif - SARQ $0x03, R9 - LEAL (R10)(R9*1), R10 - JMP match_nolit_end_calcBlockSizeSmall + SARQ $0x03, R9 + LEAL (R10)(R9*1), R10 + JMP match_nolit_end_calcBlockSizeSmall matchlen_match4_match_nolit_calcBlockSizeSmall: CMPL DI, $0x04 @@ -19737,16 +19329,10 @@ matchlen_loopback_16_standalone: JMP matchlen_loopback_16_standalone matchlen_bsf_16standalone: -#ifdef GOAMD64_v3 TZCNTQ DI, DI - -#else - BSFQ DI, DI - -#endif - SARQ $0x03, DI - LEAL 8(SI)(DI*1), SI - JMP gen_match_len_end + SARQ $0x03, DI + LEAL 8(SI)(DI*1), SI + JMP gen_match_len_end matchlen_match8_standalone: CMPL DX, $0x08 @@ -19759,16 +19345,10 @@ matchlen_match8_standalone: JMP matchlen_match4_standalone matchlen_bsf_8_standalone: -#ifdef GOAMD64_v3 TZCNTQ BX, BX - -#else - BSFQ BX, BX - -#endif - SARQ $0x03, BX - LEAL (SI)(BX*1), SI - JMP gen_match_len_end + SARQ $0x03, BX + LEAL (SI)(BX*1), SI + JMP gen_match_len_end matchlen_match4_standalone: CMPL DX, $0x04 @@ -19802,7 +19382,7 @@ gen_match_len_end: // func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 -TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64 +TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $8-64 XORQ SI, SI MOVQ dst_base+0(FP), AX MOVQ dst_len+8(FP), CX @@ -19810,457 +19390,464 @@ TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64 MOVQ src_len+32(FP), BX LEAQ (DX)(BX*1), BX LEAQ -8(AX)(CX*1), CX - XORQ DI, DI + LEAQ -4(CX), DI + XORQ R8, R8 lz4_s2_loop: CMPQ DX, BX JAE lz4_s2_corrupt CMPQ AX, CX JAE lz4_s2_dstfull - MOVBQZX (DX), R8 - MOVQ R8, R9 - MOVQ R8, R10 - SHRQ $0x04, R9 - ANDQ $0x0f, R10 - CMPQ R8, $0xf0 + MOVBQZX (DX), R9 + MOVQ R9, R10 + MOVQ R9, R11 + SHRQ $0x04, R10 + ANDQ $0x0f, R11 + CMPQ R9, $0xf0 JB lz4_s2_ll_end lz4_s2_ll_loop: INCQ DX CMPQ DX, BX JAE lz4_s2_corrupt - MOVBQZX (DX), R8 - ADDQ R8, R9 - CMPQ R8, $0xff + MOVBQZX (DX), R9 + ADDQ R9, R10 + CMPQ R9, $0xff JEQ lz4_s2_ll_loop lz4_s2_ll_end: - LEAQ (DX)(R9*1), R8 - ADDQ $0x04, R10 - CMPQ R8, BX + LEAQ (DX)(R10*1), R9 + ADDQ $0x04, R11 + CMPQ R9, BX JAE lz4_s2_corrupt INCQ DX - INCQ R8 - TESTQ R9, R9 + INCQ R9 + TESTQ R10, R10 JZ lz4_s2_lits_done - LEAQ (AX)(R9*1), R11 - CMPQ R11, CX + LEAQ (AX)(R10*1), R12 + CMPQ R12, DI JAE lz4_s2_dstfull - ADDQ R9, SI - LEAL -1(R9), R11 - CMPL R11, $0x3c + ADDQ R10, SI + LEAL -1(R10), R12 + CMPL R12, $0x3c JB one_byte_lz4_s2 - CMPL R11, $0x00000100 + CMPL R12, $0x00000100 JB two_bytes_lz4_s2 - CMPL R11, $0x00010000 + CMPL R12, $0x00010000 JB three_bytes_lz4_s2 - CMPL R11, $0x01000000 + CMPL R12, $0x01000000 JB four_bytes_lz4_s2 MOVB $0xfc, (AX) - MOVL R11, 1(AX) + MOVL R12, 1(AX) ADDQ $0x05, AX JMP memmove_long_lz4_s2 four_bytes_lz4_s2: - MOVL R11, R12 - SHRL $0x10, R12 + MOVL R12, R13 + SHRL $0x10, R13 MOVB $0xf8, (AX) - MOVW R11, 1(AX) - MOVB R12, 3(AX) + MOVW R12, 1(AX) + MOVB R13, 3(AX) ADDQ $0x04, AX JMP memmove_long_lz4_s2 three_bytes_lz4_s2: MOVB $0xf4, (AX) - MOVW R11, 1(AX) + MOVW R12, 1(AX) ADDQ $0x03, AX JMP memmove_long_lz4_s2 two_bytes_lz4_s2: MOVB $0xf0, (AX) - MOVB R11, 1(AX) + MOVB R12, 1(AX) ADDQ $0x02, AX - CMPL R11, $0x40 + CMPL R12, $0x40 JB memmove_lz4_s2 JMP memmove_long_lz4_s2 one_byte_lz4_s2: - SHLB $0x02, R11 - MOVB R11, (AX) + SHLB $0x02, R12 + MOVB R12, (AX) ADDQ $0x01, AX memmove_lz4_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveShort - CMPQ R9, $0x08 + CMPQ R10, $0x08 JBE emit_lit_memmove_lz4_s2_memmove_move_8 - CMPQ R9, $0x10 + CMPQ R10, $0x10 JBE emit_lit_memmove_lz4_s2_memmove_move_8through16 - CMPQ R9, $0x20 + CMPQ R10, $0x20 JBE emit_lit_memmove_lz4_s2_memmove_move_17through32 JMP emit_lit_memmove_lz4_s2_memmove_move_33through64 emit_lit_memmove_lz4_s2_memmove_move_8: - MOVQ (DX), R12 - MOVQ R12, (AX) + MOVQ (DX), R13 + MOVQ R13, (AX) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_8through16: - MOVQ (DX), R12 - MOVQ -8(DX)(R9*1), DX - MOVQ R12, (AX) - MOVQ DX, -8(AX)(R9*1) + MOVQ (DX), R13 + MOVQ -8(DX)(R10*1), DX + MOVQ R13, (AX) + MOVQ DX, -8(AX)(R10*1) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_17through32: MOVOU (DX), X0 - MOVOU -16(DX)(R9*1), X1 + MOVOU -16(DX)(R10*1), X1 MOVOU X0, (AX) - MOVOU X1, -16(AX)(R9*1) + MOVOU X1, -16(AX)(R10*1) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_33through64: MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) memmove_end_copy_lz4_s2: - MOVQ R11, AX + MOVQ R12, AX JMP lz4_s2_lits_emit_done memmove_long_lz4_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveLong MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ AX, R12 - ANDL $0x0000001f, R12 - MOVQ $0x00000040, R14 - SUBQ R12, R14 - DECQ R13 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 + MOVQ R10, R14 + SHRQ $0x05, R14 + MOVQ AX, R13 + ANDL $0x0000001f, R13 + MOVQ $0x00000040, R15 + SUBQ R13, R15 + DECQ R14 JA emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32 - LEAQ -32(DX)(R14*1), R12 - LEAQ -32(AX)(R14*1), R15 + LEAQ -32(DX)(R15*1), R13 + LEAQ -32(AX)(R15*1), BP emit_lit_memmove_long_lz4_s2large_big_loop_back: - MOVOU (R12), X4 - MOVOU 16(R12), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) + MOVOU (R13), X4 + MOVOU 16(R13), X5 + MOVOA X4, (BP) + MOVOA X5, 16(BP) + ADDQ $0x20, BP + ADDQ $0x20, R13 ADDQ $0x20, R15 - ADDQ $0x20, R12 - ADDQ $0x20, R14 - DECQ R13 + DECQ R14 JNA emit_lit_memmove_long_lz4_s2large_big_loop_back emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32: - MOVOU -32(DX)(R14*1), X4 - MOVOU -16(DX)(R14*1), X5 - MOVOA X4, -32(AX)(R14*1) - MOVOA X5, -16(AX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 + MOVOU -32(DX)(R15*1), X4 + MOVOU -16(DX)(R15*1), X5 + MOVOA X4, -32(AX)(R15*1) + MOVOA X5, -16(AX)(R15*1) + ADDQ $0x20, R15 + CMPQ R10, R15 JAE emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - MOVQ R11, AX + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) + MOVQ R12, AX lz4_s2_lits_emit_done: - MOVQ R8, DX + MOVQ R9, DX lz4_s2_lits_done: CMPQ DX, BX JNE lz4_s2_match - CMPQ R10, $0x04 + CMPQ R11, $0x04 JEQ lz4_s2_done JMP lz4_s2_corrupt lz4_s2_match: - LEAQ 2(DX), R8 - CMPQ R8, BX + LEAQ 2(DX), R9 + CMPQ R9, BX JAE lz4_s2_corrupt - MOVWQZX (DX), R9 - MOVQ R8, DX - TESTQ R9, R9 + MOVWQZX (DX), R10 + MOVQ R9, DX + TESTQ R10, R10 JZ lz4_s2_corrupt - CMPQ R9, SI + CMPQ R10, SI JA lz4_s2_corrupt - CMPQ R10, $0x13 + CMPQ R11, $0x13 JNE lz4_s2_ml_done lz4_s2_ml_loop: - MOVBQZX (DX), R8 + MOVBQZX (DX), R9 INCQ DX - ADDQ R8, R10 + ADDQ R9, R11 CMPQ DX, BX JAE lz4_s2_corrupt - CMPQ R8, $0xff + CMPQ R9, $0xff JEQ lz4_s2_ml_loop lz4_s2_ml_done: - ADDQ R10, SI - CMPQ R9, DI + ADDQ R11, SI + CMPQ R10, R8 JNE lz4_s2_docopy // emitRepeat emit_repeat_again_lz4_s2: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2 - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2 cant_repeat_two_offset_lz4_s2: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2 - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2 - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2 - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4_s2_loop JMP emit_repeat_again_lz4_s2 repeat_five_lz4_s2: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4_s2_loop repeat_four_lz4_s2: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4_s2_loop repeat_three_lz4_s2: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4_s2_loop repeat_two_lz4_s2: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop repeat_two_offset_lz4_s2: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop lz4_s2_docopy: - MOVQ R9, DI + MOVQ R10, R8 // emitCopy - CMPL R10, $0x40 + CMPL R11, $0x40 JBE two_byte_offset_short_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE long_offset_short_lz4_s2 - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB R9, 1(AX) - MOVL R9, R11 - SHRL $0x08, R11 - SHLL $0x05, R11 - ORL R11, R8 - MOVB R8, (AX) + MOVL $0x00000001, R9 + LEAL 16(R9), R9 + MOVB R10, 1(AX) + MOVL R10, R12 + SHRL $0x08, R12 + SHLL $0x05, R12 + ORL R12, R9 + MOVB R9, (AX) ADDQ $0x02, AX - SUBL $0x08, R10 + SUBL $0x08, R11 // emitRepeat - LEAL -4(R10), R10 + LEAL -4(R11), R11 JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b emit_repeat_again_lz4_s2_emit_copy_short_2b: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short_2b - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short_2b cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short_2b - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short_2b - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short_2b - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short_2b repeat_five_lz4_s2_emit_copy_short_2b: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4_s2_loop repeat_four_lz4_s2_emit_copy_short_2b: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4_s2_loop repeat_three_lz4_s2_emit_copy_short_2b: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4_s2_loop repeat_two_lz4_s2_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop repeat_two_offset_lz4_s2_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop long_offset_short_lz4_s2: MOVB $0xee, (AX) - MOVW R9, 1(AX) - LEAL -60(R10), R10 + MOVW R10, 1(AX) + LEAL -60(R11), R11 ADDQ $0x03, AX // emitRepeat emit_repeat_again_lz4_s2_emit_copy_short: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short cant_repeat_two_offset_lz4_s2_emit_copy_short: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short repeat_five_lz4_s2_emit_copy_short: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4_s2_loop repeat_four_lz4_s2_emit_copy_short: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4_s2_loop repeat_three_lz4_s2_emit_copy_short: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4_s2_loop repeat_two_lz4_s2_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop repeat_two_offset_lz4_s2_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4_s2_loop two_byte_offset_short_lz4_s2: - MOVL R10, R8 - SHLL $0x02, R8 - CMPL R10, $0x0c + MOVL R11, R9 + SHLL $0x02, R9 + CMPL R11, $0x0c JAE emit_copy_three_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE emit_copy_three_lz4_s2 - LEAL -15(R8), R8 - MOVB R9, 1(AX) - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R8 - MOVB R8, (AX) + LEAL -15(R9), R9 + MOVB R10, 1(AX) + SHRL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R9 + MOVB R9, (AX) ADDQ $0x02, AX JMP lz4_s2_loop emit_copy_three_lz4_s2: - LEAL -2(R8), R8 - MOVB R8, (AX) - MOVW R9, 1(AX) + LEAL -2(R9), R9 + MOVB R9, (AX) + MOVW R10, 1(AX) ADDQ $0x03, AX JMP lz4_s2_loop @@ -20285,7 +19872,7 @@ lz4_s2_dstfull: // func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 -TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64 +TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $8-64 XORQ SI, SI MOVQ dst_base+0(FP), AX MOVQ dst_len+8(FP), CX @@ -20293,459 +19880,466 @@ TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64 MOVQ src_len+32(FP), BX LEAQ (DX)(BX*1), BX LEAQ -8(AX)(CX*1), CX - XORQ DI, DI + LEAQ -4(CX), DI + XORQ R8, R8 lz4s_s2_loop: CMPQ DX, BX JAE lz4s_s2_corrupt CMPQ AX, CX JAE lz4s_s2_dstfull - MOVBQZX (DX), R8 - MOVQ R8, R9 - MOVQ R8, R10 - SHRQ $0x04, R9 - ANDQ $0x0f, R10 - CMPQ R8, $0xf0 + MOVBQZX (DX), R9 + MOVQ R9, R10 + MOVQ R9, R11 + SHRQ $0x04, R10 + ANDQ $0x0f, R11 + CMPQ R9, $0xf0 JB lz4s_s2_ll_end lz4s_s2_ll_loop: INCQ DX CMPQ DX, BX JAE lz4s_s2_corrupt - MOVBQZX (DX), R8 - ADDQ R8, R9 - CMPQ R8, $0xff + MOVBQZX (DX), R9 + ADDQ R9, R10 + CMPQ R9, $0xff JEQ lz4s_s2_ll_loop lz4s_s2_ll_end: - LEAQ (DX)(R9*1), R8 - ADDQ $0x03, R10 - CMPQ R8, BX + LEAQ (DX)(R10*1), R9 + ADDQ $0x03, R11 + CMPQ R9, BX JAE lz4s_s2_corrupt INCQ DX - INCQ R8 - TESTQ R9, R9 + INCQ R9 + TESTQ R10, R10 JZ lz4s_s2_lits_done - LEAQ (AX)(R9*1), R11 - CMPQ R11, CX + LEAQ (AX)(R10*1), R12 + CMPQ R12, DI JAE lz4s_s2_dstfull - ADDQ R9, SI - LEAL -1(R9), R11 - CMPL R11, $0x3c + ADDQ R10, SI + LEAL -1(R10), R12 + CMPL R12, $0x3c JB one_byte_lz4s_s2 - CMPL R11, $0x00000100 + CMPL R12, $0x00000100 JB two_bytes_lz4s_s2 - CMPL R11, $0x00010000 + CMPL R12, $0x00010000 JB three_bytes_lz4s_s2 - CMPL R11, $0x01000000 + CMPL R12, $0x01000000 JB four_bytes_lz4s_s2 MOVB $0xfc, (AX) - MOVL R11, 1(AX) + MOVL R12, 1(AX) ADDQ $0x05, AX JMP memmove_long_lz4s_s2 four_bytes_lz4s_s2: - MOVL R11, R12 - SHRL $0x10, R12 + MOVL R12, R13 + SHRL $0x10, R13 MOVB $0xf8, (AX) - MOVW R11, 1(AX) - MOVB R12, 3(AX) + MOVW R12, 1(AX) + MOVB R13, 3(AX) ADDQ $0x04, AX JMP memmove_long_lz4s_s2 three_bytes_lz4s_s2: MOVB $0xf4, (AX) - MOVW R11, 1(AX) + MOVW R12, 1(AX) ADDQ $0x03, AX JMP memmove_long_lz4s_s2 two_bytes_lz4s_s2: MOVB $0xf0, (AX) - MOVB R11, 1(AX) + MOVB R12, 1(AX) ADDQ $0x02, AX - CMPL R11, $0x40 + CMPL R12, $0x40 JB memmove_lz4s_s2 JMP memmove_long_lz4s_s2 one_byte_lz4s_s2: - SHLB $0x02, R11 - MOVB R11, (AX) + SHLB $0x02, R12 + MOVB R12, (AX) ADDQ $0x01, AX memmove_lz4s_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveShort - CMPQ R9, $0x08 + CMPQ R10, $0x08 JBE emit_lit_memmove_lz4s_s2_memmove_move_8 - CMPQ R9, $0x10 + CMPQ R10, $0x10 JBE emit_lit_memmove_lz4s_s2_memmove_move_8through16 - CMPQ R9, $0x20 + CMPQ R10, $0x20 JBE emit_lit_memmove_lz4s_s2_memmove_move_17through32 JMP emit_lit_memmove_lz4s_s2_memmove_move_33through64 emit_lit_memmove_lz4s_s2_memmove_move_8: - MOVQ (DX), R12 - MOVQ R12, (AX) + MOVQ (DX), R13 + MOVQ R13, (AX) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_8through16: - MOVQ (DX), R12 - MOVQ -8(DX)(R9*1), DX - MOVQ R12, (AX) - MOVQ DX, -8(AX)(R9*1) + MOVQ (DX), R13 + MOVQ -8(DX)(R10*1), DX + MOVQ R13, (AX) + MOVQ DX, -8(AX)(R10*1) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_17through32: MOVOU (DX), X0 - MOVOU -16(DX)(R9*1), X1 + MOVOU -16(DX)(R10*1), X1 MOVOU X0, (AX) - MOVOU X1, -16(AX)(R9*1) + MOVOU X1, -16(AX)(R10*1) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_33through64: MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) memmove_end_copy_lz4s_s2: - MOVQ R11, AX + MOVQ R12, AX JMP lz4s_s2_lits_emit_done memmove_long_lz4s_s2: - LEAQ (AX)(R9*1), R11 + LEAQ (AX)(R10*1), R12 // genMemMoveLong MOVOU (DX), X0 MOVOU 16(DX), X1 - MOVOU -32(DX)(R9*1), X2 - MOVOU -16(DX)(R9*1), X3 - MOVQ R9, R13 - SHRQ $0x05, R13 - MOVQ AX, R12 - ANDL $0x0000001f, R12 - MOVQ $0x00000040, R14 - SUBQ R12, R14 - DECQ R13 + MOVOU -32(DX)(R10*1), X2 + MOVOU -16(DX)(R10*1), X3 + MOVQ R10, R14 + SHRQ $0x05, R14 + MOVQ AX, R13 + ANDL $0x0000001f, R13 + MOVQ $0x00000040, R15 + SUBQ R13, R15 + DECQ R14 JA emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32 - LEAQ -32(DX)(R14*1), R12 - LEAQ -32(AX)(R14*1), R15 + LEAQ -32(DX)(R15*1), R13 + LEAQ -32(AX)(R15*1), BP emit_lit_memmove_long_lz4s_s2large_big_loop_back: - MOVOU (R12), X4 - MOVOU 16(R12), X5 - MOVOA X4, (R15) - MOVOA X5, 16(R15) + MOVOU (R13), X4 + MOVOU 16(R13), X5 + MOVOA X4, (BP) + MOVOA X5, 16(BP) + ADDQ $0x20, BP + ADDQ $0x20, R13 ADDQ $0x20, R15 - ADDQ $0x20, R12 - ADDQ $0x20, R14 - DECQ R13 + DECQ R14 JNA emit_lit_memmove_long_lz4s_s2large_big_loop_back emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32: - MOVOU -32(DX)(R14*1), X4 - MOVOU -16(DX)(R14*1), X5 - MOVOA X4, -32(AX)(R14*1) - MOVOA X5, -16(AX)(R14*1) - ADDQ $0x20, R14 - CMPQ R9, R14 + MOVOU -32(DX)(R15*1), X4 + MOVOU -16(DX)(R15*1), X5 + MOVOA X4, -32(AX)(R15*1) + MOVOA X5, -16(AX)(R15*1) + ADDQ $0x20, R15 + CMPQ R10, R15 JAE emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32 MOVOU X0, (AX) MOVOU X1, 16(AX) - MOVOU X2, -32(AX)(R9*1) - MOVOU X3, -16(AX)(R9*1) - MOVQ R11, AX + MOVOU X2, -32(AX)(R10*1) + MOVOU X3, -16(AX)(R10*1) + MOVQ R12, AX lz4s_s2_lits_emit_done: - MOVQ R8, DX + MOVQ R9, DX lz4s_s2_lits_done: CMPQ DX, BX JNE lz4s_s2_match - CMPQ R10, $0x03 + CMPQ R11, $0x03 JEQ lz4s_s2_done JMP lz4s_s2_corrupt lz4s_s2_match: - CMPQ R10, $0x03 + CMPQ R11, $0x03 JEQ lz4s_s2_loop - LEAQ 2(DX), R8 - CMPQ R8, BX + LEAQ 2(DX), R9 + CMPQ R9, BX JAE lz4s_s2_corrupt - MOVWQZX (DX), R9 - MOVQ R8, DX - TESTQ R9, R9 + MOVWQZX (DX), R10 + MOVQ R9, DX + TESTQ R10, R10 JZ lz4s_s2_corrupt - CMPQ R9, SI + CMPQ R10, SI JA lz4s_s2_corrupt - CMPQ R10, $0x12 + CMPQ R11, $0x12 JNE lz4s_s2_ml_done lz4s_s2_ml_loop: - MOVBQZX (DX), R8 + MOVBQZX (DX), R9 INCQ DX - ADDQ R8, R10 + ADDQ R9, R11 CMPQ DX, BX JAE lz4s_s2_corrupt - CMPQ R8, $0xff + CMPQ R9, $0xff JEQ lz4s_s2_ml_loop lz4s_s2_ml_done: - ADDQ R10, SI - CMPQ R9, DI + ADDQ R11, SI + CMPQ R10, R8 JNE lz4s_s2_docopy // emitRepeat emit_repeat_again_lz4_s2: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2 - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2 cant_repeat_two_offset_lz4_s2: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2 - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2 - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2 - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4s_s2_loop JMP emit_repeat_again_lz4_s2 repeat_five_lz4_s2: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4s_s2_loop repeat_four_lz4_s2: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4s_s2_loop repeat_three_lz4_s2: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4s_s2_loop repeat_two_lz4_s2: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop repeat_two_offset_lz4_s2: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop lz4s_s2_docopy: - MOVQ R9, DI + MOVQ R10, R8 // emitCopy - CMPL R10, $0x40 + CMPL R11, $0x40 JBE two_byte_offset_short_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE long_offset_short_lz4_s2 - MOVL $0x00000001, R8 - LEAL 16(R8), R8 - MOVB R9, 1(AX) - MOVL R9, R11 - SHRL $0x08, R11 - SHLL $0x05, R11 - ORL R11, R8 - MOVB R8, (AX) + MOVL $0x00000001, R9 + LEAL 16(R9), R9 + MOVB R10, 1(AX) + MOVL R10, R12 + SHRL $0x08, R12 + SHLL $0x05, R12 + ORL R12, R9 + MOVB R9, (AX) ADDQ $0x02, AX - SUBL $0x08, R10 + SUBL $0x08, R11 // emitRepeat - LEAL -4(R10), R10 + LEAL -4(R11), R11 JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b emit_repeat_again_lz4_s2_emit_copy_short_2b: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short_2b - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short_2b - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short_2b cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short_2b - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short_2b - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short_2b - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4s_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short_2b repeat_five_lz4_s2_emit_copy_short_2b: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4s_s2_loop repeat_four_lz4_s2_emit_copy_short_2b: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4s_s2_loop repeat_three_lz4_s2_emit_copy_short_2b: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4s_s2_loop repeat_two_lz4_s2_emit_copy_short_2b: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop repeat_two_offset_lz4_s2_emit_copy_short_2b: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop long_offset_short_lz4_s2: MOVB $0xee, (AX) - MOVW R9, 1(AX) - LEAL -60(R10), R10 + MOVW R10, 1(AX) + LEAL -60(R11), R11 ADDQ $0x03, AX // emitRepeat emit_repeat_again_lz4_s2_emit_copy_short: - MOVL R10, R8 - LEAL -4(R10), R10 - CMPL R8, $0x08 + MOVL R11, R9 + LEAL -4(R11), R11 + CMPL R9, $0x08 JBE repeat_two_lz4_s2_emit_copy_short - CMPL R8, $0x0c + CMPL R9, $0x0c JAE cant_repeat_two_offset_lz4_s2_emit_copy_short - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JB repeat_two_offset_lz4_s2_emit_copy_short cant_repeat_two_offset_lz4_s2_emit_copy_short: - CMPL R10, $0x00000104 + CMPL R11, $0x00000104 JB repeat_three_lz4_s2_emit_copy_short - CMPL R10, $0x00010100 + CMPL R11, $0x00010100 JB repeat_four_lz4_s2_emit_copy_short - CMPL R10, $0x0100ffff + CMPL R11, $0x0100ffff JB repeat_five_lz4_s2_emit_copy_short - LEAL -16842747(R10), R10 + LEAL -16842747(R11), R11 MOVL $0xfffb001d, (AX) MOVB $0xff, 4(AX) ADDQ $0x05, AX + CMPQ AX, CX + JAE lz4s_s2_loop JMP emit_repeat_again_lz4_s2_emit_copy_short repeat_five_lz4_s2_emit_copy_short: - LEAL -65536(R10), R10 - MOVL R10, R9 + LEAL -65536(R11), R11 + MOVL R11, R10 MOVW $0x001d, (AX) - MOVW R10, 2(AX) - SARL $0x10, R9 - MOVB R9, 4(AX) + MOVW R11, 2(AX) + SARL $0x10, R10 + MOVB R10, 4(AX) ADDQ $0x05, AX JMP lz4s_s2_loop repeat_four_lz4_s2_emit_copy_short: - LEAL -256(R10), R10 + LEAL -256(R11), R11 MOVW $0x0019, (AX) - MOVW R10, 2(AX) + MOVW R11, 2(AX) ADDQ $0x04, AX JMP lz4s_s2_loop repeat_three_lz4_s2_emit_copy_short: - LEAL -4(R10), R10 + LEAL -4(R11), R11 MOVW $0x0015, (AX) - MOVB R10, 2(AX) + MOVB R11, 2(AX) ADDQ $0x03, AX JMP lz4s_s2_loop repeat_two_lz4_s2_emit_copy_short: - SHLL $0x02, R10 - ORL $0x01, R10 - MOVW R10, (AX) + SHLL $0x02, R11 + ORL $0x01, R11 + MOVW R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop repeat_two_offset_lz4_s2_emit_copy_short: - XORQ R8, R8 - LEAL 1(R8)(R10*4), R10 - MOVB R9, 1(AX) - SARL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R10 - MOVB R10, (AX) + XORQ R9, R9 + LEAL 1(R9)(R11*4), R11 + MOVB R10, 1(AX) + SARL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R11 + MOVB R11, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop two_byte_offset_short_lz4_s2: - MOVL R10, R8 - SHLL $0x02, R8 - CMPL R10, $0x0c + MOVL R11, R9 + SHLL $0x02, R9 + CMPL R11, $0x0c JAE emit_copy_three_lz4_s2 - CMPL R9, $0x00000800 + CMPL R10, $0x00000800 JAE emit_copy_three_lz4_s2 - LEAL -15(R8), R8 - MOVB R9, 1(AX) - SHRL $0x08, R9 - SHLL $0x05, R9 - ORL R9, R8 - MOVB R8, (AX) + LEAL -15(R9), R9 + MOVB R10, 1(AX) + SHRL $0x08, R10 + SHLL $0x05, R10 + ORL R10, R9 + MOVB R9, (AX) ADDQ $0x02, AX JMP lz4s_s2_loop emit_copy_three_lz4_s2: - LEAL -2(R8), R8 - MOVB R8, (AX) - MOVW R9, 1(AX) + LEAL -2(R9), R9 + MOVB R9, (AX) + MOVW R10, 1(AX) ADDQ $0x03, AX JMP lz4s_s2_loop diff --git a/vendor/github.com/klauspost/compress/s2/encodeblock_arm64.s b/vendor/github.com/klauspost/compress/s2/encodeblock_arm64.s new file mode 100644 index 000000000..29a93289b --- /dev/null +++ b/vendor/github.com/klauspost/compress/s2/encodeblock_arm64.s @@ -0,0 +1,23576 @@ +// Code generated by command: go run gen.go -out ../encodeblock.s -arch arm64 -arm64gen -pkg=s2. DO NOT EDIT. +// EXPERIMENTAL arm64 output lowered from an amd64 avo program. + +//go:build arm64 && ((amd64 || arm64) && !appengine && !noasm && gc && !noasm) + +#include "textflag.h" + +// func encodeBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBlockAsm(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000200, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBlockAsm: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBlockAsm + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBlockAsm: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x06, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBlockAsm + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + LSL $0x10, R10, R10 + MUL R8, R10, R10 + LSR $0x32, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeBlockAsm + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R7 + MOVWU R6, R5 + MOVWU 24(RSP), R16 + SUBSW R16, R5, R5 + BEQ repeat_extend_back_end_encodeBlockAsm + +repeat_extend_back_loop_encodeBlockAsm: + CMPW R7, R6 + BLS repeat_extend_back_end_encodeBlockAsm + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeBlockAsm + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R5, R5 + BNE repeat_extend_back_loop_encodeBlockAsm + +repeat_extend_back_end_encodeBlockAsm: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $5, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeBlockAsm: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeBlockAsm + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeBlockAsm + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeBlockAsm + CMPW $0x00010000, R5 + BLO three_bytes_repeat_emit_encodeBlockAsm + CMPW $0x01000000, R5 + BLO four_bytes_repeat_emit_encodeBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R5, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_repeat_emit_encodeBlockAsm + +four_bytes_repeat_emit_encodeBlockAsm: + MOVWU R5, R10 + LSRW $0x10, R10, R10 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + MOVB R10, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_repeat_emit_encodeBlockAsm + +three_bytes_repeat_emit_encodeBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeBlockAsm + +two_bytes_repeat_emit_encodeBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeBlockAsm + JMP memmove_long_repeat_emit_encodeBlockAsm + +one_byte_repeat_emit_encodeBlockAsm: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeBlockAsm: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm + +emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm + +emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm + +emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeBlockAsm: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeBlockAsm + +memmove_long_repeat_emit_encodeBlockAsm: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R11 + MOVD R8, R12 + +emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R11) + FMOVQ F1, 16(R11) + ADD $0x20, R10, R10 + ADD $0x20, R11, R11 + SUB $0x20, R12, R12 + CMP $0x20, R12 + BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeBlockAsm: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R8 + SUBW R2, R8, R8 + ADD R2, R3, R9 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_repeat_extend_encodeBlockAsm: + CMPW $0x10, R8 + BLO matchlen_match8_repeat_extend_encodeBlockAsm + MOVD (R9)(R11), R10 + ADD R11, R9, R15 + MOVD 8(R15), R12 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm + ADD R11, R5, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16repeat_extend_encodeBlockAsm + SUB $16, R8, R8 + MOVWU R8, R8 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm + +matchlen_bsf_16repeat_extend_encodeBlockAsm: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm + +matchlen_match8_repeat_extend_encodeBlockAsm: + CMPW $0x08, R8 + BLO matchlen_match4_repeat_extend_encodeBlockAsm + MOVD (R9)(R11), R10 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm + SUB $8, R8, R8 + MOVWU R8, R8 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_repeat_extend_encodeBlockAsm + +matchlen_bsf_8_repeat_extend_encodeBlockAsm: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm + +matchlen_match4_repeat_extend_encodeBlockAsm: + CMPW $0x04, R8 + BLO matchlen_match2_repeat_extend_encodeBlockAsm + MOVWU (R9)(R11), R10 + MOVWU (R5)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_repeat_extend_encodeBlockAsm + SUB $4, R8, R8 + MOVWU R8, R8 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_repeat_extend_encodeBlockAsm: + CMPW $0x01, R8 + BEQ matchlen_match1_repeat_extend_encodeBlockAsm + BLO repeat_extend_forward_end_encodeBlockAsm + MOVHU (R9)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeBlockAsm + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R8, R8 + BEQ repeat_extend_forward_end_encodeBlockAsm + +matchlen_match1_repeat_extend_encodeBlockAsm: + MOVBU (R9)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R5, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeBlockAsm + ADD $1, R11, R11 + MOVWU R11, R11 + +repeat_extend_forward_end_encodeBlockAsm: + ADDW R11, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + TSTW R7, R7 + BEQ repeat_as_copy_encodeBlockAsm + + // emitRepeat +emit_repeat_again_match_repeat_encodeBlockAsm: + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_match_repeat_encodeBlockAsm + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm + CMPW $0x00000800, R6 + BLO repeat_two_offset_match_repeat_encodeBlockAsm + +cant_repeat_two_offset_match_repeat_encodeBlockAsm: + CMPW $0x00000104, R5 + BLO repeat_three_match_repeat_encodeBlockAsm + CMPW $0x00010100, R5 + BLO repeat_four_match_repeat_encodeBlockAsm + CMPW $0x0100ffff, R5 + BLO repeat_five_match_repeat_encodeBlockAsm + SUB $16842747, R5, R5 + MOVWU R5, R5 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_match_repeat_encodeBlockAsm + +repeat_five_match_repeat_encodeBlockAsm: + SUB $65536, R5, R5 + MOVWU R5, R5 + MOVWU R5, R6 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ASRW $0x10, R6, R6 + MOVB R6, 4(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_four_match_repeat_encodeBlockAsm: + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_three_match_repeat_encodeBlockAsm: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_two_match_repeat_encodeBlockAsm: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_two_offset_match_repeat_encodeBlockAsm: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_as_copy_encodeBlockAsm: + // emitCopy + CMPW $0x00010000, R6 + BLO two_byte_offset_repeat_as_copy_encodeBlockAsm + CMPW $0x40, R5 + BLS four_bytes_remain_repeat_as_copy_encodeBlockAsm + MOVD $0xff, R16 + MOVB R16, (R1) + MOVW R6, 1(R1) + SUB $64, R5, R5 + MOVWU R5, R5 + ADD $0x05, R1, R1 + CMPW $0x04, R5 + BLO four_bytes_remain_repeat_as_copy_encodeBlockAsm + + // emitRepeat +emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy: + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy + CMPW $0x00010100, R5 + BLO repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy + CMPW $0x0100ffff, R5 + BLO repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy + SUB $16842747, R5, R5 + MOVWU R5, R5 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy + +repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy: + SUB $65536, R5, R5 + MOVWU R5, R5 + MOVWU R5, R6 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ASRW $0x10, R6, R6 + MOVB R6, 4(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy: + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +four_bytes_remain_repeat_as_copy_encodeBlockAsm: + TSTW R5, R5 + BEQ repeat_end_emit_encodeBlockAsm + MOVD $0, R7 + ADD R5<<2, R7, R5 + SUB $1, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVW R6, 1(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +two_byte_offset_repeat_as_copy_encodeBlockAsm: + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm + CMPW $0x00000800, R6 + BHS long_offset_short_repeat_as_copy_encodeBlockAsm + MOVD $0x00000001, R7 + ADD $16, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + MOVWU R6, R8 + LSRW $0x08, R8, R8 + LSLW $0x05, R8, R8 + ORRW R8, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R5, R5 + + // emitRepeat + SUB $4, R5, R5 + MOVWU R5, R5 + JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b + +emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b + CMPW $0x00010100, R5 + BLO repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b + CMPW $0x0100ffff, R5 + BLO repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b + SUB $16842747, R5, R5 + MOVWU R5, R5 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b + +repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: + SUB $65536, R5, R5 + MOVWU R5, R5 + MOVWU R5, R6 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ASRW $0x10, R6, R6 + MOVB R6, 4(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +long_offset_short_repeat_as_copy_encodeBlockAsm: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + + // emitRepeat +emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short: + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short + CMPW $0x00010100, R5 + BLO repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short + CMPW $0x0100ffff, R5 + BLO repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short + SUB $16842747, R5, R5 + MOVWU R5, R5 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short + +repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short: + SUB $65536, R5, R5 + MOVWU R5, R5 + MOVWU R5, R6 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ASRW $0x10, R6, R6 + MOVB R6, 4(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short: + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +two_byte_offset_short_repeat_as_copy_encodeBlockAsm: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeBlockAsm + CMPW $0x00000800, R6 + BHS emit_copy_three_repeat_as_copy_encodeBlockAsm + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm + +emit_copy_three_repeat_as_copy_encodeBlockAsm: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeBlockAsm: + MOVW R2, 20(RSP) + JMP search_loop_encodeBlockAsm + +no_repeat_found_encodeBlockAsm: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBlockAsm + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeBlockAsm + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeBlockAsm + MOVWU 28(RSP), R2 + JMP search_loop_encodeBlockAsm + +candidate3_match_encodeBlockAsm: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeBlockAsm + +candidate2_match_encodeBlockAsm: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBlockAsm: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBlockAsm + +match_extend_back_loop_encodeBlockAsm: + CMPW R6, R2 + BLS match_extend_back_end_encodeBlockAsm + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBlockAsm + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBlockAsm + JMP match_extend_back_loop_encodeBlockAsm + +match_extend_back_end_encodeBlockAsm: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $5, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBlockAsm: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeBlockAsm + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeBlockAsm + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeBlockAsm + CMPW $0x00010000, R7 + BLO three_bytes_match_emit_encodeBlockAsm + CMPW $0x01000000, R7 + BLO four_bytes_match_emit_encodeBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R7, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_match_emit_encodeBlockAsm + +four_bytes_match_emit_encodeBlockAsm: + MOVWU R7, R9 + LSRW $0x10, R9, R9 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + MOVB R9, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_match_emit_encodeBlockAsm + +three_bytes_match_emit_encodeBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBlockAsm + +two_bytes_match_emit_encodeBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeBlockAsm + JMP memmove_long_match_emit_encodeBlockAsm + +one_byte_match_emit_encodeBlockAsm: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBlockAsm: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeBlockAsm + +emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm + +emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm + +emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBlockAsm: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeBlockAsm + +memmove_long_match_emit_encodeBlockAsm: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeBlockAsm: +match_nolit_loop_encodeBlockAsm: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeBlockAsm: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeBlockAsm + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeBlockAsm + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeBlockAsm + +matchlen_bsf_16match_nolit_encodeBlockAsm: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm + +matchlen_match8_match_nolit_encodeBlockAsm: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeBlockAsm + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeBlockAsm + +matchlen_bsf_8_match_nolit_encodeBlockAsm: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm + +matchlen_match4_match_nolit_encodeBlockAsm: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeBlockAsm + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeBlockAsm + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeBlockAsm: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeBlockAsm + BLO match_nolit_end_encodeBlockAsm + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBlockAsm + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeBlockAsm + +matchlen_match1_match_nolit_encodeBlockAsm: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBlockAsm + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeBlockAsm: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x00010000, R5 + BLO two_byte_offset_match_nolit_encodeBlockAsm + CMPW $0x40, R9 + BLS four_bytes_remain_match_nolit_encodeBlockAsm + MOVD $0xff, R16 + MOVB R16, (R1) + MOVW R5, 1(R1) + SUB $64, R9, R9 + MOVWU R9, R9 + ADD $0x05, R1, R1 + CMPW $0x04, R9 + BLO four_bytes_remain_match_nolit_encodeBlockAsm + + // emitRepeat +emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy: + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm_emit_copy + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy + +cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm_emit_copy + CMPW $0x00010100, R9 + BLO repeat_four_match_nolit_encodeBlockAsm_emit_copy + CMPW $0x0100ffff, R9 + BLO repeat_five_match_nolit_encodeBlockAsm_emit_copy + SUB $16842747, R9, R9 + MOVWU R9, R9 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy + +repeat_five_match_nolit_encodeBlockAsm_emit_copy: + SUB $65536, R9, R9 + MOVWU R9, R9 + MOVWU R9, R5 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ASRW $0x10, R5, R5 + MOVB R5, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_four_match_nolit_encodeBlockAsm_emit_copy: + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_three_match_nolit_encodeBlockAsm_emit_copy: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_two_match_nolit_encodeBlockAsm_emit_copy: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +four_bytes_remain_match_nolit_encodeBlockAsm: + TSTW R9, R9 + BEQ match_nolit_emitcopy_end_encodeBlockAsm + MOVD $0, R6 + ADD R9<<2, R6, R9 + SUB $1, R9, R9 + MOVWU R9, R9 + MOVB R9, (R1) + MOVW R5, 1(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +two_byte_offset_match_nolit_encodeBlockAsm: + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeBlockAsm + CMPW $0x00000800, R5 + BHS long_offset_short_match_nolit_encodeBlockAsm + MOVD $0x00000001, R6 + ADD $16, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + MOVWU R5, R7 + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R9, R9 + + // emitRepeat + SUB $4, R9, R9 + MOVWU R9, R9 + JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b + +emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b: + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b + CMPW $0x00010100, R9 + BLO repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b + CMPW $0x0100ffff, R9 + BLO repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b + SUB $16842747, R9, R9 + MOVWU R9, R9 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b + +repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b: + SUB $65536, R9, R9 + MOVWU R9, R9 + MOVWU R9, R5 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ASRW $0x10, R5, R5 + MOVB R5, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b: + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +long_offset_short_match_nolit_encodeBlockAsm: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + + // emitRepeat +emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short: + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm_emit_copy_short + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm_emit_copy_short + CMPW $0x00010100, R9 + BLO repeat_four_match_nolit_encodeBlockAsm_emit_copy_short + CMPW $0x0100ffff, R9 + BLO repeat_five_match_nolit_encodeBlockAsm_emit_copy_short + SUB $16842747, R9, R9 + MOVWU R9, R9 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short + +repeat_five_match_nolit_encodeBlockAsm_emit_copy_short: + SUB $65536, R9, R9 + MOVWU R9, R9 + MOVWU R9, R5 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ASRW $0x10, R5, R5 + MOVB R5, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_four_match_nolit_encodeBlockAsm_emit_copy_short: + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_three_match_nolit_encodeBlockAsm_emit_copy_short: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_two_match_nolit_encodeBlockAsm_emit_copy_short: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +two_byte_offset_short_match_nolit_encodeBlockAsm: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeBlockAsm + CMPW $0x00000800, R5 + BHS emit_copy_three_match_nolit_encodeBlockAsm + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm + +emit_copy_three_match_nolit_encodeBlockAsm: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeBlockAsm: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBlockAsm + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBlockAsm: + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x10, R7, R7 + MUL R8, R7, R7 + LSR $0x32, R7, R7 + LSL $0x10, R5, R5 + MUL R8, R5, R5 + LSR $0x32, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeBlockAsm + ADDW $1, R2, R2 + JMP search_loop_encodeBlockAsm + +emit_remainder_encodeBlockAsm: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $5, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBlockAsm: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBlockAsm + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBlockAsm + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBlockAsm + CMPW $0x00010000, R2 + BLO three_bytes_emit_remainder_encodeBlockAsm + CMPW $0x01000000, R2 + BLO four_bytes_emit_remainder_encodeBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R2, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_emit_remainder_encodeBlockAsm + +four_bytes_emit_remainder_encodeBlockAsm: + MOVWU R2, R3 + LSRW $0x10, R3, R3 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + MOVB R3, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_emit_remainder_encodeBlockAsm + +three_bytes_emit_remainder_encodeBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBlockAsm + +two_bytes_emit_remainder_encodeBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBlockAsm + JMP memmove_long_emit_remainder_encodeBlockAsm + +one_byte_emit_remainder_encodeBlockAsm: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBlockAsm: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm + +emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm + +emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm + +emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm + +emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm + +emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBlockAsm: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBlockAsm + +memmove_long_emit_remainder_encodeBlockAsm: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBlockAsm: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeBlockAsm4MB(dst []byte, src []byte, tmp *[65536]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBlockAsm4MB(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000200, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBlockAsm4MB: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBlockAsm4MB + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBlockAsm4MB: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x06, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBlockAsm4MB + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + LSL $0x10, R10, R10 + MUL R8, R10, R10 + LSR $0x32, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeBlockAsm4MB + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R7 + MOVWU R6, R5 + MOVWU 24(RSP), R16 + SUBSW R16, R5, R5 + BEQ repeat_extend_back_end_encodeBlockAsm4MB + +repeat_extend_back_loop_encodeBlockAsm4MB: + CMPW R7, R6 + BLS repeat_extend_back_end_encodeBlockAsm4MB + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeBlockAsm4MB + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R5, R5 + BNE repeat_extend_back_loop_encodeBlockAsm4MB + +repeat_extend_back_end_encodeBlockAsm4MB: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $4, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeBlockAsm4MB + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeBlockAsm4MB: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeBlockAsm4MB + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeBlockAsm4MB + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeBlockAsm4MB + CMPW $0x00010000, R5 + BLO three_bytes_repeat_emit_encodeBlockAsm4MB + MOVWU R5, R10 + LSRW $0x10, R10, R10 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + MOVB R10, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_repeat_emit_encodeBlockAsm4MB + +three_bytes_repeat_emit_encodeBlockAsm4MB: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeBlockAsm4MB + +two_bytes_repeat_emit_encodeBlockAsm4MB: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeBlockAsm4MB + JMP memmove_long_repeat_emit_encodeBlockAsm4MB + +one_byte_repeat_emit_encodeBlockAsm4MB: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeBlockAsm4MB: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB + +emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB + +emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB + +emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeBlockAsm4MB: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeBlockAsm4MB + +memmove_long_repeat_emit_encodeBlockAsm4MB: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R11 + MOVD R8, R12 + +emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R11) + FMOVQ F1, 16(R11) + ADD $0x20, R10, R10 + ADD $0x20, R11, R11 + SUB $0x20, R12, R12 + CMP $0x20, R12 + BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeBlockAsm4MB: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R8 + SUBW R2, R8, R8 + ADD R2, R3, R9 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB: + CMPW $0x10, R8 + BLO matchlen_match8_repeat_extend_encodeBlockAsm4MB + MOVD (R9)(R11), R10 + ADD R11, R9, R15 + MOVD 8(R15), R12 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB + ADD R11, R5, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16repeat_extend_encodeBlockAsm4MB + SUB $16, R8, R8 + MOVWU R8, R8 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB + +matchlen_bsf_16repeat_extend_encodeBlockAsm4MB: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm4MB + +matchlen_match8_repeat_extend_encodeBlockAsm4MB: + CMPW $0x08, R8 + BLO matchlen_match4_repeat_extend_encodeBlockAsm4MB + MOVD (R9)(R11), R10 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB + SUB $8, R8, R8 + MOVWU R8, R8 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_repeat_extend_encodeBlockAsm4MB + +matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm4MB + +matchlen_match4_repeat_extend_encodeBlockAsm4MB: + CMPW $0x04, R8 + BLO matchlen_match2_repeat_extend_encodeBlockAsm4MB + MOVWU (R9)(R11), R10 + MOVWU (R5)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_repeat_extend_encodeBlockAsm4MB + SUB $4, R8, R8 + MOVWU R8, R8 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_repeat_extend_encodeBlockAsm4MB: + CMPW $0x01, R8 + BEQ matchlen_match1_repeat_extend_encodeBlockAsm4MB + BLO repeat_extend_forward_end_encodeBlockAsm4MB + MOVHU (R9)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeBlockAsm4MB + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R8, R8 + BEQ repeat_extend_forward_end_encodeBlockAsm4MB + +matchlen_match1_repeat_extend_encodeBlockAsm4MB: + MOVBU (R9)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R5, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeBlockAsm4MB + ADD $1, R11, R11 + MOVWU R11, R11 + +repeat_extend_forward_end_encodeBlockAsm4MB: + ADDW R11, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + TSTW R7, R7 + BEQ repeat_as_copy_encodeBlockAsm4MB + + // emitRepeat + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_match_repeat_encodeBlockAsm4MB + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB + CMPW $0x00000800, R6 + BLO repeat_two_offset_match_repeat_encodeBlockAsm4MB + +cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB: + CMPW $0x00000104, R5 + BLO repeat_three_match_repeat_encodeBlockAsm4MB + CMPW $0x00010100, R5 + BLO repeat_four_match_repeat_encodeBlockAsm4MB + SUB $65536, R5, R5 + MOVWU R5, R5 + MOVWU R5, R6 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ASRW $0x10, R6, R6 + MOVB R6, 4(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_four_match_repeat_encodeBlockAsm4MB: + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_three_match_repeat_encodeBlockAsm4MB: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_two_match_repeat_encodeBlockAsm4MB: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_two_offset_match_repeat_encodeBlockAsm4MB: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_as_copy_encodeBlockAsm4MB: + // emitCopy + CMPW $0x00010000, R6 + BLO two_byte_offset_repeat_as_copy_encodeBlockAsm4MB + CMPW $0x40, R5 + BLS four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB + MOVD $0xff, R16 + MOVB R16, (R1) + MOVW R6, 1(R1) + SUB $64, R5, R5 + MOVWU R5, R5 + ADD $0x05, R1, R1 + CMPW $0x04, R5 + BLO four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB + + // emitRepeat + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy + CMPW $0x00010100, R5 + BLO repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy + SUB $65536, R5, R5 + MOVWU R5, R5 + MOVWU R5, R6 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ASRW $0x10, R6, R6 + MOVB R6, 4(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy: + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB: + TSTW R5, R5 + BEQ repeat_end_emit_encodeBlockAsm4MB + MOVD $0, R7 + ADD R5<<2, R7, R5 + SUB $1, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVW R6, 1(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +two_byte_offset_repeat_as_copy_encodeBlockAsm4MB: + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB + CMPW $0x00000800, R6 + BHS long_offset_short_repeat_as_copy_encodeBlockAsm4MB + MOVD $0x00000001, R7 + ADD $16, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R5, R5 + + // emitRepeat + SUB $4, R5, R5 + MOVWU R5, R5 + JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b + CMPW $0x00010100, R5 + BLO repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b + SUB $65536, R5, R5 + MOVWU R5, R5 + MOVWU R5, R6 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ASRW $0x10, R6, R6 + MOVB R6, 4(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +long_offset_short_repeat_as_copy_encodeBlockAsm4MB: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short + CMPW $0x00010100, R5 + BLO repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short + SUB $65536, R5, R5 + MOVWU R5, R5 + MOVWU R5, R6 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ASRW $0x10, R6, R6 + MOVB R6, 4(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeBlockAsm4MB + CMPW $0x00000800, R6 + BHS emit_copy_three_repeat_as_copy_encodeBlockAsm4MB + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm4MB + +emit_copy_three_repeat_as_copy_encodeBlockAsm4MB: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeBlockAsm4MB: + MOVW R2, 20(RSP) + JMP search_loop_encodeBlockAsm4MB + +no_repeat_found_encodeBlockAsm4MB: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBlockAsm4MB + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeBlockAsm4MB + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeBlockAsm4MB + MOVWU 28(RSP), R2 + JMP search_loop_encodeBlockAsm4MB + +candidate3_match_encodeBlockAsm4MB: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeBlockAsm4MB + +candidate2_match_encodeBlockAsm4MB: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBlockAsm4MB: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBlockAsm4MB + +match_extend_back_loop_encodeBlockAsm4MB: + CMPW R6, R2 + BLS match_extend_back_end_encodeBlockAsm4MB + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBlockAsm4MB + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBlockAsm4MB + JMP match_extend_back_loop_encodeBlockAsm4MB + +match_extend_back_end_encodeBlockAsm4MB: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $4, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBlockAsm4MB + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBlockAsm4MB: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeBlockAsm4MB + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeBlockAsm4MB + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeBlockAsm4MB + CMPW $0x00010000, R7 + BLO three_bytes_match_emit_encodeBlockAsm4MB + MOVWU R7, R9 + LSRW $0x10, R9, R9 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + MOVB R9, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_match_emit_encodeBlockAsm4MB + +three_bytes_match_emit_encodeBlockAsm4MB: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBlockAsm4MB + +two_bytes_match_emit_encodeBlockAsm4MB: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeBlockAsm4MB + JMP memmove_long_match_emit_encodeBlockAsm4MB + +one_byte_match_emit_encodeBlockAsm4MB: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBlockAsm4MB: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeBlockAsm4MB + +emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm4MB + +emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm4MB + +emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBlockAsm4MB: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeBlockAsm4MB + +memmove_long_match_emit_encodeBlockAsm4MB: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeBlockAsm4MB: +match_nolit_loop_encodeBlockAsm4MB: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeBlockAsm4MB: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeBlockAsm4MB + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm4MB + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeBlockAsm4MB + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeBlockAsm4MB + +matchlen_bsf_16match_nolit_encodeBlockAsm4MB: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm4MB + +matchlen_match8_match_nolit_encodeBlockAsm4MB: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeBlockAsm4MB + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm4MB + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeBlockAsm4MB + +matchlen_bsf_8_match_nolit_encodeBlockAsm4MB: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm4MB + +matchlen_match4_match_nolit_encodeBlockAsm4MB: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeBlockAsm4MB + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeBlockAsm4MB + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeBlockAsm4MB: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeBlockAsm4MB + BLO match_nolit_end_encodeBlockAsm4MB + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBlockAsm4MB + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeBlockAsm4MB + +matchlen_match1_match_nolit_encodeBlockAsm4MB: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBlockAsm4MB + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeBlockAsm4MB: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x00010000, R5 + BLO two_byte_offset_match_nolit_encodeBlockAsm4MB + CMPW $0x40, R9 + BLS four_bytes_remain_match_nolit_encodeBlockAsm4MB + MOVD $0xff, R16 + MOVB R16, (R1) + MOVW R5, 1(R1) + SUB $64, R9, R9 + MOVWU R9, R9 + ADD $0x05, R1, R1 + CMPW $0x04, R9 + BLO four_bytes_remain_match_nolit_encodeBlockAsm4MB + + // emitRepeat + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy + +cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy + CMPW $0x00010100, R9 + BLO repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy + SUB $65536, R9, R9 + MOVWU R9, R9 + MOVWU R9, R5 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ASRW $0x10, R5, R5 + MOVB R5, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy: + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +four_bytes_remain_match_nolit_encodeBlockAsm4MB: + TSTW R9, R9 + BEQ match_nolit_emitcopy_end_encodeBlockAsm4MB + MOVD $0, R6 + ADD R9<<2, R6, R9 + SUB $1, R9, R9 + MOVWU R9, R9 + MOVB R9, (R1) + MOVW R5, 1(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +two_byte_offset_match_nolit_encodeBlockAsm4MB: + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeBlockAsm4MB + CMPW $0x00000800, R5 + BHS long_offset_short_match_nolit_encodeBlockAsm4MB + MOVD $0x00000001, R6 + ADD $16, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R9, R9 + + // emitRepeat + SUB $4, R9, R9 + MOVWU R9, R9 + JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b + CMPW $0x00010100, R9 + BLO repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b + SUB $65536, R9, R9 + MOVWU R9, R9 + MOVWU R9, R5 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ASRW $0x10, R5, R5 + MOVB R5, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +long_offset_short_match_nolit_encodeBlockAsm4MB: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short + CMPW $0x00010100, R9 + BLO repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short + SUB $65536, R9, R9 + MOVWU R9, R9 + MOVWU R9, R5 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ASRW $0x10, R5, R5 + MOVB R5, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short: + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +two_byte_offset_short_match_nolit_encodeBlockAsm4MB: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeBlockAsm4MB + CMPW $0x00000800, R5 + BHS emit_copy_three_match_nolit_encodeBlockAsm4MB + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm4MB + +emit_copy_three_match_nolit_encodeBlockAsm4MB: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeBlockAsm4MB: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBlockAsm4MB + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBlockAsm4MB + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBlockAsm4MB: + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x10, R7, R7 + MUL R8, R7, R7 + LSR $0x32, R7, R7 + LSL $0x10, R5, R5 + MUL R8, R5, R5 + LSR $0x32, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeBlockAsm4MB + ADDW $1, R2, R2 + JMP search_loop_encodeBlockAsm4MB + +emit_remainder_encodeBlockAsm4MB: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $4, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBlockAsm4MB + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBlockAsm4MB: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBlockAsm4MB + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBlockAsm4MB + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBlockAsm4MB + CMPW $0x00010000, R2 + BLO three_bytes_emit_remainder_encodeBlockAsm4MB + MOVWU R2, R3 + LSRW $0x10, R3, R3 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + MOVB R3, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_emit_remainder_encodeBlockAsm4MB + +three_bytes_emit_remainder_encodeBlockAsm4MB: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBlockAsm4MB + +two_bytes_emit_remainder_encodeBlockAsm4MB: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBlockAsm4MB + JMP memmove_long_emit_remainder_encodeBlockAsm4MB + +one_byte_emit_remainder_encodeBlockAsm4MB: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBlockAsm4MB: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBlockAsm4MB: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBlockAsm4MB + +memmove_long_emit_remainder_encodeBlockAsm4MB: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBlockAsm4MB: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBlockAsm12B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000080, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBlockAsm12B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBlockAsm12B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBlockAsm12B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x05, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBlockAsm12B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x000000cf1bbcdcbb, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x18, R9, R9 + MUL R8, R9, R9 + LSR $0x34, R9, R9 + LSL $0x18, R10, R10 + MUL R8, R10, R10 + LSR $0x34, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x18, R9, R9 + MUL R8, R9, R9 + LSR $0x34, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeBlockAsm12B + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R7 + MOVWU R6, R5 + MOVWU 24(RSP), R16 + SUBSW R16, R5, R5 + BEQ repeat_extend_back_end_encodeBlockAsm12B + +repeat_extend_back_loop_encodeBlockAsm12B: + CMPW R7, R6 + BLS repeat_extend_back_end_encodeBlockAsm12B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeBlockAsm12B + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R5, R5 + BNE repeat_extend_back_loop_encodeBlockAsm12B + +repeat_extend_back_end_encodeBlockAsm12B: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $3, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeBlockAsm12B: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeBlockAsm12B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeBlockAsm12B + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeBlockAsm12B + BLO three_bytes_repeat_emit_encodeBlockAsm12B + +three_bytes_repeat_emit_encodeBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeBlockAsm12B + +two_bytes_repeat_emit_encodeBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeBlockAsm12B + JMP memmove_long_repeat_emit_encodeBlockAsm12B + +one_byte_repeat_emit_encodeBlockAsm12B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeBlockAsm12B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B + +emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B + +emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B + +emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeBlockAsm12B: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeBlockAsm12B + +memmove_long_repeat_emit_encodeBlockAsm12B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R11 + MOVD R8, R12 + +emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R11) + FMOVQ F1, 16(R11) + ADD $0x20, R10, R10 + ADD $0x20, R11, R11 + SUB $0x20, R12, R12 + CMP $0x20, R12 + BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeBlockAsm12B: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R8 + SUBW R2, R8, R8 + ADD R2, R3, R9 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_repeat_extend_encodeBlockAsm12B: + CMPW $0x10, R8 + BLO matchlen_match8_repeat_extend_encodeBlockAsm12B + MOVD (R9)(R11), R10 + ADD R11, R9, R15 + MOVD 8(R15), R12 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm12B + ADD R11, R5, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16repeat_extend_encodeBlockAsm12B + SUB $16, R8, R8 + MOVWU R8, R8 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm12B + +matchlen_bsf_16repeat_extend_encodeBlockAsm12B: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm12B + +matchlen_match8_repeat_extend_encodeBlockAsm12B: + CMPW $0x08, R8 + BLO matchlen_match4_repeat_extend_encodeBlockAsm12B + MOVD (R9)(R11), R10 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm12B + SUB $8, R8, R8 + MOVWU R8, R8 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_repeat_extend_encodeBlockAsm12B + +matchlen_bsf_8_repeat_extend_encodeBlockAsm12B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm12B + +matchlen_match4_repeat_extend_encodeBlockAsm12B: + CMPW $0x04, R8 + BLO matchlen_match2_repeat_extend_encodeBlockAsm12B + MOVWU (R9)(R11), R10 + MOVWU (R5)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_repeat_extend_encodeBlockAsm12B + SUB $4, R8, R8 + MOVWU R8, R8 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_repeat_extend_encodeBlockAsm12B: + CMPW $0x01, R8 + BEQ matchlen_match1_repeat_extend_encodeBlockAsm12B + BLO repeat_extend_forward_end_encodeBlockAsm12B + MOVHU (R9)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeBlockAsm12B + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R8, R8 + BEQ repeat_extend_forward_end_encodeBlockAsm12B + +matchlen_match1_repeat_extend_encodeBlockAsm12B: + MOVBU (R9)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R5, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeBlockAsm12B + ADD $1, R11, R11 + MOVWU R11, R11 + +repeat_extend_forward_end_encodeBlockAsm12B: + ADDW R11, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + TSTW R7, R7 + BEQ repeat_as_copy_encodeBlockAsm12B + + // emitRepeat + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_match_repeat_encodeBlockAsm12B + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm12B + CMPW $0x00000800, R6 + BLO repeat_two_offset_match_repeat_encodeBlockAsm12B + +cant_repeat_two_offset_match_repeat_encodeBlockAsm12B: + CMPW $0x00000104, R5 + BLO repeat_three_match_repeat_encodeBlockAsm12B + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_three_match_repeat_encodeBlockAsm12B: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_two_match_repeat_encodeBlockAsm12B: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_two_offset_match_repeat_encodeBlockAsm12B: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_as_copy_encodeBlockAsm12B: + // emitCopy + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B + CMPW $0x00000800, R6 + BHS long_offset_short_repeat_as_copy_encodeBlockAsm12B + MOVD $0x00000001, R7 + ADD $16, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R5, R5 + + // emitRepeat + SUB $4, R5, R5 + MOVWU R5, R5 + JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +long_offset_short_repeat_as_copy_encodeBlockAsm12B: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeBlockAsm12B + CMPW $0x00000800, R6 + BHS emit_copy_three_repeat_as_copy_encodeBlockAsm12B + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm12B + +emit_copy_three_repeat_as_copy_encodeBlockAsm12B: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeBlockAsm12B: + MOVW R2, 20(RSP) + JMP search_loop_encodeBlockAsm12B + +no_repeat_found_encodeBlockAsm12B: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBlockAsm12B + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeBlockAsm12B + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeBlockAsm12B + MOVWU 28(RSP), R2 + JMP search_loop_encodeBlockAsm12B + +candidate3_match_encodeBlockAsm12B: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeBlockAsm12B + +candidate2_match_encodeBlockAsm12B: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBlockAsm12B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBlockAsm12B + +match_extend_back_loop_encodeBlockAsm12B: + CMPW R6, R2 + BLS match_extend_back_end_encodeBlockAsm12B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBlockAsm12B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBlockAsm12B + JMP match_extend_back_loop_encodeBlockAsm12B + +match_extend_back_end_encodeBlockAsm12B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBlockAsm12B: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeBlockAsm12B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeBlockAsm12B + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeBlockAsm12B + BLO three_bytes_match_emit_encodeBlockAsm12B + +three_bytes_match_emit_encodeBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBlockAsm12B + +two_bytes_match_emit_encodeBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeBlockAsm12B + JMP memmove_long_match_emit_encodeBlockAsm12B + +one_byte_match_emit_encodeBlockAsm12B: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBlockAsm12B: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeBlockAsm12B + +emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm12B + +emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm12B + +emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBlockAsm12B: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeBlockAsm12B + +memmove_long_match_emit_encodeBlockAsm12B: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeBlockAsm12B: +match_nolit_loop_encodeBlockAsm12B: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeBlockAsm12B: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeBlockAsm12B + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm12B + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeBlockAsm12B + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeBlockAsm12B + +matchlen_bsf_16match_nolit_encodeBlockAsm12B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm12B + +matchlen_match8_match_nolit_encodeBlockAsm12B: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeBlockAsm12B + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm12B + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeBlockAsm12B + +matchlen_bsf_8_match_nolit_encodeBlockAsm12B: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm12B + +matchlen_match4_match_nolit_encodeBlockAsm12B: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeBlockAsm12B + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeBlockAsm12B + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeBlockAsm12B: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeBlockAsm12B + BLO match_nolit_end_encodeBlockAsm12B + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBlockAsm12B + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeBlockAsm12B + +matchlen_match1_match_nolit_encodeBlockAsm12B: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBlockAsm12B + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeBlockAsm12B: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeBlockAsm12B + CMPW $0x00000800, R5 + BHS long_offset_short_match_nolit_encodeBlockAsm12B + MOVD $0x00000001, R6 + ADD $16, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R9, R9 + + // emitRepeat + SUB $4, R9, R9 + MOVWU R9, R9 + JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm12B + +repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm12B + +repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm12B + +repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm12B + +long_offset_short_match_nolit_encodeBlockAsm12B: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm12B + +repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm12B + +repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm12B + +repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm12B + +two_byte_offset_short_match_nolit_encodeBlockAsm12B: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeBlockAsm12B + CMPW $0x00000800, R5 + BHS emit_copy_three_match_nolit_encodeBlockAsm12B + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm12B + +emit_copy_three_match_nolit_encodeBlockAsm12B: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeBlockAsm12B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBlockAsm12B + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBlockAsm12B: + MOVD $0x000000cf1bbcdcbb, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x18, R7, R7 + MUL R8, R7, R7 + LSR $0x34, R7, R7 + LSL $0x18, R5, R5 + MUL R8, R5, R5 + LSR $0x34, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeBlockAsm12B + ADDW $1, R2, R2 + JMP search_loop_encodeBlockAsm12B + +emit_remainder_encodeBlockAsm12B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBlockAsm12B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBlockAsm12B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBlockAsm12B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBlockAsm12B + BLO three_bytes_emit_remainder_encodeBlockAsm12B + +three_bytes_emit_remainder_encodeBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBlockAsm12B + +two_bytes_emit_remainder_encodeBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBlockAsm12B + JMP memmove_long_emit_remainder_encodeBlockAsm12B + +one_byte_emit_remainder_encodeBlockAsm12B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBlockAsm12B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBlockAsm12B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBlockAsm12B + +memmove_long_emit_remainder_encodeBlockAsm12B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBlockAsm12B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBlockAsm10B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000020, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBlockAsm10B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBlockAsm10B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBlockAsm10B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x05, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBlockAsm10B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x9e3779b1, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x36, R9, R9 + LSL $0x20, R10, R10 + MUL R8, R10, R10 + LSR $0x36, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x36, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeBlockAsm10B + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R7 + MOVWU R6, R5 + MOVWU 24(RSP), R16 + SUBSW R16, R5, R5 + BEQ repeat_extend_back_end_encodeBlockAsm10B + +repeat_extend_back_loop_encodeBlockAsm10B: + CMPW R7, R6 + BLS repeat_extend_back_end_encodeBlockAsm10B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeBlockAsm10B + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R5, R5 + BNE repeat_extend_back_loop_encodeBlockAsm10B + +repeat_extend_back_end_encodeBlockAsm10B: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $3, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeBlockAsm10B: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeBlockAsm10B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeBlockAsm10B + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeBlockAsm10B + BLO three_bytes_repeat_emit_encodeBlockAsm10B + +three_bytes_repeat_emit_encodeBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeBlockAsm10B + +two_bytes_repeat_emit_encodeBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeBlockAsm10B + JMP memmove_long_repeat_emit_encodeBlockAsm10B + +one_byte_repeat_emit_encodeBlockAsm10B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeBlockAsm10B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B + +emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B + +emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B + +emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeBlockAsm10B: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeBlockAsm10B + +memmove_long_repeat_emit_encodeBlockAsm10B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R11 + MOVD R8, R12 + +emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R11) + FMOVQ F1, 16(R11) + ADD $0x20, R10, R10 + ADD $0x20, R11, R11 + SUB $0x20, R12, R12 + CMP $0x20, R12 + BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeBlockAsm10B: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R8 + SUBW R2, R8, R8 + ADD R2, R3, R9 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_repeat_extend_encodeBlockAsm10B: + CMPW $0x10, R8 + BLO matchlen_match8_repeat_extend_encodeBlockAsm10B + MOVD (R9)(R11), R10 + ADD R11, R9, R15 + MOVD 8(R15), R12 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm10B + ADD R11, R5, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16repeat_extend_encodeBlockAsm10B + SUB $16, R8, R8 + MOVWU R8, R8 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm10B + +matchlen_bsf_16repeat_extend_encodeBlockAsm10B: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm10B + +matchlen_match8_repeat_extend_encodeBlockAsm10B: + CMPW $0x08, R8 + BLO matchlen_match4_repeat_extend_encodeBlockAsm10B + MOVD (R9)(R11), R10 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm10B + SUB $8, R8, R8 + MOVWU R8, R8 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_repeat_extend_encodeBlockAsm10B + +matchlen_bsf_8_repeat_extend_encodeBlockAsm10B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm10B + +matchlen_match4_repeat_extend_encodeBlockAsm10B: + CMPW $0x04, R8 + BLO matchlen_match2_repeat_extend_encodeBlockAsm10B + MOVWU (R9)(R11), R10 + MOVWU (R5)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_repeat_extend_encodeBlockAsm10B + SUB $4, R8, R8 + MOVWU R8, R8 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_repeat_extend_encodeBlockAsm10B: + CMPW $0x01, R8 + BEQ matchlen_match1_repeat_extend_encodeBlockAsm10B + BLO repeat_extend_forward_end_encodeBlockAsm10B + MOVHU (R9)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeBlockAsm10B + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R8, R8 + BEQ repeat_extend_forward_end_encodeBlockAsm10B + +matchlen_match1_repeat_extend_encodeBlockAsm10B: + MOVBU (R9)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R5, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeBlockAsm10B + ADD $1, R11, R11 + MOVWU R11, R11 + +repeat_extend_forward_end_encodeBlockAsm10B: + ADDW R11, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + TSTW R7, R7 + BEQ repeat_as_copy_encodeBlockAsm10B + + // emitRepeat + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_match_repeat_encodeBlockAsm10B + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm10B + CMPW $0x00000800, R6 + BLO repeat_two_offset_match_repeat_encodeBlockAsm10B + +cant_repeat_two_offset_match_repeat_encodeBlockAsm10B: + CMPW $0x00000104, R5 + BLO repeat_three_match_repeat_encodeBlockAsm10B + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_three_match_repeat_encodeBlockAsm10B: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_two_match_repeat_encodeBlockAsm10B: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_two_offset_match_repeat_encodeBlockAsm10B: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_as_copy_encodeBlockAsm10B: + // emitCopy + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B + CMPW $0x00000800, R6 + BHS long_offset_short_repeat_as_copy_encodeBlockAsm10B + MOVD $0x00000001, R7 + ADD $16, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R5, R5 + + // emitRepeat + SUB $4, R5, R5 + MOVWU R5, R5 + JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +long_offset_short_repeat_as_copy_encodeBlockAsm10B: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R5, R7 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R7 + BLS repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short + CMPW $0x0c, R7 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short + CMPW $0x00000800, R6 + BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeBlockAsm10B + CMPW $0x00000800, R6 + BHS emit_copy_three_repeat_as_copy_encodeBlockAsm10B + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm10B + +emit_copy_three_repeat_as_copy_encodeBlockAsm10B: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeBlockAsm10B: + MOVW R2, 20(RSP) + JMP search_loop_encodeBlockAsm10B + +no_repeat_found_encodeBlockAsm10B: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBlockAsm10B + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeBlockAsm10B + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeBlockAsm10B + MOVWU 28(RSP), R2 + JMP search_loop_encodeBlockAsm10B + +candidate3_match_encodeBlockAsm10B: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeBlockAsm10B + +candidate2_match_encodeBlockAsm10B: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBlockAsm10B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBlockAsm10B + +match_extend_back_loop_encodeBlockAsm10B: + CMPW R6, R2 + BLS match_extend_back_end_encodeBlockAsm10B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBlockAsm10B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBlockAsm10B + JMP match_extend_back_loop_encodeBlockAsm10B + +match_extend_back_end_encodeBlockAsm10B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBlockAsm10B: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeBlockAsm10B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeBlockAsm10B + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeBlockAsm10B + BLO three_bytes_match_emit_encodeBlockAsm10B + +three_bytes_match_emit_encodeBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBlockAsm10B + +two_bytes_match_emit_encodeBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeBlockAsm10B + JMP memmove_long_match_emit_encodeBlockAsm10B + +one_byte_match_emit_encodeBlockAsm10B: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBlockAsm10B: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeBlockAsm10B + +emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm10B + +emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm10B + +emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBlockAsm10B: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeBlockAsm10B + +memmove_long_match_emit_encodeBlockAsm10B: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeBlockAsm10B: +match_nolit_loop_encodeBlockAsm10B: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeBlockAsm10B: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeBlockAsm10B + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm10B + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeBlockAsm10B + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeBlockAsm10B + +matchlen_bsf_16match_nolit_encodeBlockAsm10B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm10B + +matchlen_match8_match_nolit_encodeBlockAsm10B: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeBlockAsm10B + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm10B + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeBlockAsm10B + +matchlen_bsf_8_match_nolit_encodeBlockAsm10B: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm10B + +matchlen_match4_match_nolit_encodeBlockAsm10B: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeBlockAsm10B + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeBlockAsm10B + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeBlockAsm10B: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeBlockAsm10B + BLO match_nolit_end_encodeBlockAsm10B + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBlockAsm10B + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeBlockAsm10B + +matchlen_match1_match_nolit_encodeBlockAsm10B: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBlockAsm10B + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeBlockAsm10B: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeBlockAsm10B + CMPW $0x00000800, R5 + BHS long_offset_short_match_nolit_encodeBlockAsm10B + MOVD $0x00000001, R6 + ADD $16, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R9, R9 + + // emitRepeat + SUB $4, R9, R9 + MOVWU R9, R9 + JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm10B + +repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm10B + +repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm10B + +repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm10B + +long_offset_short_match_nolit_encodeBlockAsm10B: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R9, R6 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R6 + BLS repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short + CMPW $0x00000800, R5 + BLO repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm10B + +repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm10B + +repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm10B + +repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short: + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm10B + +two_byte_offset_short_match_nolit_encodeBlockAsm10B: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeBlockAsm10B + CMPW $0x00000800, R5 + BHS emit_copy_three_match_nolit_encodeBlockAsm10B + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm10B + +emit_copy_three_match_nolit_encodeBlockAsm10B: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeBlockAsm10B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBlockAsm10B + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBlockAsm10B: + MOVD $0x9e3779b1, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x20, R7, R7 + MUL R8, R7, R7 + LSR $0x36, R7, R7 + LSL $0x20, R5, R5 + MUL R8, R5, R5 + LSR $0x36, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeBlockAsm10B + ADDW $1, R2, R2 + JMP search_loop_encodeBlockAsm10B + +emit_remainder_encodeBlockAsm10B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBlockAsm10B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBlockAsm10B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBlockAsm10B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBlockAsm10B + BLO three_bytes_emit_remainder_encodeBlockAsm10B + +three_bytes_emit_remainder_encodeBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBlockAsm10B + +two_bytes_emit_remainder_encodeBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBlockAsm10B + JMP memmove_long_emit_remainder_encodeBlockAsm10B + +one_byte_emit_remainder_encodeBlockAsm10B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBlockAsm10B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBlockAsm10B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBlockAsm10B + +memmove_long_emit_remainder_encodeBlockAsm10B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBlockAsm10B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBlockAsm8B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000008, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBlockAsm8B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBlockAsm8B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBlockAsm8B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x04, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBlockAsm8B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x9e3779b1, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x38, R9, R9 + LSL $0x20, R10, R10 + MUL R8, R10, R10 + LSR $0x38, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x38, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeBlockAsm8B + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R7 + MOVWU R6, R5 + MOVWU 24(RSP), R16 + SUBSW R16, R5, R5 + BEQ repeat_extend_back_end_encodeBlockAsm8B + +repeat_extend_back_loop_encodeBlockAsm8B: + CMPW R7, R6 + BLS repeat_extend_back_end_encodeBlockAsm8B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeBlockAsm8B + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R5, R5 + BNE repeat_extend_back_loop_encodeBlockAsm8B + +repeat_extend_back_end_encodeBlockAsm8B: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $3, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeBlockAsm8B: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeBlockAsm8B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeBlockAsm8B + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeBlockAsm8B + BLO three_bytes_repeat_emit_encodeBlockAsm8B + +three_bytes_repeat_emit_encodeBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeBlockAsm8B + +two_bytes_repeat_emit_encodeBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeBlockAsm8B + JMP memmove_long_repeat_emit_encodeBlockAsm8B + +one_byte_repeat_emit_encodeBlockAsm8B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeBlockAsm8B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B + +emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B + +emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B + +emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeBlockAsm8B: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeBlockAsm8B + +memmove_long_repeat_emit_encodeBlockAsm8B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R11 + MOVD R8, R12 + +emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R11) + FMOVQ F1, 16(R11) + ADD $0x20, R10, R10 + ADD $0x20, R11, R11 + SUB $0x20, R12, R12 + CMP $0x20, R12 + BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeBlockAsm8B: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R8 + SUBW R2, R8, R8 + ADD R2, R3, R9 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_repeat_extend_encodeBlockAsm8B: + CMPW $0x10, R8 + BLO matchlen_match8_repeat_extend_encodeBlockAsm8B + MOVD (R9)(R11), R10 + ADD R11, R9, R15 + MOVD 8(R15), R12 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm8B + ADD R11, R5, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16repeat_extend_encodeBlockAsm8B + SUB $16, R8, R8 + MOVWU R8, R8 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm8B + +matchlen_bsf_16repeat_extend_encodeBlockAsm8B: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm8B + +matchlen_match8_repeat_extend_encodeBlockAsm8B: + CMPW $0x08, R8 + BLO matchlen_match4_repeat_extend_encodeBlockAsm8B + MOVD (R9)(R11), R10 + MOVD (R5)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm8B + SUB $8, R8, R8 + MOVWU R8, R8 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_repeat_extend_encodeBlockAsm8B + +matchlen_bsf_8_repeat_extend_encodeBlockAsm8B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP repeat_extend_forward_end_encodeBlockAsm8B + +matchlen_match4_repeat_extend_encodeBlockAsm8B: + CMPW $0x04, R8 + BLO matchlen_match2_repeat_extend_encodeBlockAsm8B + MOVWU (R9)(R11), R10 + MOVWU (R5)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_repeat_extend_encodeBlockAsm8B + SUB $4, R8, R8 + MOVWU R8, R8 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_repeat_extend_encodeBlockAsm8B: + CMPW $0x01, R8 + BEQ matchlen_match1_repeat_extend_encodeBlockAsm8B + BLO repeat_extend_forward_end_encodeBlockAsm8B + MOVHU (R9)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeBlockAsm8B + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R8, R8 + BEQ repeat_extend_forward_end_encodeBlockAsm8B + +matchlen_match1_repeat_extend_encodeBlockAsm8B: + MOVBU (R9)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R5, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeBlockAsm8B + ADD $1, R11, R11 + MOVWU R11, R11 + +repeat_extend_forward_end_encodeBlockAsm8B: + ADDW R11, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + TSTW R7, R7 + BEQ repeat_as_copy_encodeBlockAsm8B + + // emitRepeat + MOVWU R5, R6 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R6 + BLS repeat_two_match_repeat_encodeBlockAsm8B + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm8B + +cant_repeat_two_offset_match_repeat_encodeBlockAsm8B: + CMPW $0x00000104, R5 + BLO repeat_three_match_repeat_encodeBlockAsm8B + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +repeat_three_match_repeat_encodeBlockAsm8B: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +repeat_two_match_repeat_encodeBlockAsm8B: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +repeat_as_copy_encodeBlockAsm8B: + // emitCopy + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B + CMPW $0x00000800, R6 + BHS long_offset_short_repeat_as_copy_encodeBlockAsm8B + MOVD $0x00000001, R7 + ADD $16, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R5, R5 + + // emitRepeat + SUB $4, R5, R5 + MOVWU R5, R5 + JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b + MOVWU R5, R6 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R6 + BLS repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +long_offset_short_repeat_as_copy_encodeBlockAsm8B: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R5, R6 + SUB $4, R5, R5 + MOVWU R5, R5 + CMPW $0x08, R6 + BLS repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short + CMPW $0x0c, R6 + BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short + +cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short: + CMPW $0x00000104, R5 + BLO repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short + SUB $256, R5, R5 + MOVWU R5, R5 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R5, 2(R1) + ADD $0x04, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short: + SUB $4, R5, R5 + MOVWU R5, R5 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R5, 2(R1) + ADD $0x03, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short: + LSLW $0x02, R5, R5 + ORRW $0x01, R5, R5 + MOVH R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + MOVD $0, R7 + ADD R5<<2, R7, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVB R6, 1(R1) + ASRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeBlockAsm8B + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeBlockAsm8B + +emit_copy_three_repeat_as_copy_encodeBlockAsm8B: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeBlockAsm8B: + MOVW R2, 20(RSP) + JMP search_loop_encodeBlockAsm8B + +no_repeat_found_encodeBlockAsm8B: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBlockAsm8B + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeBlockAsm8B + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeBlockAsm8B + MOVWU 28(RSP), R2 + JMP search_loop_encodeBlockAsm8B + +candidate3_match_encodeBlockAsm8B: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeBlockAsm8B + +candidate2_match_encodeBlockAsm8B: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBlockAsm8B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBlockAsm8B + +match_extend_back_loop_encodeBlockAsm8B: + CMPW R6, R2 + BLS match_extend_back_end_encodeBlockAsm8B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBlockAsm8B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBlockAsm8B + JMP match_extend_back_loop_encodeBlockAsm8B + +match_extend_back_end_encodeBlockAsm8B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBlockAsm8B: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeBlockAsm8B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeBlockAsm8B + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeBlockAsm8B + BLO three_bytes_match_emit_encodeBlockAsm8B + +three_bytes_match_emit_encodeBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBlockAsm8B + +two_bytes_match_emit_encodeBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeBlockAsm8B + JMP memmove_long_match_emit_encodeBlockAsm8B + +one_byte_match_emit_encodeBlockAsm8B: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBlockAsm8B: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeBlockAsm8B + +emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm8B + +emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBlockAsm8B + +emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBlockAsm8B: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeBlockAsm8B + +memmove_long_match_emit_encodeBlockAsm8B: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeBlockAsm8B: +match_nolit_loop_encodeBlockAsm8B: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeBlockAsm8B: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeBlockAsm8B + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm8B + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeBlockAsm8B + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeBlockAsm8B + +matchlen_bsf_16match_nolit_encodeBlockAsm8B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm8B + +matchlen_match8_match_nolit_encodeBlockAsm8B: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeBlockAsm8B + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeBlockAsm8B + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeBlockAsm8B + +matchlen_bsf_8_match_nolit_encodeBlockAsm8B: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeBlockAsm8B + +matchlen_match4_match_nolit_encodeBlockAsm8B: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeBlockAsm8B + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeBlockAsm8B + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeBlockAsm8B: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeBlockAsm8B + BLO match_nolit_end_encodeBlockAsm8B + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBlockAsm8B + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeBlockAsm8B + +matchlen_match1_match_nolit_encodeBlockAsm8B: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBlockAsm8B + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeBlockAsm8B: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeBlockAsm8B + CMPW $0x00000800, R5 + BHS long_offset_short_match_nolit_encodeBlockAsm8B + MOVD $0x00000001, R6 + ADD $16, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R9, R9 + + // emitRepeat + SUB $4, R9, R9 + MOVWU R9, R9 + JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b + MOVWU R9, R5 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm8B + +repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm8B + +repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm8B + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm8B + +long_offset_short_match_nolit_encodeBlockAsm8B: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R9, R5 + SUB $4, R9, R9 + MOVWU R9, R9 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short: + CMPW $0x00000104, R9 + BLO repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short + SUB $256, R9, R9 + MOVWU R9, R9 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R9, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm8B + +repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short: + SUB $4, R9, R9 + MOVWU R9, R9 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R9, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm8B + +repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short: + LSLW $0x02, R9, R9 + ORRW $0x01, R9, R9 + MOVH R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm8B + MOVD $0, R6 + ADD R9<<2, R6, R9 + ADD $1, R9, R9 + MOVWU R9, R9 + MOVB R5, 1(R1) + ASRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R9, R9 + MOVB R9, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm8B + +two_byte_offset_short_match_nolit_encodeBlockAsm8B: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeBlockAsm8B + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBlockAsm8B + +emit_copy_three_match_nolit_encodeBlockAsm8B: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeBlockAsm8B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBlockAsm8B + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBlockAsm8B: + MOVD $0x9e3779b1, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x20, R7, R7 + MUL R8, R7, R7 + LSR $0x38, R7, R7 + LSL $0x20, R5, R5 + MUL R8, R5, R5 + LSR $0x38, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeBlockAsm8B + ADDW $1, R2, R2 + JMP search_loop_encodeBlockAsm8B + +emit_remainder_encodeBlockAsm8B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBlockAsm8B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBlockAsm8B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBlockAsm8B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBlockAsm8B + BLO three_bytes_emit_remainder_encodeBlockAsm8B + +three_bytes_emit_remainder_encodeBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBlockAsm8B + +two_bytes_emit_remainder_encodeBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBlockAsm8B + JMP memmove_long_emit_remainder_encodeBlockAsm8B + +one_byte_emit_remainder_encodeBlockAsm8B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBlockAsm8B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBlockAsm8B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBlockAsm8B + +memmove_long_emit_remainder_encodeBlockAsm8B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBlockAsm8B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBetterBlockAsm(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00001200, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBetterBlockAsm: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBetterBlockAsm + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $6, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBetterBlockAsm: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x07, R5, R5 + CMPW $0x63, R5 + BLS check_maxskip_ok_encodeBetterBlockAsm + ADD $100, R2, R5 + MOVWU R5, R5 + JMP check_maxskip_cont_encodeBetterBlockAsm + +check_maxskip_ok_encodeBetterBlockAsm: + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + +check_maxskip_cont_encodeBetterBlockAsm: + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBetterBlockAsm + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x00cf1bbcdcbfa563, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x08, R9, R9 + MUL R8, R9, R9 + LSR $0x2f, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x32, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 524288(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 524288(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeBetterBlockAsm + CMP R6, R10 + BNE no_short_found_encodeBetterBlockAsm + MOVWU R7, R5 + JMP candidate_match_encodeBetterBlockAsm + +no_short_found_encodeBetterBlockAsm: + CMPW R6, R9 + BEQ candidate_match_encodeBetterBlockAsm + CMPW R6, R10 + BEQ candidateS_match_encodeBetterBlockAsm + MOVWU 28(RSP), R2 + JMP search_loop_encodeBetterBlockAsm + +candidateS_match_encodeBetterBlockAsm: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x08, R9, R9 + MUL R8, R9, R9 + LSR $0x2f, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBetterBlockAsm + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBetterBlockAsm: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm + +match_extend_back_loop_encodeBetterBlockAsm: + CMPW R6, R2 + BLS match_extend_back_end_encodeBetterBlockAsm + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBetterBlockAsm + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm + JMP match_extend_back_loop_encodeBetterBlockAsm + +match_extend_back_end_encodeBetterBlockAsm: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $5, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBetterBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBetterBlockAsm: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeBetterBlockAsm: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeBetterBlockAsm + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm + +matchlen_bsf_16match_nolit_encodeBetterBlockAsm: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm + +matchlen_match8_match_nolit_encodeBetterBlockAsm: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeBetterBlockAsm + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeBetterBlockAsm + +matchlen_bsf_8_match_nolit_encodeBetterBlockAsm: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm + +matchlen_match4_match_nolit_encodeBetterBlockAsm: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeBetterBlockAsm + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeBetterBlockAsm + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeBetterBlockAsm: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm + BLO match_nolit_end_encodeBetterBlockAsm + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBetterBlockAsm + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeBetterBlockAsm + +matchlen_match1_match_nolit_encodeBetterBlockAsm: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBetterBlockAsm + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeBetterBlockAsm: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + MOVWU 24(RSP), R16 + CMPW R7, R16 + BEQ match_is_repeat_encodeBetterBlockAsm + CMPW $0x01, R11 + BHI match_length_ok_encodeBetterBlockAsm + CMPW $0x0000ffff, R7 + BLS match_length_ok_encodeBetterBlockAsm + MOVWU 28(RSP), R2 + ADDW $1, R2, R2 + JMP search_loop_encodeBetterBlockAsm + +match_length_ok_encodeBetterBlockAsm: + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeBetterBlockAsm + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeBetterBlockAsm + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeBetterBlockAsm + CMPW $0x00010000, R5 + BLO three_bytes_match_emit_encodeBetterBlockAsm + CMPW $0x01000000, R5 + BLO four_bytes_match_emit_encodeBetterBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R5, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_match_emit_encodeBetterBlockAsm + +four_bytes_match_emit_encodeBetterBlockAsm: + MOVWU R5, R10 + LSRW $0x10, R10, R10 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + MOVB R10, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_match_emit_encodeBetterBlockAsm + +three_bytes_match_emit_encodeBetterBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBetterBlockAsm + +two_bytes_match_emit_encodeBetterBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeBetterBlockAsm + JMP memmove_long_match_emit_encodeBetterBlockAsm + +one_byte_match_emit_encodeBetterBlockAsm: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBetterBlockAsm: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x04, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4 + CMP $0x08, R8 + BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4: + MOVWU (R9), R10 + MOVW R10, (R1) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm + +emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7: + MOVWU (R9), R10 + ADD R8, R9, R15 + MOVWU -4(R15), R9 + MOVW R10, (R1) + ADD R8, R1, R15 + MOVW R9, -4(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm + +emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm + +emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm + +emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBetterBlockAsm: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeBetterBlockAsm + +memmove_long_match_emit_encodeBetterBlockAsm: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeBetterBlockAsm: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x00010000, R7 + BLO two_byte_offset_match_nolit_encodeBetterBlockAsm + CMPW $0x40, R11 + BLS four_bytes_remain_match_nolit_encodeBetterBlockAsm + MOVD $0xff, R16 + MOVB R16, (R1) + MOVW R7, 1(R1) + SUB $64, R11, R11 + MOVWU R11, R11 + ADD $0x05, R1, R1 + CMPW $0x04, R11 + BLO four_bytes_remain_match_nolit_encodeBetterBlockAsm + + // emitRepeat +emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy: + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy + CMPW $0x00010100, R11 + BLO repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy + CMPW $0x0100ffff, R11 + BLO repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy + SUB $16842747, R11, R11 + MOVWU R11, R11 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy + +repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy: + SUB $65536, R11, R11 + MOVWU R11, R11 + MOVWU R11, R7 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ASRW $0x10, R7, R7 + MOVB R7, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy: + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +four_bytes_remain_match_nolit_encodeBetterBlockAsm: + TSTW R11, R11 + BEQ match_nolit_emitcopy_end_encodeBetterBlockAsm + MOVD $0, R5 + ADD R11<<2, R5, R11 + SUB $1, R11, R11 + MOVWU R11, R11 + MOVB R11, (R1) + MOVW R7, 1(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +two_byte_offset_match_nolit_encodeBetterBlockAsm: + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm + CMPW $0x00000800, R7 + BHS long_offset_short_match_nolit_encodeBetterBlockAsm + MOVD $0x00000001, R5 + ADD $16, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + MOVWU R7, R8 + LSRW $0x08, R8, R8 + LSLW $0x05, R8, R8 + ORRW R8, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R11, R11 + + // emitRepeat + SUB $4, R11, R11 + MOVWU R11, R11 + JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b + +emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b + CMPW $0x00010100, R11 + BLO repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b + CMPW $0x0100ffff, R11 + BLO repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b + SUB $16842747, R11, R11 + MOVWU R11, R11 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b + +repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: + SUB $65536, R11, R11 + MOVWU R11, R11 + MOVWU R11, R7 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ASRW $0x10, R7, R7 + MOVB R7, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +long_offset_short_match_nolit_encodeBetterBlockAsm: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + + // emitRepeat +emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short: + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short + CMPW $0x00010100, R11 + BLO repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short + CMPW $0x0100ffff, R11 + BLO repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short + SUB $16842747, R11, R11 + MOVWU R11, R11 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short + +repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short: + SUB $65536, R11, R11 + MOVWU R11, R11 + MOVWU R11, R7 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ASRW $0x10, R7, R7 + MOVB R7, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short: + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +two_byte_offset_short_match_nolit_encodeBetterBlockAsm: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeBetterBlockAsm + CMPW $0x00000800, R7 + BHS emit_copy_three_match_nolit_encodeBetterBlockAsm + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +emit_copy_three_match_nolit_encodeBetterBlockAsm: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +match_is_repeat_encodeBetterBlockAsm: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_repeat_encodeBetterBlockAsm + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm + CMPW $0x00010000, R5 + BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm + CMPW $0x01000000, R5 + BLO four_bytes_match_emit_repeat_encodeBetterBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R5, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm + +four_bytes_match_emit_repeat_encodeBetterBlockAsm: + MOVWU R5, R10 + LSRW $0x10, R10, R10 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + MOVB R10, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm + +three_bytes_match_emit_repeat_encodeBetterBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm + +two_bytes_match_emit_repeat_encodeBetterBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_repeat_encodeBetterBlockAsm + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm + +one_byte_match_emit_repeat_encodeBetterBlockAsm: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_repeat_encodeBetterBlockAsm: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x04, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4 + CMP $0x08, R8 + BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64 + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4: + MOVWU (R9), R10 + MOVW R10, (R1) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7: + MOVWU (R9), R10 + ADD R8, R9, R15 + MOVWU -4(R15), R9 + MOVW R10, (R1) + ADD R8, R1, R15 + MOVW R9, -4(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm: + MOVD R5, R1 + JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm + +memmove_long_match_emit_repeat_encodeBetterBlockAsm: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_repeat_encodeBetterBlockAsm: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitRepeat +emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm: + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm + +cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm + CMPW $0x00010100, R11 + BLO repeat_four_match_nolit_repeat_encodeBetterBlockAsm + CMPW $0x0100ffff, R11 + BLO repeat_five_match_nolit_repeat_encodeBetterBlockAsm + SUB $16842747, R11, R11 + MOVWU R11, R11 + MOVD $0xfffb001d, R16 + MOVW R16, (R1) + MOVD $0xff, R16 + MOVB R16, 4(R1) + ADD $0x05, R1, R1 + JMP emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm + +repeat_five_match_nolit_repeat_encodeBetterBlockAsm: + SUB $65536, R11, R11 + MOVWU R11, R11 + MOVWU R11, R7 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ASRW $0x10, R7, R7 + MOVB R7, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_four_match_nolit_repeat_encodeBetterBlockAsm: + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_three_match_nolit_repeat_encodeBetterBlockAsm: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_two_match_nolit_repeat_encodeBetterBlockAsm: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm + +repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + +match_nolit_emitcopy_end_encodeBetterBlockAsm: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBetterBlockAsm + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBetterBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBetterBlockAsm: + MOVD $0x00cf1bbcdcbfa563, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x08, R9, R9 + MUL R5, R9, R9 + LSR $0x2f, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x32, R10, R10 + LSL $0x08, R11, R11 + MUL R5, R11, R11 + LSR $0x2f, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x32, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 524288(R15) + ADD R12<<2, R0, R15 + MOVW R13, 524288(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeBetterBlockAsm: + CMP R8, R7 + BHS search_loop_encodeBetterBlockAsm + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x08, R9, R9 + MUL R5, R9, R9 + LSR $0x2f, R9, R9 + LSL $0x08, R10, R10 + MUL R5, R10, R10 + LSR $0x2f, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeBetterBlockAsm + +emit_remainder_encodeBetterBlockAsm: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $5, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBetterBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBetterBlockAsm: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBetterBlockAsm + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBetterBlockAsm + CMPW $0x00010000, R2 + BLO three_bytes_emit_remainder_encodeBetterBlockAsm + CMPW $0x01000000, R2 + BLO four_bytes_emit_remainder_encodeBetterBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R2, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_emit_remainder_encodeBetterBlockAsm + +four_bytes_emit_remainder_encodeBetterBlockAsm: + MOVWU R2, R3 + LSRW $0x10, R3, R3 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + MOVB R3, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_emit_remainder_encodeBetterBlockAsm + +three_bytes_emit_remainder_encodeBetterBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBetterBlockAsm + +two_bytes_emit_remainder_encodeBetterBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBetterBlockAsm + JMP memmove_long_emit_remainder_encodeBetterBlockAsm + +one_byte_emit_remainder_encodeBetterBlockAsm: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBetterBlockAsm: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBetterBlockAsm: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm + +memmove_long_emit_remainder_encodeBetterBlockAsm: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBetterBlockAsm: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeBetterBlockAsm4MB(dst []byte, src []byte, tmp *[589824]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBetterBlockAsm4MB(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00001200, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBetterBlockAsm4MB: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBetterBlockAsm4MB + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $6, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBetterBlockAsm4MB: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x07, R5, R5 + CMPW $0x63, R5 + BLS check_maxskip_ok_encodeBetterBlockAsm4MB + ADD $100, R2, R5 + MOVWU R5, R5 + JMP check_maxskip_cont_encodeBetterBlockAsm4MB + +check_maxskip_ok_encodeBetterBlockAsm4MB: + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + +check_maxskip_cont_encodeBetterBlockAsm4MB: + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBetterBlockAsm4MB + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x00cf1bbcdcbfa563, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x08, R9, R9 + MUL R8, R9, R9 + LSR $0x2f, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x32, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 524288(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 524288(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeBetterBlockAsm4MB + CMP R6, R10 + BNE no_short_found_encodeBetterBlockAsm4MB + MOVWU R7, R5 + JMP candidate_match_encodeBetterBlockAsm4MB + +no_short_found_encodeBetterBlockAsm4MB: + CMPW R6, R9 + BEQ candidate_match_encodeBetterBlockAsm4MB + CMPW R6, R10 + BEQ candidateS_match_encodeBetterBlockAsm4MB + MOVWU 28(RSP), R2 + JMP search_loop_encodeBetterBlockAsm4MB + +candidateS_match_encodeBetterBlockAsm4MB: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x08, R9, R9 + MUL R8, R9, R9 + LSR $0x2f, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBetterBlockAsm4MB + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBetterBlockAsm4MB: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm4MB + +match_extend_back_loop_encodeBetterBlockAsm4MB: + CMPW R6, R2 + BLS match_extend_back_end_encodeBetterBlockAsm4MB + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBetterBlockAsm4MB + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm4MB + JMP match_extend_back_loop_encodeBetterBlockAsm4MB + +match_extend_back_end_encodeBetterBlockAsm4MB: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $4, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBetterBlockAsm4MB + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBetterBlockAsm4MB: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeBetterBlockAsm4MB + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB + +matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm4MB + +matchlen_match8_match_nolit_encodeBetterBlockAsm4MB: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeBetterBlockAsm4MB + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeBetterBlockAsm4MB + +matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm4MB + +matchlen_match4_match_nolit_encodeBetterBlockAsm4MB: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeBetterBlockAsm4MB + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeBetterBlockAsm4MB + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeBetterBlockAsm4MB: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm4MB + BLO match_nolit_end_encodeBetterBlockAsm4MB + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBetterBlockAsm4MB + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeBetterBlockAsm4MB + +matchlen_match1_match_nolit_encodeBetterBlockAsm4MB: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBetterBlockAsm4MB + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeBetterBlockAsm4MB: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + MOVWU 24(RSP), R16 + CMPW R7, R16 + BEQ match_is_repeat_encodeBetterBlockAsm4MB + CMPW $0x01, R11 + BHI match_length_ok_encodeBetterBlockAsm4MB + CMPW $0x0000ffff, R7 + BLS match_length_ok_encodeBetterBlockAsm4MB + MOVWU 28(RSP), R2 + ADDW $1, R2, R2 + JMP search_loop_encodeBetterBlockAsm4MB + +match_length_ok_encodeBetterBlockAsm4MB: + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeBetterBlockAsm4MB + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeBetterBlockAsm4MB + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeBetterBlockAsm4MB + CMPW $0x00010000, R5 + BLO three_bytes_match_emit_encodeBetterBlockAsm4MB + MOVWU R5, R10 + LSRW $0x10, R10, R10 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + MOVB R10, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_match_emit_encodeBetterBlockAsm4MB + +three_bytes_match_emit_encodeBetterBlockAsm4MB: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBetterBlockAsm4MB + +two_bytes_match_emit_encodeBetterBlockAsm4MB: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeBetterBlockAsm4MB + JMP memmove_long_match_emit_encodeBetterBlockAsm4MB + +one_byte_match_emit_encodeBetterBlockAsm4MB: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBetterBlockAsm4MB: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x04, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4 + CMP $0x08, R8 + BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4: + MOVWU (R9), R10 + MOVW R10, (R1) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB + +emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7: + MOVWU (R9), R10 + ADD R8, R9, R15 + MOVWU -4(R15), R9 + MOVW R10, (R1) + ADD R8, R1, R15 + MOVW R9, -4(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB + +emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB + +emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB + +emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBetterBlockAsm4MB: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeBetterBlockAsm4MB + +memmove_long_match_emit_encodeBetterBlockAsm4MB: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeBetterBlockAsm4MB: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x00010000, R7 + BLO two_byte_offset_match_nolit_encodeBetterBlockAsm4MB + CMPW $0x40, R11 + BLS four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB + MOVD $0xff, R16 + MOVB R16, (R1) + MOVW R7, 1(R1) + SUB $64, R11, R11 + MOVWU R11, R11 + ADD $0x05, R1, R1 + CMPW $0x04, R11 + BLO four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB + + // emitRepeat + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy + CMPW $0x00010100, R11 + BLO repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy + SUB $65536, R11, R11 + MOVWU R11, R11 + MOVWU R11, R7 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ASRW $0x10, R7, R7 + MOVB R7, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy: + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB: + TSTW R11, R11 + BEQ match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + MOVD $0, R5 + ADD R11<<2, R5, R11 + SUB $1, R11, R11 + MOVWU R11, R11 + MOVB R11, (R1) + MOVW R7, 1(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +two_byte_offset_match_nolit_encodeBetterBlockAsm4MB: + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB + CMPW $0x00000800, R7 + BHS long_offset_short_match_nolit_encodeBetterBlockAsm4MB + MOVD $0x00000001, R5 + ADD $16, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R11, R11 + + // emitRepeat + SUB $4, R11, R11 + MOVWU R11, R11 + JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b + CMPW $0x00010100, R11 + BLO repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b + SUB $65536, R11, R11 + MOVWU R11, R11 + MOVWU R11, R7 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ASRW $0x10, R7, R7 + MOVB R7, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +long_offset_short_match_nolit_encodeBetterBlockAsm4MB: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short + CMPW $0x00010100, R11 + BLO repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short + SUB $65536, R11, R11 + MOVWU R11, R11 + MOVWU R11, R7 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ASRW $0x10, R7, R7 + MOVB R7, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeBetterBlockAsm4MB + CMPW $0x00000800, R7 + BHS emit_copy_three_match_nolit_encodeBetterBlockAsm4MB + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +emit_copy_three_match_nolit_encodeBetterBlockAsm4MB: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +match_is_repeat_encodeBetterBlockAsm4MB: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_repeat_encodeBetterBlockAsm4MB + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB + CMPW $0x00010000, R5 + BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB + MOVWU R5, R10 + LSRW $0x10, R10, R10 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + MOVB R10, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB + +three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB + +two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_repeat_encodeBetterBlockAsm4MB + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB + +one_byte_match_emit_repeat_encodeBetterBlockAsm4MB: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_repeat_encodeBetterBlockAsm4MB: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x04, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4 + CMP $0x08, R8 + BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64 + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4: + MOVWU (R9), R10 + MOVW R10, (R1) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7: + MOVWU (R9), R10 + ADD R8, R9, R15 + MOVWU -4(R15), R9 + MOVW R10, (R1) + ADD R8, R1, R15 + MOVW R9, -4(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB: + MOVD R5, R1 + JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB + +memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitRepeat + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB + +cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB + CMPW $0x00010100, R11 + BLO repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB + SUB $65536, R11, R11 + MOVWU R11, R11 + MOVWU R11, R7 + MOVD $0x001d, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ASRW $0x10, R7, R7 + MOVB R7, 4(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB: + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB + +repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + +match_nolit_emitcopy_end_encodeBetterBlockAsm4MB: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBetterBlockAsm4MB + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBetterBlockAsm4MB + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBetterBlockAsm4MB: + MOVD $0x00cf1bbcdcbfa563, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x08, R9, R9 + MUL R5, R9, R9 + LSR $0x2f, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x32, R10, R10 + LSL $0x08, R11, R11 + MUL R5, R11, R11 + LSR $0x2f, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x32, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 524288(R15) + ADD R12<<2, R0, R15 + MOVW R13, 524288(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeBetterBlockAsm4MB: + CMP R8, R7 + BHS search_loop_encodeBetterBlockAsm4MB + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x08, R9, R9 + MUL R5, R9, R9 + LSR $0x2f, R9, R9 + LSL $0x08, R10, R10 + MUL R5, R10, R10 + LSR $0x2f, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeBetterBlockAsm4MB + +emit_remainder_encodeBetterBlockAsm4MB: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $4, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBetterBlockAsm4MB + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBetterBlockAsm4MB: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBetterBlockAsm4MB + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBetterBlockAsm4MB + CMPW $0x00010000, R2 + BLO three_bytes_emit_remainder_encodeBetterBlockAsm4MB + MOVWU R2, R3 + LSRW $0x10, R3, R3 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + MOVB R3, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB + +three_bytes_emit_remainder_encodeBetterBlockAsm4MB: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB + +two_bytes_emit_remainder_encodeBetterBlockAsm4MB: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBetterBlockAsm4MB + JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB + +one_byte_emit_remainder_encodeBetterBlockAsm4MB: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBetterBlockAsm4MB: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB + +memmove_long_emit_remainder_encodeBetterBlockAsm4MB: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBetterBlockAsm12B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000280, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBetterBlockAsm12B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBetterBlockAsm12B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $6, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBetterBlockAsm12B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x06, R5, R5 + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBetterBlockAsm12B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x34, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 65536(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 65536(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeBetterBlockAsm12B + CMP R6, R10 + BNE no_short_found_encodeBetterBlockAsm12B + MOVWU R7, R5 + JMP candidate_match_encodeBetterBlockAsm12B + +no_short_found_encodeBetterBlockAsm12B: + CMPW R6, R9 + BEQ candidate_match_encodeBetterBlockAsm12B + CMPW R6, R10 + BEQ candidateS_match_encodeBetterBlockAsm12B + MOVWU 28(RSP), R2 + JMP search_loop_encodeBetterBlockAsm12B + +candidateS_match_encodeBetterBlockAsm12B: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBetterBlockAsm12B + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBetterBlockAsm12B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm12B + +match_extend_back_loop_encodeBetterBlockAsm12B: + CMPW R6, R2 + BLS match_extend_back_end_encodeBetterBlockAsm12B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBetterBlockAsm12B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm12B + JMP match_extend_back_loop_encodeBetterBlockAsm12B + +match_extend_back_end_encodeBetterBlockAsm12B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBetterBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBetterBlockAsm12B: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeBetterBlockAsm12B + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B + +matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm12B + +matchlen_match8_match_nolit_encodeBetterBlockAsm12B: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeBetterBlockAsm12B + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeBetterBlockAsm12B + +matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm12B + +matchlen_match4_match_nolit_encodeBetterBlockAsm12B: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeBetterBlockAsm12B + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeBetterBlockAsm12B + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeBetterBlockAsm12B: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm12B + BLO match_nolit_end_encodeBetterBlockAsm12B + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBetterBlockAsm12B + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeBetterBlockAsm12B + +matchlen_match1_match_nolit_encodeBetterBlockAsm12B: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBetterBlockAsm12B + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeBetterBlockAsm12B: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + MOVWU 24(RSP), R16 + CMPW R7, R16 + BEQ match_is_repeat_encodeBetterBlockAsm12B + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeBetterBlockAsm12B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeBetterBlockAsm12B + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeBetterBlockAsm12B + BLO three_bytes_match_emit_encodeBetterBlockAsm12B + +three_bytes_match_emit_encodeBetterBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBetterBlockAsm12B + +two_bytes_match_emit_encodeBetterBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeBetterBlockAsm12B + JMP memmove_long_match_emit_encodeBetterBlockAsm12B + +one_byte_match_emit_encodeBetterBlockAsm12B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBetterBlockAsm12B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x04, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4 + CMP $0x08, R8 + BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4: + MOVWU (R9), R10 + MOVW R10, (R1) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7: + MOVWU (R9), R10 + ADD R8, R9, R15 + MOVWU -4(R15), R9 + MOVW R10, (R1) + ADD R8, R1, R15 + MOVW R9, -4(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBetterBlockAsm12B: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeBetterBlockAsm12B + +memmove_long_match_emit_encodeBetterBlockAsm12B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeBetterBlockAsm12B: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B + CMPW $0x00000800, R7 + BHS long_offset_short_match_nolit_encodeBetterBlockAsm12B + MOVD $0x00000001, R5 + ADD $16, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R11, R11 + + // emitRepeat + SUB $4, R11, R11 + MOVWU R11, R11 + JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +long_offset_short_match_nolit_encodeBetterBlockAsm12B: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeBetterBlockAsm12B + CMPW $0x00000800, R7 + BHS emit_copy_three_match_nolit_encodeBetterBlockAsm12B + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +emit_copy_three_match_nolit_encodeBetterBlockAsm12B: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +match_is_repeat_encodeBetterBlockAsm12B: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_repeat_encodeBetterBlockAsm12B + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm12B + BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm12B + +three_bytes_match_emit_repeat_encodeBetterBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B + +two_bytes_match_emit_repeat_encodeBetterBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_repeat_encodeBetterBlockAsm12B + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B + +one_byte_match_emit_repeat_encodeBetterBlockAsm12B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_repeat_encodeBetterBlockAsm12B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x04, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4 + CMP $0x08, R8 + BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4: + MOVWU (R9), R10 + MOVW R10, (R1) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7: + MOVWU (R9), R10 + ADD R8, R9, R15 + MOVWU -4(R15), R9 + MOVW R10, (R1) + ADD R8, R1, R15 + MOVW R9, -4(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B: + MOVD R5, R1 + JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B + +memmove_long_match_emit_repeat_encodeBetterBlockAsm12B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitRepeat + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B + +cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B + +repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + +match_nolit_emitcopy_end_encodeBetterBlockAsm12B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBetterBlockAsm12B + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBetterBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBetterBlockAsm12B: + MOVD $0x0000cf1bbcdcbf9b, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x32, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x34, R10, R10 + LSL $0x10, R11, R11 + MUL R5, R11, R11 + LSR $0x32, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x34, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 65536(R15) + ADD R12<<2, R0, R15 + MOVW R13, 65536(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeBetterBlockAsm12B: + CMP R8, R7 + BHS search_loop_encodeBetterBlockAsm12B + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x32, R9, R9 + LSL $0x10, R10, R10 + MUL R5, R10, R10 + LSR $0x32, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeBetterBlockAsm12B + +emit_remainder_encodeBetterBlockAsm12B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBetterBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBetterBlockAsm12B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm12B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBetterBlockAsm12B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBetterBlockAsm12B + BLO three_bytes_emit_remainder_encodeBetterBlockAsm12B + +three_bytes_emit_remainder_encodeBetterBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B + +two_bytes_emit_remainder_encodeBetterBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBetterBlockAsm12B + JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B + +one_byte_emit_remainder_encodeBetterBlockAsm12B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBetterBlockAsm12B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm12B + +memmove_long_emit_remainder_encodeBetterBlockAsm12B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBetterBlockAsm12B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBetterBlockAsm10B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x000000a0, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBetterBlockAsm10B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBetterBlockAsm10B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $6, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBetterBlockAsm10B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x05, R5, R5 + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBetterBlockAsm10B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x34, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x36, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 16384(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 16384(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeBetterBlockAsm10B + CMP R6, R10 + BNE no_short_found_encodeBetterBlockAsm10B + MOVWU R7, R5 + JMP candidate_match_encodeBetterBlockAsm10B + +no_short_found_encodeBetterBlockAsm10B: + CMPW R6, R9 + BEQ candidate_match_encodeBetterBlockAsm10B + CMPW R6, R10 + BEQ candidateS_match_encodeBetterBlockAsm10B + MOVWU 28(RSP), R2 + JMP search_loop_encodeBetterBlockAsm10B + +candidateS_match_encodeBetterBlockAsm10B: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x34, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBetterBlockAsm10B + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBetterBlockAsm10B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm10B + +match_extend_back_loop_encodeBetterBlockAsm10B: + CMPW R6, R2 + BLS match_extend_back_end_encodeBetterBlockAsm10B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBetterBlockAsm10B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm10B + JMP match_extend_back_loop_encodeBetterBlockAsm10B + +match_extend_back_end_encodeBetterBlockAsm10B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBetterBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBetterBlockAsm10B: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeBetterBlockAsm10B + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B + +matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm10B + +matchlen_match8_match_nolit_encodeBetterBlockAsm10B: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeBetterBlockAsm10B + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeBetterBlockAsm10B + +matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm10B + +matchlen_match4_match_nolit_encodeBetterBlockAsm10B: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeBetterBlockAsm10B + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeBetterBlockAsm10B + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeBetterBlockAsm10B: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm10B + BLO match_nolit_end_encodeBetterBlockAsm10B + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBetterBlockAsm10B + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeBetterBlockAsm10B + +matchlen_match1_match_nolit_encodeBetterBlockAsm10B: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBetterBlockAsm10B + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeBetterBlockAsm10B: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + MOVWU 24(RSP), R16 + CMPW R7, R16 + BEQ match_is_repeat_encodeBetterBlockAsm10B + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeBetterBlockAsm10B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeBetterBlockAsm10B + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeBetterBlockAsm10B + BLO three_bytes_match_emit_encodeBetterBlockAsm10B + +three_bytes_match_emit_encodeBetterBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBetterBlockAsm10B + +two_bytes_match_emit_encodeBetterBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeBetterBlockAsm10B + JMP memmove_long_match_emit_encodeBetterBlockAsm10B + +one_byte_match_emit_encodeBetterBlockAsm10B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBetterBlockAsm10B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x04, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4 + CMP $0x08, R8 + BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4: + MOVWU (R9), R10 + MOVW R10, (R1) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7: + MOVWU (R9), R10 + ADD R8, R9, R15 + MOVWU -4(R15), R9 + MOVW R10, (R1) + ADD R8, R1, R15 + MOVW R9, -4(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBetterBlockAsm10B: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeBetterBlockAsm10B + +memmove_long_match_emit_encodeBetterBlockAsm10B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeBetterBlockAsm10B: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B + CMPW $0x00000800, R7 + BHS long_offset_short_match_nolit_encodeBetterBlockAsm10B + MOVD $0x00000001, R5 + ADD $16, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R11, R11 + + // emitRepeat + SUB $4, R11, R11 + MOVWU R11, R11 + JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +long_offset_short_match_nolit_encodeBetterBlockAsm10B: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeBetterBlockAsm10B + CMPW $0x00000800, R7 + BHS emit_copy_three_match_nolit_encodeBetterBlockAsm10B + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +emit_copy_three_match_nolit_encodeBetterBlockAsm10B: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +match_is_repeat_encodeBetterBlockAsm10B: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_repeat_encodeBetterBlockAsm10B + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm10B + BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm10B + +three_bytes_match_emit_repeat_encodeBetterBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B + +two_bytes_match_emit_repeat_encodeBetterBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_repeat_encodeBetterBlockAsm10B + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B + +one_byte_match_emit_repeat_encodeBetterBlockAsm10B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_repeat_encodeBetterBlockAsm10B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x04, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4 + CMP $0x08, R8 + BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4: + MOVWU (R9), R10 + MOVW R10, (R1) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7: + MOVWU (R9), R10 + ADD R8, R9, R15 + MOVWU -4(R15), R9 + MOVW R10, (R1) + ADD R8, R1, R15 + MOVW R9, -4(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B: + MOVD R5, R1 + JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B + +memmove_long_match_emit_repeat_encodeBetterBlockAsm10B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitRepeat + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B + CMPW $0x00000800, R7 + BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B + +cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B + +repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B: + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + +match_nolit_emitcopy_end_encodeBetterBlockAsm10B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBetterBlockAsm10B + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBetterBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBetterBlockAsm10B: + MOVD $0x0000cf1bbcdcbf9b, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x34, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x36, R10, R10 + LSL $0x10, R11, R11 + MUL R5, R11, R11 + LSR $0x34, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x36, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 16384(R15) + ADD R12<<2, R0, R15 + MOVW R13, 16384(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeBetterBlockAsm10B: + CMP R8, R7 + BHS search_loop_encodeBetterBlockAsm10B + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x34, R9, R9 + LSL $0x10, R10, R10 + MUL R5, R10, R10 + LSR $0x34, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeBetterBlockAsm10B + +emit_remainder_encodeBetterBlockAsm10B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBetterBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBetterBlockAsm10B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm10B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBetterBlockAsm10B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBetterBlockAsm10B + BLO three_bytes_emit_remainder_encodeBetterBlockAsm10B + +three_bytes_emit_remainder_encodeBetterBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B + +two_bytes_emit_remainder_encodeBetterBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBetterBlockAsm10B + JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B + +one_byte_emit_remainder_encodeBetterBlockAsm10B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBetterBlockAsm10B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm10B + +memmove_long_emit_remainder_encodeBetterBlockAsm10B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBetterBlockAsm10B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeBetterBlockAsm8B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000028, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeBetterBlockAsm8B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeBetterBlockAsm8B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $6, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeBetterBlockAsm8B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x04, R5, R5 + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeBetterBlockAsm8B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x36, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x38, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 4096(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 4096(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeBetterBlockAsm8B + CMP R6, R10 + BNE no_short_found_encodeBetterBlockAsm8B + MOVWU R7, R5 + JMP candidate_match_encodeBetterBlockAsm8B + +no_short_found_encodeBetterBlockAsm8B: + CMPW R6, R9 + BEQ candidate_match_encodeBetterBlockAsm8B + CMPW R6, R10 + BEQ candidateS_match_encodeBetterBlockAsm8B + MOVWU 28(RSP), R2 + JMP search_loop_encodeBetterBlockAsm8B + +candidateS_match_encodeBetterBlockAsm8B: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x36, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeBetterBlockAsm8B + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeBetterBlockAsm8B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm8B + +match_extend_back_loop_encodeBetterBlockAsm8B: + CMPW R6, R2 + BLS match_extend_back_end_encodeBetterBlockAsm8B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeBetterBlockAsm8B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeBetterBlockAsm8B + JMP match_extend_back_loop_encodeBetterBlockAsm8B + +match_extend_back_end_encodeBetterBlockAsm8B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeBetterBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeBetterBlockAsm8B: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeBetterBlockAsm8B + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B + +matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm8B + +matchlen_match8_match_nolit_encodeBetterBlockAsm8B: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeBetterBlockAsm8B + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeBetterBlockAsm8B + +matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeBetterBlockAsm8B + +matchlen_match4_match_nolit_encodeBetterBlockAsm8B: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeBetterBlockAsm8B + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeBetterBlockAsm8B + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeBetterBlockAsm8B: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm8B + BLO match_nolit_end_encodeBetterBlockAsm8B + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeBetterBlockAsm8B + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeBetterBlockAsm8B + +matchlen_match1_match_nolit_encodeBetterBlockAsm8B: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeBetterBlockAsm8B + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeBetterBlockAsm8B: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + MOVWU 24(RSP), R16 + CMPW R7, R16 + BEQ match_is_repeat_encodeBetterBlockAsm8B + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeBetterBlockAsm8B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeBetterBlockAsm8B + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeBetterBlockAsm8B + BLO three_bytes_match_emit_encodeBetterBlockAsm8B + +three_bytes_match_emit_encodeBetterBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeBetterBlockAsm8B + +two_bytes_match_emit_encodeBetterBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeBetterBlockAsm8B + JMP memmove_long_match_emit_encodeBetterBlockAsm8B + +one_byte_match_emit_encodeBetterBlockAsm8B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeBetterBlockAsm8B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x04, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4 + CMP $0x08, R8 + BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4: + MOVWU (R9), R10 + MOVW R10, (R1) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7: + MOVWU (R9), R10 + ADD R8, R9, R15 + MOVWU -4(R15), R9 + MOVW R10, (R1) + ADD R8, R1, R15 + MOVW R9, -4(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B + +emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeBetterBlockAsm8B: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeBetterBlockAsm8B + +memmove_long_match_emit_encodeBetterBlockAsm8B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeBetterBlockAsm8B: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B + CMPW $0x00000800, R7 + BHS long_offset_short_match_nolit_encodeBetterBlockAsm8B + MOVD $0x00000001, R5 + ADD $16, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + SUBW $0x08, R11, R11 + + // emitRepeat + SUB $4, R11, R11 + MOVWU R11, R11 + JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +long_offset_short_match_nolit_encodeBetterBlockAsm8B: + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + + // emitRepeat + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short + +cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeBetterBlockAsm8B + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +emit_copy_three_match_nolit_encodeBetterBlockAsm8B: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +match_is_repeat_encodeBetterBlockAsm8B: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B + MOVWU R6, R7 + MOVW R6, 20(RSP) + ADD R5, R3, R8 + SUBW R5, R7, R7 + SUB $1, R7, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_repeat_encodeBetterBlockAsm8B + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm8B + BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm8B + +three_bytes_match_emit_repeat_encodeBetterBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B + +two_bytes_match_emit_repeat_encodeBetterBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_repeat_encodeBetterBlockAsm8B + JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B + +one_byte_match_emit_repeat_encodeBetterBlockAsm8B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_repeat_encodeBetterBlockAsm8B: + ADD R7, R1, R5 + + // genMemMoveShort + CMP $0x04, R7 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4 + CMP $0x08, R7 + BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7 + CMP $0x10, R7 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16 + CMP $0x20, R7 + BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4: + MOVWU (R8), R9 + MOVW R9, (R1) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7: + MOVWU (R8), R9 + ADD R7, R8, R15 + MOVWU -4(R15), R8 + MOVW R9, (R1) + ADD R7, R1, R15 + MOVW R8, -4(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16: + MOVD (R8), R9 + ADD R7, R8, R15 + MOVD -8(R15), R8 + MOVD R9, (R1) + ADD R7, R1, R15 + MOVD R8, -8(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32: + FMOVQ (R8), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B + +emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64: + FMOVQ (R8), F0 + FMOVQ 16(R8), F1 + ADD R7, R8, R15 + FMOVQ -32(R15), F2 + ADD R7, R8, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R7, R1, R15 + FMOVQ F2, -32(R15) + ADD R7, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B: + MOVD R5, R1 + JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B + +memmove_long_match_emit_repeat_encodeBetterBlockAsm8B: + ADD R7, R1, R5 + + // genMemMoveLong + MOVD R8, R9 + MOVD R1, R10 + MOVD R7, R12 + +emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R12, R12 + CMP $0x20, R12 + BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back + ADD R7, R8, R15 + FMOVQ -32(R15), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + ADD R7, R1, R15 + FMOVQ F0, -32(R15) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitRepeat + MOVWU R11, R5 + SUB $4, R11, R11 + MOVWU R11, R11 + CMPW $0x08, R5 + BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B + +cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B: + CMPW $0x00000104, R11 + BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B + SUB $256, R11, R11 + MOVWU R11, R11 + MOVD $0x0019, R16 + MOVH R16, (R1) + MOVH R11, 2(R1) + ADD $0x04, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B: + SUB $4, R11, R11 + MOVWU R11, R11 + MOVD $0x0015, R16 + MOVH R16, (R1) + MOVB R11, 2(R1) + ADD $0x03, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + +repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B: + LSLW $0x02, R11, R11 + ORRW $0x01, R11, R11 + MOVH R11, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B + MOVD $0, R5 + ADD R11<<2, R5, R11 + ADD $1, R11, R11 + MOVWU R11, R11 + MOVB R7, 1(R1) + ASRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R11, R11 + MOVB R11, (R1) + ADD $0x02, R1, R1 + +match_nolit_emitcopy_end_encodeBetterBlockAsm8B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeBetterBlockAsm8B + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeBetterBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeBetterBlockAsm8B: + MOVD $0x0000cf1bbcdcbf9b, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x36, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x38, R10, R10 + LSL $0x10, R11, R11 + MUL R5, R11, R11 + LSR $0x36, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x38, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 4096(R15) + ADD R12<<2, R0, R15 + MOVW R13, 4096(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeBetterBlockAsm8B: + CMP R8, R7 + BHS search_loop_encodeBetterBlockAsm8B + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x36, R9, R9 + LSL $0x10, R10, R10 + MUL R5, R10, R10 + LSR $0x36, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeBetterBlockAsm8B + +emit_remainder_encodeBetterBlockAsm8B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeBetterBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeBetterBlockAsm8B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm8B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeBetterBlockAsm8B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeBetterBlockAsm8B + BLO three_bytes_emit_remainder_encodeBetterBlockAsm8B + +three_bytes_emit_remainder_encodeBetterBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B + +two_bytes_emit_remainder_encodeBetterBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeBetterBlockAsm8B + JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B + +one_byte_emit_remainder_encodeBetterBlockAsm8B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeBetterBlockAsm8B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm8B + +memmove_long_emit_remainder_encodeBetterBlockAsm8B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeBetterBlockAsm8B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBlockAsm(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000200, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBlockAsm: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBlockAsm + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBlockAsm: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x06, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBlockAsm + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + LSL $0x10, R10, R10 + MUL R8, R10, R10 + LSR $0x32, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeSnappyBlockAsm + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R5 + MOVWU R6, R7 + MOVWU 24(RSP), R16 + SUBSW R16, R7, R7 + BEQ repeat_extend_back_end_encodeSnappyBlockAsm + +repeat_extend_back_loop_encodeSnappyBlockAsm: + CMPW R5, R6 + BLS repeat_extend_back_end_encodeSnappyBlockAsm + ADD R7, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeSnappyBlockAsm + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R7, R7 + BNE repeat_extend_back_loop_encodeSnappyBlockAsm + +repeat_extend_back_end_encodeSnappyBlockAsm: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $5, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeSnappyBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeSnappyBlockAsm: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm + MOVWU R6, R7 + MOVW R6, 20(RSP) + ADD R5, R3, R8 + SUBW R5, R7, R7 + SUB $1, R7, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeSnappyBlockAsm + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeSnappyBlockAsm + CMPW $0x00010000, R5 + BLO three_bytes_repeat_emit_encodeSnappyBlockAsm + CMPW $0x01000000, R5 + BLO four_bytes_repeat_emit_encodeSnappyBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R5, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm + +four_bytes_repeat_emit_encodeSnappyBlockAsm: + MOVWU R5, R9 + LSRW $0x10, R9, R9 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + MOVB R9, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm + +three_bytes_repeat_emit_encodeSnappyBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm + +two_bytes_repeat_emit_encodeSnappyBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeSnappyBlockAsm + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm + +one_byte_repeat_emit_encodeSnappyBlockAsm: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeSnappyBlockAsm: + ADD R7, R1, R5 + + // genMemMoveShort + CMP $0x08, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8 + CMP $0x10, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16 + CMP $0x20, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8: + MOVD (R8), R9 + MOVD R9, (R1) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16: + MOVD (R8), R9 + ADD R7, R8, R15 + MOVD -8(R15), R8 + MOVD R9, (R1) + ADD R7, R1, R15 + MOVD R8, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32: + FMOVQ (R8), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64: + FMOVQ (R8), F0 + FMOVQ 16(R8), F1 + ADD R7, R8, R15 + FMOVQ -32(R15), F2 + ADD R7, R8, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R7, R1, R15 + FMOVQ F2, -32(R15) + ADD R7, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeSnappyBlockAsm: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm + +memmove_long_repeat_emit_encodeSnappyBlockAsm: + ADD R7, R1, R5 + + // genMemMoveLong + MOVD R8, R9 + MOVD R1, R10 + MOVD R7, R11 + +emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back + ADD R7, R8, R15 + FMOVQ -32(R15), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + ADD R7, R1, R15 + FMOVQ F0, -32(R15) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeSnappyBlockAsm: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R10 + +matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm: + CMPW $0x10, R7 + BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm + MOVD (R8)(R10), R9 + ADD R10, R8, R15 + MOVD 8(R15), R11 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm + ADD R10, R5, R15 + MOVD 8(R15), R16 + EOR R16, R11, R11 + TST R11, R11 + BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R10, R10 + MOVWU R10, R10 + JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm + +matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm: + RBIT R11, R11 + CLZ R11, R11 + ASR $0x03, R11, R11 + ADD R11, R10, R10 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm + +matchlen_match8_repeat_extend_encodeSnappyBlockAsm: + CMPW $0x08, R7 + BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm + MOVD (R8)(R10), R9 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm + +matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm: + RBIT R9, R9 + CLZ R9, R9 + ASR $0x03, R9, R9 + ADD R9, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm + +matchlen_match4_repeat_extend_encodeSnappyBlockAsm: + CMPW $0x04, R7 + BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm + MOVWU (R8)(R10), R9 + MOVWU (R5)(R10), R16 + CMPW R9, R16 + BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R10, R10 + MOVWU R10, R10 + +matchlen_match2_repeat_extend_encodeSnappyBlockAsm: + CMPW $0x01, R7 + BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm + BLO repeat_extend_forward_end_encodeSnappyBlockAsm + MOVHU (R8)(R10), R16 + BFI $0, R16, $16, R9 + ADD R10, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R9, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm + ADD $2, R10, R10 + MOVWU R10, R10 + SUBSW $0x02, R7, R7 + BEQ repeat_extend_forward_end_encodeSnappyBlockAsm + +matchlen_match1_repeat_extend_encodeSnappyBlockAsm: + MOVBU (R8)(R10), R16 + BFI $0, R16, $8, R9 + ADD R10, R5, R15 + MOVBU (R15), R15 + AND $0xff, R9, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeSnappyBlockAsm + ADD $1, R10, R10 + MOVWU R10, R10 + +repeat_extend_forward_end_encodeSnappyBlockAsm: + ADDW R10, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + + // emitCopy + CMPW $0x00010000, R6 + BLO two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm + +four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm: + CMPW $0x40, R5 + BLS four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm + MOVD $0xff, R16 + MOVB R16, (R1) + MOVW R6, 1(R1) + SUB $64, R5, R5 + MOVWU R5, R5 + ADD $0x05, R1, R1 + CMPW $0x04, R5 + BLO four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm + JMP four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm + +four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm: + TSTW R5, R5 + BEQ repeat_end_emit_encodeSnappyBlockAsm + MOVD $0, R7 + ADD R5<<2, R7, R5 + SUB $1, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVW R6, 1(R1) + ADD $0x05, R1, R1 + JMP repeat_end_emit_encodeSnappyBlockAsm + +two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm: + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm + +two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm + CMPW $0x00000800, R6 + BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeSnappyBlockAsm + +emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeSnappyBlockAsm: + MOVW R2, 20(RSP) + JMP search_loop_encodeSnappyBlockAsm + +no_repeat_found_encodeSnappyBlockAsm: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBlockAsm + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeSnappyBlockAsm + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeSnappyBlockAsm + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBlockAsm + +candidate3_match_encodeSnappyBlockAsm: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeSnappyBlockAsm + +candidate2_match_encodeSnappyBlockAsm: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBlockAsm: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm + +match_extend_back_loop_encodeSnappyBlockAsm: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBlockAsm + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBlockAsm + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm + JMP match_extend_back_loop_encodeSnappyBlockAsm + +match_extend_back_end_encodeSnappyBlockAsm: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $5, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBlockAsm: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeSnappyBlockAsm + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeSnappyBlockAsm + CMPW $0x00010000, R7 + BLO three_bytes_match_emit_encodeSnappyBlockAsm + CMPW $0x01000000, R7 + BLO four_bytes_match_emit_encodeSnappyBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R7, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBlockAsm + +four_bytes_match_emit_encodeSnappyBlockAsm: + MOVWU R7, R9 + LSRW $0x10, R9, R9 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + MOVB R9, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBlockAsm + +three_bytes_match_emit_encodeSnappyBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBlockAsm + +two_bytes_match_emit_encodeSnappyBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeSnappyBlockAsm + JMP memmove_long_match_emit_encodeSnappyBlockAsm + +one_byte_match_emit_encodeSnappyBlockAsm: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBlockAsm: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBlockAsm: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeSnappyBlockAsm + +memmove_long_match_emit_encodeSnappyBlockAsm: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeSnappyBlockAsm: +match_nolit_loop_encodeSnappyBlockAsm: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm + +matchlen_bsf_16match_nolit_encodeSnappyBlockAsm: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm + +matchlen_match8_match_nolit_encodeSnappyBlockAsm: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm + +matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm + +matchlen_match4_match_nolit_encodeSnappyBlockAsm: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeSnappyBlockAsm: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm + BLO match_nolit_end_encodeSnappyBlockAsm + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeSnappyBlockAsm + +matchlen_match1_match_nolit_encodeSnappyBlockAsm: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBlockAsm + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeSnappyBlockAsm: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x00010000, R5 + BLO two_byte_offset_match_nolit_encodeSnappyBlockAsm + +four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm: + CMPW $0x40, R9 + BLS four_bytes_remain_match_nolit_encodeSnappyBlockAsm + MOVD $0xff, R16 + MOVB R16, (R1) + MOVW R5, 1(R1) + SUB $64, R9, R9 + MOVWU R9, R9 + ADD $0x05, R1, R1 + CMPW $0x04, R9 + BLO four_bytes_remain_match_nolit_encodeSnappyBlockAsm + JMP four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm + +four_bytes_remain_match_nolit_encodeSnappyBlockAsm: + TSTW R9, R9 + BEQ match_nolit_emitcopy_end_encodeSnappyBlockAsm + MOVD $0, R6 + ADD R9<<2, R6, R9 + SUB $1, R9, R9 + MOVWU R9, R9 + MOVB R9, (R1) + MOVW R5, 1(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm + +two_byte_offset_match_nolit_encodeSnappyBlockAsm: + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm + +two_byte_offset_short_match_nolit_encodeSnappyBlockAsm: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm + CMPW $0x00000800, R5 + BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm + +emit_copy_three_match_nolit_encodeSnappyBlockAsm: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBlockAsm: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBlockAsm + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBlockAsm: + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x10, R7, R7 + MUL R8, R7, R7 + LSR $0x32, R7, R7 + LSL $0x10, R5, R5 + MUL R8, R5, R5 + LSR $0x32, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeSnappyBlockAsm + ADDW $1, R2, R2 + JMP search_loop_encodeSnappyBlockAsm + +emit_remainder_encodeSnappyBlockAsm: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $5, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBlockAsm: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBlockAsm + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBlockAsm + CMPW $0x00010000, R2 + BLO three_bytes_emit_remainder_encodeSnappyBlockAsm + CMPW $0x01000000, R2 + BLO four_bytes_emit_remainder_encodeSnappyBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R2, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm + +four_bytes_emit_remainder_encodeSnappyBlockAsm: + MOVWU R2, R3 + LSRW $0x10, R3, R3 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + MOVB R3, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm + +three_bytes_emit_remainder_encodeSnappyBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm + +two_bytes_emit_remainder_encodeSnappyBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBlockAsm + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm + +one_byte_emit_remainder_encodeSnappyBlockAsm: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBlockAsm: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBlockAsm: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm + +memmove_long_emit_remainder_encodeSnappyBlockAsm: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBlockAsm: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBlockAsm64K(dst []byte, src []byte, tmp *[65536]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBlockAsm64K(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000200, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBlockAsm64K: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBlockAsm64K + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBlockAsm64K: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x06, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBlockAsm64K + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + LSL $0x10, R10, R10 + MUL R8, R10, R10 + LSR $0x32, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeSnappyBlockAsm64K + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R5 + MOVWU R6, R7 + MOVWU 24(RSP), R16 + SUBSW R16, R7, R7 + BEQ repeat_extend_back_end_encodeSnappyBlockAsm64K + +repeat_extend_back_loop_encodeSnappyBlockAsm64K: + CMPW R5, R6 + BLS repeat_extend_back_end_encodeSnappyBlockAsm64K + ADD R7, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeSnappyBlockAsm64K + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R7, R7 + BNE repeat_extend_back_loop_encodeSnappyBlockAsm64K + +repeat_extend_back_end_encodeSnappyBlockAsm64K: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $3, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeSnappyBlockAsm64K + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeSnappyBlockAsm64K: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K + MOVWU R6, R7 + MOVW R6, 20(RSP) + ADD R5, R3, R8 + SUBW R5, R7, R7 + SUB $1, R7, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeSnappyBlockAsm64K + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeSnappyBlockAsm64K + BLO three_bytes_repeat_emit_encodeSnappyBlockAsm64K + +three_bytes_repeat_emit_encodeSnappyBlockAsm64K: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K + +two_bytes_repeat_emit_encodeSnappyBlockAsm64K: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeSnappyBlockAsm64K + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K + +one_byte_repeat_emit_encodeSnappyBlockAsm64K: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeSnappyBlockAsm64K: + ADD R7, R1, R5 + + // genMemMoveShort + CMP $0x08, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8 + CMP $0x10, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16 + CMP $0x20, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8: + MOVD (R8), R9 + MOVD R9, (R1) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16: + MOVD (R8), R9 + ADD R7, R8, R15 + MOVD -8(R15), R8 + MOVD R9, (R1) + ADD R7, R1, R15 + MOVD R8, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32: + FMOVQ (R8), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64: + FMOVQ (R8), F0 + FMOVQ 16(R8), F1 + ADD R7, R8, R15 + FMOVQ -32(R15), F2 + ADD R7, R8, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R7, R1, R15 + FMOVQ F2, -32(R15) + ADD R7, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K + +memmove_long_repeat_emit_encodeSnappyBlockAsm64K: + ADD R7, R1, R5 + + // genMemMoveLong + MOVD R8, R9 + MOVD R1, R10 + MOVD R7, R11 + +emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back + ADD R7, R8, R15 + FMOVQ -32(R15), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + ADD R7, R1, R15 + FMOVQ F0, -32(R15) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R10 + +matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K: + CMPW $0x10, R7 + BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K + MOVD (R8)(R10), R9 + ADD R10, R8, R15 + MOVD 8(R15), R11 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K + ADD R10, R5, R15 + MOVD 8(R15), R16 + EOR R16, R11, R11 + TST R11, R11 + BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R10, R10 + MOVWU R10, R10 + JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K + +matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K: + RBIT R11, R11 + CLZ R11, R11 + ASR $0x03, R11, R11 + ADD R11, R10, R10 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K + +matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K: + CMPW $0x08, R7 + BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K + MOVD (R8)(R10), R9 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K + +matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K: + RBIT R9, R9 + CLZ R9, R9 + ASR $0x03, R9, R9 + ADD R9, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K + +matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K: + CMPW $0x04, R7 + BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K + MOVWU (R8)(R10), R9 + MOVWU (R5)(R10), R16 + CMPW R9, R16 + BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R10, R10 + MOVWU R10, R10 + +matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K: + CMPW $0x01, R7 + BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K + BLO repeat_extend_forward_end_encodeSnappyBlockAsm64K + MOVHU (R8)(R10), R16 + BFI $0, R16, $16, R9 + ADD R10, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R9, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K + ADD $2, R10, R10 + MOVWU R10, R10 + SUBSW $0x02, R7, R7 + BEQ repeat_extend_forward_end_encodeSnappyBlockAsm64K + +matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K: + MOVBU (R8)(R10), R16 + BFI $0, R16, $8, R9 + ADD R10, R5, R15 + MOVBU (R15), R15 + AND $0xff, R9, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeSnappyBlockAsm64K + ADD $1, R10, R10 + MOVWU R10, R10 + +repeat_extend_forward_end_encodeSnappyBlockAsm64K: + ADDW R10, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + + // emitCopy +two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K: + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K + +two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K + CMPW $0x00000800, R6 + BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeSnappyBlockAsm64K + +emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeSnappyBlockAsm64K: + MOVW R2, 20(RSP) + JMP search_loop_encodeSnappyBlockAsm64K + +no_repeat_found_encodeSnappyBlockAsm64K: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBlockAsm64K + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeSnappyBlockAsm64K + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeSnappyBlockAsm64K + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBlockAsm64K + +candidate3_match_encodeSnappyBlockAsm64K: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeSnappyBlockAsm64K + +candidate2_match_encodeSnappyBlockAsm64K: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBlockAsm64K: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm64K + +match_extend_back_loop_encodeSnappyBlockAsm64K: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBlockAsm64K + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBlockAsm64K + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm64K + JMP match_extend_back_loop_encodeSnappyBlockAsm64K + +match_extend_back_end_encodeSnappyBlockAsm64K: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBlockAsm64K + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBlockAsm64K: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm64K + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeSnappyBlockAsm64K + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeSnappyBlockAsm64K + BLO three_bytes_match_emit_encodeSnappyBlockAsm64K + +three_bytes_match_emit_encodeSnappyBlockAsm64K: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBlockAsm64K + +two_bytes_match_emit_encodeSnappyBlockAsm64K: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeSnappyBlockAsm64K + JMP memmove_long_match_emit_encodeSnappyBlockAsm64K + +one_byte_match_emit_encodeSnappyBlockAsm64K: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBlockAsm64K: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBlockAsm64K: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeSnappyBlockAsm64K + +memmove_long_match_emit_encodeSnappyBlockAsm64K: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeSnappyBlockAsm64K: +match_nolit_loop_encodeSnappyBlockAsm64K: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm64K + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K + +matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm64K + +matchlen_match8_match_nolit_encodeSnappyBlockAsm64K: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm64K + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm64K + +matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm64K + +matchlen_match4_match_nolit_encodeSnappyBlockAsm64K: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm64K + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm64K + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeSnappyBlockAsm64K: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm64K + BLO match_nolit_end_encodeSnappyBlockAsm64K + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm64K + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeSnappyBlockAsm64K + +matchlen_match1_match_nolit_encodeSnappyBlockAsm64K: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBlockAsm64K + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeSnappyBlockAsm64K: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy +two_byte_offset_match_nolit_encodeSnappyBlockAsm64K: + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm64K + +two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm64K + CMPW $0x00000800, R5 + BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm64K + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm64K + +emit_copy_three_match_nolit_encodeSnappyBlockAsm64K: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBlockAsm64K: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBlockAsm64K + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBlockAsm64K + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBlockAsm64K: + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x10, R7, R7 + MUL R8, R7, R7 + LSR $0x32, R7, R7 + LSL $0x10, R5, R5 + MUL R8, R5, R5 + LSR $0x32, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeSnappyBlockAsm64K + ADDW $1, R2, R2 + JMP search_loop_encodeSnappyBlockAsm64K + +emit_remainder_encodeSnappyBlockAsm64K: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBlockAsm64K + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBlockAsm64K: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBlockAsm64K + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBlockAsm64K + BLO three_bytes_emit_remainder_encodeSnappyBlockAsm64K + +three_bytes_emit_remainder_encodeSnappyBlockAsm64K: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K + +two_bytes_emit_remainder_encodeSnappyBlockAsm64K: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBlockAsm64K + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K + +one_byte_emit_remainder_encodeSnappyBlockAsm64K: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBlockAsm64K: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K + +memmove_long_emit_remainder_encodeSnappyBlockAsm64K: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBlockAsm12B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000080, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBlockAsm12B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBlockAsm12B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBlockAsm12B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x05, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBlockAsm12B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x000000cf1bbcdcbb, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x18, R9, R9 + MUL R8, R9, R9 + LSR $0x34, R9, R9 + LSL $0x18, R10, R10 + MUL R8, R10, R10 + LSR $0x34, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x18, R9, R9 + MUL R8, R9, R9 + LSR $0x34, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeSnappyBlockAsm12B + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R5 + MOVWU R6, R7 + MOVWU 24(RSP), R16 + SUBSW R16, R7, R7 + BEQ repeat_extend_back_end_encodeSnappyBlockAsm12B + +repeat_extend_back_loop_encodeSnappyBlockAsm12B: + CMPW R5, R6 + BLS repeat_extend_back_end_encodeSnappyBlockAsm12B + ADD R7, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeSnappyBlockAsm12B + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R7, R7 + BNE repeat_extend_back_loop_encodeSnappyBlockAsm12B + +repeat_extend_back_end_encodeSnappyBlockAsm12B: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $3, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeSnappyBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeSnappyBlockAsm12B: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B + MOVWU R6, R7 + MOVW R6, 20(RSP) + ADD R5, R3, R8 + SUBW R5, R7, R7 + SUB $1, R7, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeSnappyBlockAsm12B + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeSnappyBlockAsm12B + BLO three_bytes_repeat_emit_encodeSnappyBlockAsm12B + +three_bytes_repeat_emit_encodeSnappyBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B + +two_bytes_repeat_emit_encodeSnappyBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeSnappyBlockAsm12B + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B + +one_byte_repeat_emit_encodeSnappyBlockAsm12B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeSnappyBlockAsm12B: + ADD R7, R1, R5 + + // genMemMoveShort + CMP $0x08, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8 + CMP $0x10, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16 + CMP $0x20, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8: + MOVD (R8), R9 + MOVD R9, (R1) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16: + MOVD (R8), R9 + ADD R7, R8, R15 + MOVD -8(R15), R8 + MOVD R9, (R1) + ADD R7, R1, R15 + MOVD R8, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32: + FMOVQ (R8), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64: + FMOVQ (R8), F0 + FMOVQ 16(R8), F1 + ADD R7, R8, R15 + FMOVQ -32(R15), F2 + ADD R7, R8, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R7, R1, R15 + FMOVQ F2, -32(R15) + ADD R7, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B + +memmove_long_repeat_emit_encodeSnappyBlockAsm12B: + ADD R7, R1, R5 + + // genMemMoveLong + MOVD R8, R9 + MOVD R1, R10 + MOVD R7, R11 + +emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back + ADD R7, R8, R15 + FMOVQ -32(R15), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + ADD R7, R1, R15 + FMOVQ F0, -32(R15) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R10 + +matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B: + CMPW $0x10, R7 + BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B + MOVD (R8)(R10), R9 + ADD R10, R8, R15 + MOVD 8(R15), R11 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B + ADD R10, R5, R15 + MOVD 8(R15), R16 + EOR R16, R11, R11 + TST R11, R11 + BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R10, R10 + MOVWU R10, R10 + JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B + +matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B: + RBIT R11, R11 + CLZ R11, R11 + ASR $0x03, R11, R11 + ADD R11, R10, R10 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B + +matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B: + CMPW $0x08, R7 + BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B + MOVD (R8)(R10), R9 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B + +matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B: + RBIT R9, R9 + CLZ R9, R9 + ASR $0x03, R9, R9 + ADD R9, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B + +matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B: + CMPW $0x04, R7 + BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B + MOVWU (R8)(R10), R9 + MOVWU (R5)(R10), R16 + CMPW R9, R16 + BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R10, R10 + MOVWU R10, R10 + +matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B: + CMPW $0x01, R7 + BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B + BLO repeat_extend_forward_end_encodeSnappyBlockAsm12B + MOVHU (R8)(R10), R16 + BFI $0, R16, $16, R9 + ADD R10, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R9, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B + ADD $2, R10, R10 + MOVWU R10, R10 + SUBSW $0x02, R7, R7 + BEQ repeat_extend_forward_end_encodeSnappyBlockAsm12B + +matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B: + MOVBU (R8)(R10), R16 + BFI $0, R16, $8, R9 + ADD R10, R5, R15 + MOVBU (R15), R15 + AND $0xff, R9, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeSnappyBlockAsm12B + ADD $1, R10, R10 + MOVWU R10, R10 + +repeat_extend_forward_end_encodeSnappyBlockAsm12B: + ADDW R10, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + + // emitCopy +two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B: + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B + +two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B + CMPW $0x00000800, R6 + BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeSnappyBlockAsm12B + +emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeSnappyBlockAsm12B: + MOVW R2, 20(RSP) + JMP search_loop_encodeSnappyBlockAsm12B + +no_repeat_found_encodeSnappyBlockAsm12B: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBlockAsm12B + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeSnappyBlockAsm12B + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeSnappyBlockAsm12B + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBlockAsm12B + +candidate3_match_encodeSnappyBlockAsm12B: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeSnappyBlockAsm12B + +candidate2_match_encodeSnappyBlockAsm12B: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBlockAsm12B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm12B + +match_extend_back_loop_encodeSnappyBlockAsm12B: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBlockAsm12B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBlockAsm12B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm12B + JMP match_extend_back_loop_encodeSnappyBlockAsm12B + +match_extend_back_end_encodeSnappyBlockAsm12B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBlockAsm12B: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm12B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeSnappyBlockAsm12B + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeSnappyBlockAsm12B + BLO three_bytes_match_emit_encodeSnappyBlockAsm12B + +three_bytes_match_emit_encodeSnappyBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBlockAsm12B + +two_bytes_match_emit_encodeSnappyBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeSnappyBlockAsm12B + JMP memmove_long_match_emit_encodeSnappyBlockAsm12B + +one_byte_match_emit_encodeSnappyBlockAsm12B: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBlockAsm12B: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBlockAsm12B: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeSnappyBlockAsm12B + +memmove_long_match_emit_encodeSnappyBlockAsm12B: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeSnappyBlockAsm12B: +match_nolit_loop_encodeSnappyBlockAsm12B: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm12B + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B + +matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm12B + +matchlen_match8_match_nolit_encodeSnappyBlockAsm12B: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm12B + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm12B + +matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm12B + +matchlen_match4_match_nolit_encodeSnappyBlockAsm12B: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm12B + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm12B + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeSnappyBlockAsm12B: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm12B + BLO match_nolit_end_encodeSnappyBlockAsm12B + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm12B + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeSnappyBlockAsm12B + +matchlen_match1_match_nolit_encodeSnappyBlockAsm12B: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBlockAsm12B + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeSnappyBlockAsm12B: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy +two_byte_offset_match_nolit_encodeSnappyBlockAsm12B: + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm12B + +two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm12B + CMPW $0x00000800, R5 + BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm12B + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm12B + +emit_copy_three_match_nolit_encodeSnappyBlockAsm12B: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBlockAsm12B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBlockAsm12B + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBlockAsm12B: + MOVD $0x000000cf1bbcdcbb, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x18, R7, R7 + MUL R8, R7, R7 + LSR $0x34, R7, R7 + LSL $0x18, R5, R5 + MUL R8, R5, R5 + LSR $0x34, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeSnappyBlockAsm12B + ADDW $1, R2, R2 + JMP search_loop_encodeSnappyBlockAsm12B + +emit_remainder_encodeSnappyBlockAsm12B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBlockAsm12B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBlockAsm12B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBlockAsm12B + BLO three_bytes_emit_remainder_encodeSnappyBlockAsm12B + +three_bytes_emit_remainder_encodeSnappyBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B + +two_bytes_emit_remainder_encodeSnappyBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBlockAsm12B + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B + +one_byte_emit_remainder_encodeSnappyBlockAsm12B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBlockAsm12B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B + +memmove_long_emit_remainder_encodeSnappyBlockAsm12B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBlockAsm10B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000020, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBlockAsm10B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBlockAsm10B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBlockAsm10B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x05, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBlockAsm10B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x9e3779b1, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x36, R9, R9 + LSL $0x20, R10, R10 + MUL R8, R10, R10 + LSR $0x36, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x36, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeSnappyBlockAsm10B + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R5 + MOVWU R6, R7 + MOVWU 24(RSP), R16 + SUBSW R16, R7, R7 + BEQ repeat_extend_back_end_encodeSnappyBlockAsm10B + +repeat_extend_back_loop_encodeSnappyBlockAsm10B: + CMPW R5, R6 + BLS repeat_extend_back_end_encodeSnappyBlockAsm10B + ADD R7, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeSnappyBlockAsm10B + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R7, R7 + BNE repeat_extend_back_loop_encodeSnappyBlockAsm10B + +repeat_extend_back_end_encodeSnappyBlockAsm10B: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $3, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeSnappyBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeSnappyBlockAsm10B: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B + MOVWU R6, R7 + MOVW R6, 20(RSP) + ADD R5, R3, R8 + SUBW R5, R7, R7 + SUB $1, R7, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeSnappyBlockAsm10B + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeSnappyBlockAsm10B + BLO three_bytes_repeat_emit_encodeSnappyBlockAsm10B + +three_bytes_repeat_emit_encodeSnappyBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B + +two_bytes_repeat_emit_encodeSnappyBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeSnappyBlockAsm10B + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B + +one_byte_repeat_emit_encodeSnappyBlockAsm10B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeSnappyBlockAsm10B: + ADD R7, R1, R5 + + // genMemMoveShort + CMP $0x08, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8 + CMP $0x10, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16 + CMP $0x20, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8: + MOVD (R8), R9 + MOVD R9, (R1) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16: + MOVD (R8), R9 + ADD R7, R8, R15 + MOVD -8(R15), R8 + MOVD R9, (R1) + ADD R7, R1, R15 + MOVD R8, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32: + FMOVQ (R8), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64: + FMOVQ (R8), F0 + FMOVQ 16(R8), F1 + ADD R7, R8, R15 + FMOVQ -32(R15), F2 + ADD R7, R8, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R7, R1, R15 + FMOVQ F2, -32(R15) + ADD R7, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B + +memmove_long_repeat_emit_encodeSnappyBlockAsm10B: + ADD R7, R1, R5 + + // genMemMoveLong + MOVD R8, R9 + MOVD R1, R10 + MOVD R7, R11 + +emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back + ADD R7, R8, R15 + FMOVQ -32(R15), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + ADD R7, R1, R15 + FMOVQ F0, -32(R15) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R10 + +matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B: + CMPW $0x10, R7 + BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B + MOVD (R8)(R10), R9 + ADD R10, R8, R15 + MOVD 8(R15), R11 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B + ADD R10, R5, R15 + MOVD 8(R15), R16 + EOR R16, R11, R11 + TST R11, R11 + BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R10, R10 + MOVWU R10, R10 + JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B + +matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B: + RBIT R11, R11 + CLZ R11, R11 + ASR $0x03, R11, R11 + ADD R11, R10, R10 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B + +matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B: + CMPW $0x08, R7 + BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B + MOVD (R8)(R10), R9 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B + +matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B: + RBIT R9, R9 + CLZ R9, R9 + ASR $0x03, R9, R9 + ADD R9, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B + +matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B: + CMPW $0x04, R7 + BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B + MOVWU (R8)(R10), R9 + MOVWU (R5)(R10), R16 + CMPW R9, R16 + BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R10, R10 + MOVWU R10, R10 + +matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B: + CMPW $0x01, R7 + BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B + BLO repeat_extend_forward_end_encodeSnappyBlockAsm10B + MOVHU (R8)(R10), R16 + BFI $0, R16, $16, R9 + ADD R10, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R9, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B + ADD $2, R10, R10 + MOVWU R10, R10 + SUBSW $0x02, R7, R7 + BEQ repeat_extend_forward_end_encodeSnappyBlockAsm10B + +matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B: + MOVBU (R8)(R10), R16 + BFI $0, R16, $8, R9 + ADD R10, R5, R15 + MOVBU (R15), R15 + AND $0xff, R9, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeSnappyBlockAsm10B + ADD $1, R10, R10 + MOVWU R10, R10 + +repeat_extend_forward_end_encodeSnappyBlockAsm10B: + ADDW R10, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + + // emitCopy +two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B: + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B + +two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B + CMPW $0x00000800, R6 + BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeSnappyBlockAsm10B + +emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeSnappyBlockAsm10B: + MOVW R2, 20(RSP) + JMP search_loop_encodeSnappyBlockAsm10B + +no_repeat_found_encodeSnappyBlockAsm10B: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBlockAsm10B + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeSnappyBlockAsm10B + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeSnappyBlockAsm10B + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBlockAsm10B + +candidate3_match_encodeSnappyBlockAsm10B: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeSnappyBlockAsm10B + +candidate2_match_encodeSnappyBlockAsm10B: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBlockAsm10B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm10B + +match_extend_back_loop_encodeSnappyBlockAsm10B: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBlockAsm10B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBlockAsm10B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm10B + JMP match_extend_back_loop_encodeSnappyBlockAsm10B + +match_extend_back_end_encodeSnappyBlockAsm10B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBlockAsm10B: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm10B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeSnappyBlockAsm10B + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeSnappyBlockAsm10B + BLO three_bytes_match_emit_encodeSnappyBlockAsm10B + +three_bytes_match_emit_encodeSnappyBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBlockAsm10B + +two_bytes_match_emit_encodeSnappyBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeSnappyBlockAsm10B + JMP memmove_long_match_emit_encodeSnappyBlockAsm10B + +one_byte_match_emit_encodeSnappyBlockAsm10B: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBlockAsm10B: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBlockAsm10B: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeSnappyBlockAsm10B + +memmove_long_match_emit_encodeSnappyBlockAsm10B: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeSnappyBlockAsm10B: +match_nolit_loop_encodeSnappyBlockAsm10B: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm10B + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B + +matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm10B + +matchlen_match8_match_nolit_encodeSnappyBlockAsm10B: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm10B + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm10B + +matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm10B + +matchlen_match4_match_nolit_encodeSnappyBlockAsm10B: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm10B + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm10B + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeSnappyBlockAsm10B: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm10B + BLO match_nolit_end_encodeSnappyBlockAsm10B + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm10B + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeSnappyBlockAsm10B + +matchlen_match1_match_nolit_encodeSnappyBlockAsm10B: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBlockAsm10B + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeSnappyBlockAsm10B: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy +two_byte_offset_match_nolit_encodeSnappyBlockAsm10B: + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm10B + +two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm10B + CMPW $0x00000800, R5 + BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm10B + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm10B + +emit_copy_three_match_nolit_encodeSnappyBlockAsm10B: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBlockAsm10B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBlockAsm10B + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBlockAsm10B: + MOVD $0x9e3779b1, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x20, R7, R7 + MUL R8, R7, R7 + LSR $0x36, R7, R7 + LSL $0x20, R5, R5 + MUL R8, R5, R5 + LSR $0x36, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeSnappyBlockAsm10B + ADDW $1, R2, R2 + JMP search_loop_encodeSnappyBlockAsm10B + +emit_remainder_encodeSnappyBlockAsm10B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBlockAsm10B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBlockAsm10B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBlockAsm10B + BLO three_bytes_emit_remainder_encodeSnappyBlockAsm10B + +three_bytes_emit_remainder_encodeSnappyBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B + +two_bytes_emit_remainder_encodeSnappyBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBlockAsm10B + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B + +one_byte_emit_remainder_encodeSnappyBlockAsm10B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBlockAsm10B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B + +memmove_long_emit_remainder_encodeSnappyBlockAsm10B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBlockAsm8B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000008, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBlockAsm8B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBlockAsm8B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBlockAsm8B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x04, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBlockAsm8B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x9e3779b1, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x38, R9, R9 + LSL $0x20, R10, R10 + MUL R8, R10, R10 + LSR $0x38, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x38, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_encodeSnappyBlockAsm8B + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R5 + MOVWU R6, R7 + MOVWU 24(RSP), R16 + SUBSW R16, R7, R7 + BEQ repeat_extend_back_end_encodeSnappyBlockAsm8B + +repeat_extend_back_loop_encodeSnappyBlockAsm8B: + CMPW R5, R6 + BLS repeat_extend_back_end_encodeSnappyBlockAsm8B + ADD R7, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_encodeSnappyBlockAsm8B + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R7, R7 + BNE repeat_extend_back_loop_encodeSnappyBlockAsm8B + +repeat_extend_back_end_encodeSnappyBlockAsm8B: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $3, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_encodeSnappyBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +repeat_dst_size_check_encodeSnappyBlockAsm8B: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B + MOVWU R6, R7 + MOVW R6, 20(RSP) + ADD R5, R3, R8 + SUBW R5, R7, R7 + SUB $1, R7, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_encodeSnappyBlockAsm8B + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_encodeSnappyBlockAsm8B + BLO three_bytes_repeat_emit_encodeSnappyBlockAsm8B + +three_bytes_repeat_emit_encodeSnappyBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B + +two_bytes_repeat_emit_encodeSnappyBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_encodeSnappyBlockAsm8B + JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B + +one_byte_repeat_emit_encodeSnappyBlockAsm8B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_repeat_emit_encodeSnappyBlockAsm8B: + ADD R7, R1, R5 + + // genMemMoveShort + CMP $0x08, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8 + CMP $0x10, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16 + CMP $0x20, R7 + BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8: + MOVD (R8), R9 + MOVD R9, (R1) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16: + MOVD (R8), R9 + ADD R7, R8, R15 + MOVD -8(R15), R8 + MOVD R9, (R1) + ADD R7, R1, R15 + MOVD R8, -8(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32: + FMOVQ (R8), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B + +emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64: + FMOVQ (R8), F0 + FMOVQ 16(R8), F1 + ADD R7, R8, R15 + FMOVQ -32(R15), F2 + ADD R7, R8, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R7, R1, R15 + FMOVQ F2, -32(R15) + ADD R7, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B: + MOVD R5, R1 + JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B + +memmove_long_repeat_emit_encodeSnappyBlockAsm8B: + ADD R7, R1, R5 + + // genMemMoveLong + MOVD R8, R9 + MOVD R1, R10 + MOVD R7, R11 + +emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back + ADD R7, R8, R15 + FMOVQ -32(R15), F0 + ADD R7, R8, R15 + FMOVQ -16(R15), F1 + ADD R7, R1, R15 + FMOVQ F0, -32(R15) + ADD R7, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R10 + +matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B: + CMPW $0x10, R7 + BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B + MOVD (R8)(R10), R9 + ADD R10, R8, R15 + MOVD 8(R15), R11 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B + ADD R10, R5, R15 + MOVD 8(R15), R16 + EOR R16, R11, R11 + TST R11, R11 + BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R10, R10 + MOVWU R10, R10 + JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B + +matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B: + RBIT R11, R11 + CLZ R11, R11 + ASR $0x03, R11, R11 + ADD R11, R10, R10 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B + +matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B: + CMPW $0x08, R7 + BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B + MOVD (R8)(R10), R9 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B + +matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B: + RBIT R9, R9 + CLZ R9, R9 + ASR $0x03, R9, R9 + ADD R9, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B + +matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B: + CMPW $0x04, R7 + BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B + MOVWU (R8)(R10), R9 + MOVWU (R5)(R10), R16 + CMPW R9, R16 + BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R10, R10 + MOVWU R10, R10 + +matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B: + CMPW $0x01, R7 + BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B + BLO repeat_extend_forward_end_encodeSnappyBlockAsm8B + MOVHU (R8)(R10), R16 + BFI $0, R16, $16, R9 + ADD R10, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R9, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B + ADD $2, R10, R10 + MOVWU R10, R10 + SUBSW $0x02, R7, R7 + BEQ repeat_extend_forward_end_encodeSnappyBlockAsm8B + +matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B: + MOVBU (R8)(R10), R16 + BFI $0, R16, $8, R9 + ADD R10, R5, R15 + MOVBU (R15), R15 + AND $0xff, R9, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_encodeSnappyBlockAsm8B + ADD $1, R10, R10 + MOVWU R10, R10 + +repeat_extend_forward_end_encodeSnappyBlockAsm8B: + ADDW R10, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + + // emitCopy +two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B: + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R6, 1(R1) + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B + +two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B + SUB $15, R7, R7 + MOVWU R7, R7 + MOVB R6, 1(R1) + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R7, R7 + MOVB R7, (R1) + ADD $0x02, R1, R1 + JMP repeat_end_emit_encodeSnappyBlockAsm8B + +emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B: + SUB $2, R7, R7 + MOVWU R7, R7 + MOVB R7, (R1) + MOVH R6, 1(R1) + ADD $0x03, R1, R1 + +repeat_end_emit_encodeSnappyBlockAsm8B: + MOVW R2, 20(RSP) + JMP search_loop_encodeSnappyBlockAsm8B + +no_repeat_found_encodeSnappyBlockAsm8B: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBlockAsm8B + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_encodeSnappyBlockAsm8B + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_encodeSnappyBlockAsm8B + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBlockAsm8B + +candidate3_match_encodeSnappyBlockAsm8B: + ADDW $0x02, R2, R2 + JMP candidate_match_encodeSnappyBlockAsm8B + +candidate2_match_encodeSnappyBlockAsm8B: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBlockAsm8B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm8B + +match_extend_back_loop_encodeSnappyBlockAsm8B: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBlockAsm8B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBlockAsm8B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBlockAsm8B + JMP match_extend_back_loop_encodeSnappyBlockAsm8B + +match_extend_back_end_encodeSnappyBlockAsm8B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBlockAsm8B: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm8B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R7 + MOVWU R7, R7 + CMPW $0x3c, R7 + BLO one_byte_match_emit_encodeSnappyBlockAsm8B + CMPW $0x00000100, R7 + BLO two_bytes_match_emit_encodeSnappyBlockAsm8B + BLO three_bytes_match_emit_encodeSnappyBlockAsm8B + +three_bytes_match_emit_encodeSnappyBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBlockAsm8B + +two_bytes_match_emit_encodeSnappyBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R7, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R7 + BLO memmove_match_emit_encodeSnappyBlockAsm8B + JMP memmove_long_match_emit_encodeSnappyBlockAsm8B + +one_byte_match_emit_encodeSnappyBlockAsm8B: + LSLW $0x02, R7, R16 + BFI $0, R16, $8, R7 + MOVB R7, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBlockAsm8B: + ADD R8, R1, R7 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8: + MOVD (R6), R9 + MOVD R9, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16: + MOVD (R6), R9 + ADD R8, R6, R15 + MOVD -8(R15), R6 + MOVD R9, (R1) + ADD R8, R1, R15 + MOVD R6, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32: + FMOVQ (R6), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B + +emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64: + FMOVQ (R6), F0 + FMOVQ 16(R6), F1 + ADD R8, R6, R15 + FMOVQ -32(R15), F2 + ADD R8, R6, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBlockAsm8B: + MOVD R7, R1 + JMP emit_literal_done_match_emit_encodeSnappyBlockAsm8B + +memmove_long_match_emit_encodeSnappyBlockAsm8B: + ADD R8, R1, R7 + + // genMemMoveLong + MOVD R6, R9 + MOVD R1, R10 + MOVD R8, R11 + +emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + FMOVQ F0, (R10) + FMOVQ F1, 16(R10) + ADD $0x20, R9, R9 + ADD $0x20, R10, R10 + SUB $0x20, R11, R11 + CMP $0x20, R11 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back + ADD R8, R6, R15 + FMOVQ -32(R15), F0 + ADD R8, R6, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R7, R1 + +emit_literal_done_match_emit_encodeSnappyBlockAsm8B: +match_nolit_loop_encodeSnappyBlockAsm8B: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm8B + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B + +matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm8B + +matchlen_match8_match_nolit_encodeSnappyBlockAsm8B: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm8B + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm8B + +matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_encodeSnappyBlockAsm8B + +matchlen_match4_match_nolit_encodeSnappyBlockAsm8B: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm8B + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm8B + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_encodeSnappyBlockAsm8B: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm8B + BLO match_nolit_end_encodeSnappyBlockAsm8B + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm8B + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_encodeSnappyBlockAsm8B + +matchlen_match1_match_nolit_encodeSnappyBlockAsm8B: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBlockAsm8B + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_encodeSnappyBlockAsm8B: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy +two_byte_offset_match_nolit_encodeSnappyBlockAsm8B: + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm8B + +two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm8B + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R5, 1(R1) + LSRW $0x08, R5, R5 + LSLW $0x05, R5, R5 + ORRW R5, R6, R6 + MOVB R6, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm8B + +emit_copy_three_match_nolit_encodeSnappyBlockAsm8B: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBlockAsm8B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBlockAsm8B + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBlockAsm8B: + MOVD $0x9e3779b1, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x20, R7, R7 + MUL R8, R7, R7 + LSR $0x38, R7, R7 + LSL $0x20, R5, R5 + MUL R8, R5, R5 + LSR $0x38, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_encodeSnappyBlockAsm8B + ADDW $1, R2, R2 + JMP search_loop_encodeSnappyBlockAsm8B + +emit_remainder_encodeSnappyBlockAsm8B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBlockAsm8B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBlockAsm8B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBlockAsm8B + BLO three_bytes_emit_remainder_encodeSnappyBlockAsm8B + +three_bytes_emit_remainder_encodeSnappyBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B + +two_bytes_emit_remainder_encodeSnappyBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBlockAsm8B + JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B + +one_byte_emit_remainder_encodeSnappyBlockAsm8B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBlockAsm8B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B + +memmove_long_emit_remainder_encodeSnappyBlockAsm8B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBetterBlockAsm(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00001200, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBetterBlockAsm: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBetterBlockAsm + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBetterBlockAsm: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x07, R5, R5 + CMPW $0x63, R5 + BLS check_maxskip_ok_encodeSnappyBetterBlockAsm + ADD $100, R2, R5 + MOVWU R5, R5 + JMP check_maxskip_cont_encodeSnappyBetterBlockAsm + +check_maxskip_ok_encodeSnappyBetterBlockAsm: + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + +check_maxskip_cont_encodeSnappyBetterBlockAsm: + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBetterBlockAsm + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x00cf1bbcdcbfa563, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x08, R9, R9 + MUL R8, R9, R9 + LSR $0x2f, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x32, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 524288(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 524288(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm + CMP R6, R10 + BNE no_short_found_encodeSnappyBetterBlockAsm + MOVWU R7, R5 + JMP candidate_match_encodeSnappyBetterBlockAsm + +no_short_found_encodeSnappyBetterBlockAsm: + CMPW R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm + CMPW R6, R10 + BEQ candidateS_match_encodeSnappyBetterBlockAsm + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBetterBlockAsm + +candidateS_match_encodeSnappyBetterBlockAsm: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x08, R9, R9 + MUL R8, R9, R9 + LSR $0x2f, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBetterBlockAsm + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBetterBlockAsm: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm + +match_extend_back_loop_encodeSnappyBetterBlockAsm: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBetterBlockAsm + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBetterBlockAsm + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm + JMP match_extend_back_loop_encodeSnappyBetterBlockAsm + +match_extend_back_end_encodeSnappyBetterBlockAsm: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $5, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBetterBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBetterBlockAsm: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm + +matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm + +matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm + +matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm + +matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm + BLO match_nolit_end_encodeSnappyBetterBlockAsm + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeSnappyBetterBlockAsm + +matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBetterBlockAsm + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeSnappyBetterBlockAsm: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + CMPW $0x01, R11 + BHI match_length_ok_encodeSnappyBetterBlockAsm + CMPW $0x0000ffff, R7 + BLS match_length_ok_encodeSnappyBetterBlockAsm + MOVWU 28(RSP), R2 + ADDW $1, R2, R2 + JMP search_loop_encodeSnappyBetterBlockAsm + +match_length_ok_encodeSnappyBetterBlockAsm: + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeSnappyBetterBlockAsm + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm + CMPW $0x00010000, R5 + BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm + CMPW $0x01000000, R5 + BLO four_bytes_match_emit_encodeSnappyBetterBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R5, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm + +four_bytes_match_emit_encodeSnappyBetterBlockAsm: + MOVWU R5, R10 + LSRW $0x10, R10, R10 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + MOVB R10, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm + +three_bytes_match_emit_encodeSnappyBetterBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm + +two_bytes_match_emit_encodeSnappyBetterBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeSnappyBetterBlockAsm + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm + +one_byte_match_emit_encodeSnappyBetterBlockAsm: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBetterBlockAsm: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm + +memmove_long_match_emit_encodeSnappyBetterBlockAsm: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeSnappyBetterBlockAsm: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x00010000, R7 + BLO two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm + +four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm: + CMPW $0x40, R11 + BLS four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm + MOVD $0xff, R16 + MOVB R16, (R1) + MOVW R7, 1(R1) + SUB $64, R11, R11 + MOVWU R11, R11 + ADD $0x05, R1, R1 + CMPW $0x04, R11 + BLO four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm + JMP four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm + +four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm: + TSTW R11, R11 + BEQ match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm + MOVD $0, R5 + ADD R11<<2, R5, R11 + SUB $1, R11, R11 + MOVWU R11, R11 + MOVB R11, (R1) + MOVW R7, 1(R1) + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm + +two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm: + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm + +two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm + CMPW $0x00000800, R7 + BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm + +emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBetterBlockAsm + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBetterBlockAsm: + MOVD $0x00cf1bbcdcbfa563, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x08, R9, R9 + MUL R5, R9, R9 + LSR $0x2f, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x32, R10, R10 + LSL $0x08, R11, R11 + MUL R5, R11, R11 + LSR $0x2f, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x32, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 524288(R15) + ADD R12<<2, R0, R15 + MOVW R13, 524288(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeSnappyBetterBlockAsm: + CMP R8, R7 + BHS search_loop_encodeSnappyBetterBlockAsm + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x08, R9, R9 + MUL R5, R9, R9 + LSR $0x2f, R9, R9 + LSL $0x08, R10, R10 + MUL R5, R10, R10 + LSR $0x2f, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeSnappyBetterBlockAsm + +emit_remainder_encodeSnappyBetterBlockAsm: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $5, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBetterBlockAsm + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBetterBlockAsm: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm + CMPW $0x00010000, R2 + BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm + CMPW $0x01000000, R2 + BLO four_bytes_emit_remainder_encodeSnappyBetterBlockAsm + MOVD $0xfc, R16 + MOVB R16, (R1) + MOVW R2, 1(R1) + ADD $0x05, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm + +four_bytes_emit_remainder_encodeSnappyBetterBlockAsm: + MOVWU R2, R3 + LSRW $0x10, R3, R3 + MOVD $0xf8, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + MOVB R3, 3(R1) + ADD $0x04, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm + +three_bytes_emit_remainder_encodeSnappyBetterBlockAsm: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm + +two_bytes_emit_remainder_encodeSnappyBetterBlockAsm: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm + +one_byte_emit_remainder_encodeSnappyBetterBlockAsm: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBetterBlockAsm: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm + +memmove_long_emit_remainder_encodeSnappyBetterBlockAsm: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte, tmp *[294912]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBetterBlockAsm64K(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000900, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBetterBlockAsm64K: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBetterBlockAsm64K + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBetterBlockAsm64K: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x07, R5, R5 + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBetterBlockAsm64K + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x00cf1bbcdcbfa563, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x08, R9, R9 + MUL R8, R9, R9 + LSR $0x30, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x33, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 262144(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 262144(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm64K + CMP R6, R10 + BNE no_short_found_encodeSnappyBetterBlockAsm64K + MOVWU R7, R5 + JMP candidate_match_encodeSnappyBetterBlockAsm64K + +no_short_found_encodeSnappyBetterBlockAsm64K: + CMPW R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm64K + CMPW R6, R10 + BEQ candidateS_match_encodeSnappyBetterBlockAsm64K + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBetterBlockAsm64K + +candidateS_match_encodeSnappyBetterBlockAsm64K: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x08, R9, R9 + MUL R8, R9, R9 + LSR $0x30, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBetterBlockAsm64K + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBetterBlockAsm64K: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm64K + +match_extend_back_loop_encodeSnappyBetterBlockAsm64K: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBetterBlockAsm64K + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBetterBlockAsm64K + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm64K + JMP match_extend_back_loop_encodeSnappyBetterBlockAsm64K + +match_extend_back_end_encodeSnappyBetterBlockAsm64K: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBetterBlockAsm64K + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBetterBlockAsm64K: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K + +matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm64K + +matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K + +matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm64K + +matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K + BLO match_nolit_end_encodeSnappyBetterBlockAsm64K + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeSnappyBetterBlockAsm64K + +matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBetterBlockAsm64K + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeSnappyBetterBlockAsm64K: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeSnappyBetterBlockAsm64K + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm64K + BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm64K + +three_bytes_match_emit_encodeSnappyBetterBlockAsm64K: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K + +two_bytes_match_emit_encodeSnappyBetterBlockAsm64K: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeSnappyBetterBlockAsm64K + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K + +one_byte_match_emit_encodeSnappyBetterBlockAsm64K: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBetterBlockAsm64K: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K + +memmove_long_match_emit_encodeSnappyBetterBlockAsm64K: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy +two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K: + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K + +two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K + CMPW $0x00000800, R7 + BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K + +emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBetterBlockAsm64K + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K: + MOVD $0x00cf1bbcdcbfa563, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x08, R9, R9 + MUL R5, R9, R9 + LSR $0x30, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x33, R10, R10 + LSL $0x08, R11, R11 + MUL R5, R11, R11 + LSR $0x30, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x33, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 262144(R15) + ADD R12<<2, R0, R15 + MOVW R13, 262144(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeSnappyBetterBlockAsm64K: + CMP R8, R7 + BHS search_loop_encodeSnappyBetterBlockAsm64K + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x08, R9, R9 + MUL R5, R9, R9 + LSR $0x30, R9, R9 + LSL $0x08, R10, R10 + MUL R5, R10, R10 + LSR $0x30, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeSnappyBetterBlockAsm64K + +emit_remainder_encodeSnappyBetterBlockAsm64K: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBetterBlockAsm64K + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBetterBlockAsm64K: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K + BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K + +three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K + +two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm64K + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K + +one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBetterBlockAsm64K: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K + +memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBetterBlockAsm12B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000280, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBetterBlockAsm12B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBetterBlockAsm12B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBetterBlockAsm12B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x06, R5, R5 + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBetterBlockAsm12B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x34, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 65536(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 65536(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm12B + CMP R6, R10 + BNE no_short_found_encodeSnappyBetterBlockAsm12B + MOVWU R7, R5 + JMP candidate_match_encodeSnappyBetterBlockAsm12B + +no_short_found_encodeSnappyBetterBlockAsm12B: + CMPW R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm12B + CMPW R6, R10 + BEQ candidateS_match_encodeSnappyBetterBlockAsm12B + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBetterBlockAsm12B + +candidateS_match_encodeSnappyBetterBlockAsm12B: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x32, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBetterBlockAsm12B + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBetterBlockAsm12B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm12B + +match_extend_back_loop_encodeSnappyBetterBlockAsm12B: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBetterBlockAsm12B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBetterBlockAsm12B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm12B + JMP match_extend_back_loop_encodeSnappyBetterBlockAsm12B + +match_extend_back_end_encodeSnappyBetterBlockAsm12B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBetterBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBetterBlockAsm12B: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B + +matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm12B + +matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B + +matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm12B + +matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B + BLO match_nolit_end_encodeSnappyBetterBlockAsm12B + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeSnappyBetterBlockAsm12B + +matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBetterBlockAsm12B + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeSnappyBetterBlockAsm12B: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeSnappyBetterBlockAsm12B + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm12B + BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm12B + +three_bytes_match_emit_encodeSnappyBetterBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B + +two_bytes_match_emit_encodeSnappyBetterBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeSnappyBetterBlockAsm12B + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B + +one_byte_match_emit_encodeSnappyBetterBlockAsm12B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBetterBlockAsm12B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B + +memmove_long_match_emit_encodeSnappyBetterBlockAsm12B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy +two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B: + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B + +two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B + CMPW $0x00000800, R7 + BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B + +emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBetterBlockAsm12B + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B: + MOVD $0x0000cf1bbcdcbf9b, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x32, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x34, R10, R10 + LSL $0x10, R11, R11 + MUL R5, R11, R11 + LSR $0x32, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x34, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 65536(R15) + ADD R12<<2, R0, R15 + MOVW R13, 65536(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeSnappyBetterBlockAsm12B: + CMP R8, R7 + BHS search_loop_encodeSnappyBetterBlockAsm12B + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x32, R9, R9 + LSL $0x10, R10, R10 + MUL R5, R10, R10 + LSR $0x32, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeSnappyBetterBlockAsm12B + +emit_remainder_encodeSnappyBetterBlockAsm12B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBetterBlockAsm12B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBetterBlockAsm12B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B + BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B + +three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B + +two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm12B + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B + +one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBetterBlockAsm12B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B + +memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBetterBlockAsm10B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x000000a0, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBetterBlockAsm10B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBetterBlockAsm10B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBetterBlockAsm10B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x05, R5, R5 + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBetterBlockAsm10B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x34, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x36, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 16384(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 16384(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm10B + CMP R6, R10 + BNE no_short_found_encodeSnappyBetterBlockAsm10B + MOVWU R7, R5 + JMP candidate_match_encodeSnappyBetterBlockAsm10B + +no_short_found_encodeSnappyBetterBlockAsm10B: + CMPW R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm10B + CMPW R6, R10 + BEQ candidateS_match_encodeSnappyBetterBlockAsm10B + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBetterBlockAsm10B + +candidateS_match_encodeSnappyBetterBlockAsm10B: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x34, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBetterBlockAsm10B + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBetterBlockAsm10B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm10B + +match_extend_back_loop_encodeSnappyBetterBlockAsm10B: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBetterBlockAsm10B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBetterBlockAsm10B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm10B + JMP match_extend_back_loop_encodeSnappyBetterBlockAsm10B + +match_extend_back_end_encodeSnappyBetterBlockAsm10B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBetterBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBetterBlockAsm10B: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B + +matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm10B + +matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B + +matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm10B + +matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B + BLO match_nolit_end_encodeSnappyBetterBlockAsm10B + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeSnappyBetterBlockAsm10B + +matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBetterBlockAsm10B + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeSnappyBetterBlockAsm10B: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeSnappyBetterBlockAsm10B + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm10B + BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm10B + +three_bytes_match_emit_encodeSnappyBetterBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B + +two_bytes_match_emit_encodeSnappyBetterBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeSnappyBetterBlockAsm10B + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B + +one_byte_match_emit_encodeSnappyBetterBlockAsm10B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBetterBlockAsm10B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B + +memmove_long_match_emit_encodeSnappyBetterBlockAsm10B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy +two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B: + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B + +two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B + CMPW $0x00000800, R7 + BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B + +emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBetterBlockAsm10B + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B: + MOVD $0x0000cf1bbcdcbf9b, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x34, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x36, R10, R10 + LSL $0x10, R11, R11 + MUL R5, R11, R11 + LSR $0x34, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x36, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 16384(R15) + ADD R12<<2, R0, R15 + MOVW R13, 16384(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeSnappyBetterBlockAsm10B: + CMP R8, R7 + BHS search_loop_encodeSnappyBetterBlockAsm10B + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x34, R9, R9 + LSL $0x10, R10, R10 + MUL R5, R10, R10 + LSR $0x34, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeSnappyBetterBlockAsm10B + +emit_remainder_encodeSnappyBetterBlockAsm10B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBetterBlockAsm10B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBetterBlockAsm10B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B + BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B + +three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B + +two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm10B + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B + +one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBetterBlockAsm10B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B + +memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int +// Requires: BMI, SSE2 +TEXT ·encodeSnappyBetterBlockAsm8B(SB), $24-64 + MOVD tmp+48(FP), R0 + MOVD dst_base+0(FP), R1 + MOVD $0x00000028, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_encodeSnappyBetterBlockAsm8B: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_encodeSnappyBetterBlockAsm8B + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+32(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVD $0x00000000, R16 + MOVW R16, 24(RSP) + MOVD src_base+24(FP), R3 + +search_loop_encodeSnappyBetterBlockAsm8B: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x04, R5, R5 + ADD R5, R2, R5 + ADD $1, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_encodeSnappyBetterBlockAsm8B + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD $0x9e3779b1, R5 + MOVD R6, R9 + MOVD R6, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x36, R9, R9 + LSL $0x20, R10, R10 + MUL R5, R10, R10 + LSR $0x38, R10, R10 + MOVWU (R0)(R9<<2), R5 + ADD R10<<2, R0, R15 + MOVWU 4096(R15), R7 + MOVW R2, (R0)(R9<<2) + ADD R10<<2, R0, R15 + MOVW R2, 4096(R15) + MOVD (R3)(R5), R9 + MOVD (R3)(R7), R10 + CMP R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm8B + CMP R6, R10 + BNE no_short_found_encodeSnappyBetterBlockAsm8B + MOVWU R7, R5 + JMP candidate_match_encodeSnappyBetterBlockAsm8B + +no_short_found_encodeSnappyBetterBlockAsm8B: + CMPW R6, R9 + BEQ candidate_match_encodeSnappyBetterBlockAsm8B + CMPW R6, R10 + BEQ candidateS_match_encodeSnappyBetterBlockAsm8B + MOVWU 28(RSP), R2 + JMP search_loop_encodeSnappyBetterBlockAsm8B + +candidateS_match_encodeSnappyBetterBlockAsm8B: + LSR $0x08, R6, R6 + MOVD R6, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x36, R9, R9 + MOVWU (R0)(R9<<2), R5 + ADDW $1, R2, R2 + MOVW R2, (R0)(R9<<2) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_encodeSnappyBetterBlockAsm8B + SUBW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_encodeSnappyBetterBlockAsm8B: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm8B + +match_extend_back_loop_encodeSnappyBetterBlockAsm8B: + CMPW R6, R2 + BLS match_extend_back_end_encodeSnappyBetterBlockAsm8B + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_encodeSnappyBetterBlockAsm8B + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_encodeSnappyBetterBlockAsm8B + JMP match_extend_back_loop_encodeSnappyBetterBlockAsm8B + +match_extend_back_end_encodeSnappyBetterBlockAsm8B: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_encodeSnappyBetterBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_dst_size_check_encodeSnappyBetterBlockAsm8B: + MOVWU R2, R6 + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+32(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R9 + + // matchLen + MOVD $0, R11 + +matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B: + CMPW $0x10, R7 + BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B + MOVD (R8)(R11), R10 + ADD R11, R8, R15 + MOVD 8(R15), R12 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B + ADD R11, R9, R15 + MOVD 8(R15), R16 + EOR R16, R12, R12 + TST R12, R12 + BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R11, R11 + MOVWU R11, R11 + JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B + +matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B: + RBIT R12, R12 + CLZ R12, R12 + ASR $0x03, R12, R12 + ADD R12, R11, R11 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm8B + +matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B: + CMPW $0x08, R7 + BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B + MOVD (R8)(R11), R10 + MOVD (R9)(R11), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R11, R11 + MOVWU R11, R11 + JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B + +matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R11, R11 + MOVWU R11, R11 + JMP match_nolit_end_encodeSnappyBetterBlockAsm8B + +matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B: + CMPW $0x04, R7 + BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B + MOVWU (R8)(R11), R10 + MOVWU (R9)(R11), R16 + CMPW R10, R16 + BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R11, R11 + MOVWU R11, R11 + +matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B: + CMPW $0x01, R7 + BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B + BLO match_nolit_end_encodeSnappyBetterBlockAsm8B + MOVHU (R8)(R11), R16 + BFI $0, R16, $16, R10 + ADD R11, R9, R15 + MOVHU (R15), R15 + AND $0xffff, R10, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B + ADD $2, R11, R11 + MOVWU R11, R11 + SUBSW $0x02, R7, R7 + BEQ match_nolit_end_encodeSnappyBetterBlockAsm8B + +matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B: + MOVBU (R8)(R11), R16 + BFI $0, R16, $8, R10 + ADD R11, R9, R15 + MOVBU (R15), R15 + AND $0xff, R10, R16 + CMP R16, R15 + BNE match_nolit_end_encodeSnappyBetterBlockAsm8B + ADD $1, R11, R11 + MOVWU R11, R11 + +match_nolit_end_encodeSnappyBetterBlockAsm8B: + MOVWU R2, R7 + SUBW R5, R7, R7 + + // Check if repeat + MOVW R7, 24(RSP) + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R5, R3, R9 + SUBW R5, R8, R8 + SUB $1, R8, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_match_emit_encodeSnappyBetterBlockAsm8B + CMPW $0x00000100, R5 + BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm8B + BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm8B + +three_bytes_match_emit_encodeSnappyBetterBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R5, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B + +two_bytes_match_emit_encodeSnappyBetterBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R5, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_match_emit_encodeSnappyBetterBlockAsm8B + JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B + +one_byte_match_emit_encodeSnappyBetterBlockAsm8B: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R1) + ADD $0x01, R1, R1 + +memmove_match_emit_encodeSnappyBetterBlockAsm8B: + ADD R8, R1, R5 + + // genMemMoveShort + CMP $0x08, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8 + CMP $0x10, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16 + CMP $0x20, R8 + BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8: + MOVD (R9), R10 + MOVD R10, (R1) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16: + MOVD (R9), R10 + ADD R8, R9, R15 + MOVD -8(R15), R9 + MOVD R10, (R1) + ADD R8, R1, R15 + MOVD R9, -8(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32: + FMOVQ (R9), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B + +emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64: + FMOVQ (R9), F0 + FMOVQ 16(R9), F1 + ADD R8, R9, R15 + FMOVQ -32(R15), F2 + ADD R8, R9, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R8, R1, R15 + FMOVQ F2, -32(R15) + ADD R8, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B: + MOVD R5, R1 + JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B + +memmove_long_match_emit_encodeSnappyBetterBlockAsm8B: + ADD R8, R1, R5 + + // genMemMoveLong + MOVD R9, R10 + MOVD R1, R12 + MOVD R8, R13 + +emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R12) + FMOVQ F1, 16(R12) + ADD $0x20, R10, R10 + ADD $0x20, R12, R12 + SUB $0x20, R13, R13 + CMP $0x20, R13 + BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back + ADD R8, R9, R15 + FMOVQ -32(R15), F0 + ADD R8, R9, R15 + FMOVQ -16(R15), F1 + ADD R8, R1, R15 + FMOVQ F0, -32(R15) + ADD R8, R1, R15 + FMOVQ F1, -16(R15) + MOVD R5, R1 + +emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B: + ADDW R11, R2, R2 + ADDW $0x04, R11, R11 + MOVW R2, 20(RSP) + + // emitCopy +two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B: + CMPW $0x40, R11 + BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B + MOVD $0xee, R16 + MOVB R16, (R1) + MOVH R7, 1(R1) + SUB $60, R11, R11 + MOVWU R11, R11 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B + +two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B: + MOVWU R11, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R11 + BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R7, 1(R1) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R5, R5 + MOVB R5, (R1) + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B + +emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R1) + MOVH R7, 1(R1) + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_encodeSnappyBetterBlockAsm8B + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B: + MOVD $0x0000cf1bbcdcbf9b, R5 + MOVD $0x9e3779b1, R7 + ADD $1, R6, R6 + SUB $2, R2, R8 + MOVD (R3)(R6), R9 + ADD R6, R3, R15 + MOVD 1(R15), R10 + MOVD (R3)(R8), R11 + ADD R8, R3, R15 + MOVD 1(R15), R12 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x36, R9, R9 + LSL $0x20, R10, R10 + MUL R7, R10, R10 + LSR $0x38, R10, R10 + LSL $0x10, R11, R11 + MUL R5, R11, R11 + LSR $0x36, R11, R11 + LSL $0x20, R12, R12 + MUL R7, R12, R12 + LSR $0x38, R12, R12 + ADD $1, R6, R7 + ADD $1, R8, R13 + MOVW R6, (R0)(R9<<2) + MOVW R8, (R0)(R11<<2) + ADD R10<<2, R0, R15 + MOVW R7, 4096(R15) + ADD R12<<2, R0, R15 + MOVW R13, 4096(R15) + ADD R6, R8, R7 + ADD $1, R7, R7 + LSR $0x01, R7, R7 + ADD $0x01, R6, R6 + SUB $0x01, R8, R8 + +index_loop_encodeSnappyBetterBlockAsm8B: + CMP R8, R7 + BHS search_loop_encodeSnappyBetterBlockAsm8B + MOVD (R3)(R6), R9 + MOVD (R3)(R7), R10 + LSL $0x10, R9, R9 + MUL R5, R9, R9 + LSR $0x36, R9, R9 + LSL $0x10, R10, R10 + MUL R5, R10, R10 + LSR $0x36, R10, R10 + MOVW R6, (R0)(R9<<2) + MOVW R7, (R0)(R10<<2) + ADD $0x02, R6, R6 + ADD $0x02, R7, R7 + JMP index_loop_encodeSnappyBetterBlockAsm8B + +emit_remainder_encodeSnappyBetterBlockAsm8B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_encodeSnappyBetterBlockAsm8B + MOVD $0x00000000, R16 + MOVD R16, ret+56(FP) + RET + +emit_remainder_ok_encodeSnappyBetterBlockAsm8B: + MOVD src_len+32(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R2 + MOVWU R2, R2 + CMPW $0x3c, R2 + BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B + CMPW $0x00000100, R2 + BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B + BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B + +three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B: + MOVD $0xf4, R16 + MOVB R16, (R1) + MOVH R2, 1(R1) + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B + +two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B: + MOVD $0xf0, R16 + MOVB R16, (R1) + MOVB R2, 1(R1) + ADD $0x02, R1, R1 + CMPW $0x40, R2 + BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm8B + JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B + +one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B: + LSLW $0x02, R2, R16 + BFI $0, R16, $8, R2 + MOVB R2, (R1) + ADD $0x01, R1, R1 + +memmove_emit_remainder_encodeSnappyBetterBlockAsm8B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveShort + CMP $0x03, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2 + BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3 + CMP $0x08, R3 + BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7 + CMP $0x10, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16 + CMP $0x20, R3 + BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32 + JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64 + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2: + MOVBU (R0), R16 + BFI $0, R16, $8, R5 + ADD R3, R0, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R0 + MOVB R5, (R1) + ADD R3, R1, R15 + MOVB R0, -1(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3: + MOVHU (R0), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R0), R16 + BFI $0, R16, $8, R0 + MOVH R5, (R1) + MOVB R0, 2(R1) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7: + MOVWU (R0), R5 + ADD R3, R0, R15 + MOVWU -4(R15), R0 + MOVW R5, (R1) + ADD R3, R1, R15 + MOVW R0, -4(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16: + MOVD (R0), R5 + ADD R3, R0, R15 + MOVD -8(R15), R0 + MOVD R5, (R1) + ADD R3, R1, R15 + MOVD R0, -8(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32: + FMOVQ (R0), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R1) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B + +emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64: + FMOVQ (R0), F0 + FMOVQ 16(R0), F1 + ADD R3, R0, R15 + FMOVQ -32(R15), F2 + ADD R3, R0, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R1) + FMOVQ F1, 16(R1) + ADD R3, R1, R15 + FMOVQ F2, -32(R15) + ADD R3, R1, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B: + MOVD R2, R1 + JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B + +memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B: + ADD R5, R1, R2 + MOVWU R5, R3 + + // genMemMoveLong + MOVD R0, R5 + MOVD R1, R6 + MOVD R3, R7 + +emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back + ADD R3, R0, R15 + FMOVQ -32(R15), F0 + ADD R3, R0, R15 + FMOVQ -16(R15), F1 + ADD R3, R1, R15 + FMOVQ F0, -32(R15) + ADD R3, R1, R15 + FMOVQ F1, -16(R15) + MOVD R2, R1 + +emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B: + MOVD dst_base+0(FP), R0 + SUB R0, R1, R1 + MOVD R1, ret+56(FP) + RET + +// func calcBlockSize(src []byte, tmp *[32768]byte) int +// Requires: BMI, SSE2 +TEXT ·calcBlockSize(SB), $24-40 + MOVD tmp+24(FP), R0 + MOVD $0, R1 + MOVD $0x00000100, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_calcBlockSize: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_calcBlockSize + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+8(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+0(FP), R3 + +search_loop_calcBlockSize: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x05, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_calcBlockSize + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x33, R9, R9 + LSL $0x10, R10, R10 + MUL R8, R10, R10 + LSR $0x33, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x10, R9, R9 + MUL R8, R9, R9 + LSR $0x33, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_calcBlockSize + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R5 + MOVWU R6, R7 + MOVWU 24(RSP), R16 + SUBSW R16, R7, R7 + BEQ repeat_extend_back_end_calcBlockSize + +repeat_extend_back_loop_calcBlockSize: + CMPW R5, R6 + BLS repeat_extend_back_end_calcBlockSize + ADD R7, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_calcBlockSize + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R7, R7 + BNE repeat_extend_back_loop_calcBlockSize + +repeat_extend_back_end_calcBlockSize: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $5, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_calcBlockSize + MOVD $0x00000000, R16 + MOVD R16, ret+32(FP) + RET + +repeat_dst_size_check_calcBlockSize: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_calcBlockSize + MOVWU R6, R7 + MOVW R6, 20(RSP) + ADD R5, R3, R8 + SUBW R5, R7, R7 + SUB $1, R7, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_calcBlockSize + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_calcBlockSize + CMPW $0x00010000, R5 + BLO three_bytes_repeat_emit_calcBlockSize + CMPW $0x01000000, R5 + BLO four_bytes_repeat_emit_calcBlockSize + ADD $0x05, R1, R1 + JMP memmove_long_repeat_emit_calcBlockSize + +four_bytes_repeat_emit_calcBlockSize: + ADD $0x04, R1, R1 + JMP memmove_long_repeat_emit_calcBlockSize + +three_bytes_repeat_emit_calcBlockSize: + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_calcBlockSize + +two_bytes_repeat_emit_calcBlockSize: + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_calcBlockSize + JMP memmove_long_repeat_emit_calcBlockSize + +one_byte_repeat_emit_calcBlockSize: + ADD $0x01, R1, R1 + +memmove_repeat_emit_calcBlockSize: + ADD R7, R1, R1 + JMP emit_literal_done_repeat_emit_calcBlockSize + +memmove_long_repeat_emit_calcBlockSize: + ADD R7, R1, R1 + +emit_literal_done_repeat_emit_calcBlockSize: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+8(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R10 + +matchlen_loopback_16_repeat_extend_calcBlockSize: + CMPW $0x10, R7 + BLO matchlen_match8_repeat_extend_calcBlockSize + MOVD (R8)(R10), R9 + ADD R10, R8, R15 + MOVD 8(R15), R11 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_calcBlockSize + ADD R10, R5, R15 + MOVD 8(R15), R16 + EOR R16, R11, R11 + TST R11, R11 + BNE matchlen_bsf_16repeat_extend_calcBlockSize + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R10, R10 + MOVWU R10, R10 + JMP matchlen_loopback_16_repeat_extend_calcBlockSize + +matchlen_bsf_16repeat_extend_calcBlockSize: + RBIT R11, R11 + CLZ R11, R11 + ASR $0x03, R11, R11 + ADD R11, R10, R10 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_calcBlockSize + +matchlen_match8_repeat_extend_calcBlockSize: + CMPW $0x08, R7 + BLO matchlen_match4_repeat_extend_calcBlockSize + MOVD (R8)(R10), R9 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_calcBlockSize + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP matchlen_match4_repeat_extend_calcBlockSize + +matchlen_bsf_8_repeat_extend_calcBlockSize: + RBIT R9, R9 + CLZ R9, R9 + ASR $0x03, R9, R9 + ADD R9, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_calcBlockSize + +matchlen_match4_repeat_extend_calcBlockSize: + CMPW $0x04, R7 + BLO matchlen_match2_repeat_extend_calcBlockSize + MOVWU (R8)(R10), R9 + MOVWU (R5)(R10), R16 + CMPW R9, R16 + BNE matchlen_match2_repeat_extend_calcBlockSize + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R10, R10 + MOVWU R10, R10 + +matchlen_match2_repeat_extend_calcBlockSize: + CMPW $0x01, R7 + BEQ matchlen_match1_repeat_extend_calcBlockSize + BLO repeat_extend_forward_end_calcBlockSize + MOVHU (R8)(R10), R16 + BFI $0, R16, $16, R9 + ADD R10, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R9, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_calcBlockSize + ADD $2, R10, R10 + MOVWU R10, R10 + SUBSW $0x02, R7, R7 + BEQ repeat_extend_forward_end_calcBlockSize + +matchlen_match1_repeat_extend_calcBlockSize: + MOVBU (R8)(R10), R16 + BFI $0, R16, $8, R9 + ADD R10, R5, R15 + MOVBU (R15), R15 + AND $0xff, R9, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_calcBlockSize + ADD $1, R10, R10 + MOVWU R10, R10 + +repeat_extend_forward_end_calcBlockSize: + ADDW R10, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + + // emitCopy + CMPW $0x00010000, R6 + BLO two_byte_offset_repeat_as_copy_calcBlockSize + +four_bytes_loop_back_repeat_as_copy_calcBlockSize: + CMPW $0x40, R5 + BLS four_bytes_remain_repeat_as_copy_calcBlockSize + SUB $64, R5, R5 + MOVWU R5, R5 + ADD $0x05, R1, R1 + CMPW $0x04, R5 + BLO four_bytes_remain_repeat_as_copy_calcBlockSize + JMP four_bytes_loop_back_repeat_as_copy_calcBlockSize + +four_bytes_remain_repeat_as_copy_calcBlockSize: + TSTW R5, R5 + BEQ repeat_end_emit_calcBlockSize + MOVD $0, R5 + ADD $0x05, R1, R1 + JMP repeat_end_emit_calcBlockSize + +two_byte_offset_repeat_as_copy_calcBlockSize: + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_calcBlockSize + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + JMP two_byte_offset_repeat_as_copy_calcBlockSize + +two_byte_offset_short_repeat_as_copy_calcBlockSize: + MOVWU R5, R7 + LSLW $0x02, R7, R7 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_calcBlockSize + CMPW $0x00000800, R6 + BHS emit_copy_three_repeat_as_copy_calcBlockSize + ADD $0x02, R1, R1 + JMP repeat_end_emit_calcBlockSize + +emit_copy_three_repeat_as_copy_calcBlockSize: + ADD $0x03, R1, R1 + +repeat_end_emit_calcBlockSize: + MOVW R2, 20(RSP) + JMP search_loop_calcBlockSize + +no_repeat_found_calcBlockSize: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_calcBlockSize + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_calcBlockSize + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_calcBlockSize + MOVWU 28(RSP), R2 + JMP search_loop_calcBlockSize + +candidate3_match_calcBlockSize: + ADDW $0x02, R2, R2 + JMP candidate_match_calcBlockSize + +candidate2_match_calcBlockSize: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_calcBlockSize: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_calcBlockSize + +match_extend_back_loop_calcBlockSize: + CMPW R6, R2 + BLS match_extend_back_end_calcBlockSize + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_calcBlockSize + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_calcBlockSize + JMP match_extend_back_loop_calcBlockSize + +match_extend_back_end_calcBlockSize: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $5, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_calcBlockSize + MOVD $0x00000000, R16 + MOVD R16, ret+32(FP) + RET + +match_dst_size_check_calcBlockSize: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_calcBlockSize + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R6 + MOVWU R6, R6 + CMPW $0x3c, R6 + BLO one_byte_match_emit_calcBlockSize + CMPW $0x00000100, R6 + BLO two_bytes_match_emit_calcBlockSize + CMPW $0x00010000, R6 + BLO three_bytes_match_emit_calcBlockSize + CMPW $0x01000000, R6 + BLO four_bytes_match_emit_calcBlockSize + ADD $0x05, R1, R1 + JMP memmove_long_match_emit_calcBlockSize + +four_bytes_match_emit_calcBlockSize: + ADD $0x04, R1, R1 + JMP memmove_long_match_emit_calcBlockSize + +three_bytes_match_emit_calcBlockSize: + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_calcBlockSize + +two_bytes_match_emit_calcBlockSize: + ADD $0x02, R1, R1 + CMPW $0x40, R6 + BLO memmove_match_emit_calcBlockSize + JMP memmove_long_match_emit_calcBlockSize + +one_byte_match_emit_calcBlockSize: + ADD $0x01, R1, R1 + +memmove_match_emit_calcBlockSize: + ADD R8, R1, R1 + JMP emit_literal_done_match_emit_calcBlockSize + +memmove_long_match_emit_calcBlockSize: + ADD R8, R1, R1 + +emit_literal_done_match_emit_calcBlockSize: +match_nolit_loop_calcBlockSize: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+8(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_calcBlockSize: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_calcBlockSize + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_calcBlockSize + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_calcBlockSize + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_calcBlockSize + +matchlen_bsf_16match_nolit_calcBlockSize: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_calcBlockSize + +matchlen_match8_match_nolit_calcBlockSize: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_calcBlockSize + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_calcBlockSize + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_calcBlockSize + +matchlen_bsf_8_match_nolit_calcBlockSize: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_calcBlockSize + +matchlen_match4_match_nolit_calcBlockSize: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_calcBlockSize + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_calcBlockSize + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_calcBlockSize: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_calcBlockSize + BLO match_nolit_end_calcBlockSize + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_calcBlockSize + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_calcBlockSize + +matchlen_match1_match_nolit_calcBlockSize: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_calcBlockSize + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_calcBlockSize: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy + CMPW $0x00010000, R5 + BLO two_byte_offset_match_nolit_calcBlockSize + +four_bytes_loop_back_match_nolit_calcBlockSize: + CMPW $0x40, R9 + BLS four_bytes_remain_match_nolit_calcBlockSize + SUB $64, R9, R9 + MOVWU R9, R9 + ADD $0x05, R1, R1 + CMPW $0x04, R9 + BLO four_bytes_remain_match_nolit_calcBlockSize + JMP four_bytes_loop_back_match_nolit_calcBlockSize + +four_bytes_remain_match_nolit_calcBlockSize: + TSTW R9, R9 + BEQ match_nolit_emitcopy_end_calcBlockSize + MOVD $0, R5 + ADD $0x05, R1, R1 + JMP match_nolit_emitcopy_end_calcBlockSize + +two_byte_offset_match_nolit_calcBlockSize: + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_calcBlockSize + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_calcBlockSize + +two_byte_offset_short_match_nolit_calcBlockSize: + MOVWU R9, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_calcBlockSize + CMPW $0x00000800, R5 + BHS emit_copy_three_match_nolit_calcBlockSize + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_calcBlockSize + +emit_copy_three_match_nolit_calcBlockSize: + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_calcBlockSize: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_calcBlockSize + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_calcBlockSize + MOVD $0x00000000, R16 + MOVD R16, ret+32(FP) + RET + +match_nolit_dst_ok_calcBlockSize: + MOVD $0x0000cf1bbcdcbf9b, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x10, R7, R7 + MUL R8, R7, R7 + LSR $0x33, R7, R7 + LSL $0x10, R5, R5 + MUL R8, R5, R5 + LSR $0x33, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_calcBlockSize + ADDW $1, R2, R2 + JMP search_loop_calcBlockSize + +emit_remainder_calcBlockSize: + MOVD src_len+8(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $5, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_calcBlockSize + MOVD $0x00000000, R16 + MOVD R16, ret+32(FP) + RET + +emit_remainder_ok_calcBlockSize: + MOVD src_len+8(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_calcBlockSize + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R0 + MOVWU R0, R0 + CMPW $0x3c, R0 + BLO one_byte_emit_remainder_calcBlockSize + CMPW $0x00000100, R0 + BLO two_bytes_emit_remainder_calcBlockSize + CMPW $0x00010000, R0 + BLO three_bytes_emit_remainder_calcBlockSize + CMPW $0x01000000, R0 + BLO four_bytes_emit_remainder_calcBlockSize + ADD $0x05, R1, R1 + JMP memmove_long_emit_remainder_calcBlockSize + +four_bytes_emit_remainder_calcBlockSize: + ADD $0x04, R1, R1 + JMP memmove_long_emit_remainder_calcBlockSize + +three_bytes_emit_remainder_calcBlockSize: + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_calcBlockSize + +two_bytes_emit_remainder_calcBlockSize: + ADD $0x02, R1, R1 + CMPW $0x40, R0 + BLO memmove_emit_remainder_calcBlockSize + JMP memmove_long_emit_remainder_calcBlockSize + +one_byte_emit_remainder_calcBlockSize: + ADD $0x01, R1, R1 + +memmove_emit_remainder_calcBlockSize: + ADD R5, R1, R0 + MOVD R0, R1 + JMP emit_literal_done_emit_remainder_calcBlockSize + +memmove_long_emit_remainder_calcBlockSize: + ADD R5, R1, R0 + MOVD R0, R1 + +emit_literal_done_emit_remainder_calcBlockSize: + MOVD R1, ret+32(FP) + RET + +// func calcBlockSizeSmall(src []byte, tmp *[2048]byte) int +// Requires: BMI, SSE2 +TEXT ·calcBlockSizeSmall(SB), $24-40 + MOVD tmp+24(FP), R0 + MOVD $0, R1 + MOVD $0x00000010, R2 + MOVD R0, R3 + VEOR V0.B16, V0.B16, V0.B16 + +zero_loop_calcBlockSizeSmall: + FMOVQ F0, (R3) + FMOVQ F0, 16(R3) + FMOVQ F0, 32(R3) + FMOVQ F0, 48(R3) + FMOVQ F0, 64(R3) + FMOVQ F0, 80(R3) + FMOVQ F0, 96(R3) + FMOVQ F0, 112(R3) + ADD $0x80, R3, R3 + SUBS $1, R2, R2 + BNE zero_loop_calcBlockSizeSmall + MOVD $0x00000000, R16 + MOVW R16, 20(RSP) + MOVD src_len+8(FP), R2 + SUB $9, R2, R3 + SUB $8, R2, R5 + MOVW R5, 16(RSP) + LSR $0x05, R2, R2 + SUBW R2, R3, R3 + ADD R3, R1, R3 + MOVD R3, 8(RSP) + MOVD $0x00000001, R2 + MOVW R2, 24(RSP) + MOVD src_base+0(FP), R3 + +search_loop_calcBlockSizeSmall: + MOVWU R2, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + LSRW $0x04, R5, R5 + ADD R5, R2, R5 + ADD $4, R5, R5 + MOVWU R5, R5 + MOVWU 16(RSP), R16 + CMPW R16, R5 + BHS emit_remainder_calcBlockSizeSmall + MOVD (R3)(R2), R6 + MOVW R5, 28(RSP) + MOVD $0x9e3779b1, R8 + MOVD R6, R9 + MOVD R6, R10 + LSR $0x08, R10, R10 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x37, R9, R9 + LSL $0x20, R10, R10 + MUL R8, R10, R10 + LSR $0x37, R10, R10 + MOVWU (R0)(R9<<2), R5 + MOVWU (R0)(R10<<2), R7 + MOVW R2, (R0)(R9<<2) + ADD $1, R2, R9 + MOVWU R9, R9 + MOVW R9, (R0)(R10<<2) + MOVD R6, R9 + LSR $0x10, R9, R9 + LSL $0x20, R9, R9 + MUL R8, R9, R9 + LSR $0x37, R9, R9 + MOVWU R2, R8 + MOVWU 24(RSP), R16 + SUBW R16, R8, R8 + ADD R8, R3, R15 + MOVWU 1(R15), R10 + MOVD R6, R8 + LSR $0x08, R8, R8 + CMPW R10, R8 + BNE no_repeat_found_calcBlockSizeSmall + ADD $1, R2, R6 + MOVWU R6, R6 + MOVWU 20(RSP), R5 + MOVWU R6, R7 + MOVWU 24(RSP), R16 + SUBSW R16, R7, R7 + BEQ repeat_extend_back_end_calcBlockSizeSmall + +repeat_extend_back_loop_calcBlockSizeSmall: + CMPW R5, R6 + BLS repeat_extend_back_end_calcBlockSizeSmall + ADD R7, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + ADD R6, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R9 + AND $0xff, R9, R16 + AND $0xff, R8, R15 + CMP R16, R15 + BNE repeat_extend_back_end_calcBlockSizeSmall + SUB $1, R6, R6 + MOVWU R6, R6 + SUBSW $1, R7, R7 + BNE repeat_extend_back_loop_calcBlockSizeSmall + +repeat_extend_back_end_calcBlockSizeSmall: + MOVWU R6, R5 + MOVWU 20(RSP), R16 + SUBW R16, R5, R5 + ADD R5, R1, R5 + ADD $3, R5, R5 + MOVD 8(RSP), R16 + CMP R16, R5 + BLO repeat_dst_size_check_calcBlockSizeSmall + MOVD $0x00000000, R16 + MOVD R16, ret+32(FP) + RET + +repeat_dst_size_check_calcBlockSizeSmall: + MOVWU 20(RSP), R5 + CMPW R6, R5 + BEQ emit_literal_done_repeat_emit_calcBlockSizeSmall + MOVWU R6, R7 + MOVW R6, 20(RSP) + ADD R5, R3, R8 + SUBW R5, R7, R7 + SUB $1, R7, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_repeat_emit_calcBlockSizeSmall + CMPW $0x00000100, R5 + BLO two_bytes_repeat_emit_calcBlockSizeSmall + BLO three_bytes_repeat_emit_calcBlockSizeSmall + +three_bytes_repeat_emit_calcBlockSizeSmall: + ADD $0x03, R1, R1 + JMP memmove_long_repeat_emit_calcBlockSizeSmall + +two_bytes_repeat_emit_calcBlockSizeSmall: + ADD $0x02, R1, R1 + CMPW $0x40, R5 + BLO memmove_repeat_emit_calcBlockSizeSmall + JMP memmove_long_repeat_emit_calcBlockSizeSmall + +one_byte_repeat_emit_calcBlockSizeSmall: + ADD $0x01, R1, R1 + +memmove_repeat_emit_calcBlockSizeSmall: + ADD R7, R1, R1 + JMP emit_literal_done_repeat_emit_calcBlockSizeSmall + +memmove_long_repeat_emit_calcBlockSizeSmall: + ADD R7, R1, R1 + +emit_literal_done_repeat_emit_calcBlockSizeSmall: + ADDW $0x05, R2, R2 + MOVWU R2, R5 + MOVWU 24(RSP), R16 + SUBW R16, R5, R5 + MOVD src_len+8(FP), R7 + SUBW R2, R7, R7 + ADD R2, R3, R8 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R10 + +matchlen_loopback_16_repeat_extend_calcBlockSizeSmall: + CMPW $0x10, R7 + BLO matchlen_match8_repeat_extend_calcBlockSizeSmall + MOVD (R8)(R10), R9 + ADD R10, R8, R15 + MOVD 8(R15), R11 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_calcBlockSizeSmall + ADD R10, R5, R15 + MOVD 8(R15), R16 + EOR R16, R11, R11 + TST R11, R11 + BNE matchlen_bsf_16repeat_extend_calcBlockSizeSmall + SUB $16, R7, R7 + MOVWU R7, R7 + ADD $16, R10, R10 + MOVWU R10, R10 + JMP matchlen_loopback_16_repeat_extend_calcBlockSizeSmall + +matchlen_bsf_16repeat_extend_calcBlockSizeSmall: + RBIT R11, R11 + CLZ R11, R11 + ASR $0x03, R11, R11 + ADD R11, R10, R10 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_calcBlockSizeSmall + +matchlen_match8_repeat_extend_calcBlockSizeSmall: + CMPW $0x08, R7 + BLO matchlen_match4_repeat_extend_calcBlockSizeSmall + MOVD (R8)(R10), R9 + MOVD (R5)(R10), R16 + EOR R16, R9, R9 + TST R9, R9 + BNE matchlen_bsf_8_repeat_extend_calcBlockSizeSmall + SUB $8, R7, R7 + MOVWU R7, R7 + ADD $8, R10, R10 + MOVWU R10, R10 + JMP matchlen_match4_repeat_extend_calcBlockSizeSmall + +matchlen_bsf_8_repeat_extend_calcBlockSizeSmall: + RBIT R9, R9 + CLZ R9, R9 + ASR $0x03, R9, R9 + ADD R9, R10, R10 + MOVWU R10, R10 + JMP repeat_extend_forward_end_calcBlockSizeSmall + +matchlen_match4_repeat_extend_calcBlockSizeSmall: + CMPW $0x04, R7 + BLO matchlen_match2_repeat_extend_calcBlockSizeSmall + MOVWU (R8)(R10), R9 + MOVWU (R5)(R10), R16 + CMPW R9, R16 + BNE matchlen_match2_repeat_extend_calcBlockSizeSmall + SUB $4, R7, R7 + MOVWU R7, R7 + ADD $4, R10, R10 + MOVWU R10, R10 + +matchlen_match2_repeat_extend_calcBlockSizeSmall: + CMPW $0x01, R7 + BEQ matchlen_match1_repeat_extend_calcBlockSizeSmall + BLO repeat_extend_forward_end_calcBlockSizeSmall + MOVHU (R8)(R10), R16 + BFI $0, R16, $16, R9 + ADD R10, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R9, R16 + CMP R16, R15 + BNE matchlen_match1_repeat_extend_calcBlockSizeSmall + ADD $2, R10, R10 + MOVWU R10, R10 + SUBSW $0x02, R7, R7 + BEQ repeat_extend_forward_end_calcBlockSizeSmall + +matchlen_match1_repeat_extend_calcBlockSizeSmall: + MOVBU (R8)(R10), R16 + BFI $0, R16, $8, R9 + ADD R10, R5, R15 + MOVBU (R15), R15 + AND $0xff, R9, R16 + CMP R16, R15 + BNE repeat_extend_forward_end_calcBlockSizeSmall + ADD $1, R10, R10 + MOVWU R10, R10 + +repeat_extend_forward_end_calcBlockSizeSmall: + ADDW R10, R2, R2 + MOVWU R2, R5 + SUBW R6, R5, R5 + MOVWU 24(RSP), R6 + + // emitCopy +two_byte_offset_repeat_as_copy_calcBlockSizeSmall: + CMPW $0x40, R5 + BLS two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall + SUB $60, R5, R5 + MOVWU R5, R5 + ADD $0x03, R1, R1 + JMP two_byte_offset_repeat_as_copy_calcBlockSizeSmall + +two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall: + MOVWU R5, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R5 + BHS emit_copy_three_repeat_as_copy_calcBlockSizeSmall + ADD $0x02, R1, R1 + JMP repeat_end_emit_calcBlockSizeSmall + +emit_copy_three_repeat_as_copy_calcBlockSizeSmall: + ADD $0x03, R1, R1 + +repeat_end_emit_calcBlockSizeSmall: + MOVW R2, 20(RSP) + JMP search_loop_calcBlockSizeSmall + +no_repeat_found_calcBlockSizeSmall: + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate_match_calcBlockSizeSmall + LSR $0x08, R6, R6 + MOVWU (R0)(R9<<2), R5 + ADD $2, R2, R8 + MOVWU R8, R8 + MOVWU (R3)(R7), R16 + CMPW R6, R16 + BEQ candidate2_match_calcBlockSizeSmall + MOVW R8, (R0)(R9<<2) + LSR $0x08, R6, R6 + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ candidate3_match_calcBlockSizeSmall + MOVWU 28(RSP), R2 + JMP search_loop_calcBlockSizeSmall + +candidate3_match_calcBlockSizeSmall: + ADDW $0x02, R2, R2 + JMP candidate_match_calcBlockSizeSmall + +candidate2_match_calcBlockSizeSmall: + MOVW R8, (R0)(R9<<2) + ADDW $1, R2, R2 + MOVWU R7, R5 + +candidate_match_calcBlockSizeSmall: + MOVWU 20(RSP), R6 + TSTW R5, R5 + BEQ match_extend_back_end_calcBlockSizeSmall + +match_extend_back_loop_calcBlockSizeSmall: + CMPW R6, R2 + BLS match_extend_back_end_calcBlockSizeSmall + ADD R5, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R7 + ADD R2, R3, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R8 + AND $0xff, R8, R16 + AND $0xff, R7, R15 + CMP R16, R15 + BNE match_extend_back_end_calcBlockSizeSmall + SUB $1, R2, R2 + MOVWU R2, R2 + SUBSW $1, R5, R5 + BEQ match_extend_back_end_calcBlockSizeSmall + JMP match_extend_back_loop_calcBlockSizeSmall + +match_extend_back_end_calcBlockSizeSmall: + MOVWU R2, R6 + MOVWU 20(RSP), R16 + SUBW R16, R6, R6 + ADD R6, R1, R6 + ADD $3, R6, R6 + MOVD 8(RSP), R16 + CMP R16, R6 + BLO match_dst_size_check_calcBlockSizeSmall + MOVD $0x00000000, R16 + MOVD R16, ret+32(FP) + RET + +match_dst_size_check_calcBlockSizeSmall: + MOVWU R2, R6 + MOVWU 20(RSP), R7 + CMPW R6, R7 + BEQ emit_literal_done_match_emit_calcBlockSizeSmall + MOVWU R6, R8 + MOVW R6, 20(RSP) + ADD R7, R3, R6 + SUBW R7, R8, R8 + SUB $1, R8, R6 + MOVWU R6, R6 + CMPW $0x3c, R6 + BLO one_byte_match_emit_calcBlockSizeSmall + CMPW $0x00000100, R6 + BLO two_bytes_match_emit_calcBlockSizeSmall + BLO three_bytes_match_emit_calcBlockSizeSmall + +three_bytes_match_emit_calcBlockSizeSmall: + ADD $0x03, R1, R1 + JMP memmove_long_match_emit_calcBlockSizeSmall + +two_bytes_match_emit_calcBlockSizeSmall: + ADD $0x02, R1, R1 + CMPW $0x40, R6 + BLO memmove_match_emit_calcBlockSizeSmall + JMP memmove_long_match_emit_calcBlockSizeSmall + +one_byte_match_emit_calcBlockSizeSmall: + ADD $0x01, R1, R1 + +memmove_match_emit_calcBlockSizeSmall: + ADD R8, R1, R1 + JMP emit_literal_done_match_emit_calcBlockSizeSmall + +memmove_long_match_emit_calcBlockSizeSmall: + ADD R8, R1, R1 + +emit_literal_done_match_emit_calcBlockSizeSmall: +match_nolit_loop_calcBlockSizeSmall: + MOVWU R2, R6 + SUBW R5, R6, R6 + MOVW R6, 24(RSP) + ADDW $0x04, R2, R2 + ADDW $0x04, R5, R5 + MOVD src_len+8(FP), R6 + SUBW R2, R6, R6 + ADD R2, R3, R7 + ADD R5, R3, R5 + + // matchLen + MOVD $0, R9 + +matchlen_loopback_16_match_nolit_calcBlockSizeSmall: + CMPW $0x10, R6 + BLO matchlen_match8_match_nolit_calcBlockSizeSmall + MOVD (R7)(R9), R8 + ADD R9, R7, R15 + MOVD 8(R15), R10 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_calcBlockSizeSmall + ADD R9, R5, R15 + MOVD 8(R15), R16 + EOR R16, R10, R10 + TST R10, R10 + BNE matchlen_bsf_16match_nolit_calcBlockSizeSmall + SUB $16, R6, R6 + MOVWU R6, R6 + ADD $16, R9, R9 + MOVWU R9, R9 + JMP matchlen_loopback_16_match_nolit_calcBlockSizeSmall + +matchlen_bsf_16match_nolit_calcBlockSizeSmall: + RBIT R10, R10 + CLZ R10, R10 + ASR $0x03, R10, R10 + ADD R10, R9, R9 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_calcBlockSizeSmall + +matchlen_match8_match_nolit_calcBlockSizeSmall: + CMPW $0x08, R6 + BLO matchlen_match4_match_nolit_calcBlockSizeSmall + MOVD (R7)(R9), R8 + MOVD (R5)(R9), R16 + EOR R16, R8, R8 + TST R8, R8 + BNE matchlen_bsf_8_match_nolit_calcBlockSizeSmall + SUB $8, R6, R6 + MOVWU R6, R6 + ADD $8, R9, R9 + MOVWU R9, R9 + JMP matchlen_match4_match_nolit_calcBlockSizeSmall + +matchlen_bsf_8_match_nolit_calcBlockSizeSmall: + RBIT R8, R8 + CLZ R8, R8 + ASR $0x03, R8, R8 + ADD R8, R9, R9 + MOVWU R9, R9 + JMP match_nolit_end_calcBlockSizeSmall + +matchlen_match4_match_nolit_calcBlockSizeSmall: + CMPW $0x04, R6 + BLO matchlen_match2_match_nolit_calcBlockSizeSmall + MOVWU (R7)(R9), R8 + MOVWU (R5)(R9), R16 + CMPW R8, R16 + BNE matchlen_match2_match_nolit_calcBlockSizeSmall + SUB $4, R6, R6 + MOVWU R6, R6 + ADD $4, R9, R9 + MOVWU R9, R9 + +matchlen_match2_match_nolit_calcBlockSizeSmall: + CMPW $0x01, R6 + BEQ matchlen_match1_match_nolit_calcBlockSizeSmall + BLO match_nolit_end_calcBlockSizeSmall + MOVHU (R7)(R9), R16 + BFI $0, R16, $16, R8 + ADD R9, R5, R15 + MOVHU (R15), R15 + AND $0xffff, R8, R16 + CMP R16, R15 + BNE matchlen_match1_match_nolit_calcBlockSizeSmall + ADD $2, R9, R9 + MOVWU R9, R9 + SUBSW $0x02, R6, R6 + BEQ match_nolit_end_calcBlockSizeSmall + +matchlen_match1_match_nolit_calcBlockSizeSmall: + MOVBU (R7)(R9), R16 + BFI $0, R16, $8, R8 + ADD R9, R5, R15 + MOVBU (R15), R15 + AND $0xff, R8, R16 + CMP R16, R15 + BNE match_nolit_end_calcBlockSizeSmall + ADD $1, R9, R9 + MOVWU R9, R9 + +match_nolit_end_calcBlockSizeSmall: + ADDW R9, R2, R2 + MOVWU 24(RSP), R5 + ADDW $0x04, R9, R9 + MOVW R2, 20(RSP) + + // emitCopy +two_byte_offset_match_nolit_calcBlockSizeSmall: + CMPW $0x40, R9 + BLS two_byte_offset_short_match_nolit_calcBlockSizeSmall + SUB $60, R9, R9 + MOVWU R9, R9 + ADD $0x03, R1, R1 + JMP two_byte_offset_match_nolit_calcBlockSizeSmall + +two_byte_offset_short_match_nolit_calcBlockSizeSmall: + MOVWU R9, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R9 + BHS emit_copy_three_match_nolit_calcBlockSizeSmall + ADD $0x02, R1, R1 + JMP match_nolit_emitcopy_end_calcBlockSizeSmall + +emit_copy_three_match_nolit_calcBlockSizeSmall: + ADD $0x03, R1, R1 + +match_nolit_emitcopy_end_calcBlockSizeSmall: + MOVWU 16(RSP), R16 + CMPW R16, R2 + BHS emit_remainder_calcBlockSizeSmall + ADD R2, R3, R15 + MOVD -2(R15), R6 + MOVD 8(RSP), R16 + CMP R16, R1 + BLO match_nolit_dst_ok_calcBlockSizeSmall + MOVD $0x00000000, R16 + MOVD R16, ret+32(FP) + RET + +match_nolit_dst_ok_calcBlockSizeSmall: + MOVD $0x9e3779b1, R8 + MOVD R6, R7 + LSR $0x10, R6, R6 + MOVD R6, R5 + LSL $0x20, R7, R7 + MUL R8, R7, R7 + LSR $0x37, R7, R7 + LSL $0x20, R5, R5 + MUL R8, R5, R5 + LSR $0x37, R5, R5 + SUB $2, R2, R8 + MOVWU R8, R8 + ADD R5<<2, R0, R9 + MOVWU (R9), R5 + MOVW R8, (R0)(R7<<2) + MOVW R2, (R9) + MOVWU (R3)(R5), R16 + CMPW R6, R16 + BEQ match_nolit_loop_calcBlockSizeSmall + ADDW $1, R2, R2 + JMP search_loop_calcBlockSizeSmall + +emit_remainder_calcBlockSizeSmall: + MOVD src_len+8(FP), R0 + MOVWU 20(RSP), R16 + SUBW R16, R0, R0 + ADD R0, R1, R0 + ADD $3, R0, R0 + MOVD 8(RSP), R16 + CMP R16, R0 + BLO emit_remainder_ok_calcBlockSizeSmall + MOVD $0x00000000, R16 + MOVD R16, ret+32(FP) + RET + +emit_remainder_ok_calcBlockSizeSmall: + MOVD src_len+8(FP), R0 + MOVWU 20(RSP), R2 + CMPW R0, R2 + BEQ emit_literal_done_emit_remainder_calcBlockSizeSmall + MOVWU R0, R5 + MOVW R0, 20(RSP) + ADD R2, R3, R0 + SUBW R2, R5, R5 + SUB $1, R5, R0 + MOVWU R0, R0 + CMPW $0x3c, R0 + BLO one_byte_emit_remainder_calcBlockSizeSmall + CMPW $0x00000100, R0 + BLO two_bytes_emit_remainder_calcBlockSizeSmall + BLO three_bytes_emit_remainder_calcBlockSizeSmall + +three_bytes_emit_remainder_calcBlockSizeSmall: + ADD $0x03, R1, R1 + JMP memmove_long_emit_remainder_calcBlockSizeSmall + +two_bytes_emit_remainder_calcBlockSizeSmall: + ADD $0x02, R1, R1 + CMPW $0x40, R0 + BLO memmove_emit_remainder_calcBlockSizeSmall + JMP memmove_long_emit_remainder_calcBlockSizeSmall + +one_byte_emit_remainder_calcBlockSizeSmall: + ADD $0x01, R1, R1 + +memmove_emit_remainder_calcBlockSizeSmall: + ADD R5, R1, R0 + MOVD R0, R1 + JMP emit_literal_done_emit_remainder_calcBlockSizeSmall + +memmove_long_emit_remainder_calcBlockSizeSmall: + ADD R5, R1, R0 + MOVD R0, R1 + +emit_literal_done_emit_remainder_calcBlockSizeSmall: + MOVD R1, ret+32(FP) + RET + +// func emitLiteral(dst []byte, lit []byte) int +// Requires: SSE2 +TEXT ·emitLiteral(SB), NOSPLIT, $0-56 + MOVD lit_len+32(FP), R2 + MOVD dst_base+0(FP), R0 + MOVD lit_base+24(FP), R1 + TST R2, R2 + BEQ emit_literal_end_standalone_skip + MOVWU R2, R3 + SUB $1, R2, R5 + MOVWU R5, R5 + CMPW $0x3c, R5 + BLO one_byte_standalone + CMPW $0x00000100, R5 + BLO two_bytes_standalone + CMPW $0x00010000, R5 + BLO three_bytes_standalone + CMPW $0x01000000, R5 + BLO four_bytes_standalone + MOVD $0xfc, R16 + MOVB R16, (R0) + MOVW R5, 1(R0) + ADD $0x05, R3, R3 + ADD $0x05, R0, R0 + JMP memmove_long_standalone + +four_bytes_standalone: + MOVWU R5, R6 + LSRW $0x10, R6, R6 + MOVD $0xf8, R16 + MOVB R16, (R0) + MOVH R5, 1(R0) + MOVB R6, 3(R0) + ADD $0x04, R3, R3 + ADD $0x04, R0, R0 + JMP memmove_long_standalone + +three_bytes_standalone: + MOVD $0xf4, R16 + MOVB R16, (R0) + MOVH R5, 1(R0) + ADD $0x03, R3, R3 + ADD $0x03, R0, R0 + JMP memmove_long_standalone + +two_bytes_standalone: + MOVD $0xf0, R16 + MOVB R16, (R0) + MOVB R5, 1(R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + CMPW $0x40, R5 + BLO memmove_standalone + JMP memmove_long_standalone + +one_byte_standalone: + LSLW $0x02, R5, R16 + BFI $0, R16, $8, R5 + MOVB R5, (R0) + ADD $0x01, R3, R3 + ADD $0x01, R0, R0 + +memmove_standalone: + // genMemMoveShort + CMP $0x03, R2 + BLO emit_lit_memmove_standalone_memmove_move_1or2 + BEQ emit_lit_memmove_standalone_memmove_move_3 + CMP $0x08, R2 + BLO emit_lit_memmove_standalone_memmove_move_4through7 + CMP $0x10, R2 + BLS emit_lit_memmove_standalone_memmove_move_8through16 + CMP $0x20, R2 + BLS emit_lit_memmove_standalone_memmove_move_17through32 + JMP emit_lit_memmove_standalone_memmove_move_33through64 + +emit_lit_memmove_standalone_memmove_move_1or2: + MOVBU (R1), R16 + BFI $0, R16, $8, R5 + ADD R2, R1, R15 + MOVBU -1(R15), R16 + BFI $0, R16, $8, R1 + MOVB R5, (R0) + ADD R2, R0, R15 + MOVB R1, -1(R15) + JMP emit_literal_end_standalone + +emit_lit_memmove_standalone_memmove_move_3: + MOVHU (R1), R16 + BFI $0, R16, $16, R5 + MOVBU 2(R1), R16 + BFI $0, R16, $8, R1 + MOVH R5, (R0) + MOVB R1, 2(R0) + JMP emit_literal_end_standalone + +emit_lit_memmove_standalone_memmove_move_4through7: + MOVWU (R1), R5 + ADD R2, R1, R15 + MOVWU -4(R15), R1 + MOVW R5, (R0) + ADD R2, R0, R15 + MOVW R1, -4(R15) + JMP emit_literal_end_standalone + +emit_lit_memmove_standalone_memmove_move_8through16: + MOVD (R1), R5 + ADD R2, R1, R15 + MOVD -8(R15), R1 + MOVD R5, (R0) + ADD R2, R0, R15 + MOVD R1, -8(R15) + JMP emit_literal_end_standalone + +emit_lit_memmove_standalone_memmove_move_17through32: + FMOVQ (R1), F0 + ADD R2, R1, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R0) + ADD R2, R0, R15 + FMOVQ F1, -16(R15) + JMP emit_literal_end_standalone + +emit_lit_memmove_standalone_memmove_move_33through64: + FMOVQ (R1), F0 + FMOVQ 16(R1), F1 + ADD R2, R1, R15 + FMOVQ -32(R15), F2 + ADD R2, R1, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R0) + FMOVQ F1, 16(R0) + ADD R2, R0, R15 + FMOVQ F2, -32(R15) + ADD R2, R0, R15 + FMOVQ F3, -16(R15) + JMP emit_literal_end_standalone + JMP emit_literal_end_standalone + +memmove_long_standalone: + // genMemMoveLong + MOVD R1, R5 + MOVD R0, R6 + MOVD R2, R7 + +emit_lit_memmove_long_standalonelarge_big_loop_back: + FMOVQ (R5), F0 + FMOVQ 16(R5), F1 + FMOVQ F0, (R6) + FMOVQ F1, 16(R6) + ADD $0x20, R5, R5 + ADD $0x20, R6, R6 + SUB $0x20, R7, R7 + CMP $0x20, R7 + BHS emit_lit_memmove_long_standalonelarge_big_loop_back + ADD R2, R1, R15 + FMOVQ -32(R15), F0 + ADD R2, R1, R15 + FMOVQ -16(R15), F1 + ADD R2, R0, R15 + FMOVQ F0, -32(R15) + ADD R2, R0, R15 + FMOVQ F1, -16(R15) + JMP emit_literal_end_standalone + JMP emit_literal_end_standalone + +emit_literal_end_standalone_skip: + MOVD $0, R3 + +emit_literal_end_standalone: + MOVD R3, ret+48(FP) + RET + +// func emitRepeat(dst []byte, offset int, length int) int +TEXT ·emitRepeat(SB), NOSPLIT, $0-48 + MOVD $0, R3 + MOVD dst_base+0(FP), R0 + MOVD offset+24(FP), R1 + MOVD length+32(FP), R2 + + // emitRepeat +emit_repeat_again_standalone: + MOVWU R2, R5 + SUB $4, R2, R2 + MOVWU R2, R2 + CMPW $0x08, R5 + BLS repeat_two_standalone + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_standalone + CMPW $0x00000800, R1 + BLO repeat_two_offset_standalone + +cant_repeat_two_offset_standalone: + CMPW $0x00000104, R2 + BLO repeat_three_standalone + CMPW $0x00010100, R2 + BLO repeat_four_standalone + CMPW $0x0100ffff, R2 + BLO repeat_five_standalone + SUB $16842747, R2, R2 + MOVWU R2, R2 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + ADD $0x05, R3, R3 + JMP emit_repeat_again_standalone + +repeat_five_standalone: + SUB $65536, R2, R2 + MOVWU R2, R2 + MOVWU R2, R1 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R2, 2(R0) + ASRW $0x10, R1, R1 + MOVB R1, 4(R0) + ADD $0x05, R3, R3 + ADD $0x05, R0, R0 + JMP gen_emit_repeat_end + +repeat_four_standalone: + SUB $256, R2, R2 + MOVWU R2, R2 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R2, 2(R0) + ADD $0x04, R3, R3 + ADD $0x04, R0, R0 + JMP gen_emit_repeat_end + +repeat_three_standalone: + SUB $4, R2, R2 + MOVWU R2, R2 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R2, 2(R0) + ADD $0x03, R3, R3 + ADD $0x03, R0, R0 + JMP gen_emit_repeat_end + +repeat_two_standalone: + LSLW $0x02, R2, R2 + ORRW $0x01, R2, R2 + MOVH R2, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + JMP gen_emit_repeat_end + +repeat_two_offset_standalone: + MOVD $0, R5 + ADD R2<<2, R5, R2 + ADD $1, R2, R2 + MOVWU R2, R2 + MOVB R1, 1(R0) + ASRW $0x08, R1, R1 + LSLW $0x05, R1, R1 + ORRW R1, R2, R2 + MOVB R2, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + +gen_emit_repeat_end: + MOVD R3, ret+40(FP) + RET + +// func emitCopy(dst []byte, offset int, length int) int +TEXT ·emitCopy(SB), NOSPLIT, $0-48 + MOVD $0, R3 + MOVD dst_base+0(FP), R0 + MOVD offset+24(FP), R1 + MOVD length+32(FP), R2 + + // emitCopy + CMPW $0x00010000, R1 + BLO two_byte_offset_standalone + CMPW $0x40, R2 + BLS four_bytes_remain_standalone + MOVD $0xff, R16 + MOVB R16, (R0) + MOVW R1, 1(R0) + SUB $64, R2, R2 + MOVWU R2, R2 + ADD $0x05, R3, R3 + ADD $0x05, R0, R0 + CMPW $0x04, R2 + BLO four_bytes_remain_standalone + + // emitRepeat +emit_repeat_again_standalone_emit_copy: + MOVWU R2, R5 + SUB $4, R2, R2 + MOVWU R2, R2 + CMPW $0x08, R5 + BLS repeat_two_standalone_emit_copy + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_standalone_emit_copy + CMPW $0x00000800, R1 + BLO repeat_two_offset_standalone_emit_copy + +cant_repeat_two_offset_standalone_emit_copy: + CMPW $0x00000104, R2 + BLO repeat_three_standalone_emit_copy + CMPW $0x00010100, R2 + BLO repeat_four_standalone_emit_copy + CMPW $0x0100ffff, R2 + BLO repeat_five_standalone_emit_copy + SUB $16842747, R2, R2 + MOVWU R2, R2 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + ADD $0x05, R3, R3 + JMP emit_repeat_again_standalone_emit_copy + +repeat_five_standalone_emit_copy: + SUB $65536, R2, R2 + MOVWU R2, R2 + MOVWU R2, R1 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R2, 2(R0) + ASRW $0x10, R1, R1 + MOVB R1, 4(R0) + ADD $0x05, R3, R3 + ADD $0x05, R0, R0 + JMP gen_emit_copy_end + +repeat_four_standalone_emit_copy: + SUB $256, R2, R2 + MOVWU R2, R2 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R2, 2(R0) + ADD $0x04, R3, R3 + ADD $0x04, R0, R0 + JMP gen_emit_copy_end + +repeat_three_standalone_emit_copy: + SUB $4, R2, R2 + MOVWU R2, R2 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R2, 2(R0) + ADD $0x03, R3, R3 + ADD $0x03, R0, R0 + JMP gen_emit_copy_end + +repeat_two_standalone_emit_copy: + LSLW $0x02, R2, R2 + ORRW $0x01, R2, R2 + MOVH R2, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + JMP gen_emit_copy_end + +repeat_two_offset_standalone_emit_copy: + MOVD $0, R5 + ADD R2<<2, R5, R2 + ADD $1, R2, R2 + MOVWU R2, R2 + MOVB R1, 1(R0) + ASRW $0x08, R1, R1 + LSLW $0x05, R1, R1 + ORRW R1, R2, R2 + MOVB R2, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + JMP gen_emit_copy_end + +four_bytes_remain_standalone: + TSTW R2, R2 + BEQ gen_emit_copy_end + MOVD $0, R5 + ADD R2<<2, R5, R2 + SUB $1, R2, R2 + MOVWU R2, R2 + MOVB R2, (R0) + MOVW R1, 1(R0) + ADD $0x05, R3, R3 + ADD $0x05, R0, R0 + JMP gen_emit_copy_end + +two_byte_offset_standalone: + CMPW $0x40, R2 + BLS two_byte_offset_short_standalone + CMPW $0x00000800, R1 + BHS long_offset_short_standalone + MOVD $0x00000001, R5 + ADD $16, R5, R5 + MOVWU R5, R5 + MOVB R1, 1(R0) + MOVWU R1, R6 + LSRW $0x08, R6, R6 + LSLW $0x05, R6, R6 + ORRW R6, R5, R5 + MOVB R5, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + SUBW $0x08, R2, R2 + + // emitRepeat + SUB $4, R2, R2 + MOVWU R2, R2 + JMP cant_repeat_two_offset_standalone_emit_copy_short_2b + +emit_repeat_again_standalone_emit_copy_short_2b: + MOVWU R2, R5 + SUB $4, R2, R2 + MOVWU R2, R2 + CMPW $0x08, R5 + BLS repeat_two_standalone_emit_copy_short_2b + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_standalone_emit_copy_short_2b + CMPW $0x00000800, R1 + BLO repeat_two_offset_standalone_emit_copy_short_2b + +cant_repeat_two_offset_standalone_emit_copy_short_2b: + CMPW $0x00000104, R2 + BLO repeat_three_standalone_emit_copy_short_2b + CMPW $0x00010100, R2 + BLO repeat_four_standalone_emit_copy_short_2b + CMPW $0x0100ffff, R2 + BLO repeat_five_standalone_emit_copy_short_2b + SUB $16842747, R2, R2 + MOVWU R2, R2 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + ADD $0x05, R3, R3 + JMP emit_repeat_again_standalone_emit_copy_short_2b + +repeat_five_standalone_emit_copy_short_2b: + SUB $65536, R2, R2 + MOVWU R2, R2 + MOVWU R2, R1 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R2, 2(R0) + ASRW $0x10, R1, R1 + MOVB R1, 4(R0) + ADD $0x05, R3, R3 + ADD $0x05, R0, R0 + JMP gen_emit_copy_end + +repeat_four_standalone_emit_copy_short_2b: + SUB $256, R2, R2 + MOVWU R2, R2 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R2, 2(R0) + ADD $0x04, R3, R3 + ADD $0x04, R0, R0 + JMP gen_emit_copy_end + +repeat_three_standalone_emit_copy_short_2b: + SUB $4, R2, R2 + MOVWU R2, R2 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R2, 2(R0) + ADD $0x03, R3, R3 + ADD $0x03, R0, R0 + JMP gen_emit_copy_end + +repeat_two_standalone_emit_copy_short_2b: + LSLW $0x02, R2, R2 + ORRW $0x01, R2, R2 + MOVH R2, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + JMP gen_emit_copy_end + +repeat_two_offset_standalone_emit_copy_short_2b: + MOVD $0, R5 + ADD R2<<2, R5, R2 + ADD $1, R2, R2 + MOVWU R2, R2 + MOVB R1, 1(R0) + ASRW $0x08, R1, R1 + LSLW $0x05, R1, R1 + ORRW R1, R2, R2 + MOVB R2, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + JMP gen_emit_copy_end + +long_offset_short_standalone: + MOVD $0xee, R16 + MOVB R16, (R0) + MOVH R1, 1(R0) + SUB $60, R2, R2 + MOVWU R2, R2 + ADD $0x03, R0, R0 + ADD $0x03, R3, R3 + + // emitRepeat +emit_repeat_again_standalone_emit_copy_short: + MOVWU R2, R5 + SUB $4, R2, R2 + MOVWU R2, R2 + CMPW $0x08, R5 + BLS repeat_two_standalone_emit_copy_short + CMPW $0x0c, R5 + BHS cant_repeat_two_offset_standalone_emit_copy_short + CMPW $0x00000800, R1 + BLO repeat_two_offset_standalone_emit_copy_short + +cant_repeat_two_offset_standalone_emit_copy_short: + CMPW $0x00000104, R2 + BLO repeat_three_standalone_emit_copy_short + CMPW $0x00010100, R2 + BLO repeat_four_standalone_emit_copy_short + CMPW $0x0100ffff, R2 + BLO repeat_five_standalone_emit_copy_short + SUB $16842747, R2, R2 + MOVWU R2, R2 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + ADD $0x05, R3, R3 + JMP emit_repeat_again_standalone_emit_copy_short + +repeat_five_standalone_emit_copy_short: + SUB $65536, R2, R2 + MOVWU R2, R2 + MOVWU R2, R1 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R2, 2(R0) + ASRW $0x10, R1, R1 + MOVB R1, 4(R0) + ADD $0x05, R3, R3 + ADD $0x05, R0, R0 + JMP gen_emit_copy_end + +repeat_four_standalone_emit_copy_short: + SUB $256, R2, R2 + MOVWU R2, R2 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R2, 2(R0) + ADD $0x04, R3, R3 + ADD $0x04, R0, R0 + JMP gen_emit_copy_end + +repeat_three_standalone_emit_copy_short: + SUB $4, R2, R2 + MOVWU R2, R2 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R2, 2(R0) + ADD $0x03, R3, R3 + ADD $0x03, R0, R0 + JMP gen_emit_copy_end + +repeat_two_standalone_emit_copy_short: + LSLW $0x02, R2, R2 + ORRW $0x01, R2, R2 + MOVH R2, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + JMP gen_emit_copy_end + +repeat_two_offset_standalone_emit_copy_short: + MOVD $0, R5 + ADD R2<<2, R5, R2 + ADD $1, R2, R2 + MOVWU R2, R2 + MOVB R1, 1(R0) + ASRW $0x08, R1, R1 + LSLW $0x05, R1, R1 + ORRW R1, R2, R2 + MOVB R2, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + JMP gen_emit_copy_end + +two_byte_offset_short_standalone: + MOVWU R2, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R2 + BHS emit_copy_three_standalone + CMPW $0x00000800, R1 + BHS emit_copy_three_standalone + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R1, 1(R0) + LSRW $0x08, R1, R1 + LSLW $0x05, R1, R1 + ORRW R1, R5, R5 + MOVB R5, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + JMP gen_emit_copy_end + +emit_copy_three_standalone: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R0) + MOVH R1, 1(R0) + ADD $0x03, R3, R3 + ADD $0x03, R0, R0 + +gen_emit_copy_end: + MOVD R3, ret+40(FP) + RET + +// func emitCopyNoRepeat(dst []byte, offset int, length int) int +TEXT ·emitCopyNoRepeat(SB), NOSPLIT, $0-48 + MOVD $0, R3 + MOVD dst_base+0(FP), R0 + MOVD offset+24(FP), R1 + MOVD length+32(FP), R2 + + // emitCopy + CMPW $0x00010000, R1 + BLO two_byte_offset_standalone_snappy + +four_bytes_loop_back_standalone_snappy: + CMPW $0x40, R2 + BLS four_bytes_remain_standalone_snappy + MOVD $0xff, R16 + MOVB R16, (R0) + MOVW R1, 1(R0) + SUB $64, R2, R2 + MOVWU R2, R2 + ADD $0x05, R3, R3 + ADD $0x05, R0, R0 + CMPW $0x04, R2 + BLO four_bytes_remain_standalone_snappy + JMP four_bytes_loop_back_standalone_snappy + +four_bytes_remain_standalone_snappy: + TSTW R2, R2 + BEQ gen_emit_copy_end_snappy + MOVD $0, R5 + ADD R2<<2, R5, R2 + SUB $1, R2, R2 + MOVWU R2, R2 + MOVB R2, (R0) + MOVW R1, 1(R0) + ADD $0x05, R3, R3 + ADD $0x05, R0, R0 + JMP gen_emit_copy_end_snappy + +two_byte_offset_standalone_snappy: + CMPW $0x40, R2 + BLS two_byte_offset_short_standalone_snappy + MOVD $0xee, R16 + MOVB R16, (R0) + MOVH R1, 1(R0) + SUB $60, R2, R2 + MOVWU R2, R2 + ADD $0x03, R0, R0 + ADD $0x03, R3, R3 + JMP two_byte_offset_standalone_snappy + +two_byte_offset_short_standalone_snappy: + MOVWU R2, R5 + LSLW $0x02, R5, R5 + CMPW $0x0c, R2 + BHS emit_copy_three_standalone_snappy + CMPW $0x00000800, R1 + BHS emit_copy_three_standalone_snappy + SUB $15, R5, R5 + MOVWU R5, R5 + MOVB R1, 1(R0) + LSRW $0x08, R1, R1 + LSLW $0x05, R1, R1 + ORRW R1, R5, R5 + MOVB R5, (R0) + ADD $0x02, R3, R3 + ADD $0x02, R0, R0 + JMP gen_emit_copy_end_snappy + +emit_copy_three_standalone_snappy: + SUB $2, R5, R5 + MOVWU R5, R5 + MOVB R5, (R0) + MOVH R1, 1(R0) + ADD $0x03, R3, R3 + ADD $0x03, R0, R0 + +gen_emit_copy_end_snappy: + MOVD R3, ret+40(FP) + RET + +// func matchLen(a []byte, b []byte) int +// Requires: BMI +TEXT ·matchLen(SB), NOSPLIT, $0-56 + MOVD a_base+0(FP), R0 + MOVD b_base+24(FP), R1 + MOVD a_len+8(FP), R2 + + // matchLen + MOVD $0, R5 + +matchlen_loopback_16_standalone: + CMPW $0x10, R2 + BLO matchlen_match8_standalone + MOVD (R0)(R5), R3 + ADD R5, R0, R15 + MOVD 8(R15), R6 + MOVD (R1)(R5), R16 + EOR R16, R3, R3 + TST R3, R3 + BNE matchlen_bsf_8_standalone + ADD R5, R1, R15 + MOVD 8(R15), R16 + EOR R16, R6, R6 + TST R6, R6 + BNE matchlen_bsf_16standalone + SUB $16, R2, R2 + MOVWU R2, R2 + ADD $16, R5, R5 + MOVWU R5, R5 + JMP matchlen_loopback_16_standalone + +matchlen_bsf_16standalone: + RBIT R6, R6 + CLZ R6, R6 + ASR $0x03, R6, R6 + ADD R6, R5, R5 + ADD $8, R5, R5 + MOVWU R5, R5 + JMP gen_match_len_end + +matchlen_match8_standalone: + CMPW $0x08, R2 + BLO matchlen_match4_standalone + MOVD (R0)(R5), R3 + MOVD (R1)(R5), R16 + EOR R16, R3, R3 + TST R3, R3 + BNE matchlen_bsf_8_standalone + SUB $8, R2, R2 + MOVWU R2, R2 + ADD $8, R5, R5 + MOVWU R5, R5 + JMP matchlen_match4_standalone + +matchlen_bsf_8_standalone: + RBIT R3, R3 + CLZ R3, R3 + ASR $0x03, R3, R3 + ADD R3, R5, R5 + MOVWU R5, R5 + JMP gen_match_len_end + +matchlen_match4_standalone: + CMPW $0x04, R2 + BLO matchlen_match2_standalone + MOVWU (R0)(R5), R3 + MOVWU (R1)(R5), R16 + CMPW R3, R16 + BNE matchlen_match2_standalone + SUB $4, R2, R2 + MOVWU R2, R2 + ADD $4, R5, R5 + MOVWU R5, R5 + +matchlen_match2_standalone: + CMPW $0x01, R2 + BEQ matchlen_match1_standalone + BLO gen_match_len_end + MOVHU (R0)(R5), R16 + BFI $0, R16, $16, R3 + ADD R5, R1, R15 + MOVHU (R15), R15 + AND $0xffff, R3, R16 + CMP R16, R15 + BNE matchlen_match1_standalone + ADD $2, R5, R5 + MOVWU R5, R5 + SUBSW $0x02, R2, R2 + BEQ gen_match_len_end + +matchlen_match1_standalone: + MOVBU (R0)(R5), R16 + BFI $0, R16, $8, R3 + ADD R5, R1, R15 + MOVBU (R15), R15 + AND $0xff, R3, R16 + CMP R16, R15 + BNE gen_match_len_end + ADD $1, R5, R5 + MOVWU R5, R5 + +gen_match_len_end: + MOVD R5, ret+48(FP) + RET + +// func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) +// Requires: SSE2 +TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64 + MOVD $0, R5 + MOVD dst_base+0(FP), R0 + MOVD dst_len+8(FP), R1 + MOVD src_base+24(FP), R2 + MOVD src_len+32(FP), R3 + ADD R3, R2, R3 + ADD R1, R0, R1 + SUB $8, R1, R1 + SUB $4, R1, R6 + MOVD $0, R7 + +lz4_s2_loop: + CMP R3, R2 + BHS lz4_s2_corrupt + CMP R1, R0 + BHS lz4_s2_dstfull + MOVBU (R2), R8 + MOVD R8, R9 + MOVD R8, R10 + LSR $0x04, R9, R9 + AND $0x0f, R10, R10 + CMP $0xf0, R8 + BLO lz4_s2_ll_end + +lz4_s2_ll_loop: + ADD $1, R2, R2 + CMP R3, R2 + BHS lz4_s2_corrupt + MOVBU (R2), R8 + ADD R8, R9, R9 + CMP $0xff, R8 + BEQ lz4_s2_ll_loop + +lz4_s2_ll_end: + ADD R9, R2, R8 + ADD $0x04, R10, R10 + CMP R3, R8 + BHS lz4_s2_corrupt + ADD $1, R2, R2 + ADD $1, R8, R8 + TST R9, R9 + BEQ lz4_s2_lits_done + ADD R9, R0, R11 + CMP R6, R11 + BHS lz4_s2_dstfull + ADD R9, R5, R5 + SUB $1, R9, R11 + MOVWU R11, R11 + CMPW $0x3c, R11 + BLO one_byte_lz4_s2 + CMPW $0x00000100, R11 + BLO two_bytes_lz4_s2 + CMPW $0x00010000, R11 + BLO three_bytes_lz4_s2 + CMPW $0x01000000, R11 + BLO four_bytes_lz4_s2 + MOVD $0xfc, R16 + MOVB R16, (R0) + MOVW R11, 1(R0) + ADD $0x05, R0, R0 + JMP memmove_long_lz4_s2 + +four_bytes_lz4_s2: + MOVWU R11, R12 + LSRW $0x10, R12, R12 + MOVD $0xf8, R16 + MOVB R16, (R0) + MOVH R11, 1(R0) + MOVB R12, 3(R0) + ADD $0x04, R0, R0 + JMP memmove_long_lz4_s2 + +three_bytes_lz4_s2: + MOVD $0xf4, R16 + MOVB R16, (R0) + MOVH R11, 1(R0) + ADD $0x03, R0, R0 + JMP memmove_long_lz4_s2 + +two_bytes_lz4_s2: + MOVD $0xf0, R16 + MOVB R16, (R0) + MOVB R11, 1(R0) + ADD $0x02, R0, R0 + CMPW $0x40, R11 + BLO memmove_lz4_s2 + JMP memmove_long_lz4_s2 + +one_byte_lz4_s2: + LSLW $0x02, R11, R16 + BFI $0, R16, $8, R11 + MOVB R11, (R0) + ADD $0x01, R0, R0 + +memmove_lz4_s2: + ADD R9, R0, R11 + + // genMemMoveShort + CMP $0x08, R9 + BLS emit_lit_memmove_lz4_s2_memmove_move_8 + CMP $0x10, R9 + BLS emit_lit_memmove_lz4_s2_memmove_move_8through16 + CMP $0x20, R9 + BLS emit_lit_memmove_lz4_s2_memmove_move_17through32 + JMP emit_lit_memmove_lz4_s2_memmove_move_33through64 + +emit_lit_memmove_lz4_s2_memmove_move_8: + MOVD (R2), R12 + MOVD R12, (R0) + JMP memmove_end_copy_lz4_s2 + +emit_lit_memmove_lz4_s2_memmove_move_8through16: + MOVD (R2), R12 + ADD R9, R2, R15 + MOVD -8(R15), R2 + MOVD R12, (R0) + ADD R9, R0, R15 + MOVD R2, -8(R15) + JMP memmove_end_copy_lz4_s2 + +emit_lit_memmove_lz4_s2_memmove_move_17through32: + FMOVQ (R2), F0 + ADD R9, R2, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R0) + ADD R9, R0, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_lz4_s2 + +emit_lit_memmove_lz4_s2_memmove_move_33through64: + FMOVQ (R2), F0 + FMOVQ 16(R2), F1 + ADD R9, R2, R15 + FMOVQ -32(R15), F2 + ADD R9, R2, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R0) + FMOVQ F1, 16(R0) + ADD R9, R0, R15 + FMOVQ F2, -32(R15) + ADD R9, R0, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_lz4_s2: + MOVD R11, R0 + JMP lz4_s2_lits_emit_done + +memmove_long_lz4_s2: + ADD R9, R0, R11 + + // genMemMoveLong + MOVD R2, R12 + MOVD R0, R13 + MOVD R9, R14 + +emit_lit_memmove_long_lz4_s2large_big_loop_back: + FMOVQ (R12), F0 + FMOVQ 16(R12), F1 + FMOVQ F0, (R13) + FMOVQ F1, 16(R13) + ADD $0x20, R12, R12 + ADD $0x20, R13, R13 + SUB $0x20, R14, R14 + CMP $0x20, R14 + BHS emit_lit_memmove_long_lz4_s2large_big_loop_back + ADD R9, R2, R15 + FMOVQ -32(R15), F0 + ADD R9, R2, R15 + FMOVQ -16(R15), F1 + ADD R9, R0, R15 + FMOVQ F0, -32(R15) + ADD R9, R0, R15 + FMOVQ F1, -16(R15) + MOVD R11, R0 + +lz4_s2_lits_emit_done: + MOVD R8, R2 + +lz4_s2_lits_done: + CMP R3, R2 + BNE lz4_s2_match + CMP $0x04, R10 + BEQ lz4_s2_done + JMP lz4_s2_corrupt + +lz4_s2_match: + ADD $2, R2, R8 + CMP R3, R8 + BHS lz4_s2_corrupt + MOVHU (R2), R9 + MOVD R8, R2 + TST R9, R9 + BEQ lz4_s2_corrupt + CMP R5, R9 + BHI lz4_s2_corrupt + CMP $0x13, R10 + BNE lz4_s2_ml_done + +lz4_s2_ml_loop: + MOVBU (R2), R8 + ADD $1, R2, R2 + ADD R8, R10, R10 + CMP R3, R2 + BHS lz4_s2_corrupt + CMP $0xff, R8 + BEQ lz4_s2_ml_loop + +lz4_s2_ml_done: + ADD R10, R5, R5 + CMP R7, R9 + BNE lz4_s2_docopy + + // emitRepeat +emit_repeat_again_lz4_s2: + MOVWU R10, R8 + SUB $4, R10, R10 + MOVWU R10, R10 + CMPW $0x08, R8 + BLS repeat_two_lz4_s2 + CMPW $0x0c, R8 + BHS cant_repeat_two_offset_lz4_s2 + CMPW $0x00000800, R9 + BLO repeat_two_offset_lz4_s2 + +cant_repeat_two_offset_lz4_s2: + CMPW $0x00000104, R10 + BLO repeat_three_lz4_s2 + CMPW $0x00010100, R10 + BLO repeat_four_lz4_s2 + CMPW $0x0100ffff, R10 + BLO repeat_five_lz4_s2 + SUB $16842747, R10, R10 + MOVWU R10, R10 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + CMP R1, R0 + BHS lz4_s2_loop + JMP emit_repeat_again_lz4_s2 + +repeat_five_lz4_s2: + SUB $65536, R10, R10 + MOVWU R10, R10 + MOVWU R10, R9 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ASRW $0x10, R9, R9 + MOVB R9, 4(R0) + ADD $0x05, R0, R0 + JMP lz4_s2_loop + +repeat_four_lz4_s2: + SUB $256, R10, R10 + MOVWU R10, R10 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ADD $0x04, R0, R0 + JMP lz4_s2_loop + +repeat_three_lz4_s2: + SUB $4, R10, R10 + MOVWU R10, R10 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R10, 2(R0) + ADD $0x03, R0, R0 + JMP lz4_s2_loop + +repeat_two_lz4_s2: + LSLW $0x02, R10, R10 + ORRW $0x01, R10, R10 + MOVH R10, (R0) + ADD $0x02, R0, R0 + JMP lz4_s2_loop + +repeat_two_offset_lz4_s2: + MOVD $0, R8 + ADD R10<<2, R8, R10 + ADD $1, R10, R10 + MOVWU R10, R10 + MOVB R9, 1(R0) + ASRW $0x08, R9, R9 + LSLW $0x05, R9, R9 + ORRW R9, R10, R10 + MOVB R10, (R0) + ADD $0x02, R0, R0 + JMP lz4_s2_loop + +lz4_s2_docopy: + MOVD R9, R7 + + // emitCopy + CMPW $0x40, R10 + BLS two_byte_offset_short_lz4_s2 + CMPW $0x00000800, R9 + BHS long_offset_short_lz4_s2 + MOVD $0x00000001, R8 + ADD $16, R8, R8 + MOVWU R8, R8 + MOVB R9, 1(R0) + MOVWU R9, R11 + LSRW $0x08, R11, R11 + LSLW $0x05, R11, R11 + ORRW R11, R8, R8 + MOVB R8, (R0) + ADD $0x02, R0, R0 + SUBW $0x08, R10, R10 + + // emitRepeat + SUB $4, R10, R10 + MOVWU R10, R10 + JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b + +emit_repeat_again_lz4_s2_emit_copy_short_2b: + MOVWU R10, R8 + SUB $4, R10, R10 + MOVWU R10, R10 + CMPW $0x08, R8 + BLS repeat_two_lz4_s2_emit_copy_short_2b + CMPW $0x0c, R8 + BHS cant_repeat_two_offset_lz4_s2_emit_copy_short_2b + CMPW $0x00000800, R9 + BLO repeat_two_offset_lz4_s2_emit_copy_short_2b + +cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: + CMPW $0x00000104, R10 + BLO repeat_three_lz4_s2_emit_copy_short_2b + CMPW $0x00010100, R10 + BLO repeat_four_lz4_s2_emit_copy_short_2b + CMPW $0x0100ffff, R10 + BLO repeat_five_lz4_s2_emit_copy_short_2b + SUB $16842747, R10, R10 + MOVWU R10, R10 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + CMP R1, R0 + BHS lz4_s2_loop + JMP emit_repeat_again_lz4_s2_emit_copy_short_2b + +repeat_five_lz4_s2_emit_copy_short_2b: + SUB $65536, R10, R10 + MOVWU R10, R10 + MOVWU R10, R9 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ASRW $0x10, R9, R9 + MOVB R9, 4(R0) + ADD $0x05, R0, R0 + JMP lz4_s2_loop + +repeat_four_lz4_s2_emit_copy_short_2b: + SUB $256, R10, R10 + MOVWU R10, R10 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ADD $0x04, R0, R0 + JMP lz4_s2_loop + +repeat_three_lz4_s2_emit_copy_short_2b: + SUB $4, R10, R10 + MOVWU R10, R10 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R10, 2(R0) + ADD $0x03, R0, R0 + JMP lz4_s2_loop + +repeat_two_lz4_s2_emit_copy_short_2b: + LSLW $0x02, R10, R10 + ORRW $0x01, R10, R10 + MOVH R10, (R0) + ADD $0x02, R0, R0 + JMP lz4_s2_loop + +repeat_two_offset_lz4_s2_emit_copy_short_2b: + MOVD $0, R8 + ADD R10<<2, R8, R10 + ADD $1, R10, R10 + MOVWU R10, R10 + MOVB R9, 1(R0) + ASRW $0x08, R9, R9 + LSLW $0x05, R9, R9 + ORRW R9, R10, R10 + MOVB R10, (R0) + ADD $0x02, R0, R0 + JMP lz4_s2_loop + +long_offset_short_lz4_s2: + MOVD $0xee, R16 + MOVB R16, (R0) + MOVH R9, 1(R0) + SUB $60, R10, R10 + MOVWU R10, R10 + ADD $0x03, R0, R0 + + // emitRepeat +emit_repeat_again_lz4_s2_emit_copy_short: + MOVWU R10, R8 + SUB $4, R10, R10 + MOVWU R10, R10 + CMPW $0x08, R8 + BLS repeat_two_lz4_s2_emit_copy_short + CMPW $0x0c, R8 + BHS cant_repeat_two_offset_lz4_s2_emit_copy_short + CMPW $0x00000800, R9 + BLO repeat_two_offset_lz4_s2_emit_copy_short + +cant_repeat_two_offset_lz4_s2_emit_copy_short: + CMPW $0x00000104, R10 + BLO repeat_three_lz4_s2_emit_copy_short + CMPW $0x00010100, R10 + BLO repeat_four_lz4_s2_emit_copy_short + CMPW $0x0100ffff, R10 + BLO repeat_five_lz4_s2_emit_copy_short + SUB $16842747, R10, R10 + MOVWU R10, R10 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + CMP R1, R0 + BHS lz4_s2_loop + JMP emit_repeat_again_lz4_s2_emit_copy_short + +repeat_five_lz4_s2_emit_copy_short: + SUB $65536, R10, R10 + MOVWU R10, R10 + MOVWU R10, R9 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ASRW $0x10, R9, R9 + MOVB R9, 4(R0) + ADD $0x05, R0, R0 + JMP lz4_s2_loop + +repeat_four_lz4_s2_emit_copy_short: + SUB $256, R10, R10 + MOVWU R10, R10 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ADD $0x04, R0, R0 + JMP lz4_s2_loop + +repeat_three_lz4_s2_emit_copy_short: + SUB $4, R10, R10 + MOVWU R10, R10 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R10, 2(R0) + ADD $0x03, R0, R0 + JMP lz4_s2_loop + +repeat_two_lz4_s2_emit_copy_short: + LSLW $0x02, R10, R10 + ORRW $0x01, R10, R10 + MOVH R10, (R0) + ADD $0x02, R0, R0 + JMP lz4_s2_loop + +repeat_two_offset_lz4_s2_emit_copy_short: + MOVD $0, R8 + ADD R10<<2, R8, R10 + ADD $1, R10, R10 + MOVWU R10, R10 + MOVB R9, 1(R0) + ASRW $0x08, R9, R9 + LSLW $0x05, R9, R9 + ORRW R9, R10, R10 + MOVB R10, (R0) + ADD $0x02, R0, R0 + JMP lz4_s2_loop + +two_byte_offset_short_lz4_s2: + MOVWU R10, R8 + LSLW $0x02, R8, R8 + CMPW $0x0c, R10 + BHS emit_copy_three_lz4_s2 + CMPW $0x00000800, R9 + BHS emit_copy_three_lz4_s2 + SUB $15, R8, R8 + MOVWU R8, R8 + MOVB R9, 1(R0) + LSRW $0x08, R9, R9 + LSLW $0x05, R9, R9 + ORRW R9, R8, R8 + MOVB R8, (R0) + ADD $0x02, R0, R0 + JMP lz4_s2_loop + +emit_copy_three_lz4_s2: + SUB $2, R8, R8 + MOVWU R8, R8 + MOVB R8, (R0) + MOVH R9, 1(R0) + ADD $0x03, R0, R0 + JMP lz4_s2_loop + +lz4_s2_done: + MOVD dst_base+0(FP), R1 + SUB R1, R0, R0 + MOVD R5, uncompressed+48(FP) + MOVD R0, dstUsed+56(FP) + RET + +lz4_s2_corrupt: + MOVD $0, R0 + SUB $1, R0, R5 + MOVD R5, uncompressed+48(FP) + RET + +lz4_s2_dstfull: + MOVD $0, R0 + SUB $2, R0, R5 + MOVD R5, uncompressed+48(FP) + RET + +// func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) +// Requires: SSE2 +TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64 + MOVD $0, R5 + MOVD dst_base+0(FP), R0 + MOVD dst_len+8(FP), R1 + MOVD src_base+24(FP), R2 + MOVD src_len+32(FP), R3 + ADD R3, R2, R3 + ADD R1, R0, R1 + SUB $8, R1, R1 + SUB $4, R1, R6 + MOVD $0, R7 + +lz4s_s2_loop: + CMP R3, R2 + BHS lz4s_s2_corrupt + CMP R1, R0 + BHS lz4s_s2_dstfull + MOVBU (R2), R8 + MOVD R8, R9 + MOVD R8, R10 + LSR $0x04, R9, R9 + AND $0x0f, R10, R10 + CMP $0xf0, R8 + BLO lz4s_s2_ll_end + +lz4s_s2_ll_loop: + ADD $1, R2, R2 + CMP R3, R2 + BHS lz4s_s2_corrupt + MOVBU (R2), R8 + ADD R8, R9, R9 + CMP $0xff, R8 + BEQ lz4s_s2_ll_loop + +lz4s_s2_ll_end: + ADD R9, R2, R8 + ADD $0x03, R10, R10 + CMP R3, R8 + BHS lz4s_s2_corrupt + ADD $1, R2, R2 + ADD $1, R8, R8 + TST R9, R9 + BEQ lz4s_s2_lits_done + ADD R9, R0, R11 + CMP R6, R11 + BHS lz4s_s2_dstfull + ADD R9, R5, R5 + SUB $1, R9, R11 + MOVWU R11, R11 + CMPW $0x3c, R11 + BLO one_byte_lz4s_s2 + CMPW $0x00000100, R11 + BLO two_bytes_lz4s_s2 + CMPW $0x00010000, R11 + BLO three_bytes_lz4s_s2 + CMPW $0x01000000, R11 + BLO four_bytes_lz4s_s2 + MOVD $0xfc, R16 + MOVB R16, (R0) + MOVW R11, 1(R0) + ADD $0x05, R0, R0 + JMP memmove_long_lz4s_s2 + +four_bytes_lz4s_s2: + MOVWU R11, R12 + LSRW $0x10, R12, R12 + MOVD $0xf8, R16 + MOVB R16, (R0) + MOVH R11, 1(R0) + MOVB R12, 3(R0) + ADD $0x04, R0, R0 + JMP memmove_long_lz4s_s2 + +three_bytes_lz4s_s2: + MOVD $0xf4, R16 + MOVB R16, (R0) + MOVH R11, 1(R0) + ADD $0x03, R0, R0 + JMP memmove_long_lz4s_s2 + +two_bytes_lz4s_s2: + MOVD $0xf0, R16 + MOVB R16, (R0) + MOVB R11, 1(R0) + ADD $0x02, R0, R0 + CMPW $0x40, R11 + BLO memmove_lz4s_s2 + JMP memmove_long_lz4s_s2 + +one_byte_lz4s_s2: + LSLW $0x02, R11, R16 + BFI $0, R16, $8, R11 + MOVB R11, (R0) + ADD $0x01, R0, R0 + +memmove_lz4s_s2: + ADD R9, R0, R11 + + // genMemMoveShort + CMP $0x08, R9 + BLS emit_lit_memmove_lz4s_s2_memmove_move_8 + CMP $0x10, R9 + BLS emit_lit_memmove_lz4s_s2_memmove_move_8through16 + CMP $0x20, R9 + BLS emit_lit_memmove_lz4s_s2_memmove_move_17through32 + JMP emit_lit_memmove_lz4s_s2_memmove_move_33through64 + +emit_lit_memmove_lz4s_s2_memmove_move_8: + MOVD (R2), R12 + MOVD R12, (R0) + JMP memmove_end_copy_lz4s_s2 + +emit_lit_memmove_lz4s_s2_memmove_move_8through16: + MOVD (R2), R12 + ADD R9, R2, R15 + MOVD -8(R15), R2 + MOVD R12, (R0) + ADD R9, R0, R15 + MOVD R2, -8(R15) + JMP memmove_end_copy_lz4s_s2 + +emit_lit_memmove_lz4s_s2_memmove_move_17through32: + FMOVQ (R2), F0 + ADD R9, R2, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R0) + ADD R9, R0, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_lz4s_s2 + +emit_lit_memmove_lz4s_s2_memmove_move_33through64: + FMOVQ (R2), F0 + FMOVQ 16(R2), F1 + ADD R9, R2, R15 + FMOVQ -32(R15), F2 + ADD R9, R2, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R0) + FMOVQ F1, 16(R0) + ADD R9, R0, R15 + FMOVQ F2, -32(R15) + ADD R9, R0, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_lz4s_s2: + MOVD R11, R0 + JMP lz4s_s2_lits_emit_done + +memmove_long_lz4s_s2: + ADD R9, R0, R11 + + // genMemMoveLong + MOVD R2, R12 + MOVD R0, R13 + MOVD R9, R14 + +emit_lit_memmove_long_lz4s_s2large_big_loop_back: + FMOVQ (R12), F0 + FMOVQ 16(R12), F1 + FMOVQ F0, (R13) + FMOVQ F1, 16(R13) + ADD $0x20, R12, R12 + ADD $0x20, R13, R13 + SUB $0x20, R14, R14 + CMP $0x20, R14 + BHS emit_lit_memmove_long_lz4s_s2large_big_loop_back + ADD R9, R2, R15 + FMOVQ -32(R15), F0 + ADD R9, R2, R15 + FMOVQ -16(R15), F1 + ADD R9, R0, R15 + FMOVQ F0, -32(R15) + ADD R9, R0, R15 + FMOVQ F1, -16(R15) + MOVD R11, R0 + +lz4s_s2_lits_emit_done: + MOVD R8, R2 + +lz4s_s2_lits_done: + CMP R3, R2 + BNE lz4s_s2_match + CMP $0x03, R10 + BEQ lz4s_s2_done + JMP lz4s_s2_corrupt + +lz4s_s2_match: + CMP $0x03, R10 + BEQ lz4s_s2_loop + ADD $2, R2, R8 + CMP R3, R8 + BHS lz4s_s2_corrupt + MOVHU (R2), R9 + MOVD R8, R2 + TST R9, R9 + BEQ lz4s_s2_corrupt + CMP R5, R9 + BHI lz4s_s2_corrupt + CMP $0x12, R10 + BNE lz4s_s2_ml_done + +lz4s_s2_ml_loop: + MOVBU (R2), R8 + ADD $1, R2, R2 + ADD R8, R10, R10 + CMP R3, R2 + BHS lz4s_s2_corrupt + CMP $0xff, R8 + BEQ lz4s_s2_ml_loop + +lz4s_s2_ml_done: + ADD R10, R5, R5 + CMP R7, R9 + BNE lz4s_s2_docopy + + // emitRepeat +emit_repeat_again_lz4_s2: + MOVWU R10, R8 + SUB $4, R10, R10 + MOVWU R10, R10 + CMPW $0x08, R8 + BLS repeat_two_lz4_s2 + CMPW $0x0c, R8 + BHS cant_repeat_two_offset_lz4_s2 + CMPW $0x00000800, R9 + BLO repeat_two_offset_lz4_s2 + +cant_repeat_two_offset_lz4_s2: + CMPW $0x00000104, R10 + BLO repeat_three_lz4_s2 + CMPW $0x00010100, R10 + BLO repeat_four_lz4_s2 + CMPW $0x0100ffff, R10 + BLO repeat_five_lz4_s2 + SUB $16842747, R10, R10 + MOVWU R10, R10 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + CMP R1, R0 + BHS lz4s_s2_loop + JMP emit_repeat_again_lz4_s2 + +repeat_five_lz4_s2: + SUB $65536, R10, R10 + MOVWU R10, R10 + MOVWU R10, R9 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ASRW $0x10, R9, R9 + MOVB R9, 4(R0) + ADD $0x05, R0, R0 + JMP lz4s_s2_loop + +repeat_four_lz4_s2: + SUB $256, R10, R10 + MOVWU R10, R10 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ADD $0x04, R0, R0 + JMP lz4s_s2_loop + +repeat_three_lz4_s2: + SUB $4, R10, R10 + MOVWU R10, R10 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R10, 2(R0) + ADD $0x03, R0, R0 + JMP lz4s_s2_loop + +repeat_two_lz4_s2: + LSLW $0x02, R10, R10 + ORRW $0x01, R10, R10 + MOVH R10, (R0) + ADD $0x02, R0, R0 + JMP lz4s_s2_loop + +repeat_two_offset_lz4_s2: + MOVD $0, R8 + ADD R10<<2, R8, R10 + ADD $1, R10, R10 + MOVWU R10, R10 + MOVB R9, 1(R0) + ASRW $0x08, R9, R9 + LSLW $0x05, R9, R9 + ORRW R9, R10, R10 + MOVB R10, (R0) + ADD $0x02, R0, R0 + JMP lz4s_s2_loop + +lz4s_s2_docopy: + MOVD R9, R7 + + // emitCopy + CMPW $0x40, R10 + BLS two_byte_offset_short_lz4_s2 + CMPW $0x00000800, R9 + BHS long_offset_short_lz4_s2 + MOVD $0x00000001, R8 + ADD $16, R8, R8 + MOVWU R8, R8 + MOVB R9, 1(R0) + MOVWU R9, R11 + LSRW $0x08, R11, R11 + LSLW $0x05, R11, R11 + ORRW R11, R8, R8 + MOVB R8, (R0) + ADD $0x02, R0, R0 + SUBW $0x08, R10, R10 + + // emitRepeat + SUB $4, R10, R10 + MOVWU R10, R10 + JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b + +emit_repeat_again_lz4_s2_emit_copy_short_2b: + MOVWU R10, R8 + SUB $4, R10, R10 + MOVWU R10, R10 + CMPW $0x08, R8 + BLS repeat_two_lz4_s2_emit_copy_short_2b + CMPW $0x0c, R8 + BHS cant_repeat_two_offset_lz4_s2_emit_copy_short_2b + CMPW $0x00000800, R9 + BLO repeat_two_offset_lz4_s2_emit_copy_short_2b + +cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: + CMPW $0x00000104, R10 + BLO repeat_three_lz4_s2_emit_copy_short_2b + CMPW $0x00010100, R10 + BLO repeat_four_lz4_s2_emit_copy_short_2b + CMPW $0x0100ffff, R10 + BLO repeat_five_lz4_s2_emit_copy_short_2b + SUB $16842747, R10, R10 + MOVWU R10, R10 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + CMP R1, R0 + BHS lz4s_s2_loop + JMP emit_repeat_again_lz4_s2_emit_copy_short_2b + +repeat_five_lz4_s2_emit_copy_short_2b: + SUB $65536, R10, R10 + MOVWU R10, R10 + MOVWU R10, R9 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ASRW $0x10, R9, R9 + MOVB R9, 4(R0) + ADD $0x05, R0, R0 + JMP lz4s_s2_loop + +repeat_four_lz4_s2_emit_copy_short_2b: + SUB $256, R10, R10 + MOVWU R10, R10 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ADD $0x04, R0, R0 + JMP lz4s_s2_loop + +repeat_three_lz4_s2_emit_copy_short_2b: + SUB $4, R10, R10 + MOVWU R10, R10 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R10, 2(R0) + ADD $0x03, R0, R0 + JMP lz4s_s2_loop + +repeat_two_lz4_s2_emit_copy_short_2b: + LSLW $0x02, R10, R10 + ORRW $0x01, R10, R10 + MOVH R10, (R0) + ADD $0x02, R0, R0 + JMP lz4s_s2_loop + +repeat_two_offset_lz4_s2_emit_copy_short_2b: + MOVD $0, R8 + ADD R10<<2, R8, R10 + ADD $1, R10, R10 + MOVWU R10, R10 + MOVB R9, 1(R0) + ASRW $0x08, R9, R9 + LSLW $0x05, R9, R9 + ORRW R9, R10, R10 + MOVB R10, (R0) + ADD $0x02, R0, R0 + JMP lz4s_s2_loop + +long_offset_short_lz4_s2: + MOVD $0xee, R16 + MOVB R16, (R0) + MOVH R9, 1(R0) + SUB $60, R10, R10 + MOVWU R10, R10 + ADD $0x03, R0, R0 + + // emitRepeat +emit_repeat_again_lz4_s2_emit_copy_short: + MOVWU R10, R8 + SUB $4, R10, R10 + MOVWU R10, R10 + CMPW $0x08, R8 + BLS repeat_two_lz4_s2_emit_copy_short + CMPW $0x0c, R8 + BHS cant_repeat_two_offset_lz4_s2_emit_copy_short + CMPW $0x00000800, R9 + BLO repeat_two_offset_lz4_s2_emit_copy_short + +cant_repeat_two_offset_lz4_s2_emit_copy_short: + CMPW $0x00000104, R10 + BLO repeat_three_lz4_s2_emit_copy_short + CMPW $0x00010100, R10 + BLO repeat_four_lz4_s2_emit_copy_short + CMPW $0x0100ffff, R10 + BLO repeat_five_lz4_s2_emit_copy_short + SUB $16842747, R10, R10 + MOVWU R10, R10 + MOVD $0xfffb001d, R16 + MOVW R16, (R0) + MOVD $0xff, R16 + MOVB R16, 4(R0) + ADD $0x05, R0, R0 + CMP R1, R0 + BHS lz4s_s2_loop + JMP emit_repeat_again_lz4_s2_emit_copy_short + +repeat_five_lz4_s2_emit_copy_short: + SUB $65536, R10, R10 + MOVWU R10, R10 + MOVWU R10, R9 + MOVD $0x001d, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ASRW $0x10, R9, R9 + MOVB R9, 4(R0) + ADD $0x05, R0, R0 + JMP lz4s_s2_loop + +repeat_four_lz4_s2_emit_copy_short: + SUB $256, R10, R10 + MOVWU R10, R10 + MOVD $0x0019, R16 + MOVH R16, (R0) + MOVH R10, 2(R0) + ADD $0x04, R0, R0 + JMP lz4s_s2_loop + +repeat_three_lz4_s2_emit_copy_short: + SUB $4, R10, R10 + MOVWU R10, R10 + MOVD $0x0015, R16 + MOVH R16, (R0) + MOVB R10, 2(R0) + ADD $0x03, R0, R0 + JMP lz4s_s2_loop + +repeat_two_lz4_s2_emit_copy_short: + LSLW $0x02, R10, R10 + ORRW $0x01, R10, R10 + MOVH R10, (R0) + ADD $0x02, R0, R0 + JMP lz4s_s2_loop + +repeat_two_offset_lz4_s2_emit_copy_short: + MOVD $0, R8 + ADD R10<<2, R8, R10 + ADD $1, R10, R10 + MOVWU R10, R10 + MOVB R9, 1(R0) + ASRW $0x08, R9, R9 + LSLW $0x05, R9, R9 + ORRW R9, R10, R10 + MOVB R10, (R0) + ADD $0x02, R0, R0 + JMP lz4s_s2_loop + +two_byte_offset_short_lz4_s2: + MOVWU R10, R8 + LSLW $0x02, R8, R8 + CMPW $0x0c, R10 + BHS emit_copy_three_lz4_s2 + CMPW $0x00000800, R9 + BHS emit_copy_three_lz4_s2 + SUB $15, R8, R8 + MOVWU R8, R8 + MOVB R9, 1(R0) + LSRW $0x08, R9, R9 + LSLW $0x05, R9, R9 + ORRW R9, R8, R8 + MOVB R8, (R0) + ADD $0x02, R0, R0 + JMP lz4s_s2_loop + +emit_copy_three_lz4_s2: + SUB $2, R8, R8 + MOVWU R8, R8 + MOVB R8, (R0) + MOVH R9, 1(R0) + ADD $0x03, R0, R0 + JMP lz4s_s2_loop + +lz4s_s2_done: + MOVD dst_base+0(FP), R1 + SUB R1, R0, R0 + MOVD R5, uncompressed+48(FP) + MOVD R0, dstUsed+56(FP) + RET + +lz4s_s2_corrupt: + MOVD $0, R0 + SUB $1, R0, R5 + MOVD R5, uncompressed+48(FP) + RET + +lz4s_s2_dstfull: + MOVD $0, R0 + SUB $2, R0, R5 + MOVD R5, uncompressed+48(FP) + RET + +// func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) +// Requires: SSE2 +TEXT ·cvtLZ4BlockSnappyAsm(SB), NOSPLIT, $0-64 + MOVD $0, R5 + MOVD dst_base+0(FP), R0 + MOVD dst_len+8(FP), R1 + MOVD src_base+24(FP), R2 + MOVD src_len+32(FP), R3 + ADD R3, R2, R3 + ADD R1, R0, R1 + SUB $8, R1, R1 + +lz4_snappy_loop: + CMP R3, R2 + BHS lz4_snappy_corrupt + CMP R1, R0 + BHS lz4_snappy_dstfull + MOVBU (R2), R6 + MOVD R6, R7 + MOVD R6, R8 + LSR $0x04, R7, R7 + AND $0x0f, R8, R8 + CMP $0xf0, R6 + BLO lz4_snappy_ll_end + +lz4_snappy_ll_loop: + ADD $1, R2, R2 + CMP R3, R2 + BHS lz4_snappy_corrupt + MOVBU (R2), R6 + ADD R6, R7, R7 + CMP $0xff, R6 + BEQ lz4_snappy_ll_loop + +lz4_snappy_ll_end: + ADD R7, R2, R6 + ADD $0x04, R8, R8 + CMP R3, R6 + BHS lz4_snappy_corrupt + ADD $1, R2, R2 + ADD $1, R6, R6 + TST R7, R7 + BEQ lz4_snappy_lits_done + ADD R7, R0, R9 + CMP R1, R9 + BHS lz4_snappy_dstfull + ADD R7, R5, R5 + SUB $1, R7, R9 + MOVWU R9, R9 + CMPW $0x3c, R9 + BLO one_byte_lz4_snappy + CMPW $0x00000100, R9 + BLO two_bytes_lz4_snappy + CMPW $0x00010000, R9 + BLO three_bytes_lz4_snappy + CMPW $0x01000000, R9 + BLO four_bytes_lz4_snappy + MOVD $0xfc, R16 + MOVB R16, (R0) + MOVW R9, 1(R0) + ADD $0x05, R0, R0 + JMP memmove_long_lz4_snappy + +four_bytes_lz4_snappy: + MOVWU R9, R10 + LSRW $0x10, R10, R10 + MOVD $0xf8, R16 + MOVB R16, (R0) + MOVH R9, 1(R0) + MOVB R10, 3(R0) + ADD $0x04, R0, R0 + JMP memmove_long_lz4_snappy + +three_bytes_lz4_snappy: + MOVD $0xf4, R16 + MOVB R16, (R0) + MOVH R9, 1(R0) + ADD $0x03, R0, R0 + JMP memmove_long_lz4_snappy + +two_bytes_lz4_snappy: + MOVD $0xf0, R16 + MOVB R16, (R0) + MOVB R9, 1(R0) + ADD $0x02, R0, R0 + CMPW $0x40, R9 + BLO memmove_lz4_snappy + JMP memmove_long_lz4_snappy + +one_byte_lz4_snappy: + LSLW $0x02, R9, R16 + BFI $0, R16, $8, R9 + MOVB R9, (R0) + ADD $0x01, R0, R0 + +memmove_lz4_snappy: + ADD R7, R0, R9 + + // genMemMoveShort + CMP $0x08, R7 + BLS emit_lit_memmove_lz4_snappy_memmove_move_8 + CMP $0x10, R7 + BLS emit_lit_memmove_lz4_snappy_memmove_move_8through16 + CMP $0x20, R7 + BLS emit_lit_memmove_lz4_snappy_memmove_move_17through32 + JMP emit_lit_memmove_lz4_snappy_memmove_move_33through64 + +emit_lit_memmove_lz4_snappy_memmove_move_8: + MOVD (R2), R10 + MOVD R10, (R0) + JMP memmove_end_copy_lz4_snappy + +emit_lit_memmove_lz4_snappy_memmove_move_8through16: + MOVD (R2), R10 + ADD R7, R2, R15 + MOVD -8(R15), R2 + MOVD R10, (R0) + ADD R7, R0, R15 + MOVD R2, -8(R15) + JMP memmove_end_copy_lz4_snappy + +emit_lit_memmove_lz4_snappy_memmove_move_17through32: + FMOVQ (R2), F0 + ADD R7, R2, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R0) + ADD R7, R0, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_lz4_snappy + +emit_lit_memmove_lz4_snappy_memmove_move_33through64: + FMOVQ (R2), F0 + FMOVQ 16(R2), F1 + ADD R7, R2, R15 + FMOVQ -32(R15), F2 + ADD R7, R2, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R0) + FMOVQ F1, 16(R0) + ADD R7, R0, R15 + FMOVQ F2, -32(R15) + ADD R7, R0, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_lz4_snappy: + MOVD R9, R0 + JMP lz4_snappy_lits_emit_done + +memmove_long_lz4_snappy: + ADD R7, R0, R9 + + // genMemMoveLong + MOVD R2, R10 + MOVD R0, R11 + MOVD R7, R12 + +emit_lit_memmove_long_lz4_snappylarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R11) + FMOVQ F1, 16(R11) + ADD $0x20, R10, R10 + ADD $0x20, R11, R11 + SUB $0x20, R12, R12 + CMP $0x20, R12 + BHS emit_lit_memmove_long_lz4_snappylarge_big_loop_back + ADD R7, R2, R15 + FMOVQ -32(R15), F0 + ADD R7, R2, R15 + FMOVQ -16(R15), F1 + ADD R7, R0, R15 + FMOVQ F0, -32(R15) + ADD R7, R0, R15 + FMOVQ F1, -16(R15) + MOVD R9, R0 + +lz4_snappy_lits_emit_done: + MOVD R6, R2 + +lz4_snappy_lits_done: + CMP R3, R2 + BNE lz4_snappy_match + CMP $0x04, R8 + BEQ lz4_snappy_done + JMP lz4_snappy_corrupt + +lz4_snappy_match: + ADD $2, R2, R6 + CMP R3, R6 + BHS lz4_snappy_corrupt + MOVHU (R2), R7 + MOVD R6, R2 + TST R7, R7 + BEQ lz4_snappy_corrupt + CMP R5, R7 + BHI lz4_snappy_corrupt + CMP $0x13, R8 + BNE lz4_snappy_ml_done + +lz4_snappy_ml_loop: + MOVBU (R2), R6 + ADD $1, R2, R2 + ADD R6, R8, R8 + CMP R3, R2 + BHS lz4_snappy_corrupt + CMP $0xff, R6 + BEQ lz4_snappy_ml_loop + +lz4_snappy_ml_done: + ADD R8, R5, R5 + + // emitCopy +two_byte_offset_lz4_s2: + CMPW $0x40, R8 + BLS two_byte_offset_short_lz4_s2 + MOVD $0xee, R16 + MOVB R16, (R0) + MOVH R7, 1(R0) + SUB $60, R8, R8 + MOVWU R8, R8 + ADD $0x03, R0, R0 + CMP R1, R0 + BHS lz4_snappy_loop + JMP two_byte_offset_lz4_s2 + +two_byte_offset_short_lz4_s2: + MOVWU R8, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R8 + BHS emit_copy_three_lz4_s2 + CMPW $0x00000800, R7 + BHS emit_copy_three_lz4_s2 + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R7, 1(R0) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R6, R6 + MOVB R6, (R0) + ADD $0x02, R0, R0 + JMP lz4_snappy_loop + +emit_copy_three_lz4_s2: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R0) + MOVH R7, 1(R0) + ADD $0x03, R0, R0 + JMP lz4_snappy_loop + +lz4_snappy_done: + MOVD dst_base+0(FP), R1 + SUB R1, R0, R0 + MOVD R5, uncompressed+48(FP) + MOVD R0, dstUsed+56(FP) + RET + +lz4_snappy_corrupt: + MOVD $0, R0 + SUB $1, R0, R5 + MOVD R5, uncompressed+48(FP) + RET + +lz4_snappy_dstfull: + MOVD $0, R0 + SUB $2, R0, R5 + MOVD R5, uncompressed+48(FP) + RET + +// func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) +// Requires: SSE2 +TEXT ·cvtLZ4sBlockSnappyAsm(SB), NOSPLIT, $0-64 + MOVD $0, R5 + MOVD dst_base+0(FP), R0 + MOVD dst_len+8(FP), R1 + MOVD src_base+24(FP), R2 + MOVD src_len+32(FP), R3 + ADD R3, R2, R3 + ADD R1, R0, R1 + SUB $8, R1, R1 + +lz4s_snappy_loop: + CMP R3, R2 + BHS lz4s_snappy_corrupt + CMP R1, R0 + BHS lz4s_snappy_dstfull + MOVBU (R2), R6 + MOVD R6, R7 + MOVD R6, R8 + LSR $0x04, R7, R7 + AND $0x0f, R8, R8 + CMP $0xf0, R6 + BLO lz4s_snappy_ll_end + +lz4s_snappy_ll_loop: + ADD $1, R2, R2 + CMP R3, R2 + BHS lz4s_snappy_corrupt + MOVBU (R2), R6 + ADD R6, R7, R7 + CMP $0xff, R6 + BEQ lz4s_snappy_ll_loop + +lz4s_snappy_ll_end: + ADD R7, R2, R6 + ADD $0x03, R8, R8 + CMP R3, R6 + BHS lz4s_snappy_corrupt + ADD $1, R2, R2 + ADD $1, R6, R6 + TST R7, R7 + BEQ lz4s_snappy_lits_done + ADD R7, R0, R9 + CMP R1, R9 + BHS lz4s_snappy_dstfull + ADD R7, R5, R5 + SUB $1, R7, R9 + MOVWU R9, R9 + CMPW $0x3c, R9 + BLO one_byte_lz4s_snappy + CMPW $0x00000100, R9 + BLO two_bytes_lz4s_snappy + CMPW $0x00010000, R9 + BLO three_bytes_lz4s_snappy + CMPW $0x01000000, R9 + BLO four_bytes_lz4s_snappy + MOVD $0xfc, R16 + MOVB R16, (R0) + MOVW R9, 1(R0) + ADD $0x05, R0, R0 + JMP memmove_long_lz4s_snappy + +four_bytes_lz4s_snappy: + MOVWU R9, R10 + LSRW $0x10, R10, R10 + MOVD $0xf8, R16 + MOVB R16, (R0) + MOVH R9, 1(R0) + MOVB R10, 3(R0) + ADD $0x04, R0, R0 + JMP memmove_long_lz4s_snappy + +three_bytes_lz4s_snappy: + MOVD $0xf4, R16 + MOVB R16, (R0) + MOVH R9, 1(R0) + ADD $0x03, R0, R0 + JMP memmove_long_lz4s_snappy + +two_bytes_lz4s_snappy: + MOVD $0xf0, R16 + MOVB R16, (R0) + MOVB R9, 1(R0) + ADD $0x02, R0, R0 + CMPW $0x40, R9 + BLO memmove_lz4s_snappy + JMP memmove_long_lz4s_snappy + +one_byte_lz4s_snappy: + LSLW $0x02, R9, R16 + BFI $0, R16, $8, R9 + MOVB R9, (R0) + ADD $0x01, R0, R0 + +memmove_lz4s_snappy: + ADD R7, R0, R9 + + // genMemMoveShort + CMP $0x08, R7 + BLS emit_lit_memmove_lz4s_snappy_memmove_move_8 + CMP $0x10, R7 + BLS emit_lit_memmove_lz4s_snappy_memmove_move_8through16 + CMP $0x20, R7 + BLS emit_lit_memmove_lz4s_snappy_memmove_move_17through32 + JMP emit_lit_memmove_lz4s_snappy_memmove_move_33through64 + +emit_lit_memmove_lz4s_snappy_memmove_move_8: + MOVD (R2), R10 + MOVD R10, (R0) + JMP memmove_end_copy_lz4s_snappy + +emit_lit_memmove_lz4s_snappy_memmove_move_8through16: + MOVD (R2), R10 + ADD R7, R2, R15 + MOVD -8(R15), R2 + MOVD R10, (R0) + ADD R7, R0, R15 + MOVD R2, -8(R15) + JMP memmove_end_copy_lz4s_snappy + +emit_lit_memmove_lz4s_snappy_memmove_move_17through32: + FMOVQ (R2), F0 + ADD R7, R2, R15 + FMOVQ -16(R15), F1 + FMOVQ F0, (R0) + ADD R7, R0, R15 + FMOVQ F1, -16(R15) + JMP memmove_end_copy_lz4s_snappy + +emit_lit_memmove_lz4s_snappy_memmove_move_33through64: + FMOVQ (R2), F0 + FMOVQ 16(R2), F1 + ADD R7, R2, R15 + FMOVQ -32(R15), F2 + ADD R7, R2, R15 + FMOVQ -16(R15), F3 + FMOVQ F0, (R0) + FMOVQ F1, 16(R0) + ADD R7, R0, R15 + FMOVQ F2, -32(R15) + ADD R7, R0, R15 + FMOVQ F3, -16(R15) + +memmove_end_copy_lz4s_snappy: + MOVD R9, R0 + JMP lz4s_snappy_lits_emit_done + +memmove_long_lz4s_snappy: + ADD R7, R0, R9 + + // genMemMoveLong + MOVD R2, R10 + MOVD R0, R11 + MOVD R7, R12 + +emit_lit_memmove_long_lz4s_snappylarge_big_loop_back: + FMOVQ (R10), F0 + FMOVQ 16(R10), F1 + FMOVQ F0, (R11) + FMOVQ F1, 16(R11) + ADD $0x20, R10, R10 + ADD $0x20, R11, R11 + SUB $0x20, R12, R12 + CMP $0x20, R12 + BHS emit_lit_memmove_long_lz4s_snappylarge_big_loop_back + ADD R7, R2, R15 + FMOVQ -32(R15), F0 + ADD R7, R2, R15 + FMOVQ -16(R15), F1 + ADD R7, R0, R15 + FMOVQ F0, -32(R15) + ADD R7, R0, R15 + FMOVQ F1, -16(R15) + MOVD R9, R0 + +lz4s_snappy_lits_emit_done: + MOVD R6, R2 + +lz4s_snappy_lits_done: + CMP R3, R2 + BNE lz4s_snappy_match + CMP $0x03, R8 + BEQ lz4s_snappy_done + JMP lz4s_snappy_corrupt + +lz4s_snappy_match: + CMP $0x03, R8 + BEQ lz4s_snappy_loop + ADD $2, R2, R6 + CMP R3, R6 + BHS lz4s_snappy_corrupt + MOVHU (R2), R7 + MOVD R6, R2 + TST R7, R7 + BEQ lz4s_snappy_corrupt + CMP R5, R7 + BHI lz4s_snappy_corrupt + CMP $0x12, R8 + BNE lz4s_snappy_ml_done + +lz4s_snappy_ml_loop: + MOVBU (R2), R6 + ADD $1, R2, R2 + ADD R6, R8, R8 + CMP R3, R2 + BHS lz4s_snappy_corrupt + CMP $0xff, R6 + BEQ lz4s_snappy_ml_loop + +lz4s_snappy_ml_done: + ADD R8, R5, R5 + + // emitCopy +two_byte_offset_lz4_s2: + CMPW $0x40, R8 + BLS two_byte_offset_short_lz4_s2 + MOVD $0xee, R16 + MOVB R16, (R0) + MOVH R7, 1(R0) + SUB $60, R8, R8 + MOVWU R8, R8 + ADD $0x03, R0, R0 + CMP R1, R0 + BHS lz4s_snappy_loop + JMP two_byte_offset_lz4_s2 + +two_byte_offset_short_lz4_s2: + MOVWU R8, R6 + LSLW $0x02, R6, R6 + CMPW $0x0c, R8 + BHS emit_copy_three_lz4_s2 + CMPW $0x00000800, R7 + BHS emit_copy_three_lz4_s2 + SUB $15, R6, R6 + MOVWU R6, R6 + MOVB R7, 1(R0) + LSRW $0x08, R7, R7 + LSLW $0x05, R7, R7 + ORRW R7, R6, R6 + MOVB R6, (R0) + ADD $0x02, R0, R0 + JMP lz4s_snappy_loop + +emit_copy_three_lz4_s2: + SUB $2, R6, R6 + MOVWU R6, R6 + MOVB R6, (R0) + MOVH R7, 1(R0) + ADD $0x03, R0, R0 + JMP lz4s_snappy_loop + +lz4s_snappy_done: + MOVD dst_base+0(FP), R1 + SUB R1, R0, R0 + MOVD R5, uncompressed+48(FP) + MOVD R0, dstUsed+56(FP) + RET + +lz4s_snappy_corrupt: + MOVD $0, R0 + SUB $1, R0, R5 + MOVD R5, uncompressed+48(FP) + RET + +lz4s_snappy_dstfull: + MOVD $0, R0 + SUB $2, R0, R5 + MOVD R5, uncompressed+48(FP) + RET diff --git a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go b/vendor/github.com/klauspost/compress/s2/encodeblock_asm.go similarity index 98% rename from vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go rename to vendor/github.com/klauspost/compress/s2/encodeblock_asm.go index f43aa8154..fb602efd5 100644 --- a/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go +++ b/vendor/github.com/klauspost/compress/s2/encodeblock_asm.go @@ -1,6 +1,6 @@ -// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_asm.go -pkg=s2. DO NOT EDIT. -//go:build !appengine && !noasm && gc && !noasm +//go:build (amd64 || arm64) && !appengine && !noasm && gc && !noasm package s2 diff --git a/vendor/github.com/klauspost/compress/s2/index.go b/vendor/github.com/klauspost/compress/s2/index.go index fb7db2531..a5e751a8a 100644 --- a/vendor/github.com/klauspost/compress/s2/index.go +++ b/vendor/github.com/klauspost/compress/s2/index.go @@ -151,7 +151,7 @@ func (i *Index) reduce() { i.estBlockUncomp += i.estBlockUncomp * int64(removeN) } -func (i *Index) appendTo(b []byte, uncompTotal, compTotal int64) []byte { +func (i *Index) appendTo(b []byte, compTotal, uncompTotal int64) []byte { i.reduce() var tmp [binary.MaxVarintLen64]byte @@ -425,7 +425,7 @@ func IndexStream(r io.Reader) ([]byte, error) { _, err := io.ReadFull(r, buf[:4]) if err != nil { if err == io.EOF { - return i.appendTo(nil, i.TotalUncompressed, i.TotalCompressed), nil + return i.appendTo(nil, i.TotalCompressed, i.TotalUncompressed), nil } return nil, err } diff --git a/vendor/github.com/klauspost/compress/s2/lz4convert.go b/vendor/github.com/klauspost/compress/s2/lz4convert.go index 46ed908e3..23bbc77e2 100644 --- a/vendor/github.com/klauspost/compress/s2/lz4convert.go +++ b/vendor/github.com/klauspost/compress/s2/lz4convert.go @@ -13,6 +13,7 @@ import ( // LZ4Converter provides conversion from LZ4 blocks as defined here: // https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md type LZ4Converter struct { + noAsm bool } // ErrDstTooSmall is returned when provided destination is too small. @@ -32,7 +33,7 @@ func (l *LZ4Converter) ConvertBlock(dst, src []byte) ([]byte, int, error) { s, d := 0, len(dst) dst = dst[:cap(dst)] - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4BlockAsm(dst[d:], src) if res < 0 { const ( @@ -96,7 +97,8 @@ func (l *LZ4Converter) ConvertBlock(dst, src []byte) ([]byte, int, error) { } s++ if ll > 0 { - if d+ll > dLimit { + // Leave room for a 5-byte literal header and an 8-byte copy. + if d+ll > len(dst)-13 { return nil, 0, ErrDstTooSmall } if debug { @@ -209,11 +211,13 @@ func (l *LZ4Converter) ConvertBlock(dst, src []byte) ([]byte, int, error) { dst[2] = uint8(length >> 0) dst[1] = 0 dst[0] = 7<<2 | tagCopy1 + d += 5 if left > 0 { - d += 5 + emitRepeat16(dst[5:], offset, left) - break + // The loop condition checks room for the next chunk. + dst = dst[5:] + length = left + continue } - d += 5 break } } @@ -288,7 +292,7 @@ func (l *LZ4Converter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) s, d := 0, len(dst) dst = dst[:cap(dst)] // Use assembly when possible - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4BlockSnappyAsm(dst[d:], src) if res < 0 { const ( @@ -488,6 +492,11 @@ func emitRepeat16(dst []byte, offset uint16, length int) int { length -= 1 << 16 left := 0 if length > maxRepeat { + if len(dst) < 10 { + // No room for this and the next chunk. + // Report more than len(dst) so the caller sees dst is full. + return len(dst) + 1 + } left = length - maxRepeat + 4 length = maxRepeat - 4 } diff --git a/vendor/github.com/klauspost/compress/s2/lz4sconvert.go b/vendor/github.com/klauspost/compress/s2/lz4sconvert.go index 000f39719..ca01a2c2b 100644 --- a/vendor/github.com/klauspost/compress/s2/lz4sconvert.go +++ b/vendor/github.com/klauspost/compress/s2/lz4sconvert.go @@ -21,6 +21,7 @@ import ( // For Min Match of 4 bytes, Copy length value 1-15 means length 4-18 with 18 bytes adding an extra byte. // ONLY "Min match of 4 bytes" is supported. type LZ4sConverter struct { + noAsm bool } // ConvertBlock will convert an LZ4s block and append it as an S2 @@ -37,7 +38,7 @@ func (l *LZ4sConverter) ConvertBlock(dst, src []byte) ([]byte, int, error) { s, d := 0, len(dst) dst = dst[:cap(dst)] - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4sBlockAsm(dst[d:], src) if res < 0 { const ( @@ -101,7 +102,8 @@ func (l *LZ4sConverter) ConvertBlock(dst, src []byte) ([]byte, int, error) { } s++ if ll > 0 { - if d+ll > dLimit { + // Leave room for a 5-byte literal header and an 8-byte copy. + if d+ll > len(dst)-13 { return nil, 0, ErrDstTooSmall } if debug { @@ -218,11 +220,13 @@ func (l *LZ4sConverter) ConvertBlock(dst, src []byte) ([]byte, int, error) { dst[2] = uint8(length >> 0) dst[1] = 0 dst[0] = 7<<2 | tagCopy1 + d += 5 if left > 0 { - d += 5 + emitRepeat16(dst[5:], offset, left) - break + // The loop condition checks room for the next chunk. + dst = dst[5:] + length = left + continue } - d += 5 break } } @@ -297,7 +301,7 @@ func (l *LZ4sConverter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) s, d := 0, len(dst) dst = dst[:cap(dst)] // Use assembly when possible - if !debug && hasAmd64Asm { + if !debug && hasAsm && !l.noAsm { res, sz := cvtLZ4sBlockSnappyAsm(dst[d:], src) if res < 0 { const ( diff --git a/vendor/github.com/klauspost/compress/s2/reader.go b/vendor/github.com/klauspost/compress/s2/reader.go index 17443e2c1..5c6cb4cfd 100644 --- a/vendor/github.com/klauspost/compress/s2/reader.go +++ b/vendor/github.com/klauspost/compress/s2/reader.go @@ -451,9 +451,7 @@ func (r *Reader) DecodeConcurrent(w io.Writer, concurrent int) (written int64, e } // Writer var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for toWrite := range queue { entry := <-toWrite reUse <- toWrite @@ -480,7 +478,7 @@ func (r *Reader) DecodeConcurrent(w io.Writer, concurrent int) (written int64, e } aWritten += int64(n) } - }() + }) defer func() { if r.err != nil { diff --git a/vendor/github.com/klauspost/compress/s2/writer.go b/vendor/github.com/klauspost/compress/s2/writer.go index 09f1cff3a..8077f2f68 100644 --- a/vendor/github.com/klauspost/compress/s2/writer.go +++ b/vendor/github.com/klauspost/compress/s2/writer.go @@ -139,11 +139,9 @@ func (w *Writer) Reset(writer io.Writer) { toWrite := make(chan chan result, w.concurrency) w.output = toWrite - w.writerWg.Add(1) // Start a writer goroutine that will write all output in order. - go func() { - defer w.writerWg.Done() + w.writerWg.Go(func() { // Get a queued write. for write := range toWrite { @@ -175,7 +173,7 @@ func (w *Writer) Reset(writer io.Writer) { // This can be used for synchronizing flushes. close(write) } - }() + }) } // Write satisfies the io.Writer interface. @@ -811,7 +809,7 @@ func (w *Writer) closeIndex(idx bool) ([]byte, error) { if w.pad <= 1 { compSize = w.written } - index = w.index.appendTo(w.ibuf[:0], w.uncompWritten, compSize) + index = w.index.appendTo(w.ibuf[:0], compSize, w.uncompWritten) // Count as written for padding. if w.appendIndex { w.written += int64(len(index)) diff --git a/vendor/modules.txt b/vendor/modules.txt index 63e84b3fb..77a93f528 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -522,8 +522,8 @@ github.com/kballard/go-shellquote # github.com/kevinburke/ssh_config v1.6.0 ## explicit; go 1.18 github.com/kevinburke/ssh_config -# github.com/klauspost/compress v1.19.2 -## explicit; go 1.24 +# github.com/klauspost/compress v1.20.2 +## explicit; go 1.25 github.com/klauspost/compress/internal/le github.com/klauspost/compress/internal/race github.com/klauspost/compress/s2