// Code generated by command: go run gen.go -out ../asm_amd64.s -stubs ../asm_amd64.go -pkg=minlz. DO NOT EDIT.

//go:build !appengine && !noasm && gc && !purego

#include "textflag.h"

// func encodeBlockAsm(dst []byte, src []byte, tmp *[131072]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBlockAsm(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000400, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x06, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBlockAsm
	MOVQ  (BX)(DX*1), DI
	LEAL  -2162685(DX), R8
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R10
	MOVQ  DI, R9
	SHLQ  $0x10, R9
	IMULQ R10, R9
	SHRQ  $0x31, R9
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x10, R11
	IMULQ R10, R11
	SHRQ  $0x31, R11
	MOVL  (AX)(R9*4), SI
	MOVL  DX, (AX)(R9*4)
	MOVL  (AX)(R11*4), R9
	MOVL  DX, (AX)(R11*4)
	MOVQ  DI, R11
	SHRQ  $0x10, R11
	SHLQ  $0x10, R11
	IMULQ R10, R11
	SHRQ  $0x31, R11
	MOVL  DX, R10
	SUBL  16(SP), R10
	MOVL  1(BX)(R10*1), R12
	MOVQ  DI, R10
	SHRQ  $0x08, R10
	CMPL  R10, R12
	JNE   no_repeat_found_encodeBlockAsm
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_encodeBlockAsm

repeat_extend_back_loop_encodeBlockAsm:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeBlockAsm
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeBlockAsm

repeat_extend_back_end_encodeBlockAsm:
	MOVL DI, SI
	MOVL 12(SP), R8
	SUBL R8, SI
	LEAQ 4(CX)(SI*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_1
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeBlockAsm
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeBlockAsm
	CMPL R9, $0x00010000
	JB   three_bytes_repeat_emit_lits_encodeBlockAsm
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm

three_bytes_repeat_emit_lits_encodeBlockAsm:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm

two_bytes_repeat_emit_lits_encodeBlockAsm:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeBlockAsm
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm

one_byte_repeat_emit_lits_encodeBlockAsm:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeBlockAsm:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm

memmove_midrepeat_emit_lits_encodeBlockAsm:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm

memmove_long_repeat_emit_lits_encodeBlockAsm:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsmlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsmlarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeBlockAsm:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBlockAsm
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBlockAsm:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeBlockAsm:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBlockAsm
	JMP  matchlen_match8_repeat_extend_encodeBlockAsm
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBlockAsm:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeBlockAsm

matchlen_match8_repeat_extend_encodeBlockAsm:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBlockAsm:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeBlockAsm

matchlen_match4_repeat_extend_encodeBlockAsm:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeBlockAsm:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm
	JB   repeat_extend_forward_end_encodeBlockAsm
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeBlockAsm

matchlen_match1_repeat_extend_encodeBlockAsm:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeBlockAsm
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeBlockAsm
	PCALIGN $0x10

repeat_extend_forward_end_encodeBlockAsm:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBlockAsm
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBlockAsm
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBlockAsm
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_three_match_repeat_encodeBlockAsm:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_two_match_repeat_encodeBlockAsm:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_one_match_repeat_encodeBlockAsm:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBlockAsm:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm

no_repeat_found_encodeBlockAsm:
	CMPL    SI, R8
	CMOVLLE R8, SI
	CMPL    (BX)(SI*1), DI
	JEQ     candidate_match_encodeBlockAsm
	SHRQ    $0x08, DI
	MOVL    (AX)(R11*4), SI
	LEAL    2(DX), R10
	CMPL    R9, R8
	CMOVLLE R8, R9
	CMPL    (BX)(R9*1), DI
	JEQ     candidate2_match_encodeBlockAsm
	MOVL    R10, (AX)(R11*4)
	SHRQ    $0x08, DI
	CMPL    SI, R8
	CMOVLLE R8, SI
	CMPL    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeBlockAsm
	MOVL    20(SP), DX
	JMP     search_loop_encodeBlockAsm

candidate3_match_encodeBlockAsm:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm

candidate2_match_encodeBlockAsm:
	MOVL R10, (AX)(R11*4)
	INCL DX
	MOVL R9, SI

candidate_match_encodeBlockAsm:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm

match_extend_back_loop_encodeBlockAsm:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm
	JMP  match_extend_back_loop_encodeBlockAsm

match_extend_back_end_encodeBlockAsm:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBlockAsm
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBlockAsm:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBlockAsm:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBlockAsm
	JMP  matchlen_match8_match_nolit_encodeBlockAsm
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBlockAsm:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBlockAsm

matchlen_match8_match_nolit_encodeBlockAsm:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBlockAsm
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBlockAsm:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBlockAsm

matchlen_match4_match_nolit_encodeBlockAsm:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBlockAsm
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBlockAsm:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm
	JB   match_nolit_end_encodeBlockAsm
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBlockAsm
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm

matchlen_match1_match_nolit_encodeBlockAsm:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeBlockAsm
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBlockAsm
	PCALIGN $0x10

match_nolit_end_encodeBlockAsm:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeBlockAsm
	LEAQ (BX)(DI*1), DI
	CMPL R8, $0x03
	JA   match_emit_lits_copy_encodeBlockAsm
	CMPL SI, $0x40
	JB   match_emit_lits_copy_encodeBlockAsm
	MOVL (DI), DI
	CMPL SI, $0x0001003f
	JBE  match_emit_copy2lits_encodeBlockAsm

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(SI), SI
	SHLL $0x0b, SI
	LEAL 7(SI)(R8*8), SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_emit_lits_encodeBlockAsm
	LEAL -60(R11), R9
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_emit_lits_encodeBlockAsm
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_emit_lits_encodeBlockAsm
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL R9, 4(CX)
	ADDQ $0x07, CX
	JMP  match_emit_copy_litsencodeBlockAsm

emit_copy3_2_match_emit_lits_encodeBlockAsm:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW R9, 4(CX)
	ADDQ $0x06, CX
	JMP  match_emit_copy_litsencodeBlockAsm

emit_copy3_1_match_emit_lits_encodeBlockAsm:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB R9, 4(CX)
	ADDQ $0x05, CX
	JMP  match_emit_copy_litsencodeBlockAsm

emit_copy3_0_match_emit_lits_encodeBlockAsm:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX

match_emit_copy_litsencodeBlockAsm:
	MOVL DI, (CX)
	ADDQ R8, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

match_emit_copy2lits_encodeBlockAsm:
	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, SI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    SI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, SI
	CMOVLLT R11, SI
	LEAL    -1(R8)(SI*4), SI
	MOVL    $0x00000003, R10
	LEAL    (R10)(SI*8), SI
	MOVB    SI, (CX)
	ADDQ    $0x03, CX
	MOVL    DI, (CX)
	ADDQ    R8, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBlockAsm

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBlockAsm
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBlockAsm
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_three_match_emit_repeat_copy2_encodeBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_two_match_emit_repeat_copy2_encodeBlockAsm:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_one_match_emit_repeat_copy2_encodeBlockAsm:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

match_emit_lits_copy_encodeBlockAsm:
	LEAQ 4(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBlockAsm
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm
	CMPL R9, $0x00010000
	JB   three_bytes_match_emit_encodeBlockAsm
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm

three_bytes_match_emit_encodeBlockAsm:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm

two_bytes_match_emit_encodeBlockAsm:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBlockAsm
	JMP  memmove_long_match_emit_encodeBlockAsm

one_byte_match_emit_encodeBlockAsm:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm

emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm

emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeBlockAsm:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm

memmove_midmatch_emit_encodeBlockAsm:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBlockAsm_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBlockAsm

emit_lit_memmove_mid_match_emit_encodeBlockAsm_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeBlockAsm:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm

memmove_long_match_emit_encodeBlockAsm:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeBlockAsm
	PCALIGN $0x10

match_nolits_copy_encodeBlockAsm:
	// emitCopy
	CMPL SI, $0x0001003f
	JBE  two_byte_offset_match_nolit_encodeBlockAsm

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(SI), SI
	SHLL $0x0b, SI
	ADDL $0x07, SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_nolit_encodeBlockAsm_emit3
	LEAL -60(R11), DI
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_nolit_encodeBlockAsm_emit3
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_nolit_encodeBlockAsm_emit3
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL DI, 4(CX)
	ADDQ $0x07, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

emit_copy3_2_match_nolit_encodeBlockAsm_emit3:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW DI, 4(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

emit_copy3_1_match_nolit_encodeBlockAsm_emit3:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

emit_copy3_0_match_nolit_encodeBlockAsm_emit3:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

two_byte_offset_match_nolit_encodeBlockAsm:
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeBlockAsm
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBlockAsm
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

emit_one_longer_match_nolit_encodeBlockAsm:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBlockAsm
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

emit_copy1_repeat_match_nolit_encodeBlockAsm:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

two_byte_match_nolit_encodeBlockAsm:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBlockAsm_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBlockAsm_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBlockAsm_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

emit_copy2_2_match_nolit_encodeBlockAsm_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

emit_copy2_1_match_nolit_encodeBlockAsm_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

emit_copy2_0_match_nolit_encodeBlockAsm_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm:
	MOVQ  $0x0000cf1bbcdcbf9b, SI
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, R9
	SHLQ  $0x10, R8
	IMULQ SI, R8
	SHRQ  $0x31, R8
	SHLQ  $0x10, R9
	IMULQ SI, R9
	SHRQ  $0x31, R9
	LEAL  -2(DX), R10
	MOVL  (AX)(R9*4), SI
	MOVL  R10, (AX)(R8*4)
	MOVL  DX, (AX)(R9*4)
	MOVL  DX, R8
	INCL  DX
	LEAL  -2162687(R8), R9
	CMPL  SI, R9
	JA    match_nolit_len_okencodeBlockAsm
	JMP   search_loop_encodeBlockAsm

match_nolit_len_okencodeBlockAsm:
	CMPL (BX)(SI*1), DI
	JNE  search_loop_encodeBlockAsm
	MOVL R8, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	CMPQ CX, (SP)
	JB   dst_size_check_ok_4
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x03, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeBlockAsm
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeBlockAsm:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeBlockAsm
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeBlockAsm:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeBlockAsm
	JMP  matchlen_match8_match_nolit2_encodeBlockAsm
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeBlockAsm:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeBlockAsm

matchlen_match8_match_nolit2_encodeBlockAsm:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeBlockAsm
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeBlockAsm
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeBlockAsm:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeBlockAsm

matchlen_match4_match_nolit2_encodeBlockAsm:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeBlockAsm
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeBlockAsm
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeBlockAsm:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeBlockAsm
	JB   match_nolit2_end_encodeBlockAsm
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeBlockAsm
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeBlockAsm

matchlen_match1_match_nolit2_encodeBlockAsm:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeBlockAsm
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeBlockAsm
	PCALIGN $0x10

match_nolit2_end_encodeBlockAsm:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeBlockAsm

emit_remainder_encodeBlockAsm:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeBlockAsm
	LEAQ (BX)(DX*1), DX
	LEAQ 4(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeBlockAsm
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm
	CMPL BX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBlockAsm
	MOVL BX, SI
	SHRL $0x10, SI
	MOVB $0xf8, (CX)
	MOVW BX, 1(CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm

three_bytes_emit_remainder_encodeBlockAsm:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm

two_bytes_emit_remainder_encodeBlockAsm:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeBlockAsm
	JMP  memmove_long_emit_remainder_encodeBlockAsm

one_byte_emit_remainder_encodeBlockAsm:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm

memmove_midemit_remainder_encodeBlockAsm:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeBlockAsm:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm

memmove_long_emit_remainder_encodeBlockAsm:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeBlockAsm:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm2MB(dst []byte, src []byte, tmp *[131072]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBlockAsm2MB(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000400, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm2MB:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm2MB
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm2MB:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x06, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBlockAsm2MB
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R8
	SHLQ  $0x10, R8
	IMULQ R9, R8
	SHRQ  $0x31, R8
	MOVQ  DI, R10
	SHRQ  $0x08, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x31, R10
	MOVL  (AX)(R8*4), SI
	MOVL  DX, (AX)(R8*4)
	MOVL  (AX)(R10*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x31, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeBlockAsm2MB
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_encodeBlockAsm2MB

repeat_extend_back_loop_encodeBlockAsm2MB:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeBlockAsm2MB
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm2MB
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeBlockAsm2MB

repeat_extend_back_end_encodeBlockAsm2MB:
	MOVL DI, SI
	MOVL 12(SP), R8
	SUBL R8, SI
	LEAQ 4(CX)(SI*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_1
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeBlockAsm2MB
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeBlockAsm2MB
	CMPL R9, $0x00010000
	JB   three_bytes_repeat_emit_lits_encodeBlockAsm2MB
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm2MB

three_bytes_repeat_emit_lits_encodeBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm2MB

two_bytes_repeat_emit_lits_encodeBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeBlockAsm2MB
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm2MB

one_byte_repeat_emit_lits_encodeBlockAsm2MB:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm2MB

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm2MB

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeBlockAsm2MB:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm2MB

memmove_midrepeat_emit_lits_encodeBlockAsm2MB:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm2MB

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm2MB:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm2MB

memmove_long_repeat_emit_lits_encodeBlockAsm2MB:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm2MBlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeBlockAsm2MB:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBlockAsm2MB
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBlockAsm2MB:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm2MB
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm2MB
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeBlockAsm2MB:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBlockAsm2MB
	JMP  matchlen_match8_repeat_extend_encodeBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBlockAsm2MB:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeBlockAsm2MB

matchlen_match8_repeat_extend_encodeBlockAsm2MB:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm2MB
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm2MB
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBlockAsm2MB:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeBlockAsm2MB

matchlen_match4_repeat_extend_encodeBlockAsm2MB:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm2MB
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm2MB
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeBlockAsm2MB:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm2MB
	JB   repeat_extend_forward_end_encodeBlockAsm2MB
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm2MB
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeBlockAsm2MB

matchlen_match1_repeat_extend_encodeBlockAsm2MB:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeBlockAsm2MB
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeBlockAsm2MB
	PCALIGN $0x10

repeat_extend_forward_end_encodeBlockAsm2MB:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBlockAsm2MB
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBlockAsm2MB
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBlockAsm2MB
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm2MB

repeat_three_match_repeat_encodeBlockAsm2MB:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm2MB

repeat_two_match_repeat_encodeBlockAsm2MB:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm2MB

repeat_one_match_repeat_encodeBlockAsm2MB:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBlockAsm2MB:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm2MB

no_repeat_found_encodeBlockAsm2MB:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeBlockAsm2MB
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeBlockAsm2MB
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeBlockAsm2MB
	MOVL 20(SP), DX
	JMP  search_loop_encodeBlockAsm2MB

candidate3_match_encodeBlockAsm2MB:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm2MB

candidate2_match_encodeBlockAsm2MB:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm2MB:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm2MB

match_extend_back_loop_encodeBlockAsm2MB:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm2MB
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm2MB
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm2MB
	JMP  match_extend_back_loop_encodeBlockAsm2MB

match_extend_back_end_encodeBlockAsm2MB:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBlockAsm2MB
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBlockAsm2MB:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm2MB
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm2MB
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBlockAsm2MB:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBlockAsm2MB
	JMP  matchlen_match8_match_nolit_encodeBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBlockAsm2MB:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBlockAsm2MB

matchlen_match8_match_nolit_encodeBlockAsm2MB:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm2MB
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm2MB
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBlockAsm2MB:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBlockAsm2MB

matchlen_match4_match_nolit_encodeBlockAsm2MB:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm2MB
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBlockAsm2MB
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBlockAsm2MB:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm2MB
	JB   match_nolit_end_encodeBlockAsm2MB
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBlockAsm2MB
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm2MB

matchlen_match1_match_nolit_encodeBlockAsm2MB:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeBlockAsm2MB
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBlockAsm2MB
	PCALIGN $0x10

match_nolit_end_encodeBlockAsm2MB:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeBlockAsm2MB
	LEAQ (BX)(DI*1), DI
	CMPL R8, $0x03
	JA   match_emit_lits_copy_encodeBlockAsm2MB
	CMPL SI, $0x40
	JB   match_emit_lits_copy_encodeBlockAsm2MB
	MOVL (DI), DI
	CMPL SI, $0x0001003f
	JBE  match_emit_copy2lits_encodeBlockAsm2MB

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(SI), SI
	SHLL $0x0b, SI
	LEAL 7(SI)(R8*8), SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_emit_lits_encodeBlockAsm2MB
	LEAL -60(R11), R9
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_emit_lits_encodeBlockAsm2MB
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_emit_lits_encodeBlockAsm2MB
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL R9, 4(CX)
	ADDQ $0x07, CX
	JMP  match_emit_copy_litsencodeBlockAsm2MB

emit_copy3_2_match_emit_lits_encodeBlockAsm2MB:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW R9, 4(CX)
	ADDQ $0x06, CX
	JMP  match_emit_copy_litsencodeBlockAsm2MB

emit_copy3_1_match_emit_lits_encodeBlockAsm2MB:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB R9, 4(CX)
	ADDQ $0x05, CX
	JMP  match_emit_copy_litsencodeBlockAsm2MB

emit_copy3_0_match_emit_lits_encodeBlockAsm2MB:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX

match_emit_copy_litsencodeBlockAsm2MB:
	MOVL DI, (CX)
	ADDQ R8, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

match_emit_copy2lits_encodeBlockAsm2MB:
	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, SI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    SI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, SI
	CMOVLLT R11, SI
	LEAL    -1(R8)(SI*4), SI
	MOVL    $0x00000003, R10
	LEAL    (R10)(SI*8), SI
	MOVB    SI, (CX)
	ADDQ    $0x03, CX
	MOVL    DI, (CX)
	ADDQ    R8, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBlockAsm2MB

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBlockAsm2MB
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBlockAsm2MB
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBlockAsm2MB
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

repeat_three_match_emit_repeat_copy2_encodeBlockAsm2MB:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

repeat_two_match_emit_repeat_copy2_encodeBlockAsm2MB:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

repeat_one_match_emit_repeat_copy2_encodeBlockAsm2MB:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

match_emit_lits_copy_encodeBlockAsm2MB:
	LEAQ 4(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBlockAsm2MB
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm2MB
	CMPL R9, $0x00010000
	JB   three_bytes_match_emit_encodeBlockAsm2MB
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm2MB

three_bytes_match_emit_encodeBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm2MB

two_bytes_match_emit_encodeBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBlockAsm2MB
	JMP  memmove_long_match_emit_encodeBlockAsm2MB

one_byte_match_emit_encodeBlockAsm2MB:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm2MB_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm2MB_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBlockAsm2MB_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm2MB

emit_lit_memmove_match_emit_encodeBlockAsm2MB_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm2MB

emit_lit_memmove_match_emit_encodeBlockAsm2MB_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeBlockAsm2MB:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm2MB

memmove_midmatch_emit_encodeBlockAsm2MB:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBlockAsm2MB_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBlockAsm2MB

emit_lit_memmove_mid_match_emit_encodeBlockAsm2MB_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeBlockAsm2MB:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm2MB

memmove_long_match_emit_encodeBlockAsm2MB:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBlockAsm2MBlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeBlockAsm2MB
	PCALIGN $0x10

match_nolits_copy_encodeBlockAsm2MB:
	// emitCopy
	CMPL SI, $0x0001003f
	JBE  two_byte_offset_match_nolit_encodeBlockAsm2MB

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(SI), SI
	SHLL $0x0b, SI
	ADDL $0x07, SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_nolit_encodeBlockAsm2MB_emit3
	LEAL -60(R11), DI
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_nolit_encodeBlockAsm2MB_emit3
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_nolit_encodeBlockAsm2MB_emit3
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL DI, 4(CX)
	ADDQ $0x07, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

emit_copy3_2_match_nolit_encodeBlockAsm2MB_emit3:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW DI, 4(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

emit_copy3_1_match_nolit_encodeBlockAsm2MB_emit3:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

emit_copy3_0_match_nolit_encodeBlockAsm2MB_emit3:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

two_byte_offset_match_nolit_encodeBlockAsm2MB:
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeBlockAsm2MB
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBlockAsm2MB
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

emit_one_longer_match_nolit_encodeBlockAsm2MB:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBlockAsm2MB
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

emit_copy1_repeat_match_nolit_encodeBlockAsm2MB:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm2MB
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm2MB
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm2MB
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm2MB:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm2MB:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm2MB:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

two_byte_match_nolit_encodeBlockAsm2MB:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBlockAsm2MB_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBlockAsm2MB_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBlockAsm2MB_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

emit_copy2_2_match_nolit_encodeBlockAsm2MB_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

emit_copy2_1_match_nolit_encodeBlockAsm2MB_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm2MB

emit_copy2_0_match_nolit_encodeBlockAsm2MB_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm2MB:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm2MB
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm2MB
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm2MB:
	MOVQ  $0x0000cf1bbcdcbf9b, SI
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, R9
	SHLQ  $0x10, R8
	IMULQ SI, R8
	SHRQ  $0x31, R8
	SHLQ  $0x10, R9
	IMULQ SI, R9
	SHRQ  $0x31, R9
	LEAL  -2(DX), R10
	MOVL  (AX)(R9*4), SI
	MOVL  R10, (AX)(R8*4)
	MOVL  DX, (AX)(R9*4)
	MOVL  DX, R8
	INCL  DX
	CMPL  (BX)(SI*1), DI
	JNE   search_loop_encodeBlockAsm2MB
	MOVL  R8, DI
	SUBL  SI, DI
	MOVL  DI, 16(SP)
	CMPQ  CX, (SP)
	JB    dst_size_check_ok_4
	MOVQ  $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x03, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeBlockAsm2MB
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeBlockAsm2MB:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm2MB
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeBlockAsm2MB
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeBlockAsm2MB:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeBlockAsm2MB
	JMP  matchlen_match8_match_nolit2_encodeBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeBlockAsm2MB:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeBlockAsm2MB

matchlen_match8_match_nolit2_encodeBlockAsm2MB:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeBlockAsm2MB
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm2MB
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeBlockAsm2MB:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeBlockAsm2MB

matchlen_match4_match_nolit2_encodeBlockAsm2MB:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeBlockAsm2MB
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeBlockAsm2MB
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeBlockAsm2MB:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeBlockAsm2MB
	JB   match_nolit2_end_encodeBlockAsm2MB
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeBlockAsm2MB
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeBlockAsm2MB

matchlen_match1_match_nolit2_encodeBlockAsm2MB:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeBlockAsm2MB
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeBlockAsm2MB
	PCALIGN $0x10

match_nolit2_end_encodeBlockAsm2MB:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeBlockAsm2MB

emit_remainder_encodeBlockAsm2MB:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeBlockAsm2MB
	LEAQ (BX)(DX*1), DX
	LEAQ 4(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeBlockAsm2MB
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm2MB
	CMPL BX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBlockAsm2MB
	MOVL BX, SI
	SHRL $0x10, SI
	MOVB $0xf8, (CX)
	MOVW BX, 1(CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm2MB

three_bytes_emit_remainder_encodeBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm2MB

two_bytes_emit_remainder_encodeBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeBlockAsm2MB
	JMP  memmove_long_emit_remainder_encodeBlockAsm2MB

one_byte_emit_remainder_encodeBlockAsm2MB:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm2MB
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeBlockAsm2MB_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm2MB:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm2MB

memmove_midemit_remainder_encodeBlockAsm2MB:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm2MB_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBlockAsm2MB

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm2MB_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeBlockAsm2MB:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm2MB

memmove_long_emit_remainder_encodeBlockAsm2MB:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBlockAsm2MBlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeBlockAsm2MB:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm512K(dst []byte, src []byte, tmp *[65536]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBlockAsm512K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000200, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm512K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm512K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm512K:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x06, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBlockAsm512K
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R8
	SHLQ  $0x10, R8
	IMULQ R9, R8
	SHRQ  $0x32, R8
	MOVQ  DI, R10
	SHRQ  $0x08, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	MOVL  (AX)(R8*4), SI
	MOVL  DX, (AX)(R8*4)
	MOVL  (AX)(R10*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeBlockAsm512K
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_encodeBlockAsm512K

repeat_extend_back_loop_encodeBlockAsm512K:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeBlockAsm512K
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm512K
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeBlockAsm512K

repeat_extend_back_end_encodeBlockAsm512K:
	MOVL DI, SI
	MOVL 12(SP), R8
	SUBL R8, SI
	LEAQ 4(CX)(SI*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_1
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeBlockAsm512K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeBlockAsm512K
	CMPL R9, $0x00010000
	JB   three_bytes_repeat_emit_lits_encodeBlockAsm512K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm512K

three_bytes_repeat_emit_lits_encodeBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm512K

two_bytes_repeat_emit_lits_encodeBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeBlockAsm512K
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm512K

one_byte_repeat_emit_lits_encodeBlockAsm512K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm512K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm512K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm512K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm512K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm512K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeBlockAsm512K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm512K

memmove_midrepeat_emit_lits_encodeBlockAsm512K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm512K

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm512K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm512K

memmove_long_repeat_emit_lits_encodeBlockAsm512K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm512Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeBlockAsm512K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBlockAsm512K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBlockAsm512K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm512K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm512K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeBlockAsm512K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBlockAsm512K
	JMP  matchlen_match8_repeat_extend_encodeBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBlockAsm512K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeBlockAsm512K

matchlen_match8_repeat_extend_encodeBlockAsm512K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm512K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm512K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBlockAsm512K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeBlockAsm512K

matchlen_match4_repeat_extend_encodeBlockAsm512K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm512K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm512K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeBlockAsm512K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm512K
	JB   repeat_extend_forward_end_encodeBlockAsm512K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm512K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeBlockAsm512K

matchlen_match1_repeat_extend_encodeBlockAsm512K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeBlockAsm512K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeBlockAsm512K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBlockAsm512K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBlockAsm512K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBlockAsm512K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBlockAsm512K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm512K

repeat_three_match_repeat_encodeBlockAsm512K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm512K

repeat_two_match_repeat_encodeBlockAsm512K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm512K

repeat_one_match_repeat_encodeBlockAsm512K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBlockAsm512K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm512K

no_repeat_found_encodeBlockAsm512K:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeBlockAsm512K
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeBlockAsm512K
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeBlockAsm512K
	MOVL 20(SP), DX
	JMP  search_loop_encodeBlockAsm512K

candidate3_match_encodeBlockAsm512K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm512K

candidate2_match_encodeBlockAsm512K:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm512K:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm512K

match_extend_back_loop_encodeBlockAsm512K:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm512K
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm512K
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm512K
	JMP  match_extend_back_loop_encodeBlockAsm512K

match_extend_back_end_encodeBlockAsm512K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBlockAsm512K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBlockAsm512K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm512K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm512K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBlockAsm512K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBlockAsm512K
	JMP  matchlen_match8_match_nolit_encodeBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBlockAsm512K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBlockAsm512K

matchlen_match8_match_nolit_encodeBlockAsm512K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm512K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm512K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBlockAsm512K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBlockAsm512K

matchlen_match4_match_nolit_encodeBlockAsm512K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm512K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBlockAsm512K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBlockAsm512K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm512K
	JB   match_nolit_end_encodeBlockAsm512K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBlockAsm512K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm512K

matchlen_match1_match_nolit_encodeBlockAsm512K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeBlockAsm512K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBlockAsm512K
	PCALIGN $0x10

match_nolit_end_encodeBlockAsm512K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeBlockAsm512K
	LEAQ (BX)(DI*1), DI
	CMPL R8, $0x03
	JA   match_emit_lits_copy_encodeBlockAsm512K
	CMPL SI, $0x40
	JB   match_emit_lits_copy_encodeBlockAsm512K
	MOVL (DI), DI
	CMPL SI, $0x0001003f
	JBE  match_emit_copy2lits_encodeBlockAsm512K

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(SI), SI
	SHLL $0x0b, SI
	LEAL 7(SI)(R8*8), SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_emit_lits_encodeBlockAsm512K
	LEAL -60(R11), R9
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_emit_lits_encodeBlockAsm512K
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_emit_lits_encodeBlockAsm512K
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL R9, 4(CX)
	ADDQ $0x07, CX
	JMP  match_emit_copy_litsencodeBlockAsm512K

emit_copy3_2_match_emit_lits_encodeBlockAsm512K:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW R9, 4(CX)
	ADDQ $0x06, CX
	JMP  match_emit_copy_litsencodeBlockAsm512K

emit_copy3_1_match_emit_lits_encodeBlockAsm512K:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB R9, 4(CX)
	ADDQ $0x05, CX
	JMP  match_emit_copy_litsencodeBlockAsm512K

emit_copy3_0_match_emit_lits_encodeBlockAsm512K:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX

match_emit_copy_litsencodeBlockAsm512K:
	MOVL DI, (CX)
	ADDQ R8, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

match_emit_copy2lits_encodeBlockAsm512K:
	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, SI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    SI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, SI
	CMOVLLT R11, SI
	LEAL    -1(R8)(SI*4), SI
	MOVL    $0x00000003, R10
	LEAL    (R10)(SI*8), SI
	MOVB    SI, (CX)
	ADDQ    $0x03, CX
	MOVL    DI, (CX)
	ADDQ    R8, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBlockAsm512K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBlockAsm512K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBlockAsm512K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBlockAsm512K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

repeat_three_match_emit_repeat_copy2_encodeBlockAsm512K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

repeat_two_match_emit_repeat_copy2_encodeBlockAsm512K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

repeat_one_match_emit_repeat_copy2_encodeBlockAsm512K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

match_emit_lits_copy_encodeBlockAsm512K:
	LEAQ 4(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBlockAsm512K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm512K
	CMPL R9, $0x00010000
	JB   three_bytes_match_emit_encodeBlockAsm512K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm512K

three_bytes_match_emit_encodeBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm512K

two_bytes_match_emit_encodeBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBlockAsm512K
	JMP  memmove_long_match_emit_encodeBlockAsm512K

one_byte_match_emit_encodeBlockAsm512K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm512K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm512K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBlockAsm512K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm512K

emit_lit_memmove_match_emit_encodeBlockAsm512K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm512K

emit_lit_memmove_match_emit_encodeBlockAsm512K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeBlockAsm512K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm512K

memmove_midmatch_emit_encodeBlockAsm512K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBlockAsm512K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBlockAsm512K

emit_lit_memmove_mid_match_emit_encodeBlockAsm512K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeBlockAsm512K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm512K

memmove_long_match_emit_encodeBlockAsm512K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBlockAsm512Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeBlockAsm512K
	PCALIGN $0x10

match_nolits_copy_encodeBlockAsm512K:
	// emitCopy
	CMPL SI, $0x0001003f
	JBE  two_byte_offset_match_nolit_encodeBlockAsm512K

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(SI), SI
	SHLL $0x0b, SI
	ADDL $0x07, SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_nolit_encodeBlockAsm512K_emit3
	LEAL -60(R11), DI
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_nolit_encodeBlockAsm512K_emit3
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_nolit_encodeBlockAsm512K_emit3
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL DI, 4(CX)
	ADDQ $0x07, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

emit_copy3_2_match_nolit_encodeBlockAsm512K_emit3:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW DI, 4(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

emit_copy3_1_match_nolit_encodeBlockAsm512K_emit3:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

emit_copy3_0_match_nolit_encodeBlockAsm512K_emit3:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

two_byte_offset_match_nolit_encodeBlockAsm512K:
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeBlockAsm512K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBlockAsm512K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

emit_one_longer_match_nolit_encodeBlockAsm512K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBlockAsm512K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

emit_copy1_repeat_match_nolit_encodeBlockAsm512K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm512K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm512K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm512K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm512K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm512K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm512K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

two_byte_match_nolit_encodeBlockAsm512K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBlockAsm512K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBlockAsm512K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBlockAsm512K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

emit_copy2_2_match_nolit_encodeBlockAsm512K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

emit_copy2_1_match_nolit_encodeBlockAsm512K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm512K

emit_copy2_0_match_nolit_encodeBlockAsm512K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm512K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm512K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm512K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm512K:
	MOVQ  $0x0000cf1bbcdcbf9b, SI
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, R9
	SHLQ  $0x10, R8
	IMULQ SI, R8
	SHRQ  $0x32, R8
	SHLQ  $0x10, R9
	IMULQ SI, R9
	SHRQ  $0x32, R9
	LEAL  -2(DX), R10
	MOVL  (AX)(R9*4), SI
	MOVL  R10, (AX)(R8*4)
	MOVL  DX, (AX)(R9*4)
	MOVL  DX, R8
	INCL  DX
	CMPL  (BX)(SI*1), DI
	JNE   search_loop_encodeBlockAsm512K
	MOVL  R8, DI
	SUBL  SI, DI
	MOVL  DI, 16(SP)
	CMPQ  CX, (SP)
	JB    dst_size_check_ok_4
	MOVQ  $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x03, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeBlockAsm512K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeBlockAsm512K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm512K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeBlockAsm512K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeBlockAsm512K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeBlockAsm512K
	JMP  matchlen_match8_match_nolit2_encodeBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeBlockAsm512K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeBlockAsm512K

matchlen_match8_match_nolit2_encodeBlockAsm512K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeBlockAsm512K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm512K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeBlockAsm512K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeBlockAsm512K

matchlen_match4_match_nolit2_encodeBlockAsm512K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeBlockAsm512K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeBlockAsm512K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeBlockAsm512K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeBlockAsm512K
	JB   match_nolit2_end_encodeBlockAsm512K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeBlockAsm512K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeBlockAsm512K

matchlen_match1_match_nolit2_encodeBlockAsm512K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeBlockAsm512K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeBlockAsm512K
	PCALIGN $0x10

match_nolit2_end_encodeBlockAsm512K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeBlockAsm512K

emit_remainder_encodeBlockAsm512K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeBlockAsm512K
	LEAQ (BX)(DX*1), DX
	LEAQ 4(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeBlockAsm512K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm512K
	CMPL BX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBlockAsm512K
	MOVL BX, SI
	SHRL $0x10, SI
	MOVB $0xf8, (CX)
	MOVW BX, 1(CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm512K

three_bytes_emit_remainder_encodeBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm512K

two_bytes_emit_remainder_encodeBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeBlockAsm512K
	JMP  memmove_long_emit_remainder_encodeBlockAsm512K

one_byte_emit_remainder_encodeBlockAsm512K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm512K

emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm512K

emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm512K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm512K

emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm512K

emit_lit_memmove_emit_remainder_encodeBlockAsm512K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm512K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm512K

memmove_midemit_remainder_encodeBlockAsm512K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm512K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBlockAsm512K

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm512K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeBlockAsm512K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm512K

memmove_long_emit_remainder_encodeBlockAsm512K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBlockAsm512Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeBlockAsm512K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm64K(dst []byte, src []byte, tmp *[16384]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBlockAsm64K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000080, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm64K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm64K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm64K:
	MOVL    DX, SI
	SUBL    12(SP), SI
	SHRL    $0x05, SI
	LEAL    3(DX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeBlockAsm64K
	MOVQ    (BX)(DX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0x0000cf1bbcdcbf9b, R9
	MOVQ    DI, R8
	SHLQ    $0x10, R8
	IMULQ   R9, R8
	SHRQ    $0x33, R8
	MOVQ    DI, R10
	SHRQ    $0x08, R10
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x33, R10
	MOVWLZX (AX)(R8*2), SI
	MOVW    DX, (AX)(R8*2)
	MOVWLZX (AX)(R10*2), R8
	MOVW    DX, (AX)(R10*2)
	MOVQ    DI, R10
	SHRQ    $0x10, R10
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x33, R10
	MOVL    DX, R9
	SUBL    16(SP), R9
	MOVL    1(BX)(R9*1), R11
	MOVQ    DI, R9
	SHRQ    $0x08, R9
	CMPL    R9, R11
	JNE     no_repeat_found_encodeBlockAsm64K
	LEAL    1(DX), DI
	MOVL    12(SP), SI
	MOVL    DI, R8
	SUBL    16(SP), R8
	JZ      repeat_extend_back_end_encodeBlockAsm64K

repeat_extend_back_loop_encodeBlockAsm64K:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeBlockAsm64K
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm64K
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeBlockAsm64K

repeat_extend_back_end_encodeBlockAsm64K:
	MOVL DI, SI
	MOVL 12(SP), R8
	SUBL R8, SI
	LEAQ 4(CX)(SI*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_1
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeBlockAsm64K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeBlockAsm64K
	JB   three_bytes_repeat_emit_lits_encodeBlockAsm64K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm64K

three_bytes_repeat_emit_lits_encodeBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm64K

two_bytes_repeat_emit_lits_encodeBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeBlockAsm64K
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm64K

one_byte_repeat_emit_lits_encodeBlockAsm64K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm64K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm64K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm64K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm64K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm64K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeBlockAsm64K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm64K

memmove_midrepeat_emit_lits_encodeBlockAsm64K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm64K

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm64K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm64K

memmove_long_repeat_emit_lits_encodeBlockAsm64K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm64Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeBlockAsm64K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBlockAsm64K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBlockAsm64K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm64K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm64K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeBlockAsm64K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBlockAsm64K
	JMP  matchlen_match8_repeat_extend_encodeBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBlockAsm64K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeBlockAsm64K

matchlen_match8_repeat_extend_encodeBlockAsm64K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm64K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm64K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBlockAsm64K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeBlockAsm64K

matchlen_match4_repeat_extend_encodeBlockAsm64K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm64K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm64K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeBlockAsm64K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm64K
	JB   repeat_extend_forward_end_encodeBlockAsm64K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm64K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeBlockAsm64K

matchlen_match1_repeat_extend_encodeBlockAsm64K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeBlockAsm64K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeBlockAsm64K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBlockAsm64K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBlockAsm64K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBlockAsm64K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBlockAsm64K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm64K

repeat_three_match_repeat_encodeBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm64K

repeat_two_match_repeat_encodeBlockAsm64K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm64K

repeat_one_match_repeat_encodeBlockAsm64K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBlockAsm64K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm64K

no_repeat_found_encodeBlockAsm64K:
	CMPL    (BX)(SI*1), DI
	JEQ     candidate_match_encodeBlockAsm64K
	SHRQ    $0x08, DI
	MOVWLZX (AX)(R10*2), SI
	LEAL    2(DX), R9
	CMPL    (BX)(R8*1), DI
	JEQ     candidate2_match_encodeBlockAsm64K
	MOVW    R9, (AX)(R10*2)
	SHRQ    $0x08, DI
	CMPL    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeBlockAsm64K
	MOVL    20(SP), DX
	JMP     search_loop_encodeBlockAsm64K

candidate3_match_encodeBlockAsm64K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm64K

candidate2_match_encodeBlockAsm64K:
	MOVW R9, (AX)(R10*2)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm64K:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm64K

match_extend_back_loop_encodeBlockAsm64K:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm64K
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm64K
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm64K
	JMP  match_extend_back_loop_encodeBlockAsm64K

match_extend_back_end_encodeBlockAsm64K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBlockAsm64K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBlockAsm64K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm64K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm64K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBlockAsm64K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBlockAsm64K
	JMP  matchlen_match8_match_nolit_encodeBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBlockAsm64K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBlockAsm64K

matchlen_match8_match_nolit_encodeBlockAsm64K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm64K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm64K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBlockAsm64K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBlockAsm64K

matchlen_match4_match_nolit_encodeBlockAsm64K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm64K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBlockAsm64K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBlockAsm64K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm64K
	JB   match_nolit_end_encodeBlockAsm64K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBlockAsm64K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm64K

matchlen_match1_match_nolit_encodeBlockAsm64K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeBlockAsm64K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBlockAsm64K
	PCALIGN $0x10

match_nolit_end_encodeBlockAsm64K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeBlockAsm64K
	LEAQ (BX)(DI*1), DI
	CMPL R8, $0x03
	JA   match_emit_lits_copy_encodeBlockAsm64K
	CMPL SI, $0x40
	JB   match_emit_lits_copy_encodeBlockAsm64K
	MOVL (DI), DI

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, SI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    SI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, SI
	CMOVLLT R11, SI
	LEAL    -1(R8)(SI*4), SI
	MOVL    $0x00000003, R10
	LEAL    (R10)(SI*8), SI
	MOVB    SI, (CX)
	ADDQ    $0x03, CX
	MOVL    DI, (CX)
	ADDQ    R8, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBlockAsm64K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBlockAsm64K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBlockAsm64K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBlockAsm64K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

repeat_three_match_emit_repeat_copy2_encodeBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

repeat_two_match_emit_repeat_copy2_encodeBlockAsm64K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

repeat_one_match_emit_repeat_copy2_encodeBlockAsm64K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

match_emit_lits_copy_encodeBlockAsm64K:
	LEAQ 4(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBlockAsm64K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm64K
	JB   three_bytes_match_emit_encodeBlockAsm64K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm64K

three_bytes_match_emit_encodeBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm64K

two_bytes_match_emit_encodeBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBlockAsm64K
	JMP  memmove_long_match_emit_encodeBlockAsm64K

one_byte_match_emit_encodeBlockAsm64K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm64K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm64K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBlockAsm64K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm64K

emit_lit_memmove_match_emit_encodeBlockAsm64K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm64K

emit_lit_memmove_match_emit_encodeBlockAsm64K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeBlockAsm64K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm64K

memmove_midmatch_emit_encodeBlockAsm64K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBlockAsm64K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBlockAsm64K

emit_lit_memmove_mid_match_emit_encodeBlockAsm64K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeBlockAsm64K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm64K

memmove_long_match_emit_encodeBlockAsm64K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBlockAsm64Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeBlockAsm64K
	PCALIGN $0x10

match_nolits_copy_encodeBlockAsm64K:
	// emitCopy
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeBlockAsm64K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBlockAsm64K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

emit_one_longer_match_nolit_encodeBlockAsm64K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBlockAsm64K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

emit_copy1_repeat_match_nolit_encodeBlockAsm64K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm64K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm64K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm64K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm64K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm64K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

two_byte_match_nolit_encodeBlockAsm64K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBlockAsm64K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBlockAsm64K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBlockAsm64K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

emit_copy2_2_match_nolit_encodeBlockAsm64K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

emit_copy2_1_match_nolit_encodeBlockAsm64K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm64K

emit_copy2_0_match_nolit_encodeBlockAsm64K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm64K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm64K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm64K:
	MOVQ    $0x0000cf1bbcdcbf9b, SI
	MOVQ    DI, R8
	SHRQ    $0x10, DI
	MOVQ    DI, R9
	SHLQ    $0x10, R8
	IMULQ   SI, R8
	SHRQ    $0x33, R8
	SHLQ    $0x10, R9
	IMULQ   SI, R9
	SHRQ    $0x33, R9
	LEAL    -2(DX), R10
	MOVWLZX (AX)(R9*2), SI
	MOVW    R10, (AX)(R8*2)
	MOVW    DX, (AX)(R9*2)
	MOVL    DX, R8
	INCL    DX
	CMPL    (BX)(SI*1), DI
	JNE     search_loop_encodeBlockAsm64K
	MOVL    R8, DI
	SUBL    SI, DI
	MOVL    DI, 16(SP)
	CMPQ    CX, (SP)
	JB      dst_size_check_ok_4
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x03, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeBlockAsm64K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeBlockAsm64K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm64K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeBlockAsm64K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeBlockAsm64K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeBlockAsm64K
	JMP  matchlen_match8_match_nolit2_encodeBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeBlockAsm64K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeBlockAsm64K

matchlen_match8_match_nolit2_encodeBlockAsm64K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeBlockAsm64K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm64K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeBlockAsm64K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeBlockAsm64K

matchlen_match4_match_nolit2_encodeBlockAsm64K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeBlockAsm64K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeBlockAsm64K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeBlockAsm64K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeBlockAsm64K
	JB   match_nolit2_end_encodeBlockAsm64K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeBlockAsm64K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeBlockAsm64K

matchlen_match1_match_nolit2_encodeBlockAsm64K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeBlockAsm64K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeBlockAsm64K
	PCALIGN $0x10

match_nolit2_end_encodeBlockAsm64K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeBlockAsm64K

emit_remainder_encodeBlockAsm64K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeBlockAsm64K
	LEAQ (BX)(DX*1), DX
	LEAQ 4(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeBlockAsm64K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm64K
	JB   three_bytes_emit_remainder_encodeBlockAsm64K
	MOVL BX, SI
	SHRL $0x10, SI
	MOVB $0xf8, (CX)
	MOVW BX, 1(CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm64K

three_bytes_emit_remainder_encodeBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm64K

two_bytes_emit_remainder_encodeBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeBlockAsm64K
	JMP  memmove_long_emit_remainder_encodeBlockAsm64K

one_byte_emit_remainder_encodeBlockAsm64K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm64K

emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm64K

emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm64K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm64K

emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm64K

emit_lit_memmove_emit_remainder_encodeBlockAsm64K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm64K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm64K

memmove_midemit_remainder_encodeBlockAsm64K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm64K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBlockAsm64K

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm64K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeBlockAsm64K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm64K

memmove_long_emit_remainder_encodeBlockAsm64K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBlockAsm64Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeBlockAsm64K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm16K(dst []byte, src []byte, tmp *[8192]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBlockAsm16K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000040, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm16K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm16K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm16K:
	MOVL    DX, SI
	SUBL    12(SP), SI
	SHRL    $0x05, SI
	LEAL    3(DX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeBlockAsm16K
	MOVQ    (BX)(DX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0x000000cf1bbcdcbb, R9
	MOVQ    DI, R8
	SHLQ    $0x18, R8
	IMULQ   R9, R8
	SHRQ    $0x34, R8
	MOVQ    DI, R10
	SHRQ    $0x08, R10
	SHLQ    $0x18, R10
	IMULQ   R9, R10
	SHRQ    $0x34, R10
	MOVWLZX (AX)(R8*2), SI
	MOVW    DX, (AX)(R8*2)
	MOVWLZX (AX)(R10*2), R8
	MOVW    DX, (AX)(R10*2)
	MOVQ    DI, R10
	SHRQ    $0x10, R10
	SHLQ    $0x18, R10
	IMULQ   R9, R10
	SHRQ    $0x34, R10
	MOVL    DX, R9
	SUBL    16(SP), R9
	MOVL    1(BX)(R9*1), R11
	MOVQ    DI, R9
	SHRQ    $0x08, R9
	CMPL    R9, R11
	JNE     no_repeat_found_encodeBlockAsm16K
	LEAL    1(DX), DI
	MOVL    12(SP), SI
	MOVL    DI, R8
	SUBL    16(SP), R8
	JZ      repeat_extend_back_end_encodeBlockAsm16K

repeat_extend_back_loop_encodeBlockAsm16K:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeBlockAsm16K
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm16K
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeBlockAsm16K

repeat_extend_back_end_encodeBlockAsm16K:
	MOVL DI, SI
	MOVL 12(SP), R8
	SUBL R8, SI
	LEAQ 3(CX)(SI*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_1
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeBlockAsm16K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeBlockAsm16K
	JB   three_bytes_repeat_emit_lits_encodeBlockAsm16K

three_bytes_repeat_emit_lits_encodeBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm16K

two_bytes_repeat_emit_lits_encodeBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeBlockAsm16K
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm16K

one_byte_repeat_emit_lits_encodeBlockAsm16K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm16K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm16K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm16K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm16K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm16K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeBlockAsm16K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm16K

memmove_midrepeat_emit_lits_encodeBlockAsm16K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm16K

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm16K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm16K

memmove_long_repeat_emit_lits_encodeBlockAsm16K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm16Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeBlockAsm16K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBlockAsm16K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBlockAsm16K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm16K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm16K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeBlockAsm16K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBlockAsm16K
	JMP  matchlen_match8_repeat_extend_encodeBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBlockAsm16K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeBlockAsm16K

matchlen_match8_repeat_extend_encodeBlockAsm16K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm16K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm16K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBlockAsm16K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeBlockAsm16K

matchlen_match4_repeat_extend_encodeBlockAsm16K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm16K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm16K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeBlockAsm16K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm16K
	JB   repeat_extend_forward_end_encodeBlockAsm16K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm16K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeBlockAsm16K

matchlen_match1_repeat_extend_encodeBlockAsm16K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeBlockAsm16K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeBlockAsm16K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBlockAsm16K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBlockAsm16K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBlockAsm16K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBlockAsm16K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm16K

repeat_three_match_repeat_encodeBlockAsm16K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm16K

repeat_two_match_repeat_encodeBlockAsm16K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm16K

repeat_one_match_repeat_encodeBlockAsm16K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBlockAsm16K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm16K

no_repeat_found_encodeBlockAsm16K:
	CMPL    (BX)(SI*1), DI
	JEQ     candidate_match_encodeBlockAsm16K
	SHRQ    $0x08, DI
	MOVWLZX (AX)(R10*2), SI
	LEAL    2(DX), R9
	CMPL    (BX)(R8*1), DI
	JEQ     candidate2_match_encodeBlockAsm16K
	MOVW    R9, (AX)(R10*2)
	SHRQ    $0x08, DI
	CMPL    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeBlockAsm16K
	MOVL    20(SP), DX
	JMP     search_loop_encodeBlockAsm16K

candidate3_match_encodeBlockAsm16K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm16K

candidate2_match_encodeBlockAsm16K:
	MOVW R9, (AX)(R10*2)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm16K:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm16K

match_extend_back_loop_encodeBlockAsm16K:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm16K
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm16K
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm16K
	JMP  match_extend_back_loop_encodeBlockAsm16K

match_extend_back_end_encodeBlockAsm16K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBlockAsm16K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBlockAsm16K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm16K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm16K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBlockAsm16K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBlockAsm16K
	JMP  matchlen_match8_match_nolit_encodeBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBlockAsm16K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBlockAsm16K

matchlen_match8_match_nolit_encodeBlockAsm16K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm16K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm16K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBlockAsm16K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBlockAsm16K

matchlen_match4_match_nolit_encodeBlockAsm16K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm16K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBlockAsm16K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBlockAsm16K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm16K
	JB   match_nolit_end_encodeBlockAsm16K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBlockAsm16K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm16K

matchlen_match1_match_nolit_encodeBlockAsm16K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeBlockAsm16K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBlockAsm16K
	PCALIGN $0x10

match_nolit_end_encodeBlockAsm16K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeBlockAsm16K
	LEAQ (BX)(DI*1), DI
	CMPL R8, $0x03
	JA   match_emit_lits_copy_encodeBlockAsm16K
	CMPL SI, $0x40
	JB   match_emit_lits_copy_encodeBlockAsm16K
	MOVL (DI), DI

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, SI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    SI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, SI
	CMOVLLT R11, SI
	LEAL    -1(R8)(SI*4), SI
	MOVL    $0x00000003, R10
	LEAL    (R10)(SI*8), SI
	MOVB    SI, (CX)
	ADDQ    $0x03, CX
	MOVL    DI, (CX)
	ADDQ    R8, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBlockAsm16K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBlockAsm16K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBlockAsm16K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBlockAsm16K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

repeat_three_match_emit_repeat_copy2_encodeBlockAsm16K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

repeat_two_match_emit_repeat_copy2_encodeBlockAsm16K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

repeat_one_match_emit_repeat_copy2_encodeBlockAsm16K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

match_emit_lits_copy_encodeBlockAsm16K:
	LEAQ 3(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBlockAsm16K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm16K
	JB   three_bytes_match_emit_encodeBlockAsm16K

three_bytes_match_emit_encodeBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm16K

two_bytes_match_emit_encodeBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBlockAsm16K
	JMP  memmove_long_match_emit_encodeBlockAsm16K

one_byte_match_emit_encodeBlockAsm16K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm16K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm16K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBlockAsm16K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm16K

emit_lit_memmove_match_emit_encodeBlockAsm16K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm16K

emit_lit_memmove_match_emit_encodeBlockAsm16K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeBlockAsm16K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm16K

memmove_midmatch_emit_encodeBlockAsm16K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBlockAsm16K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBlockAsm16K

emit_lit_memmove_mid_match_emit_encodeBlockAsm16K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeBlockAsm16K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm16K

memmove_long_match_emit_encodeBlockAsm16K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBlockAsm16Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeBlockAsm16K
	PCALIGN $0x10

match_nolits_copy_encodeBlockAsm16K:
	// emitCopy
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeBlockAsm16K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBlockAsm16K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

emit_one_longer_match_nolit_encodeBlockAsm16K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBlockAsm16K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

emit_copy1_repeat_match_nolit_encodeBlockAsm16K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm16K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm16K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm16K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm16K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm16K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm16K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

two_byte_match_nolit_encodeBlockAsm16K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBlockAsm16K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBlockAsm16K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBlockAsm16K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

emit_copy2_2_match_nolit_encodeBlockAsm16K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

emit_copy2_1_match_nolit_encodeBlockAsm16K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm16K

emit_copy2_0_match_nolit_encodeBlockAsm16K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm16K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm16K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm16K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm16K:
	MOVQ    $0x000000cf1bbcdcbb, SI
	MOVQ    DI, R8
	SHRQ    $0x10, DI
	MOVQ    DI, R9
	SHLQ    $0x18, R8
	IMULQ   SI, R8
	SHRQ    $0x34, R8
	SHLQ    $0x18, R9
	IMULQ   SI, R9
	SHRQ    $0x34, R9
	LEAL    -2(DX), R10
	MOVWLZX (AX)(R9*2), SI
	MOVW    R10, (AX)(R8*2)
	MOVW    DX, (AX)(R9*2)
	MOVL    DX, R8
	INCL    DX
	CMPL    (BX)(SI*1), DI
	JNE     search_loop_encodeBlockAsm16K
	MOVL    R8, DI
	SUBL    SI, DI
	MOVL    DI, 16(SP)
	CMPQ    CX, (SP)
	JB      dst_size_check_ok_4
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x03, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeBlockAsm16K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeBlockAsm16K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm16K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeBlockAsm16K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeBlockAsm16K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeBlockAsm16K
	JMP  matchlen_match8_match_nolit2_encodeBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeBlockAsm16K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeBlockAsm16K

matchlen_match8_match_nolit2_encodeBlockAsm16K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeBlockAsm16K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm16K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeBlockAsm16K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeBlockAsm16K

matchlen_match4_match_nolit2_encodeBlockAsm16K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeBlockAsm16K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeBlockAsm16K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeBlockAsm16K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeBlockAsm16K
	JB   match_nolit2_end_encodeBlockAsm16K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeBlockAsm16K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeBlockAsm16K

matchlen_match1_match_nolit2_encodeBlockAsm16K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeBlockAsm16K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeBlockAsm16K
	PCALIGN $0x10

match_nolit2_end_encodeBlockAsm16K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeBlockAsm16K

emit_remainder_encodeBlockAsm16K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeBlockAsm16K
	LEAQ (BX)(DX*1), DX
	LEAQ 3(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeBlockAsm16K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm16K
	JB   three_bytes_emit_remainder_encodeBlockAsm16K

three_bytes_emit_remainder_encodeBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm16K

two_bytes_emit_remainder_encodeBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeBlockAsm16K
	JMP  memmove_long_emit_remainder_encodeBlockAsm16K

one_byte_emit_remainder_encodeBlockAsm16K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm16K

emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm16K

emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm16K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm16K

emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm16K

emit_lit_memmove_emit_remainder_encodeBlockAsm16K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm16K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm16K

memmove_midemit_remainder_encodeBlockAsm16K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm16K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBlockAsm16K

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm16K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeBlockAsm16K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm16K

memmove_long_emit_remainder_encodeBlockAsm16K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBlockAsm16Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeBlockAsm16K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm4K(dst []byte, src []byte, tmp *[2048]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBlockAsm4K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000010, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm4K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm4K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm4K:
	MOVL    DX, SI
	SUBL    12(SP), SI
	SHRL    $0x05, SI
	LEAL    3(DX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeBlockAsm4K
	MOVQ    (BX)(DX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0x9e3779b1, R9
	MOVQ    DI, R8
	SHLQ    $0x20, R8
	IMULQ   R9, R8
	SHRQ    $0x36, R8
	MOVQ    DI, R10
	SHRQ    $0x08, R10
	SHLQ    $0x20, R10
	IMULQ   R9, R10
	SHRQ    $0x36, R10
	MOVWLZX (AX)(R8*2), SI
	MOVW    DX, (AX)(R8*2)
	MOVWLZX (AX)(R10*2), R8
	MOVW    DX, (AX)(R10*2)
	MOVQ    DI, R10
	SHRQ    $0x10, R10
	SHLQ    $0x20, R10
	IMULQ   R9, R10
	SHRQ    $0x36, R10
	MOVL    DX, R9
	SUBL    16(SP), R9
	MOVL    1(BX)(R9*1), R11
	MOVQ    DI, R9
	SHRQ    $0x08, R9
	CMPL    R9, R11
	JNE     no_repeat_found_encodeBlockAsm4K
	LEAL    1(DX), DI
	MOVL    12(SP), SI
	MOVL    DI, R8
	SUBL    16(SP), R8
	JZ      repeat_extend_back_end_encodeBlockAsm4K

repeat_extend_back_loop_encodeBlockAsm4K:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeBlockAsm4K
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm4K
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeBlockAsm4K

repeat_extend_back_end_encodeBlockAsm4K:
	MOVL DI, SI
	MOVL 12(SP), R8
	SUBL R8, SI
	LEAQ 3(CX)(SI*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_1
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeBlockAsm4K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeBlockAsm4K
	JB   three_bytes_repeat_emit_lits_encodeBlockAsm4K

three_bytes_repeat_emit_lits_encodeBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm4K

two_bytes_repeat_emit_lits_encodeBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeBlockAsm4K
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm4K

one_byte_repeat_emit_lits_encodeBlockAsm4K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm4K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm4K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm4K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm4K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm4K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeBlockAsm4K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm4K

memmove_midrepeat_emit_lits_encodeBlockAsm4K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm4K

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm4K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm4K

memmove_long_repeat_emit_lits_encodeBlockAsm4K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm4Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeBlockAsm4K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBlockAsm4K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBlockAsm4K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm4K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm4K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeBlockAsm4K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBlockAsm4K
	JMP  matchlen_match8_repeat_extend_encodeBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBlockAsm4K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeBlockAsm4K

matchlen_match8_repeat_extend_encodeBlockAsm4K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm4K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm4K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBlockAsm4K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeBlockAsm4K

matchlen_match4_repeat_extend_encodeBlockAsm4K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm4K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm4K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeBlockAsm4K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm4K
	JB   repeat_extend_forward_end_encodeBlockAsm4K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm4K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeBlockAsm4K

matchlen_match1_repeat_extend_encodeBlockAsm4K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeBlockAsm4K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeBlockAsm4K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBlockAsm4K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBlockAsm4K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBlockAsm4K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBlockAsm4K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm4K

repeat_three_match_repeat_encodeBlockAsm4K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm4K

repeat_two_match_repeat_encodeBlockAsm4K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4K

repeat_one_match_repeat_encodeBlockAsm4K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBlockAsm4K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm4K

no_repeat_found_encodeBlockAsm4K:
	CMPL    (BX)(SI*1), DI
	JEQ     candidate_match_encodeBlockAsm4K
	SHRQ    $0x08, DI
	MOVWLZX (AX)(R10*2), SI
	LEAL    2(DX), R9
	CMPL    (BX)(R8*1), DI
	JEQ     candidate2_match_encodeBlockAsm4K
	MOVW    R9, (AX)(R10*2)
	SHRQ    $0x08, DI
	CMPL    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeBlockAsm4K
	MOVL    20(SP), DX
	JMP     search_loop_encodeBlockAsm4K

candidate3_match_encodeBlockAsm4K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm4K

candidate2_match_encodeBlockAsm4K:
	MOVW R9, (AX)(R10*2)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm4K:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm4K

match_extend_back_loop_encodeBlockAsm4K:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm4K
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm4K
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm4K
	JMP  match_extend_back_loop_encodeBlockAsm4K

match_extend_back_end_encodeBlockAsm4K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBlockAsm4K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBlockAsm4K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm4K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm4K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBlockAsm4K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBlockAsm4K
	JMP  matchlen_match8_match_nolit_encodeBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBlockAsm4K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBlockAsm4K

matchlen_match8_match_nolit_encodeBlockAsm4K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm4K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm4K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBlockAsm4K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBlockAsm4K

matchlen_match4_match_nolit_encodeBlockAsm4K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm4K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBlockAsm4K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBlockAsm4K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm4K
	JB   match_nolit_end_encodeBlockAsm4K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBlockAsm4K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm4K

matchlen_match1_match_nolit_encodeBlockAsm4K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeBlockAsm4K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBlockAsm4K
	PCALIGN $0x10

match_nolit_end_encodeBlockAsm4K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeBlockAsm4K
	LEAQ (BX)(DI*1), DI
	CMPL R8, $0x03
	JA   match_emit_lits_copy_encodeBlockAsm4K
	CMPL SI, $0x40
	JB   match_emit_lits_copy_encodeBlockAsm4K
	MOVL (DI), DI

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, SI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    SI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, SI
	CMOVLLT R11, SI
	LEAL    -1(R8)(SI*4), SI
	MOVL    $0x00000003, R10
	LEAL    (R10)(SI*8), SI
	MOVB    SI, (CX)
	ADDQ    $0x03, CX
	MOVL    DI, (CX)
	ADDQ    R8, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBlockAsm4K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBlockAsm4K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBlockAsm4K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBlockAsm4K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

repeat_three_match_emit_repeat_copy2_encodeBlockAsm4K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

repeat_two_match_emit_repeat_copy2_encodeBlockAsm4K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

repeat_one_match_emit_repeat_copy2_encodeBlockAsm4K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

match_emit_lits_copy_encodeBlockAsm4K:
	LEAQ 3(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBlockAsm4K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm4K
	JB   three_bytes_match_emit_encodeBlockAsm4K

three_bytes_match_emit_encodeBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm4K

two_bytes_match_emit_encodeBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBlockAsm4K
	JMP  memmove_long_match_emit_encodeBlockAsm4K

one_byte_match_emit_encodeBlockAsm4K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm4K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm4K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBlockAsm4K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm4K

emit_lit_memmove_match_emit_encodeBlockAsm4K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm4K

emit_lit_memmove_match_emit_encodeBlockAsm4K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeBlockAsm4K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm4K

memmove_midmatch_emit_encodeBlockAsm4K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBlockAsm4K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBlockAsm4K

emit_lit_memmove_mid_match_emit_encodeBlockAsm4K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeBlockAsm4K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm4K

memmove_long_match_emit_encodeBlockAsm4K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBlockAsm4Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeBlockAsm4K
	PCALIGN $0x10

match_nolits_copy_encodeBlockAsm4K:
	// emitCopy
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeBlockAsm4K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBlockAsm4K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

emit_one_longer_match_nolit_encodeBlockAsm4K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBlockAsm4K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

emit_copy1_repeat_match_nolit_encodeBlockAsm4K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm4K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm4K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm4K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm4K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm4K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm4K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

two_byte_match_nolit_encodeBlockAsm4K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBlockAsm4K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBlockAsm4K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBlockAsm4K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

emit_copy2_2_match_nolit_encodeBlockAsm4K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

emit_copy2_1_match_nolit_encodeBlockAsm4K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4K

emit_copy2_0_match_nolit_encodeBlockAsm4K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm4K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm4K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm4K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm4K:
	MOVQ    $0x9e3779b1, SI
	MOVQ    DI, R8
	SHRQ    $0x10, DI
	MOVQ    DI, R9
	SHLQ    $0x20, R8
	IMULQ   SI, R8
	SHRQ    $0x36, R8
	SHLQ    $0x20, R9
	IMULQ   SI, R9
	SHRQ    $0x36, R9
	LEAL    -2(DX), R10
	MOVWLZX (AX)(R9*2), SI
	MOVW    R10, (AX)(R8*2)
	MOVW    DX, (AX)(R9*2)
	MOVL    DX, R8
	INCL    DX
	CMPL    (BX)(SI*1), DI
	JNE     search_loop_encodeBlockAsm4K
	MOVL    R8, DI
	SUBL    SI, DI
	MOVL    DI, 16(SP)
	CMPQ    CX, (SP)
	JB      dst_size_check_ok_4
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x03, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeBlockAsm4K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeBlockAsm4K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm4K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeBlockAsm4K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeBlockAsm4K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeBlockAsm4K
	JMP  matchlen_match8_match_nolit2_encodeBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeBlockAsm4K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeBlockAsm4K

matchlen_match8_match_nolit2_encodeBlockAsm4K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeBlockAsm4K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm4K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeBlockAsm4K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeBlockAsm4K

matchlen_match4_match_nolit2_encodeBlockAsm4K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeBlockAsm4K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeBlockAsm4K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeBlockAsm4K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeBlockAsm4K
	JB   match_nolit2_end_encodeBlockAsm4K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeBlockAsm4K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeBlockAsm4K

matchlen_match1_match_nolit2_encodeBlockAsm4K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeBlockAsm4K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeBlockAsm4K
	PCALIGN $0x10

match_nolit2_end_encodeBlockAsm4K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeBlockAsm4K

emit_remainder_encodeBlockAsm4K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeBlockAsm4K
	LEAQ (BX)(DX*1), DX
	LEAQ 3(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeBlockAsm4K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm4K
	JB   three_bytes_emit_remainder_encodeBlockAsm4K

three_bytes_emit_remainder_encodeBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm4K

two_bytes_emit_remainder_encodeBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeBlockAsm4K
	JMP  memmove_long_emit_remainder_encodeBlockAsm4K

one_byte_emit_remainder_encodeBlockAsm4K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm4K

emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm4K

emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm4K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm4K

emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm4K

emit_lit_memmove_emit_remainder_encodeBlockAsm4K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm4K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm4K

memmove_midemit_remainder_encodeBlockAsm4K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm4K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBlockAsm4K

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm4K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeBlockAsm4K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm4K

memmove_long_emit_remainder_encodeBlockAsm4K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBlockAsm4Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeBlockAsm4K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm1K(dst []byte, src []byte, tmp *[1024]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBlockAsm1K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000008, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm1K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm1K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm1K:
	MOVL    DX, SI
	SUBL    12(SP), SI
	SHRL    $0x04, SI
	LEAL    3(DX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeBlockAsm1K
	MOVQ    (BX)(DX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0x9e3779b1, R9
	MOVQ    DI, R8
	SHLQ    $0x20, R8
	IMULQ   R9, R8
	SHRQ    $0x37, R8
	MOVQ    DI, R10
	SHRQ    $0x08, R10
	SHLQ    $0x20, R10
	IMULQ   R9, R10
	SHRQ    $0x37, R10
	MOVWLZX (AX)(R8*2), SI
	MOVW    DX, (AX)(R8*2)
	MOVWLZX (AX)(R10*2), R8
	MOVW    DX, (AX)(R10*2)
	MOVQ    DI, R10
	SHRQ    $0x10, R10
	SHLQ    $0x20, R10
	IMULQ   R9, R10
	SHRQ    $0x37, R10
	MOVL    DX, R9
	SUBL    16(SP), R9
	MOVL    1(BX)(R9*1), R11
	MOVQ    DI, R9
	SHRQ    $0x08, R9
	CMPL    R9, R11
	JNE     no_repeat_found_encodeBlockAsm1K
	LEAL    1(DX), DI
	MOVL    12(SP), SI
	MOVL    DI, R8
	SUBL    16(SP), R8
	JZ      repeat_extend_back_end_encodeBlockAsm1K

repeat_extend_back_loop_encodeBlockAsm1K:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeBlockAsm1K
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm1K
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeBlockAsm1K

repeat_extend_back_end_encodeBlockAsm1K:
	MOVL DI, SI
	MOVL 12(SP), R8
	SUBL R8, SI
	LEAQ 3(CX)(SI*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_1
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeBlockAsm1K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeBlockAsm1K
	JB   three_bytes_repeat_emit_lits_encodeBlockAsm1K

three_bytes_repeat_emit_lits_encodeBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm1K

two_bytes_repeat_emit_lits_encodeBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeBlockAsm1K
	JMP  memmove_long_repeat_emit_lits_encodeBlockAsm1K

one_byte_repeat_emit_lits_encodeBlockAsm1K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm1K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeBlockAsm1K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm1K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm1K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeBlockAsm1K

emit_lit_memmove_repeat_emit_lits_encodeBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeBlockAsm1K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm1K

memmove_midrepeat_emit_lits_encodeBlockAsm1K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm1K

emit_lit_memmove_mid_repeat_emit_lits_encodeBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeBlockAsm1K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeBlockAsm1K

memmove_long_repeat_emit_lits_encodeBlockAsm1K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm1Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeBlockAsm1K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBlockAsm1K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBlockAsm1K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm1K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm1K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeBlockAsm1K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBlockAsm1K
	JMP  matchlen_match8_repeat_extend_encodeBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBlockAsm1K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeBlockAsm1K

matchlen_match8_repeat_extend_encodeBlockAsm1K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm1K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm1K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBlockAsm1K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeBlockAsm1K

matchlen_match4_repeat_extend_encodeBlockAsm1K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm1K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm1K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeBlockAsm1K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm1K
	JB   repeat_extend_forward_end_encodeBlockAsm1K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm1K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeBlockAsm1K

matchlen_match1_repeat_extend_encodeBlockAsm1K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeBlockAsm1K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeBlockAsm1K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBlockAsm1K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBlockAsm1K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBlockAsm1K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBlockAsm1K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm1K

repeat_three_match_repeat_encodeBlockAsm1K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm1K

repeat_two_match_repeat_encodeBlockAsm1K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm1K

repeat_one_match_repeat_encodeBlockAsm1K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBlockAsm1K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm1K

no_repeat_found_encodeBlockAsm1K:
	CMPL    (BX)(SI*1), DI
	JEQ     candidate_match_encodeBlockAsm1K
	SHRQ    $0x08, DI
	MOVWLZX (AX)(R10*2), SI
	LEAL    2(DX), R9
	CMPL    (BX)(R8*1), DI
	JEQ     candidate2_match_encodeBlockAsm1K
	MOVW    R9, (AX)(R10*2)
	SHRQ    $0x08, DI
	CMPL    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeBlockAsm1K
	MOVL    20(SP), DX
	JMP     search_loop_encodeBlockAsm1K

candidate3_match_encodeBlockAsm1K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm1K

candidate2_match_encodeBlockAsm1K:
	MOVW R9, (AX)(R10*2)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm1K:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm1K

match_extend_back_loop_encodeBlockAsm1K:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm1K
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm1K
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm1K
	JMP  match_extend_back_loop_encodeBlockAsm1K

match_extend_back_end_encodeBlockAsm1K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBlockAsm1K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBlockAsm1K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm1K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm1K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBlockAsm1K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBlockAsm1K
	JMP  matchlen_match8_match_nolit_encodeBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBlockAsm1K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBlockAsm1K

matchlen_match8_match_nolit_encodeBlockAsm1K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm1K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm1K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBlockAsm1K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBlockAsm1K

matchlen_match4_match_nolit_encodeBlockAsm1K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm1K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBlockAsm1K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBlockAsm1K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm1K
	JB   match_nolit_end_encodeBlockAsm1K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBlockAsm1K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm1K

matchlen_match1_match_nolit_encodeBlockAsm1K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeBlockAsm1K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBlockAsm1K
	PCALIGN $0x10

match_nolit_end_encodeBlockAsm1K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeBlockAsm1K
	LEAQ (BX)(DI*1), DI
	CMPL R8, $0x03
	JA   match_emit_lits_copy_encodeBlockAsm1K
	CMPL SI, $0x40
	JB   match_emit_lits_copy_encodeBlockAsm1K
	MOVL (DI), DI

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, SI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    SI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, SI
	CMOVLLT R11, SI
	LEAL    -1(R8)(SI*4), SI
	MOVL    $0x00000003, R10
	LEAL    (R10)(SI*8), SI
	MOVB    SI, (CX)
	ADDQ    $0x03, CX
	MOVL    DI, (CX)
	ADDQ    R8, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBlockAsm1K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBlockAsm1K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBlockAsm1K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBlockAsm1K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

repeat_three_match_emit_repeat_copy2_encodeBlockAsm1K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

repeat_two_match_emit_repeat_copy2_encodeBlockAsm1K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

repeat_one_match_emit_repeat_copy2_encodeBlockAsm1K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

match_emit_lits_copy_encodeBlockAsm1K:
	LEAQ 3(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBlockAsm1K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm1K
	JB   three_bytes_match_emit_encodeBlockAsm1K

three_bytes_match_emit_encodeBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBlockAsm1K

two_bytes_match_emit_encodeBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBlockAsm1K
	JMP  memmove_long_match_emit_encodeBlockAsm1K

one_byte_match_emit_encodeBlockAsm1K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm1K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm1K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBlockAsm1K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm1K

emit_lit_memmove_match_emit_encodeBlockAsm1K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm1K

emit_lit_memmove_match_emit_encodeBlockAsm1K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeBlockAsm1K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm1K

memmove_midmatch_emit_encodeBlockAsm1K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBlockAsm1K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBlockAsm1K

emit_lit_memmove_mid_match_emit_encodeBlockAsm1K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeBlockAsm1K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeBlockAsm1K

memmove_long_match_emit_encodeBlockAsm1K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBlockAsm1Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeBlockAsm1K
	PCALIGN $0x10

match_nolits_copy_encodeBlockAsm1K:
	// emitCopy
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeBlockAsm1K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBlockAsm1K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

emit_one_longer_match_nolit_encodeBlockAsm1K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBlockAsm1K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

emit_copy1_repeat_match_nolit_encodeBlockAsm1K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm1K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm1K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm1K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBlockAsm1K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBlockAsm1K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBlockAsm1K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

two_byte_match_nolit_encodeBlockAsm1K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBlockAsm1K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBlockAsm1K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBlockAsm1K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

emit_copy2_2_match_nolit_encodeBlockAsm1K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

emit_copy2_1_match_nolit_encodeBlockAsm1K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm1K

emit_copy2_0_match_nolit_encodeBlockAsm1K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm1K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm1K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm1K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm1K:
	MOVQ    $0x9e3779b1, SI
	MOVQ    DI, R8
	SHRQ    $0x10, DI
	MOVQ    DI, R9
	SHLQ    $0x20, R8
	IMULQ   SI, R8
	SHRQ    $0x37, R8
	SHLQ    $0x20, R9
	IMULQ   SI, R9
	SHRQ    $0x37, R9
	LEAL    -2(DX), R10
	MOVWLZX (AX)(R9*2), SI
	MOVW    R10, (AX)(R8*2)
	MOVW    DX, (AX)(R9*2)
	MOVL    DX, R8
	INCL    DX
	CMPL    (BX)(SI*1), DI
	JNE     search_loop_encodeBlockAsm1K
	MOVL    R8, DI
	SUBL    SI, DI
	MOVL    DI, 16(SP)
	CMPQ    CX, (SP)
	JB      dst_size_check_ok_4
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x03, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeBlockAsm1K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeBlockAsm1K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm1K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeBlockAsm1K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeBlockAsm1K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeBlockAsm1K
	JMP  matchlen_match8_match_nolit2_encodeBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeBlockAsm1K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeBlockAsm1K

matchlen_match8_match_nolit2_encodeBlockAsm1K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeBlockAsm1K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeBlockAsm1K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeBlockAsm1K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeBlockAsm1K

matchlen_match4_match_nolit2_encodeBlockAsm1K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeBlockAsm1K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeBlockAsm1K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeBlockAsm1K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeBlockAsm1K
	JB   match_nolit2_end_encodeBlockAsm1K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeBlockAsm1K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeBlockAsm1K

matchlen_match1_match_nolit2_encodeBlockAsm1K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeBlockAsm1K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeBlockAsm1K
	PCALIGN $0x10

match_nolit2_end_encodeBlockAsm1K:
	ADDL R11, DX
	ADDL $0x04, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeBlockAsm1K

emit_remainder_encodeBlockAsm1K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeBlockAsm1K
	LEAQ (BX)(DX*1), DX
	LEAQ 3(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeBlockAsm1K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm1K
	JB   three_bytes_emit_remainder_encodeBlockAsm1K

three_bytes_emit_remainder_encodeBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeBlockAsm1K

two_bytes_emit_remainder_encodeBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeBlockAsm1K
	JMP  memmove_long_emit_remainder_encodeBlockAsm1K

one_byte_emit_remainder_encodeBlockAsm1K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm1K

emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm1K

emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm1K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm1K

emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm1K

emit_lit_memmove_emit_remainder_encodeBlockAsm1K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm1K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm1K

memmove_midemit_remainder_encodeBlockAsm1K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm1K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBlockAsm1K

emit_lit_memmove_mid_emit_remainder_encodeBlockAsm1K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeBlockAsm1K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeBlockAsm1K

memmove_long_emit_remainder_encodeBlockAsm1K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBlockAsm1Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeBlockAsm1K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeFastBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeFastBlockAsm(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000200, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeFastBlockAsm:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeFastBlockAsm
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x03, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeFastBlockAsm:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeFastBlockAsm
	MOVQ  (BX)(DX*1), DI
	LEAL  -2162685(DX), R8
	MOVL  SI, 20(SP)
	MOVQ  $0xcf1bbcdcb7a56463, R10
	MOVQ  DI, R9
	IMULQ R10, R9
	SHRQ  $0x32, R9
	MOVQ  1(BX)(DX*1), R11
	IMULQ R10, R11
	SHRQ  $0x32, R11
	MOVL  (AX)(R9*4), SI
	MOVL  DX, (AX)(R9*4)
	MOVL  (AX)(R11*4), R9
	MOVL  DX, (AX)(R11*4)
	MOVQ  2(BX)(DX*1), R11
	IMULQ R10, R11
	SHRQ  $0x32, R11
	MOVL  DX, R10
	SUBL  16(SP), R10
	MOVL  1(BX)(R10*1), R12
	MOVQ  DI, R10
	SHRQ  $0x08, R10
	CMPL  R10, R12
	JNE   no_repeat_found_encodeFastBlockAsm
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, SI
	MOVL  12(SP), R8
	SUBL  R8, SI
	LEAQ  4(CX)(SI*1), R9
	CMPQ  R9, (SP)
	JB    dst_size_check_ok_1
	MOVQ  $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeFastBlockAsm
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeFastBlockAsm
	CMPL R9, $0x00010000
	JB   three_bytes_repeat_emit_lits_encodeFastBlockAsm
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm

three_bytes_repeat_emit_lits_encodeFastBlockAsm:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm

two_bytes_repeat_emit_lits_encodeFastBlockAsm:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeFastBlockAsm
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm

one_byte_repeat_emit_lits_encodeFastBlockAsm:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm

memmove_midrepeat_emit_lits_encodeFastBlockAsm:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm

memmove_long_repeat_emit_lits_encodeFastBlockAsm:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsmlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsmlarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeFastBlockAsm:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeFastBlockAsm:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeFastBlockAsm
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeFastBlockAsm
	JMP  matchlen_match8_repeat_extend_encodeFastBlockAsm
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeFastBlockAsm:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm

matchlen_match8_repeat_extend_encodeFastBlockAsm:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeFastBlockAsm
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeFastBlockAsm
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeFastBlockAsm:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm

matchlen_match4_repeat_extend_encodeFastBlockAsm:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeFastBlockAsm
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeFastBlockAsm
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeFastBlockAsm:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeFastBlockAsm
	JB   repeat_extend_forward_end_encodeFastBlockAsm
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeFastBlockAsm
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeFastBlockAsm

matchlen_match1_repeat_extend_encodeFastBlockAsm:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeFastBlockAsm
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeFastBlockAsm
	PCALIGN $0x10

repeat_extend_forward_end_encodeFastBlockAsm:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeFastBlockAsm
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeFastBlockAsm
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeFastBlockAsm
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeFastBlockAsm

repeat_three_match_repeat_encodeFastBlockAsm:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeFastBlockAsm

repeat_two_match_repeat_encodeFastBlockAsm:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeFastBlockAsm

repeat_one_match_repeat_encodeFastBlockAsm:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeFastBlockAsm:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeFastBlockAsm

no_repeat_found_encodeFastBlockAsm:
	CMPL    SI, R8
	CMOVLLE R8, SI
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate_match_encodeFastBlockAsm
	MOVQ    1(BX)(DX*1), DI
	MOVL    (AX)(R11*4), SI
	LEAL    2(DX), R10
	CMPL    R9, R8
	CMOVLLE R8, R9
	CMPQ    (BX)(R9*1), DI
	JEQ     candidate2_match_encodeFastBlockAsm
	MOVL    R10, (AX)(R11*4)
	MOVQ    2(BX)(DX*1), DI
	CMPL    SI, R8
	CMOVLLE R8, SI
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeFastBlockAsm
	MOVL    20(SP), DX
	JMP     search_loop_encodeFastBlockAsm

candidate3_match_encodeFastBlockAsm:
	ADDL $0x02, DX
	JMP  candidate_match_encodeFastBlockAsm

candidate2_match_encodeFastBlockAsm:
	MOVL R10, (AX)(R11*4)
	INCL DX
	MOVL R9, SI

candidate_match_encodeFastBlockAsm:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x08, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeFastBlockAsm:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeFastBlockAsm
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeFastBlockAsm
	JMP  matchlen_match8_match_nolit_encodeFastBlockAsm
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeFastBlockAsm:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeFastBlockAsm

matchlen_match8_match_nolit_encodeFastBlockAsm:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeFastBlockAsm
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeFastBlockAsm
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeFastBlockAsm:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeFastBlockAsm

matchlen_match4_match_nolit_encodeFastBlockAsm:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeFastBlockAsm
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeFastBlockAsm
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeFastBlockAsm:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeFastBlockAsm
	JB   match_nolit_end_encodeFastBlockAsm
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeFastBlockAsm
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeFastBlockAsm

matchlen_match1_match_nolit_encodeFastBlockAsm:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeFastBlockAsm
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeFastBlockAsm
	PCALIGN $0x10

match_nolit_end_encodeFastBlockAsm:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeFastBlockAsm
	LEAQ (BX)(DI*1), DI
	LEAQ 4(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeFastBlockAsm
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeFastBlockAsm
	CMPL R9, $0x00010000
	JB   three_bytes_match_emit_encodeFastBlockAsm
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm

three_bytes_match_emit_encodeFastBlockAsm:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm

two_bytes_match_emit_encodeFastBlockAsm:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeFastBlockAsm
	JMP  memmove_long_match_emit_encodeFastBlockAsm

one_byte_match_emit_encodeFastBlockAsm:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeFastBlockAsm_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeFastBlockAsm_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm

emit_lit_memmove_match_emit_encodeFastBlockAsm_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm

emit_lit_memmove_match_emit_encodeFastBlockAsm_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeFastBlockAsm:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm

memmove_midmatch_emit_encodeFastBlockAsm:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeFastBlockAsm

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeFastBlockAsm:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm

memmove_long_match_emit_encodeFastBlockAsm:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeFastBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeFastBlockAsmlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeFastBlockAsmlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeFastBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeFastBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeFastBlockAsm
	PCALIGN $0x10

match_nolits_copy_encodeFastBlockAsm:
	// emitCopy
	CMPL SI, $0x0001003f
	JBE  two_byte_offset_match_nolit_encodeFastBlockAsm

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(SI), SI
	SHLL $0x0b, SI
	ADDL $0x07, SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_nolit_encodeFastBlockAsm_emit3
	LEAL -60(R11), DI
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_nolit_encodeFastBlockAsm_emit3
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_nolit_encodeFastBlockAsm_emit3
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL DI, 4(CX)
	ADDQ $0x07, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

emit_copy3_2_match_nolit_encodeFastBlockAsm_emit3:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW DI, 4(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

emit_copy3_1_match_nolit_encodeFastBlockAsm_emit3:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

emit_copy3_0_match_nolit_encodeFastBlockAsm_emit3:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

two_byte_offset_match_nolit_encodeFastBlockAsm:
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeFastBlockAsm
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeFastBlockAsm
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

emit_one_longer_match_nolit_encodeFastBlockAsm:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeFastBlockAsm
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

emit_copy1_repeat_match_nolit_encodeFastBlockAsm:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

two_byte_match_nolit_encodeFastBlockAsm:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeFastBlockAsm_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeFastBlockAsm_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeFastBlockAsm_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

emit_copy2_2_match_nolit_encodeFastBlockAsm_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

emit_copy2_1_match_nolit_encodeFastBlockAsm_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm

emit_copy2_0_match_nolit_encodeFastBlockAsm_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeFastBlockAsm:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeFastBlockAsm
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeFastBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeFastBlockAsm:
	MOVQ  $0xcf1bbcdcb7a56463, SI
	MOVQ  DI, R8
	MOVQ  (BX)(DX*1), DI
	MOVQ  DI, R9
	IMULQ SI, R8
	SHRQ  $0x32, R8
	IMULQ SI, R9
	SHRQ  $0x32, R9
	LEAL  -2(DX), R10
	MOVL  (AX)(R9*4), SI
	MOVL  R10, (AX)(R8*4)
	MOVL  DX, (AX)(R9*4)
	MOVL  DX, R8
	INCL  DX
	LEAL  -2162687(R8), R9
	CMPL  SI, R9
	JA    match_nolit_len_okencodeFastBlockAsm
	JMP   search_loop_encodeFastBlockAsm

match_nolit_len_okencodeFastBlockAsm:
	CMPQ (BX)(SI*1), DI
	JNE  search_loop_encodeFastBlockAsm
	MOVL R8, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	CMPQ CX, (SP)
	JB   dst_size_check_ok_4
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x07, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeFastBlockAsm:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeFastBlockAsm
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeFastBlockAsm
	JMP  matchlen_match8_match_nolit2_encodeFastBlockAsm
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeFastBlockAsm:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeFastBlockAsm

matchlen_match8_match_nolit2_encodeFastBlockAsm:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeFastBlockAsm
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeFastBlockAsm
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeFastBlockAsm:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeFastBlockAsm

matchlen_match4_match_nolit2_encodeFastBlockAsm:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeFastBlockAsm
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeFastBlockAsm
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeFastBlockAsm:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeFastBlockAsm
	JB   match_nolit2_end_encodeFastBlockAsm
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeFastBlockAsm
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeFastBlockAsm

matchlen_match1_match_nolit2_encodeFastBlockAsm:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeFastBlockAsm
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeFastBlockAsm
	PCALIGN $0x10

match_nolit2_end_encodeFastBlockAsm:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeFastBlockAsm

emit_remainder_encodeFastBlockAsm:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeFastBlockAsm
	LEAQ (BX)(DX*1), DX
	LEAQ 4(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeFastBlockAsm
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeFastBlockAsm
	CMPL BX, $0x00010000
	JB   three_bytes_emit_remainder_encodeFastBlockAsm
	MOVL BX, SI
	SHRL $0x10, SI
	MOVB $0xf8, (CX)
	MOVW BX, 1(CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm

three_bytes_emit_remainder_encodeFastBlockAsm:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm

two_bytes_emit_remainder_encodeFastBlockAsm:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeFastBlockAsm
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm

one_byte_emit_remainder_encodeFastBlockAsm:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm

emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm

emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm

emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeFastBlockAsm

emit_lit_memmove_emit_remainder_encodeFastBlockAsm_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeFastBlockAsm:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm

memmove_midemit_remainder_encodeFastBlockAsm:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm

memmove_long_emit_remainder_encodeFastBlockAsm:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeFastBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsmlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsmlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeFastBlockAsm:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeFastBlockAsm2MB(dst []byte, src []byte, tmp *[32768]byte) int
// Requires: BMI, SSE2
TEXT ·encodeFastBlockAsm2MB(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000100, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeFastBlockAsm2MB:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeFastBlockAsm2MB
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x03, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeFastBlockAsm2MB:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeFastBlockAsm2MB
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0xcf1bbcdcb7a56463, R9
	MOVQ  DI, R8
	IMULQ R9, R8
	SHRQ  $0x33, R8
	MOVQ  1(BX)(DX*1), R10
	IMULQ R9, R10
	SHRQ  $0x33, R10
	MOVL  (AX)(R8*4), SI
	MOVL  DX, (AX)(R8*4)
	MOVL  (AX)(R10*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVQ  2(BX)(DX*1), R10
	IMULQ R9, R10
	SHRQ  $0x33, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeFastBlockAsm2MB
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, SI
	MOVL  12(SP), R8
	SUBL  R8, SI
	LEAQ  4(CX)(SI*1), R9
	CMPQ  R9, (SP)
	JB    dst_size_check_ok_1
	MOVQ  $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeFastBlockAsm2MB
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeFastBlockAsm2MB
	CMPL R9, $0x00010000
	JB   three_bytes_repeat_emit_lits_encodeFastBlockAsm2MB
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm2MB

three_bytes_repeat_emit_lits_encodeFastBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm2MB

two_bytes_repeat_emit_lits_encodeFastBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeFastBlockAsm2MB
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm2MB

one_byte_repeat_emit_lits_encodeFastBlockAsm2MB:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm2MB

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm2MB

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm2MB:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm2MB

memmove_midrepeat_emit_lits_encodeFastBlockAsm2MB:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm2MB

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm2MB:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm2MB

memmove_long_repeat_emit_lits_encodeFastBlockAsm2MB:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm2MBlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeFastBlockAsm2MB:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm2MB
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeFastBlockAsm2MB:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm2MB
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeFastBlockAsm2MB
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm2MB:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeFastBlockAsm2MB
	JMP  matchlen_match8_repeat_extend_encodeFastBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeFastBlockAsm2MB:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm2MB

matchlen_match8_repeat_extend_encodeFastBlockAsm2MB:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeFastBlockAsm2MB
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm2MB
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeFastBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeFastBlockAsm2MB:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm2MB

matchlen_match4_repeat_extend_encodeFastBlockAsm2MB:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeFastBlockAsm2MB
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeFastBlockAsm2MB
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeFastBlockAsm2MB:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeFastBlockAsm2MB
	JB   repeat_extend_forward_end_encodeFastBlockAsm2MB
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeFastBlockAsm2MB
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeFastBlockAsm2MB

matchlen_match1_repeat_extend_encodeFastBlockAsm2MB:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeFastBlockAsm2MB
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeFastBlockAsm2MB
	PCALIGN $0x10

repeat_extend_forward_end_encodeFastBlockAsm2MB:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeFastBlockAsm2MB
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeFastBlockAsm2MB
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeFastBlockAsm2MB
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeFastBlockAsm2MB

repeat_three_match_repeat_encodeFastBlockAsm2MB:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeFastBlockAsm2MB

repeat_two_match_repeat_encodeFastBlockAsm2MB:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeFastBlockAsm2MB

repeat_one_match_repeat_encodeFastBlockAsm2MB:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeFastBlockAsm2MB:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeFastBlockAsm2MB

no_repeat_found_encodeFastBlockAsm2MB:
	CMPQ (BX)(SI*1), DI
	JEQ  candidate_match_encodeFastBlockAsm2MB
	MOVQ 1(BX)(DX*1), DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPQ (BX)(R8*1), DI
	JEQ  candidate2_match_encodeFastBlockAsm2MB
	MOVL R9, (AX)(R10*4)
	MOVQ 2(BX)(DX*1), DI
	CMPQ (BX)(SI*1), DI
	JEQ  candidate3_match_encodeFastBlockAsm2MB
	MOVL 20(SP), DX
	JMP  search_loop_encodeFastBlockAsm2MB

candidate3_match_encodeFastBlockAsm2MB:
	ADDL $0x02, DX
	JMP  candidate_match_encodeFastBlockAsm2MB

candidate2_match_encodeFastBlockAsm2MB:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeFastBlockAsm2MB:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x08, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm2MB
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeFastBlockAsm2MB:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm2MB
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeFastBlockAsm2MB
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm2MB:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeFastBlockAsm2MB
	JMP  matchlen_match8_match_nolit_encodeFastBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeFastBlockAsm2MB:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeFastBlockAsm2MB

matchlen_match8_match_nolit_encodeFastBlockAsm2MB:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeFastBlockAsm2MB
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm2MB
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeFastBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeFastBlockAsm2MB:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeFastBlockAsm2MB

matchlen_match4_match_nolit_encodeFastBlockAsm2MB:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeFastBlockAsm2MB
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeFastBlockAsm2MB
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeFastBlockAsm2MB:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeFastBlockAsm2MB
	JB   match_nolit_end_encodeFastBlockAsm2MB
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeFastBlockAsm2MB
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeFastBlockAsm2MB

matchlen_match1_match_nolit_encodeFastBlockAsm2MB:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeFastBlockAsm2MB
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeFastBlockAsm2MB
	PCALIGN $0x10

match_nolit_end_encodeFastBlockAsm2MB:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeFastBlockAsm2MB
	LEAQ (BX)(DI*1), DI
	LEAQ 4(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeFastBlockAsm2MB
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeFastBlockAsm2MB
	CMPL R9, $0x00010000
	JB   three_bytes_match_emit_encodeFastBlockAsm2MB
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm2MB

three_bytes_match_emit_encodeFastBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm2MB

two_bytes_match_emit_encodeFastBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeFastBlockAsm2MB
	JMP  memmove_long_match_emit_encodeFastBlockAsm2MB

one_byte_match_emit_encodeFastBlockAsm2MB:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm2MB_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeFastBlockAsm2MB_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeFastBlockAsm2MB_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm2MB

emit_lit_memmove_match_emit_encodeFastBlockAsm2MB_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm2MB

emit_lit_memmove_match_emit_encodeFastBlockAsm2MB_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeFastBlockAsm2MB:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm2MB

memmove_midmatch_emit_encodeFastBlockAsm2MB:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm2MB_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeFastBlockAsm2MB

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm2MB_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeFastBlockAsm2MB:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm2MB

memmove_long_match_emit_encodeFastBlockAsm2MB:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeFastBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeFastBlockAsm2MBlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeFastBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeFastBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeFastBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeFastBlockAsm2MB
	PCALIGN $0x10

match_nolits_copy_encodeFastBlockAsm2MB:
	// emitCopy
	CMPL SI, $0x0001003f
	JBE  two_byte_offset_match_nolit_encodeFastBlockAsm2MB

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(SI), SI
	SHLL $0x0b, SI
	ADDL $0x07, SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_nolit_encodeFastBlockAsm2MB_emit3
	LEAL -60(R11), DI
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_nolit_encodeFastBlockAsm2MB_emit3
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_nolit_encodeFastBlockAsm2MB_emit3
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL DI, 4(CX)
	ADDQ $0x07, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

emit_copy3_2_match_nolit_encodeFastBlockAsm2MB_emit3:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW DI, 4(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

emit_copy3_1_match_nolit_encodeFastBlockAsm2MB_emit3:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

emit_copy3_0_match_nolit_encodeFastBlockAsm2MB_emit3:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

two_byte_offset_match_nolit_encodeFastBlockAsm2MB:
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeFastBlockAsm2MB
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeFastBlockAsm2MB
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

emit_one_longer_match_nolit_encodeFastBlockAsm2MB:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeFastBlockAsm2MB
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

emit_copy1_repeat_match_nolit_encodeFastBlockAsm2MB:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm2MB
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm2MB
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm2MB
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm2MB:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm2MB:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm2MB:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

two_byte_match_nolit_encodeFastBlockAsm2MB:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeFastBlockAsm2MB_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeFastBlockAsm2MB_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeFastBlockAsm2MB_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

emit_copy2_2_match_nolit_encodeFastBlockAsm2MB_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

emit_copy2_1_match_nolit_encodeFastBlockAsm2MB_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm2MB

emit_copy2_0_match_nolit_encodeFastBlockAsm2MB_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeFastBlockAsm2MB:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeFastBlockAsm2MB
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeFastBlockAsm2MB
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeFastBlockAsm2MB:
	MOVQ  $0xcf1bbcdcb7a56463, SI
	MOVQ  DI, R8
	MOVQ  (BX)(DX*1), DI
	MOVQ  DI, R9
	IMULQ SI, R8
	SHRQ  $0x33, R8
	IMULQ SI, R9
	SHRQ  $0x33, R9
	LEAL  -2(DX), R10
	MOVL  (AX)(R9*4), SI
	MOVL  R10, (AX)(R8*4)
	MOVL  DX, (AX)(R9*4)
	MOVL  DX, R8
	INCL  DX
	CMPQ  (BX)(SI*1), DI
	JNE   search_loop_encodeFastBlockAsm2MB
	MOVL  R8, DI
	SUBL  SI, DI
	MOVL  DI, 16(SP)
	CMPQ  CX, (SP)
	JB    dst_size_check_ok_4
	MOVQ  $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x07, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm2MB
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeFastBlockAsm2MB:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm2MB
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeFastBlockAsm2MB
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm2MB:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeFastBlockAsm2MB
	JMP  matchlen_match8_match_nolit2_encodeFastBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeFastBlockAsm2MB:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeFastBlockAsm2MB

matchlen_match8_match_nolit2_encodeFastBlockAsm2MB:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeFastBlockAsm2MB
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm2MB
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeFastBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeFastBlockAsm2MB:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeFastBlockAsm2MB

matchlen_match4_match_nolit2_encodeFastBlockAsm2MB:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeFastBlockAsm2MB
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeFastBlockAsm2MB
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeFastBlockAsm2MB:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeFastBlockAsm2MB
	JB   match_nolit2_end_encodeFastBlockAsm2MB
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeFastBlockAsm2MB
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeFastBlockAsm2MB

matchlen_match1_match_nolit2_encodeFastBlockAsm2MB:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeFastBlockAsm2MB
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeFastBlockAsm2MB
	PCALIGN $0x10

match_nolit2_end_encodeFastBlockAsm2MB:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeFastBlockAsm2MB

emit_remainder_encodeFastBlockAsm2MB:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeFastBlockAsm2MB
	LEAQ (BX)(DX*1), DX
	LEAQ 4(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeFastBlockAsm2MB
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeFastBlockAsm2MB
	CMPL BX, $0x00010000
	JB   three_bytes_emit_remainder_encodeFastBlockAsm2MB
	MOVL BX, SI
	SHRL $0x10, SI
	MOVB $0xf8, (CX)
	MOVW BX, 1(CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm2MB

three_bytes_emit_remainder_encodeFastBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm2MB

two_bytes_emit_remainder_encodeFastBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeFastBlockAsm2MB
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm2MB

one_byte_emit_remainder_encodeFastBlockAsm2MB:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm2MB
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeFastBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeFastBlockAsm2MB_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeFastBlockAsm2MB:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm2MB

memmove_midemit_remainder_encodeFastBlockAsm2MB:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm2MB_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm2MB

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm2MB_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm2MB:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm2MB

memmove_long_emit_remainder_encodeFastBlockAsm2MB:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm2MBlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeFastBlockAsm2MB:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeFastBlockAsm512K(dst []byte, src []byte, tmp *[32768]byte) int
// Requires: BMI, SSE2
TEXT ·encodeFastBlockAsm512K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000100, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeFastBlockAsm512K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeFastBlockAsm512K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x03, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeFastBlockAsm512K:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeFastBlockAsm512K
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0xcf1bbcdcb7a56463, R9
	MOVQ  DI, R8
	IMULQ R9, R8
	SHRQ  $0x33, R8
	MOVQ  1(BX)(DX*1), R10
	IMULQ R9, R10
	SHRQ  $0x33, R10
	MOVL  (AX)(R8*4), SI
	MOVL  DX, (AX)(R8*4)
	MOVL  (AX)(R10*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVQ  2(BX)(DX*1), R10
	IMULQ R9, R10
	SHRQ  $0x33, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeFastBlockAsm512K
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, SI
	MOVL  12(SP), R8
	SUBL  R8, SI
	LEAQ  4(CX)(SI*1), R9
	CMPQ  R9, (SP)
	JB    dst_size_check_ok_1
	MOVQ  $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeFastBlockAsm512K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeFastBlockAsm512K
	CMPL R9, $0x00010000
	JB   three_bytes_repeat_emit_lits_encodeFastBlockAsm512K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm512K

three_bytes_repeat_emit_lits_encodeFastBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm512K

two_bytes_repeat_emit_lits_encodeFastBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeFastBlockAsm512K
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm512K

one_byte_repeat_emit_lits_encodeFastBlockAsm512K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm512K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm512K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm512K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm512K

memmove_midrepeat_emit_lits_encodeFastBlockAsm512K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm512K

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm512K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm512K

memmove_long_repeat_emit_lits_encodeFastBlockAsm512K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm512Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeFastBlockAsm512K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm512K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeFastBlockAsm512K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm512K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeFastBlockAsm512K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm512K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeFastBlockAsm512K
	JMP  matchlen_match8_repeat_extend_encodeFastBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeFastBlockAsm512K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm512K

matchlen_match8_repeat_extend_encodeFastBlockAsm512K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeFastBlockAsm512K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm512K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeFastBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeFastBlockAsm512K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm512K

matchlen_match4_repeat_extend_encodeFastBlockAsm512K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeFastBlockAsm512K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeFastBlockAsm512K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeFastBlockAsm512K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeFastBlockAsm512K
	JB   repeat_extend_forward_end_encodeFastBlockAsm512K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeFastBlockAsm512K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeFastBlockAsm512K

matchlen_match1_repeat_extend_encodeFastBlockAsm512K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeFastBlockAsm512K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeFastBlockAsm512K
	PCALIGN $0x10

repeat_extend_forward_end_encodeFastBlockAsm512K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeFastBlockAsm512K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeFastBlockAsm512K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeFastBlockAsm512K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeFastBlockAsm512K

repeat_three_match_repeat_encodeFastBlockAsm512K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeFastBlockAsm512K

repeat_two_match_repeat_encodeFastBlockAsm512K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeFastBlockAsm512K

repeat_one_match_repeat_encodeFastBlockAsm512K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeFastBlockAsm512K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeFastBlockAsm512K

no_repeat_found_encodeFastBlockAsm512K:
	CMPQ (BX)(SI*1), DI
	JEQ  candidate_match_encodeFastBlockAsm512K
	MOVQ 1(BX)(DX*1), DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPQ (BX)(R8*1), DI
	JEQ  candidate2_match_encodeFastBlockAsm512K
	MOVL R9, (AX)(R10*4)
	MOVQ 2(BX)(DX*1), DI
	CMPQ (BX)(SI*1), DI
	JEQ  candidate3_match_encodeFastBlockAsm512K
	MOVL 20(SP), DX
	JMP  search_loop_encodeFastBlockAsm512K

candidate3_match_encodeFastBlockAsm512K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeFastBlockAsm512K

candidate2_match_encodeFastBlockAsm512K:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeFastBlockAsm512K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x08, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm512K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeFastBlockAsm512K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm512K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeFastBlockAsm512K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm512K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeFastBlockAsm512K
	JMP  matchlen_match8_match_nolit_encodeFastBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeFastBlockAsm512K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeFastBlockAsm512K

matchlen_match8_match_nolit_encodeFastBlockAsm512K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeFastBlockAsm512K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm512K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeFastBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeFastBlockAsm512K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeFastBlockAsm512K

matchlen_match4_match_nolit_encodeFastBlockAsm512K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeFastBlockAsm512K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeFastBlockAsm512K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeFastBlockAsm512K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeFastBlockAsm512K
	JB   match_nolit_end_encodeFastBlockAsm512K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeFastBlockAsm512K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeFastBlockAsm512K

matchlen_match1_match_nolit_encodeFastBlockAsm512K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeFastBlockAsm512K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeFastBlockAsm512K
	PCALIGN $0x10

match_nolit_end_encodeFastBlockAsm512K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeFastBlockAsm512K
	LEAQ (BX)(DI*1), DI
	LEAQ 4(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeFastBlockAsm512K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeFastBlockAsm512K
	CMPL R9, $0x00010000
	JB   three_bytes_match_emit_encodeFastBlockAsm512K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm512K

three_bytes_match_emit_encodeFastBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm512K

two_bytes_match_emit_encodeFastBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeFastBlockAsm512K
	JMP  memmove_long_match_emit_encodeFastBlockAsm512K

one_byte_match_emit_encodeFastBlockAsm512K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm512K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeFastBlockAsm512K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeFastBlockAsm512K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm512K

emit_lit_memmove_match_emit_encodeFastBlockAsm512K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm512K

emit_lit_memmove_match_emit_encodeFastBlockAsm512K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeFastBlockAsm512K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm512K

memmove_midmatch_emit_encodeFastBlockAsm512K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm512K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeFastBlockAsm512K

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm512K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeFastBlockAsm512K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm512K

memmove_long_match_emit_encodeFastBlockAsm512K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeFastBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeFastBlockAsm512Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeFastBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeFastBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeFastBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeFastBlockAsm512K
	PCALIGN $0x10

match_nolits_copy_encodeFastBlockAsm512K:
	// emitCopy
	CMPL SI, $0x0001003f
	JBE  two_byte_offset_match_nolit_encodeFastBlockAsm512K

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(SI), SI
	SHLL $0x0b, SI
	ADDL $0x07, SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_nolit_encodeFastBlockAsm512K_emit3
	LEAL -60(R11), DI
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_nolit_encodeFastBlockAsm512K_emit3
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_nolit_encodeFastBlockAsm512K_emit3
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL DI, 4(CX)
	ADDQ $0x07, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

emit_copy3_2_match_nolit_encodeFastBlockAsm512K_emit3:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW DI, 4(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

emit_copy3_1_match_nolit_encodeFastBlockAsm512K_emit3:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

emit_copy3_0_match_nolit_encodeFastBlockAsm512K_emit3:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

two_byte_offset_match_nolit_encodeFastBlockAsm512K:
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeFastBlockAsm512K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeFastBlockAsm512K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

emit_one_longer_match_nolit_encodeFastBlockAsm512K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeFastBlockAsm512K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

emit_copy1_repeat_match_nolit_encodeFastBlockAsm512K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm512K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm512K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm512K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm512K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm512K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm512K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

two_byte_match_nolit_encodeFastBlockAsm512K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeFastBlockAsm512K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeFastBlockAsm512K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeFastBlockAsm512K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

emit_copy2_2_match_nolit_encodeFastBlockAsm512K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

emit_copy2_1_match_nolit_encodeFastBlockAsm512K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm512K

emit_copy2_0_match_nolit_encodeFastBlockAsm512K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeFastBlockAsm512K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeFastBlockAsm512K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeFastBlockAsm512K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeFastBlockAsm512K:
	MOVQ  $0xcf1bbcdcb7a56463, SI
	MOVQ  DI, R8
	MOVQ  (BX)(DX*1), DI
	MOVQ  DI, R9
	IMULQ SI, R8
	SHRQ  $0x33, R8
	IMULQ SI, R9
	SHRQ  $0x33, R9
	LEAL  -2(DX), R10
	MOVL  (AX)(R9*4), SI
	MOVL  R10, (AX)(R8*4)
	MOVL  DX, (AX)(R9*4)
	MOVL  DX, R8
	INCL  DX
	CMPQ  (BX)(SI*1), DI
	JNE   search_loop_encodeFastBlockAsm512K
	MOVL  R8, DI
	SUBL  SI, DI
	MOVL  DI, 16(SP)
	CMPQ  CX, (SP)
	JB    dst_size_check_ok_4
	MOVQ  $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x07, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm512K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeFastBlockAsm512K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm512K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeFastBlockAsm512K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm512K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeFastBlockAsm512K
	JMP  matchlen_match8_match_nolit2_encodeFastBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeFastBlockAsm512K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeFastBlockAsm512K

matchlen_match8_match_nolit2_encodeFastBlockAsm512K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeFastBlockAsm512K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm512K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeFastBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeFastBlockAsm512K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeFastBlockAsm512K

matchlen_match4_match_nolit2_encodeFastBlockAsm512K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeFastBlockAsm512K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeFastBlockAsm512K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeFastBlockAsm512K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeFastBlockAsm512K
	JB   match_nolit2_end_encodeFastBlockAsm512K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeFastBlockAsm512K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeFastBlockAsm512K

matchlen_match1_match_nolit2_encodeFastBlockAsm512K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeFastBlockAsm512K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeFastBlockAsm512K
	PCALIGN $0x10

match_nolit2_end_encodeFastBlockAsm512K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeFastBlockAsm512K

emit_remainder_encodeFastBlockAsm512K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeFastBlockAsm512K
	LEAQ (BX)(DX*1), DX
	LEAQ 4(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeFastBlockAsm512K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeFastBlockAsm512K
	CMPL BX, $0x00010000
	JB   three_bytes_emit_remainder_encodeFastBlockAsm512K
	MOVL BX, SI
	SHRL $0x10, SI
	MOVB $0xf8, (CX)
	MOVW BX, 1(CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm512K

three_bytes_emit_remainder_encodeFastBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm512K

two_bytes_emit_remainder_encodeFastBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeFastBlockAsm512K
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm512K

one_byte_emit_remainder_encodeFastBlockAsm512K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm512K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm512K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm512K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm512K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeFastBlockAsm512K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm512K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeFastBlockAsm512K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm512K

memmove_midemit_remainder_encodeFastBlockAsm512K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm512K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm512K

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm512K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm512K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm512K

memmove_long_emit_remainder_encodeFastBlockAsm512K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm512Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeFastBlockAsm512K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeFastBlockAsm64K(dst []byte, src []byte, tmp *[8192]byte) int
// Requires: BMI, SSE2
TEXT ·encodeFastBlockAsm64K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000040, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeFastBlockAsm64K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeFastBlockAsm64K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x03, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeFastBlockAsm64K:
	MOVL    DX, SI
	SUBL    12(SP), SI
	SHRL    $0x04, SI
	LEAL    4(DX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeFastBlockAsm64K
	MOVQ    (BX)(DX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0xcf1bbcdcb7a56463, R9
	MOVQ    DI, R8
	IMULQ   R9, R8
	SHRQ    $0x34, R8
	MOVQ    1(BX)(DX*1), R10
	IMULQ   R9, R10
	SHRQ    $0x34, R10
	MOVWLZX (AX)(R8*2), SI
	MOVW    DX, (AX)(R8*2)
	MOVWLZX (AX)(R10*2), R8
	MOVW    DX, (AX)(R10*2)
	MOVQ    2(BX)(DX*1), R10
	IMULQ   R9, R10
	SHRQ    $0x34, R10
	MOVL    DX, R9
	SUBL    16(SP), R9
	MOVL    1(BX)(R9*1), R11
	MOVQ    DI, R9
	SHRQ    $0x08, R9
	CMPL    R9, R11
	JNE     no_repeat_found_encodeFastBlockAsm64K
	LEAL    1(DX), DI
	MOVL    12(SP), SI
	MOVL    DI, SI
	MOVL    12(SP), R8
	SUBL    R8, SI
	LEAQ    4(CX)(SI*1), R9
	CMPQ    R9, (SP)
	JB      dst_size_check_ok_1
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeFastBlockAsm64K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeFastBlockAsm64K
	JB   three_bytes_repeat_emit_lits_encodeFastBlockAsm64K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm64K

three_bytes_repeat_emit_lits_encodeFastBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm64K

two_bytes_repeat_emit_lits_encodeFastBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeFastBlockAsm64K
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm64K

one_byte_repeat_emit_lits_encodeFastBlockAsm64K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm64K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm64K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm64K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm64K

memmove_midrepeat_emit_lits_encodeFastBlockAsm64K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm64K

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm64K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm64K

memmove_long_repeat_emit_lits_encodeFastBlockAsm64K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm64Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeFastBlockAsm64K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm64K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeFastBlockAsm64K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm64K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeFastBlockAsm64K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm64K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeFastBlockAsm64K
	JMP  matchlen_match8_repeat_extend_encodeFastBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeFastBlockAsm64K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm64K

matchlen_match8_repeat_extend_encodeFastBlockAsm64K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeFastBlockAsm64K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm64K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeFastBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeFastBlockAsm64K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm64K

matchlen_match4_repeat_extend_encodeFastBlockAsm64K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeFastBlockAsm64K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeFastBlockAsm64K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeFastBlockAsm64K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeFastBlockAsm64K
	JB   repeat_extend_forward_end_encodeFastBlockAsm64K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeFastBlockAsm64K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeFastBlockAsm64K

matchlen_match1_repeat_extend_encodeFastBlockAsm64K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeFastBlockAsm64K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeFastBlockAsm64K
	PCALIGN $0x10

repeat_extend_forward_end_encodeFastBlockAsm64K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeFastBlockAsm64K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeFastBlockAsm64K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeFastBlockAsm64K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeFastBlockAsm64K

repeat_three_match_repeat_encodeFastBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeFastBlockAsm64K

repeat_two_match_repeat_encodeFastBlockAsm64K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeFastBlockAsm64K

repeat_one_match_repeat_encodeFastBlockAsm64K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeFastBlockAsm64K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeFastBlockAsm64K

no_repeat_found_encodeFastBlockAsm64K:
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate_match_encodeFastBlockAsm64K
	MOVQ    1(BX)(DX*1), DI
	MOVWLZX (AX)(R10*2), SI
	LEAL    2(DX), R9
	CMPQ    (BX)(R8*1), DI
	JEQ     candidate2_match_encodeFastBlockAsm64K
	MOVW    R9, (AX)(R10*2)
	MOVQ    2(BX)(DX*1), DI
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeFastBlockAsm64K
	MOVL    20(SP), DX
	JMP     search_loop_encodeFastBlockAsm64K

candidate3_match_encodeFastBlockAsm64K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeFastBlockAsm64K

candidate2_match_encodeFastBlockAsm64K:
	MOVW R9, (AX)(R10*2)
	INCL DX
	MOVL R8, SI

candidate_match_encodeFastBlockAsm64K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x08, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm64K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeFastBlockAsm64K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm64K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeFastBlockAsm64K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm64K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeFastBlockAsm64K
	JMP  matchlen_match8_match_nolit_encodeFastBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeFastBlockAsm64K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeFastBlockAsm64K

matchlen_match8_match_nolit_encodeFastBlockAsm64K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeFastBlockAsm64K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm64K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeFastBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeFastBlockAsm64K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeFastBlockAsm64K

matchlen_match4_match_nolit_encodeFastBlockAsm64K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeFastBlockAsm64K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeFastBlockAsm64K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeFastBlockAsm64K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeFastBlockAsm64K
	JB   match_nolit_end_encodeFastBlockAsm64K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeFastBlockAsm64K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeFastBlockAsm64K

matchlen_match1_match_nolit_encodeFastBlockAsm64K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeFastBlockAsm64K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeFastBlockAsm64K
	PCALIGN $0x10

match_nolit_end_encodeFastBlockAsm64K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeFastBlockAsm64K
	LEAQ (BX)(DI*1), DI
	LEAQ 4(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeFastBlockAsm64K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeFastBlockAsm64K
	JB   three_bytes_match_emit_encodeFastBlockAsm64K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm64K

three_bytes_match_emit_encodeFastBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm64K

two_bytes_match_emit_encodeFastBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeFastBlockAsm64K
	JMP  memmove_long_match_emit_encodeFastBlockAsm64K

one_byte_match_emit_encodeFastBlockAsm64K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm64K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeFastBlockAsm64K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeFastBlockAsm64K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm64K

emit_lit_memmove_match_emit_encodeFastBlockAsm64K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm64K

emit_lit_memmove_match_emit_encodeFastBlockAsm64K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeFastBlockAsm64K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm64K

memmove_midmatch_emit_encodeFastBlockAsm64K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm64K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeFastBlockAsm64K

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm64K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeFastBlockAsm64K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm64K

memmove_long_match_emit_encodeFastBlockAsm64K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeFastBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeFastBlockAsm64Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeFastBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeFastBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeFastBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeFastBlockAsm64K
	PCALIGN $0x10

match_nolits_copy_encodeFastBlockAsm64K:
	// emitCopy
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeFastBlockAsm64K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeFastBlockAsm64K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm64K

emit_one_longer_match_nolit_encodeFastBlockAsm64K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeFastBlockAsm64K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm64K

emit_copy1_repeat_match_nolit_encodeFastBlockAsm64K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm64K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm64K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm64K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm64K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm64K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm64K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm64K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm64K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm64K

two_byte_match_nolit_encodeFastBlockAsm64K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeFastBlockAsm64K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeFastBlockAsm64K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeFastBlockAsm64K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm64K

emit_copy2_2_match_nolit_encodeFastBlockAsm64K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm64K

emit_copy2_1_match_nolit_encodeFastBlockAsm64K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm64K

emit_copy2_0_match_nolit_encodeFastBlockAsm64K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeFastBlockAsm64K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeFastBlockAsm64K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeFastBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeFastBlockAsm64K:
	MOVQ    $0xcf1bbcdcb7a56463, SI
	MOVQ    DI, R8
	MOVQ    (BX)(DX*1), DI
	MOVQ    DI, R9
	IMULQ   SI, R8
	SHRQ    $0x34, R8
	IMULQ   SI, R9
	SHRQ    $0x34, R9
	LEAL    -2(DX), R10
	MOVWLZX (AX)(R9*2), SI
	MOVW    R10, (AX)(R8*2)
	MOVW    DX, (AX)(R9*2)
	MOVL    DX, R8
	INCL    DX
	CMPQ    (BX)(SI*1), DI
	JNE     search_loop_encodeFastBlockAsm64K
	MOVL    R8, DI
	SUBL    SI, DI
	MOVL    DI, 16(SP)
	CMPQ    CX, (SP)
	JB      dst_size_check_ok_4
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x07, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm64K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeFastBlockAsm64K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm64K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeFastBlockAsm64K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm64K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeFastBlockAsm64K
	JMP  matchlen_match8_match_nolit2_encodeFastBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeFastBlockAsm64K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeFastBlockAsm64K

matchlen_match8_match_nolit2_encodeFastBlockAsm64K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeFastBlockAsm64K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm64K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeFastBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeFastBlockAsm64K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeFastBlockAsm64K

matchlen_match4_match_nolit2_encodeFastBlockAsm64K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeFastBlockAsm64K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeFastBlockAsm64K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeFastBlockAsm64K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeFastBlockAsm64K
	JB   match_nolit2_end_encodeFastBlockAsm64K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeFastBlockAsm64K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeFastBlockAsm64K

matchlen_match1_match_nolit2_encodeFastBlockAsm64K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeFastBlockAsm64K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeFastBlockAsm64K
	PCALIGN $0x10

match_nolit2_end_encodeFastBlockAsm64K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeFastBlockAsm64K

emit_remainder_encodeFastBlockAsm64K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeFastBlockAsm64K
	LEAQ (BX)(DX*1), DX
	LEAQ 4(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeFastBlockAsm64K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeFastBlockAsm64K
	JB   three_bytes_emit_remainder_encodeFastBlockAsm64K
	MOVL BX, SI
	SHRL $0x10, SI
	MOVB $0xf8, (CX)
	MOVW BX, 1(CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm64K

three_bytes_emit_remainder_encodeFastBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm64K

two_bytes_emit_remainder_encodeFastBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeFastBlockAsm64K
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm64K

one_byte_emit_remainder_encodeFastBlockAsm64K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm64K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm64K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm64K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm64K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeFastBlockAsm64K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm64K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeFastBlockAsm64K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm64K

memmove_midemit_remainder_encodeFastBlockAsm64K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm64K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm64K

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm64K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm64K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm64K

memmove_long_emit_remainder_encodeFastBlockAsm64K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm64Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeFastBlockAsm64K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeFastBlockAsm16K(dst []byte, src []byte, tmp *[4096]byte) int
// Requires: BMI, SSE2
TEXT ·encodeFastBlockAsm16K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000020, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeFastBlockAsm16K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeFastBlockAsm16K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x03, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeFastBlockAsm16K:
	MOVL    DX, SI
	SUBL    12(SP), SI
	SHRL    $0x04, SI
	LEAL    4(DX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeFastBlockAsm16K
	MOVQ    (BX)(DX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0xcf1bbcdcb7a56463, R9
	MOVQ    DI, R8
	IMULQ   R9, R8
	SHRQ    $0x35, R8
	MOVQ    1(BX)(DX*1), R10
	IMULQ   R9, R10
	SHRQ    $0x35, R10
	MOVWLZX (AX)(R8*2), SI
	MOVW    DX, (AX)(R8*2)
	MOVWLZX (AX)(R10*2), R8
	MOVW    DX, (AX)(R10*2)
	MOVQ    2(BX)(DX*1), R10
	IMULQ   R9, R10
	SHRQ    $0x35, R10
	MOVL    DX, R9
	SUBL    16(SP), R9
	MOVL    1(BX)(R9*1), R11
	MOVQ    DI, R9
	SHRQ    $0x08, R9
	CMPL    R9, R11
	JNE     no_repeat_found_encodeFastBlockAsm16K
	LEAL    1(DX), DI
	MOVL    12(SP), SI
	MOVL    DI, SI
	MOVL    12(SP), R8
	SUBL    R8, SI
	LEAQ    3(CX)(SI*1), R9
	CMPQ    R9, (SP)
	JB      dst_size_check_ok_1
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeFastBlockAsm16K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeFastBlockAsm16K
	JB   three_bytes_repeat_emit_lits_encodeFastBlockAsm16K

three_bytes_repeat_emit_lits_encodeFastBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm16K

two_bytes_repeat_emit_lits_encodeFastBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeFastBlockAsm16K
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm16K

one_byte_repeat_emit_lits_encodeFastBlockAsm16K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm16K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm16K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm16K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm16K

memmove_midrepeat_emit_lits_encodeFastBlockAsm16K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm16K

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm16K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm16K

memmove_long_repeat_emit_lits_encodeFastBlockAsm16K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm16Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeFastBlockAsm16K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm16K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeFastBlockAsm16K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm16K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeFastBlockAsm16K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm16K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeFastBlockAsm16K
	JMP  matchlen_match8_repeat_extend_encodeFastBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeFastBlockAsm16K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm16K

matchlen_match8_repeat_extend_encodeFastBlockAsm16K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeFastBlockAsm16K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm16K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeFastBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeFastBlockAsm16K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm16K

matchlen_match4_repeat_extend_encodeFastBlockAsm16K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeFastBlockAsm16K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeFastBlockAsm16K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeFastBlockAsm16K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeFastBlockAsm16K
	JB   repeat_extend_forward_end_encodeFastBlockAsm16K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeFastBlockAsm16K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeFastBlockAsm16K

matchlen_match1_repeat_extend_encodeFastBlockAsm16K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeFastBlockAsm16K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeFastBlockAsm16K
	PCALIGN $0x10

repeat_extend_forward_end_encodeFastBlockAsm16K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeFastBlockAsm16K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeFastBlockAsm16K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeFastBlockAsm16K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeFastBlockAsm16K

repeat_three_match_repeat_encodeFastBlockAsm16K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeFastBlockAsm16K

repeat_two_match_repeat_encodeFastBlockAsm16K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeFastBlockAsm16K

repeat_one_match_repeat_encodeFastBlockAsm16K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeFastBlockAsm16K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeFastBlockAsm16K

no_repeat_found_encodeFastBlockAsm16K:
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate_match_encodeFastBlockAsm16K
	MOVQ    1(BX)(DX*1), DI
	MOVWLZX (AX)(R10*2), SI
	LEAL    2(DX), R9
	CMPQ    (BX)(R8*1), DI
	JEQ     candidate2_match_encodeFastBlockAsm16K
	MOVW    R9, (AX)(R10*2)
	MOVQ    2(BX)(DX*1), DI
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeFastBlockAsm16K
	MOVL    20(SP), DX
	JMP     search_loop_encodeFastBlockAsm16K

candidate3_match_encodeFastBlockAsm16K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeFastBlockAsm16K

candidate2_match_encodeFastBlockAsm16K:
	MOVW R9, (AX)(R10*2)
	INCL DX
	MOVL R8, SI

candidate_match_encodeFastBlockAsm16K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x08, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm16K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeFastBlockAsm16K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm16K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeFastBlockAsm16K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm16K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeFastBlockAsm16K
	JMP  matchlen_match8_match_nolit_encodeFastBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeFastBlockAsm16K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeFastBlockAsm16K

matchlen_match8_match_nolit_encodeFastBlockAsm16K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeFastBlockAsm16K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm16K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeFastBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeFastBlockAsm16K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeFastBlockAsm16K

matchlen_match4_match_nolit_encodeFastBlockAsm16K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeFastBlockAsm16K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeFastBlockAsm16K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeFastBlockAsm16K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeFastBlockAsm16K
	JB   match_nolit_end_encodeFastBlockAsm16K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeFastBlockAsm16K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeFastBlockAsm16K

matchlen_match1_match_nolit_encodeFastBlockAsm16K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeFastBlockAsm16K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeFastBlockAsm16K
	PCALIGN $0x10

match_nolit_end_encodeFastBlockAsm16K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeFastBlockAsm16K
	LEAQ (BX)(DI*1), DI
	LEAQ 3(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeFastBlockAsm16K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeFastBlockAsm16K
	JB   three_bytes_match_emit_encodeFastBlockAsm16K

three_bytes_match_emit_encodeFastBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm16K

two_bytes_match_emit_encodeFastBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeFastBlockAsm16K
	JMP  memmove_long_match_emit_encodeFastBlockAsm16K

one_byte_match_emit_encodeFastBlockAsm16K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm16K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeFastBlockAsm16K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeFastBlockAsm16K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm16K

emit_lit_memmove_match_emit_encodeFastBlockAsm16K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm16K

emit_lit_memmove_match_emit_encodeFastBlockAsm16K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeFastBlockAsm16K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm16K

memmove_midmatch_emit_encodeFastBlockAsm16K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm16K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeFastBlockAsm16K

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm16K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeFastBlockAsm16K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm16K

memmove_long_match_emit_encodeFastBlockAsm16K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeFastBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeFastBlockAsm16Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeFastBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeFastBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeFastBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeFastBlockAsm16K
	PCALIGN $0x10

match_nolits_copy_encodeFastBlockAsm16K:
	// emitCopy
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeFastBlockAsm16K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeFastBlockAsm16K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm16K

emit_one_longer_match_nolit_encodeFastBlockAsm16K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeFastBlockAsm16K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm16K

emit_copy1_repeat_match_nolit_encodeFastBlockAsm16K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm16K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm16K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm16K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm16K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm16K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm16K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm16K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm16K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm16K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm16K

two_byte_match_nolit_encodeFastBlockAsm16K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeFastBlockAsm16K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeFastBlockAsm16K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeFastBlockAsm16K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm16K

emit_copy2_2_match_nolit_encodeFastBlockAsm16K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm16K

emit_copy2_1_match_nolit_encodeFastBlockAsm16K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm16K

emit_copy2_0_match_nolit_encodeFastBlockAsm16K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeFastBlockAsm16K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeFastBlockAsm16K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeFastBlockAsm16K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeFastBlockAsm16K:
	MOVQ    $0xcf1bbcdcb7a56463, SI
	MOVQ    DI, R8
	MOVQ    (BX)(DX*1), DI
	MOVQ    DI, R9
	IMULQ   SI, R8
	SHRQ    $0x35, R8
	IMULQ   SI, R9
	SHRQ    $0x35, R9
	LEAL    -2(DX), R10
	MOVWLZX (AX)(R9*2), SI
	MOVW    R10, (AX)(R8*2)
	MOVW    DX, (AX)(R9*2)
	MOVL    DX, R8
	INCL    DX
	CMPQ    (BX)(SI*1), DI
	JNE     search_loop_encodeFastBlockAsm16K
	MOVL    R8, DI
	SUBL    SI, DI
	MOVL    DI, 16(SP)
	CMPQ    CX, (SP)
	JB      dst_size_check_ok_4
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x07, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm16K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeFastBlockAsm16K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm16K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeFastBlockAsm16K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm16K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeFastBlockAsm16K
	JMP  matchlen_match8_match_nolit2_encodeFastBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeFastBlockAsm16K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeFastBlockAsm16K

matchlen_match8_match_nolit2_encodeFastBlockAsm16K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeFastBlockAsm16K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm16K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeFastBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeFastBlockAsm16K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeFastBlockAsm16K

matchlen_match4_match_nolit2_encodeFastBlockAsm16K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeFastBlockAsm16K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeFastBlockAsm16K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeFastBlockAsm16K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeFastBlockAsm16K
	JB   match_nolit2_end_encodeFastBlockAsm16K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeFastBlockAsm16K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeFastBlockAsm16K

matchlen_match1_match_nolit2_encodeFastBlockAsm16K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeFastBlockAsm16K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeFastBlockAsm16K
	PCALIGN $0x10

match_nolit2_end_encodeFastBlockAsm16K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeFastBlockAsm16K

emit_remainder_encodeFastBlockAsm16K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeFastBlockAsm16K
	LEAQ (BX)(DX*1), DX
	LEAQ 3(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeFastBlockAsm16K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeFastBlockAsm16K
	JB   three_bytes_emit_remainder_encodeFastBlockAsm16K

three_bytes_emit_remainder_encodeFastBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm16K

two_bytes_emit_remainder_encodeFastBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeFastBlockAsm16K
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm16K

one_byte_emit_remainder_encodeFastBlockAsm16K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm16K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm16K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm16K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm16K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeFastBlockAsm16K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm16K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeFastBlockAsm16K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm16K

memmove_midemit_remainder_encodeFastBlockAsm16K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm16K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm16K

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm16K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm16K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm16K

memmove_long_emit_remainder_encodeFastBlockAsm16K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm16Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeFastBlockAsm16K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeFastBlockAsm4K(dst []byte, src []byte, tmp *[2048]byte) int
// Requires: BMI, SSE2
TEXT ·encodeFastBlockAsm4K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000010, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeFastBlockAsm4K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeFastBlockAsm4K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x03, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeFastBlockAsm4K:
	MOVL    DX, SI
	SUBL    12(SP), SI
	SHRL    $0x04, SI
	LEAL    4(DX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeFastBlockAsm4K
	MOVQ    (BX)(DX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0xcf1bbcdcb7a56463, R9
	MOVQ    DI, R8
	IMULQ   R9, R8
	SHRQ    $0x36, R8
	MOVQ    1(BX)(DX*1), R10
	IMULQ   R9, R10
	SHRQ    $0x36, R10
	MOVWLZX (AX)(R8*2), SI
	MOVW    DX, (AX)(R8*2)
	MOVWLZX (AX)(R10*2), R8
	MOVW    DX, (AX)(R10*2)
	MOVQ    2(BX)(DX*1), R10
	IMULQ   R9, R10
	SHRQ    $0x36, R10
	MOVL    DX, R9
	SUBL    16(SP), R9
	MOVL    1(BX)(R9*1), R11
	MOVQ    DI, R9
	SHRQ    $0x08, R9
	CMPL    R9, R11
	JNE     no_repeat_found_encodeFastBlockAsm4K
	LEAL    1(DX), DI
	MOVL    12(SP), SI
	MOVL    DI, SI
	MOVL    12(SP), R8
	SUBL    R8, SI
	LEAQ    3(CX)(SI*1), R9
	CMPQ    R9, (SP)
	JB      dst_size_check_ok_1
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeFastBlockAsm4K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeFastBlockAsm4K
	JB   three_bytes_repeat_emit_lits_encodeFastBlockAsm4K

three_bytes_repeat_emit_lits_encodeFastBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm4K

two_bytes_repeat_emit_lits_encodeFastBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeFastBlockAsm4K
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm4K

one_byte_repeat_emit_lits_encodeFastBlockAsm4K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm4K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm4K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm4K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm4K

memmove_midrepeat_emit_lits_encodeFastBlockAsm4K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm4K

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm4K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm4K

memmove_long_repeat_emit_lits_encodeFastBlockAsm4K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm4Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeFastBlockAsm4K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm4K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeFastBlockAsm4K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm4K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeFastBlockAsm4K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm4K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeFastBlockAsm4K
	JMP  matchlen_match8_repeat_extend_encodeFastBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeFastBlockAsm4K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm4K

matchlen_match8_repeat_extend_encodeFastBlockAsm4K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeFastBlockAsm4K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm4K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeFastBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeFastBlockAsm4K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm4K

matchlen_match4_repeat_extend_encodeFastBlockAsm4K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeFastBlockAsm4K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeFastBlockAsm4K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeFastBlockAsm4K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeFastBlockAsm4K
	JB   repeat_extend_forward_end_encodeFastBlockAsm4K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeFastBlockAsm4K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeFastBlockAsm4K

matchlen_match1_repeat_extend_encodeFastBlockAsm4K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeFastBlockAsm4K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeFastBlockAsm4K
	PCALIGN $0x10

repeat_extend_forward_end_encodeFastBlockAsm4K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeFastBlockAsm4K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeFastBlockAsm4K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeFastBlockAsm4K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeFastBlockAsm4K

repeat_three_match_repeat_encodeFastBlockAsm4K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeFastBlockAsm4K

repeat_two_match_repeat_encodeFastBlockAsm4K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeFastBlockAsm4K

repeat_one_match_repeat_encodeFastBlockAsm4K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeFastBlockAsm4K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeFastBlockAsm4K

no_repeat_found_encodeFastBlockAsm4K:
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate_match_encodeFastBlockAsm4K
	MOVQ    1(BX)(DX*1), DI
	MOVWLZX (AX)(R10*2), SI
	LEAL    2(DX), R9
	CMPQ    (BX)(R8*1), DI
	JEQ     candidate2_match_encodeFastBlockAsm4K
	MOVW    R9, (AX)(R10*2)
	MOVQ    2(BX)(DX*1), DI
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeFastBlockAsm4K
	MOVL    20(SP), DX
	JMP     search_loop_encodeFastBlockAsm4K

candidate3_match_encodeFastBlockAsm4K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeFastBlockAsm4K

candidate2_match_encodeFastBlockAsm4K:
	MOVW R9, (AX)(R10*2)
	INCL DX
	MOVL R8, SI

candidate_match_encodeFastBlockAsm4K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x08, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm4K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeFastBlockAsm4K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm4K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeFastBlockAsm4K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm4K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeFastBlockAsm4K
	JMP  matchlen_match8_match_nolit_encodeFastBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeFastBlockAsm4K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeFastBlockAsm4K

matchlen_match8_match_nolit_encodeFastBlockAsm4K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeFastBlockAsm4K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm4K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeFastBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeFastBlockAsm4K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeFastBlockAsm4K

matchlen_match4_match_nolit_encodeFastBlockAsm4K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeFastBlockAsm4K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeFastBlockAsm4K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeFastBlockAsm4K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeFastBlockAsm4K
	JB   match_nolit_end_encodeFastBlockAsm4K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeFastBlockAsm4K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeFastBlockAsm4K

matchlen_match1_match_nolit_encodeFastBlockAsm4K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeFastBlockAsm4K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeFastBlockAsm4K
	PCALIGN $0x10

match_nolit_end_encodeFastBlockAsm4K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeFastBlockAsm4K
	LEAQ (BX)(DI*1), DI
	LEAQ 3(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeFastBlockAsm4K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeFastBlockAsm4K
	JB   three_bytes_match_emit_encodeFastBlockAsm4K

three_bytes_match_emit_encodeFastBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm4K

two_bytes_match_emit_encodeFastBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeFastBlockAsm4K
	JMP  memmove_long_match_emit_encodeFastBlockAsm4K

one_byte_match_emit_encodeFastBlockAsm4K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm4K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeFastBlockAsm4K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeFastBlockAsm4K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm4K

emit_lit_memmove_match_emit_encodeFastBlockAsm4K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm4K

emit_lit_memmove_match_emit_encodeFastBlockAsm4K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeFastBlockAsm4K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm4K

memmove_midmatch_emit_encodeFastBlockAsm4K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm4K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeFastBlockAsm4K

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm4K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeFastBlockAsm4K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm4K

memmove_long_match_emit_encodeFastBlockAsm4K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeFastBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeFastBlockAsm4Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeFastBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeFastBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeFastBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeFastBlockAsm4K
	PCALIGN $0x10

match_nolits_copy_encodeFastBlockAsm4K:
	// emitCopy
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeFastBlockAsm4K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeFastBlockAsm4K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm4K

emit_one_longer_match_nolit_encodeFastBlockAsm4K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeFastBlockAsm4K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm4K

emit_copy1_repeat_match_nolit_encodeFastBlockAsm4K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm4K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm4K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm4K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm4K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm4K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm4K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm4K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm4K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm4K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm4K

two_byte_match_nolit_encodeFastBlockAsm4K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeFastBlockAsm4K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeFastBlockAsm4K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeFastBlockAsm4K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm4K

emit_copy2_2_match_nolit_encodeFastBlockAsm4K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm4K

emit_copy2_1_match_nolit_encodeFastBlockAsm4K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm4K

emit_copy2_0_match_nolit_encodeFastBlockAsm4K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeFastBlockAsm4K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeFastBlockAsm4K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeFastBlockAsm4K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeFastBlockAsm4K:
	MOVQ    $0xcf1bbcdcb7a56463, SI
	MOVQ    DI, R8
	MOVQ    (BX)(DX*1), DI
	MOVQ    DI, R9
	IMULQ   SI, R8
	SHRQ    $0x36, R8
	IMULQ   SI, R9
	SHRQ    $0x36, R9
	LEAL    -2(DX), R10
	MOVWLZX (AX)(R9*2), SI
	MOVW    R10, (AX)(R8*2)
	MOVW    DX, (AX)(R9*2)
	MOVL    DX, R8
	INCL    DX
	CMPQ    (BX)(SI*1), DI
	JNE     search_loop_encodeFastBlockAsm4K
	MOVL    R8, DI
	SUBL    SI, DI
	MOVL    DI, 16(SP)
	CMPQ    CX, (SP)
	JB      dst_size_check_ok_4
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x07, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm4K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeFastBlockAsm4K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm4K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeFastBlockAsm4K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm4K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeFastBlockAsm4K
	JMP  matchlen_match8_match_nolit2_encodeFastBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeFastBlockAsm4K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeFastBlockAsm4K

matchlen_match8_match_nolit2_encodeFastBlockAsm4K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeFastBlockAsm4K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm4K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeFastBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeFastBlockAsm4K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeFastBlockAsm4K

matchlen_match4_match_nolit2_encodeFastBlockAsm4K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeFastBlockAsm4K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeFastBlockAsm4K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeFastBlockAsm4K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeFastBlockAsm4K
	JB   match_nolit2_end_encodeFastBlockAsm4K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeFastBlockAsm4K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeFastBlockAsm4K

matchlen_match1_match_nolit2_encodeFastBlockAsm4K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeFastBlockAsm4K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeFastBlockAsm4K
	PCALIGN $0x10

match_nolit2_end_encodeFastBlockAsm4K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeFastBlockAsm4K

emit_remainder_encodeFastBlockAsm4K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeFastBlockAsm4K
	LEAQ (BX)(DX*1), DX
	LEAQ 3(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeFastBlockAsm4K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeFastBlockAsm4K
	JB   three_bytes_emit_remainder_encodeFastBlockAsm4K

three_bytes_emit_remainder_encodeFastBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm4K

two_bytes_emit_remainder_encodeFastBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeFastBlockAsm4K
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm4K

one_byte_emit_remainder_encodeFastBlockAsm4K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm4K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm4K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm4K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm4K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeFastBlockAsm4K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm4K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeFastBlockAsm4K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm4K

memmove_midemit_remainder_encodeFastBlockAsm4K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm4K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm4K

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm4K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm4K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm4K

memmove_long_emit_remainder_encodeFastBlockAsm4K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm4Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeFastBlockAsm4K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeFastBlockAsm1K(dst []byte, src []byte, tmp *[1024]byte) int
// Requires: BMI, SSE2
TEXT ·encodeFastBlockAsm1K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000008, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeFastBlockAsm1K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeFastBlockAsm1K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -17(DX), BX
	LEAQ  -17(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x03, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeFastBlockAsm1K:
	MOVL    DX, SI
	SUBL    12(SP), SI
	SHRL    $0x03, SI
	LEAL    4(DX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeFastBlockAsm1K
	MOVQ    (BX)(DX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0xcf1bbcdcb7a56463, R9
	MOVQ    DI, R8
	IMULQ   R9, R8
	SHRQ    $0x37, R8
	MOVQ    1(BX)(DX*1), R10
	IMULQ   R9, R10
	SHRQ    $0x37, R10
	MOVWLZX (AX)(R8*2), SI
	MOVW    DX, (AX)(R8*2)
	MOVWLZX (AX)(R10*2), R8
	MOVW    DX, (AX)(R10*2)
	MOVQ    2(BX)(DX*1), R10
	IMULQ   R9, R10
	SHRQ    $0x37, R10
	MOVL    DX, R9
	SUBL    16(SP), R9
	MOVL    1(BX)(R9*1), R11
	MOVQ    DI, R9
	SHRQ    $0x08, R9
	CMPL    R9, R11
	JNE     no_repeat_found_encodeFastBlockAsm1K
	LEAL    1(DX), DI
	MOVL    12(SP), SI
	MOVL    DI, SI
	MOVL    12(SP), R8
	SUBL    R8, SI
	LEAQ    3(CX)(SI*1), R9
	CMPQ    R9, (SP)
	JB      dst_size_check_ok_1
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_1:
	LEAQ (BX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_repeat_emit_lits_encodeFastBlockAsm1K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_repeat_emit_lits_encodeFastBlockAsm1K
	JB   three_bytes_repeat_emit_lits_encodeFastBlockAsm1K

three_bytes_repeat_emit_lits_encodeFastBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm1K

two_bytes_repeat_emit_lits_encodeFastBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midrepeat_emit_lits_encodeFastBlockAsm1K
	JMP  memmove_long_repeat_emit_lits_encodeFastBlockAsm1K

one_byte_repeat_emit_lits_encodeFastBlockAsm1K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm1K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm1K

emit_lit_memmove_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_repeat_emit_lits_encodeFastBlockAsm1K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm1K

memmove_midrepeat_emit_lits_encodeFastBlockAsm1K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm1K

emit_lit_memmove_mid_repeat_emit_lits_encodeFastBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_repeat_emit_lits_encodeFastBlockAsm1K:
	MOVQ R9, CX
	JMP  repeat_emit_lits_end_encodeFastBlockAsm1K

memmove_long_repeat_emit_lits_encodeFastBlockAsm1K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R11
	SHRQ    $0x05, R11
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R10
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm1Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  SI, R12
	JAE   emit_lit_memmove_long_repeat_emit_lits_encodeFastBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

repeat_emit_lits_end_encodeFastBlockAsm1K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm1K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeFastBlockAsm1K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm1K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeFastBlockAsm1K
	LEAL -16(R8), R8
	LEAL 16(R11), R11

matchlen_loop_16_entry_repeat_extend_encodeFastBlockAsm1K:
	CMPL R8, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeFastBlockAsm1K
	JMP  matchlen_match8_repeat_extend_encodeFastBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeFastBlockAsm1K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm1K

matchlen_match8_repeat_extend_encodeFastBlockAsm1K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeFastBlockAsm1K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeFastBlockAsm1K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeFastBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeFastBlockAsm1K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    repeat_extend_forward_end_encodeFastBlockAsm1K

matchlen_match4_repeat_extend_encodeFastBlockAsm1K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeFastBlockAsm1K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeFastBlockAsm1K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeFastBlockAsm1K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeFastBlockAsm1K
	JB   repeat_extend_forward_end_encodeFastBlockAsm1K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeFastBlockAsm1K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeFastBlockAsm1K

matchlen_match1_repeat_extend_encodeFastBlockAsm1K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeFastBlockAsm1K
	LEAL 1(R11), R11
	JMP  repeat_extend_forward_end_encodeFastBlockAsm1K
	PCALIGN $0x10

repeat_extend_forward_end_encodeFastBlockAsm1K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitRepeat
	LEAL -1(SI), DI
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeFastBlockAsm1K
	LEAL -30(SI), DI
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeFastBlockAsm1K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeFastBlockAsm1K
	MOVB $0xfc, (CX)
	MOVL DI, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeFastBlockAsm1K

repeat_three_match_repeat_encodeFastBlockAsm1K:
	MOVB $0xf4, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeFastBlockAsm1K

repeat_two_match_repeat_encodeFastBlockAsm1K:
	MOVB $0xec, (CX)
	MOVB DI, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeFastBlockAsm1K

repeat_one_match_repeat_encodeFastBlockAsm1K:
	XORL DI, DI
	LEAL -4(DI)(SI*8), DI
	MOVB DI, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeFastBlockAsm1K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeFastBlockAsm1K

no_repeat_found_encodeFastBlockAsm1K:
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate_match_encodeFastBlockAsm1K
	MOVQ    1(BX)(DX*1), DI
	MOVWLZX (AX)(R10*2), SI
	LEAL    2(DX), R9
	CMPQ    (BX)(R8*1), DI
	JEQ     candidate2_match_encodeFastBlockAsm1K
	MOVW    R9, (AX)(R10*2)
	MOVQ    2(BX)(DX*1), DI
	CMPQ    (BX)(SI*1), DI
	JEQ     candidate3_match_encodeFastBlockAsm1K
	MOVL    20(SP), DX
	JMP     search_loop_encodeFastBlockAsm1K

candidate3_match_encodeFastBlockAsm1K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeFastBlockAsm1K

candidate2_match_encodeFastBlockAsm1K:
	MOVW R9, (AX)(R10*2)
	INCL DX
	MOVL R8, SI

candidate_match_encodeFastBlockAsm1K:
	CMPQ CX, (SP)
	JB   dst_size_check_ok_2
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_2:
	MOVL DX, R8
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x08, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm1K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeFastBlockAsm1K:
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm1K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeFastBlockAsm1K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeFastBlockAsm1K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeFastBlockAsm1K
	JMP  matchlen_match8_match_nolit_encodeFastBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeFastBlockAsm1K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeFastBlockAsm1K

matchlen_match8_match_nolit_encodeFastBlockAsm1K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeFastBlockAsm1K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeFastBlockAsm1K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeFastBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeFastBlockAsm1K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeFastBlockAsm1K

matchlen_match4_match_nolit_encodeFastBlockAsm1K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeFastBlockAsm1K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeFastBlockAsm1K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeFastBlockAsm1K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeFastBlockAsm1K
	JB   match_nolit_end_encodeFastBlockAsm1K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeFastBlockAsm1K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeFastBlockAsm1K

matchlen_match1_match_nolit_encodeFastBlockAsm1K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  match_nolit_end_encodeFastBlockAsm1K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeFastBlockAsm1K
	PCALIGN $0x10

match_nolit_end_encodeFastBlockAsm1K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL 16(SP), SI
	MOVL 12(SP), DI
	MOVL DX, 12(SP)
	SUBL DI, R8
	JZ   match_nolits_copy_encodeFastBlockAsm1K
	LEAQ (BX)(DI*1), DI
	LEAQ 3(CX)(R8*1), R9
	CMPQ R9, (SP)
	JB   dst_size_check_ok_3
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_3:
	// emitLiteral
	LEAL -1(R8), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeFastBlockAsm1K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeFastBlockAsm1K
	JB   three_bytes_match_emit_encodeFastBlockAsm1K

three_bytes_match_emit_encodeFastBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeFastBlockAsm1K

two_bytes_match_emit_encodeFastBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeFastBlockAsm1K
	JMP  memmove_long_match_emit_encodeFastBlockAsm1K

one_byte_match_emit_encodeFastBlockAsm1K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm1K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_encodeFastBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeFastBlockAsm1K_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeFastBlockAsm1K_memmove_move_8through16:
	MOVOU (DI), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm1K

emit_lit_memmove_match_emit_encodeFastBlockAsm1K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_encodeFastBlockAsm1K

emit_lit_memmove_match_emit_encodeFastBlockAsm1K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_encodeFastBlockAsm1K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm1K

memmove_midmatch_emit_encodeFastBlockAsm1K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeFastBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm1K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_mid_end_copy_match_emit_encodeFastBlockAsm1K

emit_lit_memmove_mid_match_emit_encodeFastBlockAsm1K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R8*1), X2
	MOVOU -16(DI)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_mid_end_copy_match_emit_encodeFastBlockAsm1K:
	MOVQ R9, CX
	JMP  match_nolits_copy_encodeFastBlockAsm1K

memmove_long_match_emit_encodeFastBlockAsm1K:
	LEAQ (CX)(R8*1), R9

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R8*1), X2
	MOVOU   -16(DI)(R8*1), X3
	MOVQ    R8, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeFastBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(DI)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeFastBlockAsm1Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeFastBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeFastBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R13*1), X4
	MOVOU -16(DI)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_encodeFastBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  R9, CX
	JMP   match_nolits_copy_encodeFastBlockAsm1K
	PCALIGN $0x10

match_nolits_copy_encodeFastBlockAsm1K:
	// emitCopy
	CMPL SI, $0x00000400
	JA   two_byte_match_nolit_encodeFastBlockAsm1K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeFastBlockAsm1K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm1K

emit_one_longer_match_nolit_encodeFastBlockAsm1K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeFastBlockAsm1K
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm1K

emit_copy1_repeat_match_nolit_encodeFastBlockAsm1K:
	LEAL -1(SI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm1K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm1K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm1K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm1K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm1K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm1K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm1K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm1K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeFastBlockAsm1K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm1K

two_byte_match_nolit_encodeFastBlockAsm1K:
	// emitCopy2
	LEAL -64(SI), SI
	LEAL -4(R11), R11
	MOVW SI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeFastBlockAsm1K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeFastBlockAsm1K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeFastBlockAsm1K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm1K

emit_copy2_2_match_nolit_encodeFastBlockAsm1K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm1K

emit_copy2_1_match_nolit_encodeFastBlockAsm1K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeFastBlockAsm1K

emit_copy2_0_match_nolit_encodeFastBlockAsm1K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeFastBlockAsm1K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeFastBlockAsm1K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeFastBlockAsm1K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeFastBlockAsm1K:
	MOVQ    $0xcf1bbcdcb7a56463, SI
	MOVQ    DI, R8
	MOVQ    (BX)(DX*1), DI
	MOVQ    DI, R9
	IMULQ   SI, R8
	SHRQ    $0x37, R8
	IMULQ   SI, R9
	SHRQ    $0x37, R9
	LEAL    -2(DX), R10
	MOVWLZX (AX)(R9*2), SI
	MOVW    R10, (AX)(R8*2)
	MOVW    DX, (AX)(R9*2)
	MOVL    DX, R8
	INCL    DX
	CMPQ    (BX)(SI*1), DI
	JNE     search_loop_encodeFastBlockAsm1K
	MOVL    R8, DI
	SUBL    SI, DI
	MOVL    DI, 16(SP)
	CMPQ    CX, (SP)
	JB      dst_size_check_ok_4
	MOVQ    $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_4:
	ADDL $0x07, DX
	ADDL $0x08, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm1K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit2_encodeFastBlockAsm1K:
	MOVQ (R8)(R11*1), R9
	MOVQ 8(R8)(R11*1), R10
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm1K
	XORQ 8(SI)(R11*1), R10
	JNZ  matchlen_bsf_16match_nolit2_encodeFastBlockAsm1K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit2_encodeFastBlockAsm1K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit2_encodeFastBlockAsm1K
	JMP  matchlen_match8_match_nolit2_encodeFastBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_16match_nolit2_encodeFastBlockAsm1K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	LEAL   8(R11)(R10*1), R11
	JMP    match_nolit2_end_encodeFastBlockAsm1K

matchlen_match8_match_nolit2_encodeFastBlockAsm1K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit2_encodeFastBlockAsm1K
	MOVQ (R8)(R11*1), R9
	XORQ (SI)(R11*1), R9
	JNZ  matchlen_bsf_8_match_nolit2_encodeFastBlockAsm1K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit2_encodeFastBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit2_encodeFastBlockAsm1K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R11
	JMP    match_nolit2_end_encodeFastBlockAsm1K

matchlen_match4_match_nolit2_encodeFastBlockAsm1K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit2_encodeFastBlockAsm1K
	MOVL (R8)(R11*1), R9
	CMPL (SI)(R11*1), R9
	JNE  matchlen_match2_match_nolit2_encodeFastBlockAsm1K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit2_encodeFastBlockAsm1K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit2_encodeFastBlockAsm1K
	JB   match_nolit2_end_encodeFastBlockAsm1K
	MOVW (R8)(R11*1), R9
	CMPW (SI)(R11*1), R9
	JNE  matchlen_match1_match_nolit2_encodeFastBlockAsm1K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit2_end_encodeFastBlockAsm1K

matchlen_match1_match_nolit2_encodeFastBlockAsm1K:
	MOVB (R8)(R11*1), R9
	CMPB (SI)(R11*1), R9
	JNE  match_nolit2_end_encodeFastBlockAsm1K
	LEAL 1(R11), R11
	JMP  match_nolit2_end_encodeFastBlockAsm1K
	PCALIGN $0x10

match_nolit2_end_encodeFastBlockAsm1K:
	ADDL R11, DX
	ADDL $0x08, R11
	MOVL DX, 12(SP)
	MOVL 16(SP), SI
	JMP  match_nolits_copy_encodeFastBlockAsm1K

emit_remainder_encodeFastBlockAsm1K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	SUBL DX, AX
	JZ   emit_remainder_end_encodeFastBlockAsm1K
	LEAQ (BX)(DX*1), DX
	LEAQ 3(CX)(AX*1), BX
	CMPQ BX, (SP)
	JB   dst_size_check_ok_5
	MOVQ $0x00000000, ret+56(FP)
	RET

dst_size_check_ok_5:
	// emitLiteral
	LEAL -1(AX), BX
	CMPL BX, $0x1d
	JB   one_byte_emit_remainder_encodeFastBlockAsm1K
	SUBL $0x1d, BX
	CMPL BX, $0x00000100
	JB   two_bytes_emit_remainder_encodeFastBlockAsm1K
	JB   three_bytes_emit_remainder_encodeFastBlockAsm1K

three_bytes_emit_remainder_encodeFastBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, BX
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm1K

two_bytes_emit_remainder_encodeFastBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB BL, 1(CX)
	ADDL $0x1d, BX
	ADDQ $0x02, CX
	CMPL BX, $0x40
	JB   memmove_midemit_remainder_encodeFastBlockAsm1K
	JMP  memmove_long_emit_remainder_encodeFastBlockAsm1K

one_byte_emit_remainder_encodeFastBlockAsm1K:
	SHLB $0x03, BL
	MOVB BL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ AX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_3
	CMPQ AX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_4through8
	CMPQ AX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_8through16
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_1or2:
	MOVB (DX), SI
	MOVB -1(DX)(AX*1), DL
	MOVB SI, (CX)
	MOVB DL, -1(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm1K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_3:
	MOVW (DX), SI
	MOVB 2(DX), DL
	MOVW SI, (CX)
	MOVB DL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm1K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_4through8:
	MOVL (DX), SI
	MOVL -4(DX)(AX*1), DX
	MOVL SI, (CX)
	MOVL DX, -4(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm1K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_8through16:
	MOVQ (DX), SI
	MOVQ -8(DX)(AX*1), DX
	MOVQ SI, (CX)
	MOVQ DX, -8(CX)(AX*1)
	JMP  memmove_end_copy_emit_remainder_encodeFastBlockAsm1K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_end_copy_emit_remainder_encodeFastBlockAsm1K

emit_lit_memmove_emit_remainder_encodeFastBlockAsm1K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_end_copy_emit_remainder_encodeFastBlockAsm1K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm1K

memmove_midemit_remainder_encodeFastBlockAsm1K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ AX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm1K_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(AX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(AX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm1K

emit_lit_memmove_mid_emit_remainder_encodeFastBlockAsm1K_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(AX*1), X2
	MOVOU -16(DX)(AX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)

memmove_mid_end_copy_emit_remainder_encodeFastBlockAsm1K:
	MOVQ BX, CX
	JMP  emit_remainder_end_encodeFastBlockAsm1K

memmove_long_emit_remainder_encodeFastBlockAsm1K:
	LEAQ (CX)(AX*1), BX

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(AX*1), X2
	MOVOU   -16(DX)(AX*1), X3
	MOVQ    AX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(DX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm1Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(DX)(R8*1), X4
	MOVOU -16(DX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  AX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeFastBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(AX*1)
	MOVOU X3, -16(CX)(AX*1)
	MOVQ  BX, CX

emit_remainder_end_encodeFastBlockAsm1K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBetterBlockAsm(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00001200, DX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm:
	MOVOU   X0, (AX)
	MOVOU   X0, 16(AX)
	MOVOU   X0, 32(AX)
	MOVOU   X0, 48(AX)
	MOVOU   X0, 64(AX)
	MOVOU   X0, 80(AX)
	MOVOU   X0, 96(AX)
	MOVOU   X0, 112(AX)
	ADDQ    $0x80, AX
	DECQ    DX
	JNZ     zero_loop_encodeBetterBlockAsm
	MOVL    $0x00000000, 12(SP)
	MOVQ    src_len+32(FP), AX
	LEAQ    -17(AX), DX
	LEAQ    -17(AX), BX
	MOVL    BX, 8(SP)
	SHRQ    $0x05, AX
	SUBL    AX, DX
	LEAQ    (CX)(DX*1), DX
	MOVQ    DX, (SP)
	MOVL    $0x00000001, AX
	MOVL    AX, 16(SP)
	MOVQ    src_base+24(FP), DX
	PCALIGN $0x10

search_loop_encodeBetterBlockAsm:
	MOVQ tmp+48(FP), BX
	MOVL AX, SI
	SUBL 12(SP), SI
	SHRL $0x08, SI
	CMPL SI, $0x63
	JBE  check_maxskip_ok_encodeBetterBlockAsm
	LEAL 100(AX), SI
	JMP  check_maxskip_cont_encodeBetterBlockAsm

check_maxskip_ok_encodeBetterBlockAsm:
	LEAL 1(AX)(SI*1), SI

check_maxskip_cont_encodeBetterBlockAsm:
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeBetterBlockAsm
	MOVQ    (DX)(AX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0x00cf1bbcdcbfa563, R9
	MOVQ    $0x9e3779b1, SI
	MOVQ    DI, R10
	MOVQ    DI, R11
	SHLQ    $0x08, R10
	IMULQ   R9, R10
	SHRQ    $0x2f, R10
	SHLQ    $0x20, R11
	IMULQ   SI, R11
	SHRQ    $0x32, R11
	MOVL    (BX)(R10*4), SI
	MOVL    524288(BX)(R11*4), R8
	MOVL    AX, (BX)(R10*4)
	MOVL    AX, 524288(BX)(R11*4)
	LEAL    -2162685(AX), R10
	CMPL    SI, R10
	CMOVLLE R10, SI
	MOVQ    (DX)(SI*1), R11
	CMPQ    R11, DI
	JEQ     candidate_match_encodeBetterBlockAsm
	CMPL    R8, R10
	CMOVLLE R10, R8
	MOVQ    (DX)(R8*1), R12
	CMPQ    R12, DI
	MOVL    AX, R13
	SUBL    16(SP), R13
	MOVQ    (DX)(R13*1), R13
	MOVQ    $0x000000ffffffff00, R14
	XORQ    DI, R13
	TESTQ   R14, R13
	JNE     no_repeat_found_encodeBetterBlockAsm
	LEAL    1(AX), BX
	MOVL    12(SP), SI
	MOVL    BX, DI
	SUBL    16(SP), DI
	JZ      repeat_extend_back_end_encodeBetterBlockAsm

repeat_extend_back_loop_encodeBetterBlockAsm:
	CMPL BX, SI
	JBE  repeat_extend_back_end_encodeBetterBlockAsm
	MOVB -1(DX)(DI*1), R8
	MOVB -1(DX)(BX*1), R9
	CMPB R8, R9
	JNE  repeat_extend_back_end_encodeBetterBlockAsm
	LEAL -1(BX), BX
	DECL DI
	JNZ  repeat_extend_back_loop_encodeBetterBlockAsm

repeat_extend_back_end_encodeBetterBlockAsm:
	MOVL BX, SI
	SUBL 12(SP), SI
	LEAQ 4(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBetterBlockAsm:
	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_repeat_emit_encodeBetterBlockAsm
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_repeat_emit_encodeBetterBlockAsm
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBetterBlockAsm
	CMPL SI, $0x00010000
	JB   three_bytes_repeat_emit_encodeBetterBlockAsm
	MOVL SI, R9
	SHRL $0x10, R9
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R9, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm

three_bytes_repeat_emit_encodeBetterBlockAsm:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm

two_bytes_repeat_emit_encodeBetterBlockAsm:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midrepeat_emit_encodeBetterBlockAsm
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm

one_byte_repeat_emit_encodeBetterBlockAsm:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_encodeBetterBlockAsm

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_repeat_emit_encodeBetterBlockAsm

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_repeat_emit_encodeBetterBlockAsm:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm

memmove_midrepeat_emit_encodeBetterBlockAsm:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm

memmove_long_repeat_emit_encodeBetterBlockAsm:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R11
	SUBQ    R9, R11
	DECQ    R10
	JA      emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R11*1), R9
	LEAQ    -32(CX)(R11*1), R12
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsmlarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R12)
	MOVOA X5, 16(R12)
	ADDQ  $0x20, R12
	ADDQ  $0x20, R9
	ADDQ  $0x20, R11
	DECQ  R10
	JNA   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsmlarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R11*1), X4
	MOVOU -16(R8)(R11*1), X5
	MOVOA X4, -32(CX)(R11*1)
	MOVOA X5, -16(CX)(R11*1)
	ADDQ  $0x20, R11
	CMPQ  DI, R11
	JAE   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBetterBlockAsm:
	ADDL $0x05, AX
	MOVL AX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), SI

	// matchLen
	XORL R10, R10
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm:
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16repeat_extend_encodeBetterBlockAsm
	LEAL -16(DI), DI
	LEAL 16(R10), R10

matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm
	JMP  matchlen_match8_repeat_extend_encodeBetterBlockAsm
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBetterBlockAsm:
	TZCNTQ R11, R11
	SARQ   $0x03, R11
	LEAL   8(R10)(R11*1), R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm

matchlen_match8_repeat_extend_encodeBetterBlockAsm:
	CMPL DI, $0x08
	JB   matchlen_match4_repeat_extend_encodeBetterBlockAsm
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_repeat_extend_encodeBetterBlockAsm
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm

matchlen_match4_repeat_extend_encodeBetterBlockAsm:
	CMPL DI, $0x04
	JB   matchlen_match2_repeat_extend_encodeBetterBlockAsm
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_repeat_extend_encodeBetterBlockAsm
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_repeat_extend_encodeBetterBlockAsm:
	CMPL DI, $0x01
	JE   matchlen_match1_repeat_extend_encodeBetterBlockAsm
	JB   repeat_extend_forward_end_encodeBetterBlockAsm
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_repeat_extend_encodeBetterBlockAsm
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   repeat_extend_forward_end_encodeBetterBlockAsm

matchlen_match1_repeat_extend_encodeBetterBlockAsm:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  repeat_extend_forward_end_encodeBetterBlockAsm
	LEAL 1(R10), R10
	JMP  repeat_extend_forward_end_encodeBetterBlockAsm
	PCALIGN $0x10

repeat_extend_forward_end_encodeBetterBlockAsm:
	ADDL R10, AX
	MOVL AX, SI
	SUBL BX, SI
	MOVL 16(SP), BX

	// emitRepeat
	LEAL -1(SI), BX
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBetterBlockAsm
	LEAL -30(SI), BX
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBetterBlockAsm
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBetterBlockAsm
	MOVB $0xfc, (CX)
	MOVL BX, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm

repeat_three_match_repeat_encodeBetterBlockAsm:
	MOVB $0xf4, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm

repeat_two_match_repeat_encodeBetterBlockAsm:
	MOVB $0xec, (CX)
	MOVB BL, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm

repeat_one_match_repeat_encodeBetterBlockAsm:
	XORL BX, BX
	LEAL -4(BX)(SI*8), BX
	MOVB BL, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBetterBlockAsm:
	MOVL AX, 12(SP)
	JMP  search_loop_encodeBetterBlockAsm
	PCALIGN $0x10

no_repeat_found_encodeBetterBlockAsm:
	CMPL    SI, R10
	CMOVLLE R10, SI
	CMPL    R11, DI
	JEQ     candidate_match_encodeBetterBlockAsm
	CMPL    R8, R10
	CMOVLLE R10, R8
	CMPL    R12, DI
	JEQ     candidateS_match_encodeBetterBlockAsm
	MOVL    20(SP), AX
	JMP     search_loop_encodeBetterBlockAsm

candidateS_match_encodeBetterBlockAsm:
	SHRQ    $0x08, DI
	MOVQ    DI, R11
	SHLQ    $0x08, R11
	IMULQ   R9, R11
	SHRQ    $0x2f, R11
	MOVL    (BX)(R11*4), SI
	INCL    AX
	MOVL    AX, (BX)(R11*4)
	CMPL    SI, R10
	CMOVLLE R10, SI
	CMPL    (DX)(SI*1), DI
	JEQ     candidate_match_encodeBetterBlockAsm
	DECL    AX
	MOVL    R8, SI
	PCALIGN $0x10

candidate_match_encodeBetterBlockAsm:
	MOVL  12(SP), BX
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm

match_extend_back_loop_encodeBetterBlockAsm:
	CMPL AX, BX
	JBE  match_extend_back_end_encodeBetterBlockAsm
	MOVB -1(DX)(SI*1), DI
	MOVB -1(DX)(AX*1), R8
	CMPB DI, R8
	JNE  match_extend_back_end_encodeBetterBlockAsm
	LEAL -1(AX), AX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm
	JMP  match_extend_back_loop_encodeBetterBlockAsm

match_extend_back_end_encodeBetterBlockAsm:
	MOVL AX, BX
	SUBL 12(SP), BX
	LEAQ 4(CX)(BX*1), BX
	CMPQ BX, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm:
	MOVL AX, BX
	ADDL $0x04, AX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), R9

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm:
	MOVQ (R8)(R11*1), R10
	MOVQ 8(R8)(R11*1), R12
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm
	XORQ 8(R9)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm
	JMP  matchlen_match8_match_nolit_encodeBetterBlockAsm
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBetterBlockAsm:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBetterBlockAsm

matchlen_match8_match_nolit_encodeBetterBlockAsm:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm
	MOVQ (R8)(R11*1), R10
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBetterBlockAsm

matchlen_match4_match_nolit_encodeBetterBlockAsm:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm
	MOVL (R8)(R11*1), R10
	CMPL (R9)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBetterBlockAsm:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm
	JB   match_nolit_end_encodeBetterBlockAsm
	MOVW (R8)(R11*1), R10
	CMPW (R9)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBetterBlockAsm

matchlen_match1_match_nolit_encodeBetterBlockAsm:
	MOVB (R8)(R11*1), R10
	CMPB (R9)(R11*1), R10
	JNE  match_nolit_end_encodeBetterBlockAsm
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBetterBlockAsm
	PCALIGN $0x10

match_nolit_end_encodeBetterBlockAsm:
	MOVL AX, DI
	SUBL SI, DI
	CMPL R11, $0x01
	JA   match_length_ok_encodeBetterBlockAsm
	CMPL DI, $0x0001003f
	JBE  match_length_ok_encodeBetterBlockAsm
	MOVL 20(SP), AX
	INCL AX
	JMP  search_loop_encodeBetterBlockAsm

match_length_ok_encodeBetterBlockAsm:
	MOVL DI, 16(SP)

	// Check if we can combine lit+copy
	MOVLQZX 12(SP), R8
	MOVL    BX, SI
	SUBL    R8, SI
	JZ      match_emit_nolits_encodeBetterBlockAsm
	CMPL    DI, $0x00000040
	JL      match_emit_lits_encodeBetterBlockAsm
	CMPL    DI, $0x0001003f
	JA      match_emit_copy3_encodeBetterBlockAsm
	CMPL    SI, $0x04
	JA      match_emit_lits_encodeBetterBlockAsm
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, DI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    DI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, DI
	CMOVLLT R11, DI
	LEAL    -1(SI)(DI*4), DI
	MOVL    $0x00000003, R10
	LEAL    (R10)(DI*8), DI
	MOVB    DI, (CX)
	ADDQ    $0x03, CX
	MOVL    R8, (CX)
	ADDQ    SI, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBetterBlockAsm

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

match_emit_copy3_encodeBetterBlockAsm:
	CMPL    SI, $0x03
	JA      match_emit_lits_encodeBetterBlockAsm
	MOVLQZX 12(SP), R8
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(DI), DI
	SHLL $0x0b, DI
	LEAL 7(DI)(SI*8), DI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_emit_lits_encodeBetterBlockAsm
	LEAL -60(R11), R9
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_emit_lits_encodeBetterBlockAsm
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_emit_lits_encodeBetterBlockAsm
	ADDL $0x000007e0, DI
	MOVL DI, (CX)
	MOVL R9, 4(CX)
	ADDQ $0x07, CX
	JMP  match_emit_copy_litsencodeBetterBlockAsm

emit_copy3_2_match_emit_lits_encodeBetterBlockAsm:
	ADDL $0x000007c0, DI
	MOVL DI, (CX)
	MOVW R9, 4(CX)
	ADDQ $0x06, CX
	JMP  match_emit_copy_litsencodeBetterBlockAsm

emit_copy3_1_match_emit_lits_encodeBetterBlockAsm:
	ADDL $0x000007a0, DI
	MOVL DI, (CX)
	MOVB R9, 4(CX)
	ADDQ $0x05, CX
	JMP  match_emit_copy_litsencodeBetterBlockAsm

emit_copy3_0_match_emit_lits_encodeBetterBlockAsm:
	SHLL $0x05, R11
	ORL  R11, DI
	MOVL DI, (CX)
	ADDQ $0x04, CX

match_emit_copy_litsencodeBetterBlockAsm:
	MOVL R8, (CX)
	ADDQ SI, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

match_emit_lits_encodeBetterBlockAsm:
	LEAQ (DX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBetterBlockAsm
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm
	CMPL R9, $0x00010000
	JB   three_bytes_match_emit_encodeBetterBlockAsm
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm

three_bytes_match_emit_encodeBetterBlockAsm:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm

two_bytes_match_emit_encodeBetterBlockAsm:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBetterBlockAsm
	JMP  memmove_long_match_emit_encodeBetterBlockAsm

one_byte_match_emit_encodeBetterBlockAsm:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm

emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm

emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm

memmove_midmatch_emit_encodeBetterBlockAsm:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBetterBlockAsm

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_match_emit_encodeBetterBlockAsm:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm

memmove_long_match_emit_encodeBetterBlockAsm:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R13*1), X4
	MOVOU -16(R8)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  SI, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

match_emit_nolits_encodeBetterBlockAsm:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitCopy
	CMPL DI, $0x0001003f
	JBE  two_byte_offset_match_nolit_encodeBetterBlockAsm

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(DI), SI
	SHLL $0x0b, SI
	ADDL $0x07, SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_nolit_encodeBetterBlockAsm_emit3
	LEAL -60(R11), DI
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_nolit_encodeBetterBlockAsm_emit3
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_nolit_encodeBetterBlockAsm_emit3
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL DI, 4(CX)
	ADDQ $0x07, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

emit_copy3_2_match_nolit_encodeBetterBlockAsm_emit3:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW DI, 4(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

emit_copy3_1_match_nolit_encodeBetterBlockAsm_emit3:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

emit_copy3_0_match_nolit_encodeBetterBlockAsm_emit3:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

two_byte_offset_match_nolit_encodeBetterBlockAsm:
	CMPL DI, $0x00000400
	JA   two_byte_match_nolit_encodeBetterBlockAsm
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBetterBlockAsm
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

emit_one_longer_match_nolit_encodeBetterBlockAsm:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBetterBlockAsm
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

emit_copy1_repeat_match_nolit_encodeBetterBlockAsm:
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

two_byte_match_nolit_encodeBetterBlockAsm:
	// emitCopy2
	LEAL -64(DI), DI
	LEAL -4(R11), R11
	MOVW DI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBetterBlockAsm_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBetterBlockAsm_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBetterBlockAsm_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

emit_copy2_2_match_nolit_encodeBetterBlockAsm_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

emit_copy2_1_match_nolit_encodeBetterBlockAsm_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

emit_copy2_0_match_nolit_encodeBetterBlockAsm_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm
	CMPL SI, $0x00010000
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm
	MOVL SI, R9
	SHRL $0x10, R9
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R9, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm

three_bytes_match_emit_repeat_encodeBetterBlockAsm:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm

two_bytes_match_emit_repeat_encodeBetterBlockAsm:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midmatch_emit_repeat_encodeBetterBlockAsm
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm

one_byte_match_emit_repeat_encodeBetterBlockAsm:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm

memmove_midmatch_emit_repeat_encodeBetterBlockAsm:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm

memmove_long_match_emit_repeat_encodeBetterBlockAsm:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R12
	SUBQ    R9, R12
	DECQ    R10
	JA      emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R9
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R9
	ADDQ  $0x20, R12
	DECQ  R10
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  DI, R12
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_match_nolit_repeat_encodeBetterBlockAsm
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_match_nolit_repeat_encodeBetterBlockAsm
	CMPL R11, $0x0001001e
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_three_match_nolit_repeat_encodeBetterBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_match_nolit_repeat_encodeBetterBlockAsm:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_one_match_nolit_repeat_encodeBetterBlockAsm:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm:
	CMPL AX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm:
	MOVQ  tmp+48(FP), SI
	MOVQ  $0x00cf1bbcdcbfa563, DI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(BX), BX
	LEAQ  -2(AX), R9
	MOVQ  (DX)(BX*1), R10
	MOVQ  1(DX)(BX*1), R11
	MOVQ  (DX)(R9*1), R12
	MOVQ  1(DX)(R9*1), R13
	SHLQ  $0x08, R10
	IMULQ DI, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x32, R11
	SHLQ  $0x08, R12
	IMULQ DI, R12
	SHRQ  $0x2f, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x32, R13
	LEAQ  1(BX), R8
	LEAQ  1(R9), R14
	MOVL  BX, (SI)(R10*4)
	MOVL  R9, (SI)(R12*4)
	LEAQ  1(R9)(BX*1), R10
	SHRQ  $0x01, R10
	ADDQ  $0x01, BX
	SUBQ  $0x01, R9
	MOVL  R8, 524288(SI)(R11*4)
	MOVL  R14, 524288(SI)(R13*4)

index_loop_encodeBetterBlockAsm:
	CMPQ  R10, R9
	JAE   search_loop_encodeBetterBlockAsm
	MOVQ  (DX)(BX*1), R8
	MOVQ  (DX)(R10*1), R11
	SHLQ  $0x08, R8
	IMULQ DI, R8
	SHRQ  $0x2f, R8
	SHLQ  $0x08, R11
	IMULQ DI, R11
	SHRQ  $0x2f, R11
	MOVL  BX, (SI)(R8*4)
	MOVL  R9, (SI)(R11*4)
	ADDQ  $0x02, BX
	ADDQ  $0x02, R10
	JMP   index_loop_encodeBetterBlockAsm

emit_remainder_encodeBetterBlockAsm:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 4(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm:
	MOVQ src_len+32(FP), AX

	// emitLiteralsDstP
	MOVL 12(SP), BX
	CMPL BX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (DX)(BX*1), AX
	SUBL BX, SI

	// emitLiteral
	LEAL -1(SI), DX
	CMPL DX, $0x1d
	JB   one_byte_emit_remainder_encodeBetterBlockAsm
	SUBL $0x1d, DX
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm
	CMPL DX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm

three_bytes_emit_remainder_encodeBetterBlockAsm:
	MOVB $0xf0, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm

two_bytes_emit_remainder_encodeBetterBlockAsm:
	MOVB $0xe8, (CX)
	MOVB DL, 1(CX)
	ADDL $0x1d, DX
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_midemit_remainder_encodeBetterBlockAsm
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm

one_byte_emit_remainder_encodeBetterBlockAsm:
	SHLB $0x03, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -1, min move: 1
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3
	CMPQ BX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through8
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through8:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm

memmove_midemit_remainder_encodeBetterBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -2, min move: 30
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm

memmove_long_emit_remainder_encodeBetterBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU   (AX), X0
	MOVOU   16(AX), X1
	MOVOU   -32(AX)(BX*1), X2
	MOVOU   -16(AX)(BX*1), X3
	MOVQ    BX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32
	LEAQ    -32(AX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm2MB(dst []byte, src []byte, tmp *[589824]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBetterBlockAsm2MB(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00001200, DX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm2MB:
	MOVOU   X0, (AX)
	MOVOU   X0, 16(AX)
	MOVOU   X0, 32(AX)
	MOVOU   X0, 48(AX)
	MOVOU   X0, 64(AX)
	MOVOU   X0, 80(AX)
	MOVOU   X0, 96(AX)
	MOVOU   X0, 112(AX)
	ADDQ    $0x80, AX
	DECQ    DX
	JNZ     zero_loop_encodeBetterBlockAsm2MB
	MOVL    $0x00000000, 12(SP)
	MOVQ    src_len+32(FP), AX
	LEAQ    -17(AX), DX
	LEAQ    -17(AX), BX
	MOVL    BX, 8(SP)
	SHRQ    $0x05, AX
	SUBL    AX, DX
	LEAQ    (CX)(DX*1), DX
	MOVQ    DX, (SP)
	MOVL    $0x00000001, AX
	MOVL    AX, 16(SP)
	MOVQ    src_base+24(FP), DX
	PCALIGN $0x10

search_loop_encodeBetterBlockAsm2MB:
	MOVQ tmp+48(FP), BX
	MOVL AX, SI
	SUBL 12(SP), SI
	SHRL $0x07, SI
	CMPL SI, $0x63
	JBE  check_maxskip_ok_encodeBetterBlockAsm2MB
	LEAL 100(AX), SI
	JMP  check_maxskip_cont_encodeBetterBlockAsm2MB

check_maxskip_ok_encodeBetterBlockAsm2MB:
	LEAL 1(AX)(SI*1), SI

check_maxskip_cont_encodeBetterBlockAsm2MB:
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBetterBlockAsm2MB
	MOVQ  (DX)(AX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x00cf1bbcdcbfa563, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x32, R11
	MOVL  (BX)(R10*4), SI
	MOVL  524288(BX)(R11*4), R8
	MOVL  AX, (BX)(R10*4)
	MOVL  AX, 524288(BX)(R11*4)
	MOVQ  (DX)(SI*1), R10
	CMPQ  R10, DI
	JEQ   candidate_match_encodeBetterBlockAsm2MB
	MOVQ  (DX)(R8*1), R11
	CMPQ  R11, DI
	MOVL  AX, R12
	SUBL  16(SP), R12
	MOVQ  (DX)(R12*1), R12
	MOVQ  $0x000000ffffffff00, R13
	XORQ  DI, R12
	TESTQ R13, R12
	JNE   no_repeat_found_encodeBetterBlockAsm2MB
	LEAL  1(AX), BX
	MOVL  12(SP), SI
	MOVL  BX, DI
	SUBL  16(SP), DI
	JZ    repeat_extend_back_end_encodeBetterBlockAsm2MB

repeat_extend_back_loop_encodeBetterBlockAsm2MB:
	CMPL BX, SI
	JBE  repeat_extend_back_end_encodeBetterBlockAsm2MB
	MOVB -1(DX)(DI*1), R8
	MOVB -1(DX)(BX*1), R9
	CMPB R8, R9
	JNE  repeat_extend_back_end_encodeBetterBlockAsm2MB
	LEAL -1(BX), BX
	DECL DI
	JNZ  repeat_extend_back_loop_encodeBetterBlockAsm2MB

repeat_extend_back_end_encodeBetterBlockAsm2MB:
	MOVL BX, SI
	SUBL 12(SP), SI
	LEAQ 4(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBetterBlockAsm2MB
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBetterBlockAsm2MB:
	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_repeat_emit_encodeBetterBlockAsm2MB
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_repeat_emit_encodeBetterBlockAsm2MB
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBetterBlockAsm2MB
	CMPL SI, $0x00010000
	JB   three_bytes_repeat_emit_encodeBetterBlockAsm2MB
	MOVL SI, R9
	SHRL $0x10, R9
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R9, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm2MB

three_bytes_repeat_emit_encodeBetterBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm2MB

two_bytes_repeat_emit_encodeBetterBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midrepeat_emit_encodeBetterBlockAsm2MB
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm2MB

one_byte_repeat_emit_encodeBetterBlockAsm2MB:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_repeat_emit_encodeBetterBlockAsm2MB

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_repeat_emit_encodeBetterBlockAsm2MB

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_repeat_emit_encodeBetterBlockAsm2MB:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm2MB

memmove_midrepeat_emit_encodeBetterBlockAsm2MB:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm2MB

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm2MB:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm2MB

memmove_long_repeat_emit_encodeBetterBlockAsm2MB:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R11
	SUBQ    R9, R11
	DECQ    R10
	JA      emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R11*1), R9
	LEAQ    -32(CX)(R11*1), R12
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm2MBlarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R12)
	MOVOA X5, 16(R12)
	ADDQ  $0x20, R12
	ADDQ  $0x20, R9
	ADDQ  $0x20, R11
	DECQ  R10
	JNA   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R11*1), X4
	MOVOU -16(R8)(R11*1), X5
	MOVOA X4, -32(CX)(R11*1)
	MOVOA X5, -16(CX)(R11*1)
	ADDQ  $0x20, R11
	CMPQ  DI, R11
	JAE   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBetterBlockAsm2MB:
	ADDL $0x05, AX
	MOVL AX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), SI

	// matchLen
	XORL R10, R10
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm2MB
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm2MB:
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm2MB
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16repeat_extend_encodeBetterBlockAsm2MB
	LEAL -16(DI), DI
	LEAL 16(R10), R10

matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm2MB:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm2MB
	JMP  matchlen_match8_repeat_extend_encodeBetterBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBetterBlockAsm2MB:
	TZCNTQ R11, R11
	SARQ   $0x03, R11
	LEAL   8(R10)(R11*1), R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm2MB

matchlen_match8_repeat_extend_encodeBetterBlockAsm2MB:
	CMPL DI, $0x08
	JB   matchlen_match4_repeat_extend_encodeBetterBlockAsm2MB
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm2MB
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_repeat_extend_encodeBetterBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm2MB:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm2MB

matchlen_match4_repeat_extend_encodeBetterBlockAsm2MB:
	CMPL DI, $0x04
	JB   matchlen_match2_repeat_extend_encodeBetterBlockAsm2MB
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_repeat_extend_encodeBetterBlockAsm2MB
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_repeat_extend_encodeBetterBlockAsm2MB:
	CMPL DI, $0x01
	JE   matchlen_match1_repeat_extend_encodeBetterBlockAsm2MB
	JB   repeat_extend_forward_end_encodeBetterBlockAsm2MB
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_repeat_extend_encodeBetterBlockAsm2MB
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   repeat_extend_forward_end_encodeBetterBlockAsm2MB

matchlen_match1_repeat_extend_encodeBetterBlockAsm2MB:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  repeat_extend_forward_end_encodeBetterBlockAsm2MB
	LEAL 1(R10), R10
	JMP  repeat_extend_forward_end_encodeBetterBlockAsm2MB
	PCALIGN $0x10

repeat_extend_forward_end_encodeBetterBlockAsm2MB:
	ADDL R10, AX
	MOVL AX, SI
	SUBL BX, SI
	MOVL 16(SP), BX

	// emitRepeat
	LEAL -1(SI), BX
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBetterBlockAsm2MB
	LEAL -30(SI), BX
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBetterBlockAsm2MB
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBetterBlockAsm2MB
	MOVB $0xfc, (CX)
	MOVL BX, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm2MB

repeat_three_match_repeat_encodeBetterBlockAsm2MB:
	MOVB $0xf4, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm2MB

repeat_two_match_repeat_encodeBetterBlockAsm2MB:
	MOVB $0xec, (CX)
	MOVB BL, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm2MB

repeat_one_match_repeat_encodeBetterBlockAsm2MB:
	XORL BX, BX
	LEAL -4(BX)(SI*8), BX
	MOVB BL, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBetterBlockAsm2MB:
	MOVL AX, 12(SP)
	JMP  search_loop_encodeBetterBlockAsm2MB
	PCALIGN $0x10

no_repeat_found_encodeBetterBlockAsm2MB:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm2MB
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm2MB
	MOVL 20(SP), AX
	JMP  search_loop_encodeBetterBlockAsm2MB

candidateS_match_encodeBetterBlockAsm2MB:
	SHRQ    $0x08, DI
	MOVQ    DI, R10
	SHLQ    $0x08, R10
	IMULQ   R9, R10
	SHRQ    $0x2f, R10
	MOVL    (BX)(R10*4), SI
	INCL    AX
	MOVL    AX, (BX)(R10*4)
	CMPL    (DX)(SI*1), DI
	JEQ     candidate_match_encodeBetterBlockAsm2MB
	DECL    AX
	MOVL    R8, SI
	PCALIGN $0x10

candidate_match_encodeBetterBlockAsm2MB:
	MOVL  12(SP), BX
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm2MB

match_extend_back_loop_encodeBetterBlockAsm2MB:
	CMPL AX, BX
	JBE  match_extend_back_end_encodeBetterBlockAsm2MB
	MOVB -1(DX)(SI*1), DI
	MOVB -1(DX)(AX*1), R8
	CMPB DI, R8
	JNE  match_extend_back_end_encodeBetterBlockAsm2MB
	LEAL -1(AX), AX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm2MB
	JMP  match_extend_back_loop_encodeBetterBlockAsm2MB

match_extend_back_end_encodeBetterBlockAsm2MB:
	MOVL AX, BX
	SUBL 12(SP), BX
	LEAQ 4(CX)(BX*1), BX
	CMPQ BX, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm2MB
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm2MB:
	MOVL AX, BX
	ADDL $0x04, AX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), R9

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm2MB
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm2MB:
	MOVQ (R8)(R11*1), R10
	MOVQ 8(R8)(R11*1), R12
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm2MB
	XORQ 8(R9)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm2MB
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm2MB:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm2MB
	JMP  matchlen_match8_match_nolit_encodeBetterBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBetterBlockAsm2MB:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBetterBlockAsm2MB

matchlen_match8_match_nolit_encodeBetterBlockAsm2MB:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm2MB
	MOVQ (R8)(R11*1), R10
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm2MB
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm2MB
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm2MB:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBetterBlockAsm2MB

matchlen_match4_match_nolit_encodeBetterBlockAsm2MB:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm2MB
	MOVL (R8)(R11*1), R10
	CMPL (R9)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm2MB
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBetterBlockAsm2MB:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm2MB
	JB   match_nolit_end_encodeBetterBlockAsm2MB
	MOVW (R8)(R11*1), R10
	CMPW (R9)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm2MB
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBetterBlockAsm2MB

matchlen_match1_match_nolit_encodeBetterBlockAsm2MB:
	MOVB (R8)(R11*1), R10
	CMPB (R9)(R11*1), R10
	JNE  match_nolit_end_encodeBetterBlockAsm2MB
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBetterBlockAsm2MB
	PCALIGN $0x10

match_nolit_end_encodeBetterBlockAsm2MB:
	MOVL AX, DI
	SUBL SI, DI
	CMPL R11, $0x01
	JA   match_length_ok_encodeBetterBlockAsm2MB
	CMPL DI, $0x0001003f
	JBE  match_length_ok_encodeBetterBlockAsm2MB
	MOVL 20(SP), AX
	INCL AX
	JMP  search_loop_encodeBetterBlockAsm2MB

match_length_ok_encodeBetterBlockAsm2MB:
	MOVL DI, 16(SP)

	// Check if we can combine lit+copy
	MOVLQZX 12(SP), R8
	MOVL    BX, SI
	SUBL    R8, SI
	JZ      match_emit_nolits_encodeBetterBlockAsm2MB
	CMPL    DI, $0x00000040
	JL      match_emit_lits_encodeBetterBlockAsm2MB
	CMPL    DI, $0x0001003f
	JA      match_emit_copy3_encodeBetterBlockAsm2MB
	CMPL    SI, $0x04
	JA      match_emit_lits_encodeBetterBlockAsm2MB
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, DI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    DI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, DI
	CMOVLLT R11, DI
	LEAL    -1(SI)(DI*4), DI
	MOVL    $0x00000003, R10
	LEAL    (R10)(DI*8), DI
	MOVB    DI, (CX)
	ADDQ    $0x03, CX
	MOVL    R8, (CX)
	ADDQ    SI, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm2MB
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm2MB
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm2MB
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm2MB:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm2MB:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm2MB:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

match_emit_copy3_encodeBetterBlockAsm2MB:
	CMPL    SI, $0x03
	JA      match_emit_lits_encodeBetterBlockAsm2MB
	MOVLQZX 12(SP), R8
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(DI), DI
	SHLL $0x0b, DI
	LEAL 7(DI)(SI*8), DI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_emit_lits_encodeBetterBlockAsm2MB
	LEAL -60(R11), R9
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_emit_lits_encodeBetterBlockAsm2MB
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_emit_lits_encodeBetterBlockAsm2MB
	ADDL $0x000007e0, DI
	MOVL DI, (CX)
	MOVL R9, 4(CX)
	ADDQ $0x07, CX
	JMP  match_emit_copy_litsencodeBetterBlockAsm2MB

emit_copy3_2_match_emit_lits_encodeBetterBlockAsm2MB:
	ADDL $0x000007c0, DI
	MOVL DI, (CX)
	MOVW R9, 4(CX)
	ADDQ $0x06, CX
	JMP  match_emit_copy_litsencodeBetterBlockAsm2MB

emit_copy3_1_match_emit_lits_encodeBetterBlockAsm2MB:
	ADDL $0x000007a0, DI
	MOVL DI, (CX)
	MOVB R9, 4(CX)
	ADDQ $0x05, CX
	JMP  match_emit_copy_litsencodeBetterBlockAsm2MB

emit_copy3_0_match_emit_lits_encodeBetterBlockAsm2MB:
	SHLL $0x05, R11
	ORL  R11, DI
	MOVL DI, (CX)
	ADDQ $0x04, CX

match_emit_copy_litsencodeBetterBlockAsm2MB:
	MOVL R8, (CX)
	ADDQ SI, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

match_emit_lits_encodeBetterBlockAsm2MB:
	LEAQ (DX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBetterBlockAsm2MB
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm2MB
	CMPL R9, $0x00010000
	JB   three_bytes_match_emit_encodeBetterBlockAsm2MB
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm2MB

three_bytes_match_emit_encodeBetterBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm2MB

two_bytes_match_emit_encodeBetterBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBetterBlockAsm2MB
	JMP  memmove_long_match_emit_encodeBetterBlockAsm2MB

one_byte_match_emit_encodeBetterBlockAsm2MB:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm2MB_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm2MB_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBetterBlockAsm2MB_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm2MB

emit_lit_memmove_match_emit_encodeBetterBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm2MB

emit_lit_memmove_match_emit_encodeBetterBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm2MB:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm2MB

memmove_midmatch_emit_encodeBetterBlockAsm2MB:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBetterBlockAsm2MB

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_match_emit_encodeBetterBlockAsm2MB:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm2MB

memmove_long_match_emit_encodeBetterBlockAsm2MB:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm2MBlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R13*1), X4
	MOVOU -16(R8)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  SI, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

match_emit_nolits_encodeBetterBlockAsm2MB:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitCopy
	CMPL DI, $0x0001003f
	JBE  two_byte_offset_match_nolit_encodeBetterBlockAsm2MB

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(DI), SI
	SHLL $0x0b, SI
	ADDL $0x07, SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_nolit_encodeBetterBlockAsm2MB_emit3
	LEAL -60(R11), DI
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_nolit_encodeBetterBlockAsm2MB_emit3
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_nolit_encodeBetterBlockAsm2MB_emit3
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL DI, 4(CX)
	ADDQ $0x07, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

emit_copy3_2_match_nolit_encodeBetterBlockAsm2MB_emit3:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW DI, 4(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

emit_copy3_1_match_nolit_encodeBetterBlockAsm2MB_emit3:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

emit_copy3_0_match_nolit_encodeBetterBlockAsm2MB_emit3:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

two_byte_offset_match_nolit_encodeBetterBlockAsm2MB:
	CMPL DI, $0x00000400
	JA   two_byte_match_nolit_encodeBetterBlockAsm2MB
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBetterBlockAsm2MB
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

emit_one_longer_match_nolit_encodeBetterBlockAsm2MB:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBetterBlockAsm2MB
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

emit_copy1_repeat_match_nolit_encodeBetterBlockAsm2MB:
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm2MB
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm2MB
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm2MB
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm2MB:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm2MB:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm2MB:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

two_byte_match_nolit_encodeBetterBlockAsm2MB:
	// emitCopy2
	LEAL -64(DI), DI
	LEAL -4(R11), R11
	MOVW DI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBetterBlockAsm2MB_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBetterBlockAsm2MB_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBetterBlockAsm2MB_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

emit_copy2_2_match_nolit_encodeBetterBlockAsm2MB_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

emit_copy2_1_match_nolit_encodeBetterBlockAsm2MB_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

emit_copy2_0_match_nolit_encodeBetterBlockAsm2MB_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm2MB
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm2MB
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm2MB
	CMPL SI, $0x00010000
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm2MB
	MOVL SI, R9
	SHRL $0x10, R9
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R9, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm2MB

three_bytes_match_emit_repeat_encodeBetterBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm2MB

two_bytes_match_emit_repeat_encodeBetterBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midmatch_emit_repeat_encodeBetterBlockAsm2MB
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm2MB

one_byte_match_emit_repeat_encodeBetterBlockAsm2MB:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_33through64
	PCALIGN $0x10

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (CX)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm2MB

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm2MB

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm2MB:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm2MB

memmove_midmatch_emit_repeat_encodeBetterBlockAsm2MB:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 15, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm2MB

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm2MB_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm2MB:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm2MB

memmove_long_match_emit_repeat_encodeBetterBlockAsm2MB:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R12
	SUBQ    R9, R12
	DECQ    R10
	JA      emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R9
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm2MBlarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R9
	ADDQ  $0x20, R12
	DECQ  R10
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  DI, R12
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm2MB:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_match_nolit_repeat_encodeBetterBlockAsm2MB
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_match_nolit_repeat_encodeBetterBlockAsm2MB
	CMPL R11, $0x0001001e
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm2MB
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

repeat_three_match_nolit_repeat_encodeBetterBlockAsm2MB:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

repeat_two_match_nolit_repeat_encodeBetterBlockAsm2MB:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm2MB

repeat_one_match_nolit_repeat_encodeBetterBlockAsm2MB:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm2MB:
	CMPL AX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm2MB
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm2MB
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm2MB:
	MOVQ  tmp+48(FP), SI
	MOVQ  $0x00cf1bbcdcbfa563, DI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(BX), BX
	LEAQ  -2(AX), R9
	MOVQ  (DX)(BX*1), R10
	MOVQ  1(DX)(BX*1), R11
	MOVQ  (DX)(R9*1), R12
	MOVQ  1(DX)(R9*1), R13
	SHLQ  $0x08, R10
	IMULQ DI, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x32, R11
	SHLQ  $0x08, R12
	IMULQ DI, R12
	SHRQ  $0x2f, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x32, R13
	LEAQ  1(BX), R8
	LEAQ  1(R9), R14
	MOVL  BX, (SI)(R10*4)
	MOVL  R9, (SI)(R12*4)
	LEAQ  1(R9)(BX*1), R10
	SHRQ  $0x01, R10
	ADDQ  $0x01, BX
	SUBQ  $0x01, R9
	MOVL  R8, 524288(SI)(R11*4)
	MOVL  R14, 524288(SI)(R13*4)

index_loop_encodeBetterBlockAsm2MB:
	CMPQ  R10, R9
	JAE   search_loop_encodeBetterBlockAsm2MB
	MOVQ  (DX)(BX*1), R8
	MOVQ  (DX)(R10*1), R11
	SHLQ  $0x08, R8
	IMULQ DI, R8
	SHRQ  $0x2f, R8
	SHLQ  $0x08, R11
	IMULQ DI, R11
	SHRQ  $0x2f, R11
	MOVL  BX, (SI)(R8*4)
	MOVL  R9, (SI)(R11*4)
	ADDQ  $0x02, BX
	ADDQ  $0x02, R10
	JMP   index_loop_encodeBetterBlockAsm2MB

emit_remainder_encodeBetterBlockAsm2MB:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 4(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm2MB
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm2MB:
	MOVQ src_len+32(FP), AX

	// emitLiteralsDstP
	MOVL 12(SP), BX
	CMPL BX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm2MB
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (DX)(BX*1), AX
	SUBL BX, SI

	// emitLiteral
	LEAL -1(SI), DX
	CMPL DX, $0x1d
	JB   one_byte_emit_remainder_encodeBetterBlockAsm2MB
	SUBL $0x1d, DX
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm2MB
	CMPL DX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm2MB
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm2MB

three_bytes_emit_remainder_encodeBetterBlockAsm2MB:
	MOVB $0xf0, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm2MB

two_bytes_emit_remainder_encodeBetterBlockAsm2MB:
	MOVB $0xe8, (CX)
	MOVB DL, 1(CX)
	ADDL $0x1d, DX
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_midemit_remainder_encodeBetterBlockAsm2MB
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm2MB

one_byte_emit_remainder_encodeBetterBlockAsm2MB:
	SHLB $0x03, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -1, min move: 1
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_3
	CMPQ BX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_4through8
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_4through8:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm2MB
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm2MB

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm2MB:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm2MB

memmove_midemit_remainder_encodeBetterBlockAsm2MB:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -2, min move: 30
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm2MB

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm2MB_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm2MB:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm2MB

memmove_long_emit_remainder_encodeBetterBlockAsm2MB:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU   (AX), X0
	MOVOU   16(AX), X1
	MOVOU   -32(AX)(BX*1), X2
	MOVOU   -16(AX)(BX*1), X3
	MOVQ    BX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32
	LEAQ    -32(AX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm2MBlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm2MBlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm2MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm2MB:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm512K(dst []byte, src []byte, tmp *[294912]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBetterBlockAsm512K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000900, DX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm512K:
	MOVOU   X0, (AX)
	MOVOU   X0, 16(AX)
	MOVOU   X0, 32(AX)
	MOVOU   X0, 48(AX)
	MOVOU   X0, 64(AX)
	MOVOU   X0, 80(AX)
	MOVOU   X0, 96(AX)
	MOVOU   X0, 112(AX)
	ADDQ    $0x80, AX
	DECQ    DX
	JNZ     zero_loop_encodeBetterBlockAsm512K
	MOVL    $0x00000000, 12(SP)
	MOVQ    src_len+32(FP), AX
	LEAQ    -11(AX), DX
	LEAQ    -8(AX), BX
	MOVL    BX, 8(SP)
	SHRQ    $0x05, AX
	SUBL    AX, DX
	LEAQ    (CX)(DX*1), DX
	MOVQ    DX, (SP)
	MOVL    $0x00000001, AX
	MOVL    AX, 16(SP)
	MOVQ    src_base+24(FP), DX
	PCALIGN $0x10

search_loop_encodeBetterBlockAsm512K:
	MOVQ tmp+48(FP), BX
	MOVL AX, SI
	SUBL 12(SP), SI
	SHRL $0x07, SI
	CMPL SI, $0x63
	JBE  check_maxskip_ok_encodeBetterBlockAsm512K
	LEAL 100(AX), SI
	JMP  check_maxskip_cont_encodeBetterBlockAsm512K

check_maxskip_ok_encodeBetterBlockAsm512K:
	LEAL 1(AX)(SI*1), SI

check_maxskip_cont_encodeBetterBlockAsm512K:
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBetterBlockAsm512K
	MOVQ  (DX)(AX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x00cf1bbcdcbfa563, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x30, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x33, R11
	MOVL  (BX)(R10*4), SI
	MOVL  262144(BX)(R11*4), R8
	MOVL  AX, (BX)(R10*4)
	MOVL  AX, 262144(BX)(R11*4)
	MOVQ  (DX)(SI*1), R10
	CMPQ  R10, DI
	JEQ   candidate_match_encodeBetterBlockAsm512K
	MOVQ  (DX)(R8*1), R11
	CMPQ  R11, DI
	MOVL  AX, R12
	SUBL  16(SP), R12
	MOVQ  (DX)(R12*1), R12
	MOVQ  $0x000000ffffffff00, R13
	XORQ  DI, R12
	TESTQ R13, R12
	JNE   no_repeat_found_encodeBetterBlockAsm512K
	LEAL  1(AX), BX
	MOVL  12(SP), SI
	MOVL  BX, DI
	SUBL  16(SP), DI
	JZ    repeat_extend_back_end_encodeBetterBlockAsm512K

repeat_extend_back_loop_encodeBetterBlockAsm512K:
	CMPL BX, SI
	JBE  repeat_extend_back_end_encodeBetterBlockAsm512K
	MOVB -1(DX)(DI*1), R8
	MOVB -1(DX)(BX*1), R9
	CMPB R8, R9
	JNE  repeat_extend_back_end_encodeBetterBlockAsm512K
	LEAL -1(BX), BX
	DECL DI
	JNZ  repeat_extend_back_loop_encodeBetterBlockAsm512K

repeat_extend_back_end_encodeBetterBlockAsm512K:
	MOVL BX, SI
	SUBL 12(SP), SI
	LEAQ 4(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBetterBlockAsm512K
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBetterBlockAsm512K:
	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_repeat_emit_encodeBetterBlockAsm512K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_repeat_emit_encodeBetterBlockAsm512K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBetterBlockAsm512K
	CMPL SI, $0x00010000
	JB   three_bytes_repeat_emit_encodeBetterBlockAsm512K
	MOVL SI, R9
	SHRL $0x10, R9
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R9, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm512K

three_bytes_repeat_emit_encodeBetterBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm512K

two_bytes_repeat_emit_encodeBetterBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midrepeat_emit_encodeBetterBlockAsm512K
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm512K

one_byte_repeat_emit_encodeBetterBlockAsm512K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm512K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm512K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm512K_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm512K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm512K
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm512K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm512K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_repeat_emit_encodeBetterBlockAsm512K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_repeat_emit_encodeBetterBlockAsm512K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm512K

memmove_midrepeat_emit_encodeBetterBlockAsm512K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm512K

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm512K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm512K

memmove_long_repeat_emit_encodeBetterBlockAsm512K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R11
	SUBQ    R9, R11
	DECQ    R10
	JA      emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R11*1), R9
	LEAQ    -32(CX)(R11*1), R12
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm512Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R12)
	MOVOA X5, 16(R12)
	ADDQ  $0x20, R12
	ADDQ  $0x20, R9
	ADDQ  $0x20, R11
	DECQ  R10
	JNA   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R11*1), X4
	MOVOU -16(R8)(R11*1), X5
	MOVOA X4, -32(CX)(R11*1)
	MOVOA X5, -16(CX)(R11*1)
	ADDQ  $0x20, R11
	CMPQ  DI, R11
	JAE   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBetterBlockAsm512K:
	ADDL $0x05, AX
	MOVL AX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), SI

	// matchLen
	XORL R10, R10
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm512K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm512K:
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm512K
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16repeat_extend_encodeBetterBlockAsm512K
	LEAL -16(DI), DI
	LEAL 16(R10), R10

matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm512K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm512K
	JMP  matchlen_match8_repeat_extend_encodeBetterBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBetterBlockAsm512K:
	TZCNTQ R11, R11
	SARQ   $0x03, R11
	LEAL   8(R10)(R11*1), R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm512K

matchlen_match8_repeat_extend_encodeBetterBlockAsm512K:
	CMPL DI, $0x08
	JB   matchlen_match4_repeat_extend_encodeBetterBlockAsm512K
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm512K
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_repeat_extend_encodeBetterBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm512K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm512K

matchlen_match4_repeat_extend_encodeBetterBlockAsm512K:
	CMPL DI, $0x04
	JB   matchlen_match2_repeat_extend_encodeBetterBlockAsm512K
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_repeat_extend_encodeBetterBlockAsm512K
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_repeat_extend_encodeBetterBlockAsm512K:
	CMPL DI, $0x01
	JE   matchlen_match1_repeat_extend_encodeBetterBlockAsm512K
	JB   repeat_extend_forward_end_encodeBetterBlockAsm512K
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_repeat_extend_encodeBetterBlockAsm512K
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   repeat_extend_forward_end_encodeBetterBlockAsm512K

matchlen_match1_repeat_extend_encodeBetterBlockAsm512K:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  repeat_extend_forward_end_encodeBetterBlockAsm512K
	LEAL 1(R10), R10
	JMP  repeat_extend_forward_end_encodeBetterBlockAsm512K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBetterBlockAsm512K:
	ADDL R10, AX
	MOVL AX, SI
	SUBL BX, SI
	MOVL 16(SP), BX

	// emitRepeat
	LEAL -1(SI), BX
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBetterBlockAsm512K
	LEAL -30(SI), BX
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBetterBlockAsm512K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBetterBlockAsm512K
	MOVB $0xfc, (CX)
	MOVL BX, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm512K

repeat_three_match_repeat_encodeBetterBlockAsm512K:
	MOVB $0xf4, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm512K

repeat_two_match_repeat_encodeBetterBlockAsm512K:
	MOVB $0xec, (CX)
	MOVB BL, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm512K

repeat_one_match_repeat_encodeBetterBlockAsm512K:
	XORL BX, BX
	LEAL -4(BX)(SI*8), BX
	MOVB BL, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBetterBlockAsm512K:
	MOVL AX, 12(SP)
	JMP  search_loop_encodeBetterBlockAsm512K
	PCALIGN $0x10

no_repeat_found_encodeBetterBlockAsm512K:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm512K
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm512K
	MOVL 20(SP), AX
	JMP  search_loop_encodeBetterBlockAsm512K

candidateS_match_encodeBetterBlockAsm512K:
	SHRQ    $0x08, DI
	MOVQ    DI, R10
	SHLQ    $0x08, R10
	IMULQ   R9, R10
	SHRQ    $0x30, R10
	MOVL    (BX)(R10*4), SI
	INCL    AX
	MOVL    AX, (BX)(R10*4)
	CMPL    (DX)(SI*1), DI
	JEQ     candidate_match_encodeBetterBlockAsm512K
	DECL    AX
	MOVL    R8, SI
	PCALIGN $0x10

candidate_match_encodeBetterBlockAsm512K:
	MOVL  12(SP), BX
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm512K

match_extend_back_loop_encodeBetterBlockAsm512K:
	CMPL AX, BX
	JBE  match_extend_back_end_encodeBetterBlockAsm512K
	MOVB -1(DX)(SI*1), DI
	MOVB -1(DX)(AX*1), R8
	CMPB DI, R8
	JNE  match_extend_back_end_encodeBetterBlockAsm512K
	LEAL -1(AX), AX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm512K
	JMP  match_extend_back_loop_encodeBetterBlockAsm512K

match_extend_back_end_encodeBetterBlockAsm512K:
	MOVL AX, BX
	SUBL 12(SP), BX
	LEAQ 4(CX)(BX*1), BX
	CMPQ BX, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm512K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm512K:
	MOVL AX, BX
	ADDL $0x04, AX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), R9

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm512K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm512K:
	MOVQ (R8)(R11*1), R10
	MOVQ 8(R8)(R11*1), R12
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm512K
	XORQ 8(R9)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm512K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm512K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm512K
	JMP  matchlen_match8_match_nolit_encodeBetterBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBetterBlockAsm512K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBetterBlockAsm512K

matchlen_match8_match_nolit_encodeBetterBlockAsm512K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm512K
	MOVQ (R8)(R11*1), R10
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm512K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm512K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm512K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBetterBlockAsm512K

matchlen_match4_match_nolit_encodeBetterBlockAsm512K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm512K
	MOVL (R8)(R11*1), R10
	CMPL (R9)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm512K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBetterBlockAsm512K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm512K
	JB   match_nolit_end_encodeBetterBlockAsm512K
	MOVW (R8)(R11*1), R10
	CMPW (R9)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm512K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBetterBlockAsm512K

matchlen_match1_match_nolit_encodeBetterBlockAsm512K:
	MOVB (R8)(R11*1), R10
	CMPB (R9)(R11*1), R10
	JNE  match_nolit_end_encodeBetterBlockAsm512K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBetterBlockAsm512K
	PCALIGN $0x10

match_nolit_end_encodeBetterBlockAsm512K:
	MOVL AX, DI
	SUBL SI, DI
	CMPL R11, $0x01
	JA   match_length_ok_encodeBetterBlockAsm512K
	CMPL DI, $0x0001003f
	JBE  match_length_ok_encodeBetterBlockAsm512K
	MOVL 20(SP), AX
	INCL AX
	JMP  search_loop_encodeBetterBlockAsm512K

match_length_ok_encodeBetterBlockAsm512K:
	MOVL DI, 16(SP)

	// Check if we can combine lit+copy
	MOVLQZX 12(SP), R8
	MOVL    BX, SI
	SUBL    R8, SI
	JZ      match_emit_nolits_encodeBetterBlockAsm512K
	CMPL    DI, $0x00000040
	JL      match_emit_lits_encodeBetterBlockAsm512K
	CMPL    DI, $0x0001003f
	JA      match_emit_copy3_encodeBetterBlockAsm512K
	CMPL    SI, $0x04
	JA      match_emit_lits_encodeBetterBlockAsm512K
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, DI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    DI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, DI
	CMOVLLT R11, DI
	LEAL    -1(SI)(DI*4), DI
	MOVL    $0x00000003, R10
	LEAL    (R10)(DI*8), DI
	MOVB    DI, (CX)
	ADDQ    $0x03, CX
	MOVL    R8, (CX)
	ADDQ    SI, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBetterBlockAsm512K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm512K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm512K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm512K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm512K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm512K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm512K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

match_emit_copy3_encodeBetterBlockAsm512K:
	CMPL    SI, $0x03
	JA      match_emit_lits_encodeBetterBlockAsm512K
	MOVLQZX 12(SP), R8
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(DI), DI
	SHLL $0x0b, DI
	LEAL 7(DI)(SI*8), DI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_emit_lits_encodeBetterBlockAsm512K
	LEAL -60(R11), R9
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_emit_lits_encodeBetterBlockAsm512K
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_emit_lits_encodeBetterBlockAsm512K
	ADDL $0x000007e0, DI
	MOVL DI, (CX)
	MOVL R9, 4(CX)
	ADDQ $0x07, CX
	JMP  match_emit_copy_litsencodeBetterBlockAsm512K

emit_copy3_2_match_emit_lits_encodeBetterBlockAsm512K:
	ADDL $0x000007c0, DI
	MOVL DI, (CX)
	MOVW R9, 4(CX)
	ADDQ $0x06, CX
	JMP  match_emit_copy_litsencodeBetterBlockAsm512K

emit_copy3_1_match_emit_lits_encodeBetterBlockAsm512K:
	ADDL $0x000007a0, DI
	MOVL DI, (CX)
	MOVB R9, 4(CX)
	ADDQ $0x05, CX
	JMP  match_emit_copy_litsencodeBetterBlockAsm512K

emit_copy3_0_match_emit_lits_encodeBetterBlockAsm512K:
	SHLL $0x05, R11
	ORL  R11, DI
	MOVL DI, (CX)
	ADDQ $0x04, CX

match_emit_copy_litsencodeBetterBlockAsm512K:
	MOVL R8, (CX)
	ADDQ SI, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

match_emit_lits_encodeBetterBlockAsm512K:
	LEAQ (DX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBetterBlockAsm512K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm512K
	CMPL R9, $0x00010000
	JB   three_bytes_match_emit_encodeBetterBlockAsm512K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm512K

three_bytes_match_emit_encodeBetterBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm512K

two_bytes_match_emit_encodeBetterBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBetterBlockAsm512K
	JMP  memmove_long_match_emit_encodeBetterBlockAsm512K

one_byte_match_emit_encodeBetterBlockAsm512K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ SI, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm512K_memmove_move_8
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm512K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm512K_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm512K_memmove_move_8:
	MOVQ (R8), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm512K
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBetterBlockAsm512K_memmove_move_8through16:
	MOVQ (R8), R10
	MOVQ -8(R8)(SI*1), R8
	MOVQ R10, (CX)
	MOVQ R8, -8(CX)(SI*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm512K

emit_lit_memmove_match_emit_encodeBetterBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm512K

emit_lit_memmove_match_emit_encodeBetterBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm512K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm512K

memmove_midmatch_emit_encodeBetterBlockAsm512K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBetterBlockAsm512K

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_match_emit_encodeBetterBlockAsm512K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm512K

memmove_long_match_emit_encodeBetterBlockAsm512K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBetterBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm512Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R13*1), X4
	MOVOU -16(R8)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  SI, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

match_emit_nolits_encodeBetterBlockAsm512K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitCopy
	CMPL DI, $0x0001003f
	JBE  two_byte_offset_match_nolit_encodeBetterBlockAsm512K

	// emitCopy3
	LEAL -4(R11), R11
	LEAL -65536(DI), SI
	SHLL $0x0b, SI
	ADDL $0x07, SI
	CMPL R11, $0x3c
	JBE  emit_copy3_0_match_nolit_encodeBetterBlockAsm512K_emit3
	LEAL -60(R11), DI
	CMPL R11, $0x0000013c
	JB   emit_copy3_1_match_nolit_encodeBetterBlockAsm512K_emit3
	CMPL R11, $0x0001003c
	JB   emit_copy3_2_match_nolit_encodeBetterBlockAsm512K_emit3
	ADDL $0x000007e0, SI
	MOVL SI, (CX)
	MOVL DI, 4(CX)
	ADDQ $0x07, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

emit_copy3_2_match_nolit_encodeBetterBlockAsm512K_emit3:
	ADDL $0x000007c0, SI
	MOVL SI, (CX)
	MOVW DI, 4(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

emit_copy3_1_match_nolit_encodeBetterBlockAsm512K_emit3:
	ADDL $0x000007a0, SI
	MOVL SI, (CX)
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

emit_copy3_0_match_nolit_encodeBetterBlockAsm512K_emit3:
	SHLL $0x05, R11
	ORL  R11, SI
	MOVL SI, (CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

two_byte_offset_match_nolit_encodeBetterBlockAsm512K:
	CMPL DI, $0x00000400
	JA   two_byte_match_nolit_encodeBetterBlockAsm512K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBetterBlockAsm512K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

emit_one_longer_match_nolit_encodeBetterBlockAsm512K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBetterBlockAsm512K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

emit_copy1_repeat_match_nolit_encodeBetterBlockAsm512K:
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm512K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm512K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm512K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm512K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm512K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm512K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

two_byte_match_nolit_encodeBetterBlockAsm512K:
	// emitCopy2
	LEAL -64(DI), DI
	LEAL -4(R11), R11
	MOVW DI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBetterBlockAsm512K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBetterBlockAsm512K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBetterBlockAsm512K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

emit_copy2_2_match_nolit_encodeBetterBlockAsm512K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

emit_copy2_1_match_nolit_encodeBetterBlockAsm512K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

emit_copy2_0_match_nolit_encodeBetterBlockAsm512K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm512K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm512K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm512K
	CMPL SI, $0x00010000
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm512K
	MOVL SI, R9
	SHRL $0x10, R9
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R9, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm512K

three_bytes_match_emit_repeat_encodeBetterBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm512K

two_bytes_match_emit_repeat_encodeBetterBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midmatch_emit_repeat_encodeBetterBlockAsm512K
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm512K

one_byte_match_emit_repeat_encodeBetterBlockAsm512K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm512K
	PCALIGN $0x10

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm512K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm512K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm512K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm512K

memmove_midmatch_emit_repeat_encodeBetterBlockAsm512K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm512K

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm512K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm512K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm512K

memmove_long_match_emit_repeat_encodeBetterBlockAsm512K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R12
	SUBQ    R9, R12
	DECQ    R10
	JA      emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R9
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm512Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R9
	ADDQ  $0x20, R12
	DECQ  R10
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  DI, R12
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm512K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_match_nolit_repeat_encodeBetterBlockAsm512K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_match_nolit_repeat_encodeBetterBlockAsm512K
	CMPL R11, $0x0001001e
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm512K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

repeat_three_match_nolit_repeat_encodeBetterBlockAsm512K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

repeat_two_match_nolit_repeat_encodeBetterBlockAsm512K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm512K

repeat_one_match_nolit_repeat_encodeBetterBlockAsm512K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm512K:
	CMPL AX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm512K
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm512K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm512K:
	MOVQ  tmp+48(FP), SI
	MOVQ  $0x00cf1bbcdcbfa563, DI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(BX), BX
	LEAQ  -2(AX), R9
	MOVQ  (DX)(BX*1), R10
	MOVQ  1(DX)(BX*1), R11
	MOVQ  (DX)(R9*1), R12
	MOVQ  1(DX)(R9*1), R13
	SHLQ  $0x08, R10
	IMULQ DI, R10
	SHRQ  $0x30, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x33, R11
	SHLQ  $0x08, R12
	IMULQ DI, R12
	SHRQ  $0x30, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x33, R13
	LEAQ  1(BX), R8
	LEAQ  1(R9), R14
	MOVL  BX, (SI)(R10*4)
	MOVL  R9, (SI)(R12*4)
	LEAQ  1(R9)(BX*1), R10
	SHRQ  $0x01, R10
	ADDQ  $0x01, BX
	SUBQ  $0x01, R9
	MOVL  R8, 262144(SI)(R11*4)
	MOVL  R14, 262144(SI)(R13*4)

index_loop_encodeBetterBlockAsm512K:
	CMPQ  R10, R9
	JAE   search_loop_encodeBetterBlockAsm512K
	MOVQ  (DX)(BX*1), R8
	MOVQ  (DX)(R10*1), R11
	SHLQ  $0x08, R8
	IMULQ DI, R8
	SHRQ  $0x30, R8
	SHLQ  $0x08, R11
	IMULQ DI, R11
	SHRQ  $0x30, R11
	MOVL  BX, (SI)(R8*4)
	MOVL  R9, (SI)(R11*4)
	ADDQ  $0x02, BX
	ADDQ  $0x02, R10
	JMP   index_loop_encodeBetterBlockAsm512K

emit_remainder_encodeBetterBlockAsm512K:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 4(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm512K
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm512K:
	MOVQ src_len+32(FP), AX

	// emitLiteralsDstP
	MOVL 12(SP), BX
	CMPL BX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm512K
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (DX)(BX*1), AX
	SUBL BX, SI

	// emitLiteral
	LEAL -1(SI), DX
	CMPL DX, $0x1d
	JB   one_byte_emit_remainder_encodeBetterBlockAsm512K
	SUBL $0x1d, DX
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm512K
	CMPL DX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm512K
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm512K

three_bytes_emit_remainder_encodeBetterBlockAsm512K:
	MOVB $0xf0, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm512K

two_bytes_emit_remainder_encodeBetterBlockAsm512K:
	MOVB $0xe8, (CX)
	MOVB DL, 1(CX)
	ADDL $0x1d, DX
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_midemit_remainder_encodeBetterBlockAsm512K
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm512K

one_byte_emit_remainder_encodeBetterBlockAsm512K:
	SHLB $0x03, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -1, min move: 1
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_3
	CMPQ BX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_4through8
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm512K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm512K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_4through8:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm512K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm512K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm512K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm512K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm512K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm512K

memmove_midemit_remainder_encodeBetterBlockAsm512K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -2, min move: 30
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm512K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm512K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm512K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm512K

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm512K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm512K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm512K

memmove_long_emit_remainder_encodeBetterBlockAsm512K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU   (AX), X0
	MOVOU   16(AX), X1
	MOVOU   -32(AX)(BX*1), X2
	MOVOU   -16(AX)(BX*1), X3
	MOVQ    BX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm512Klarge_forward_sse_loop_32
	LEAQ    -32(AX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm512Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm512Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm512Klarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm512Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm512K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm64K(dst []byte, src []byte, tmp *[73728]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBetterBlockAsm64K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000240, DX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm64K:
	MOVOU   X0, (AX)
	MOVOU   X0, 16(AX)
	MOVOU   X0, 32(AX)
	MOVOU   X0, 48(AX)
	MOVOU   X0, 64(AX)
	MOVOU   X0, 80(AX)
	MOVOU   X0, 96(AX)
	MOVOU   X0, 112(AX)
	ADDQ    $0x80, AX
	DECQ    DX
	JNZ     zero_loop_encodeBetterBlockAsm64K
	MOVL    $0x00000000, 12(SP)
	MOVQ    src_len+32(FP), AX
	LEAQ    -11(AX), DX
	LEAQ    -8(AX), BX
	MOVL    BX, 8(SP)
	SHRQ    $0x05, AX
	SUBL    AX, DX
	LEAQ    (CX)(DX*1), DX
	MOVQ    DX, (SP)
	MOVL    $0x00000001, AX
	MOVL    AX, 16(SP)
	MOVQ    src_base+24(FP), DX
	PCALIGN $0x10

search_loop_encodeBetterBlockAsm64K:
	MOVQ    tmp+48(FP), BX
	MOVL    AX, SI
	SUBL    12(SP), SI
	SHRL    $0x06, SI
	LEAL    1(AX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeBetterBlockAsm64K
	MOVQ    (DX)(AX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0x0000cf1bbcdcbf9b, R9
	MOVQ    $0x9e3779b1, SI
	MOVQ    DI, R10
	MOVQ    DI, R11
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x31, R10
	SHLQ    $0x20, R11
	IMULQ   SI, R11
	SHRQ    $0x34, R11
	MOVWLZX (BX)(R10*2), SI
	MOVWLZX 65536(BX)(R11*2), R8
	MOVW    AX, (BX)(R10*2)
	MOVW    AX, 65536(BX)(R11*2)
	MOVQ    (DX)(SI*1), R10
	CMPQ    R10, DI
	JEQ     candidate_match_encodeBetterBlockAsm64K
	MOVQ    (DX)(R8*1), R11
	CMPQ    R11, DI
	MOVL    AX, R12
	SUBL    16(SP), R12
	MOVQ    (DX)(R12*1), R12
	MOVQ    $0x000000ffffffff00, R13
	XORQ    DI, R12
	TESTQ   R13, R12
	JNE     no_repeat_found_encodeBetterBlockAsm64K
	LEAL    1(AX), BX
	MOVL    12(SP), SI
	MOVL    BX, DI
	SUBL    16(SP), DI
	JZ      repeat_extend_back_end_encodeBetterBlockAsm64K

repeat_extend_back_loop_encodeBetterBlockAsm64K:
	CMPL BX, SI
	JBE  repeat_extend_back_end_encodeBetterBlockAsm64K
	MOVB -1(DX)(DI*1), R8
	MOVB -1(DX)(BX*1), R9
	CMPB R8, R9
	JNE  repeat_extend_back_end_encodeBetterBlockAsm64K
	LEAL -1(BX), BX
	DECL DI
	JNZ  repeat_extend_back_loop_encodeBetterBlockAsm64K

repeat_extend_back_end_encodeBetterBlockAsm64K:
	MOVL BX, SI
	SUBL 12(SP), SI
	LEAQ 4(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBetterBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBetterBlockAsm64K:
	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_repeat_emit_encodeBetterBlockAsm64K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_repeat_emit_encodeBetterBlockAsm64K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBetterBlockAsm64K
	JB   three_bytes_repeat_emit_encodeBetterBlockAsm64K
	MOVL SI, R9
	SHRL $0x10, R9
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R9, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm64K

three_bytes_repeat_emit_encodeBetterBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm64K

two_bytes_repeat_emit_encodeBetterBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midrepeat_emit_encodeBetterBlockAsm64K
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm64K

one_byte_repeat_emit_encodeBetterBlockAsm64K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm64K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm64K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm64K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm64K
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm64K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm64K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_repeat_emit_encodeBetterBlockAsm64K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_repeat_emit_encodeBetterBlockAsm64K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm64K

memmove_midrepeat_emit_encodeBetterBlockAsm64K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm64K

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm64K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm64K

memmove_long_repeat_emit_encodeBetterBlockAsm64K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R11
	SUBQ    R9, R11
	DECQ    R10
	JA      emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R11*1), R9
	LEAQ    -32(CX)(R11*1), R12
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm64Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R12)
	MOVOA X5, 16(R12)
	ADDQ  $0x20, R12
	ADDQ  $0x20, R9
	ADDQ  $0x20, R11
	DECQ  R10
	JNA   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R11*1), X4
	MOVOU -16(R8)(R11*1), X5
	MOVOA X4, -32(CX)(R11*1)
	MOVOA X5, -16(CX)(R11*1)
	ADDQ  $0x20, R11
	CMPQ  DI, R11
	JAE   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBetterBlockAsm64K:
	ADDL $0x05, AX
	MOVL AX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), SI

	// matchLen
	XORL R10, R10
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm64K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm64K:
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm64K
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16repeat_extend_encodeBetterBlockAsm64K
	LEAL -16(DI), DI
	LEAL 16(R10), R10

matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm64K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm64K
	JMP  matchlen_match8_repeat_extend_encodeBetterBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBetterBlockAsm64K:
	TZCNTQ R11, R11
	SARQ   $0x03, R11
	LEAL   8(R10)(R11*1), R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm64K

matchlen_match8_repeat_extend_encodeBetterBlockAsm64K:
	CMPL DI, $0x08
	JB   matchlen_match4_repeat_extend_encodeBetterBlockAsm64K
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm64K
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_repeat_extend_encodeBetterBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm64K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm64K

matchlen_match4_repeat_extend_encodeBetterBlockAsm64K:
	CMPL DI, $0x04
	JB   matchlen_match2_repeat_extend_encodeBetterBlockAsm64K
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_repeat_extend_encodeBetterBlockAsm64K
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_repeat_extend_encodeBetterBlockAsm64K:
	CMPL DI, $0x01
	JE   matchlen_match1_repeat_extend_encodeBetterBlockAsm64K
	JB   repeat_extend_forward_end_encodeBetterBlockAsm64K
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_repeat_extend_encodeBetterBlockAsm64K
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   repeat_extend_forward_end_encodeBetterBlockAsm64K

matchlen_match1_repeat_extend_encodeBetterBlockAsm64K:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  repeat_extend_forward_end_encodeBetterBlockAsm64K
	LEAL 1(R10), R10
	JMP  repeat_extend_forward_end_encodeBetterBlockAsm64K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBetterBlockAsm64K:
	ADDL R10, AX
	MOVL AX, SI
	SUBL BX, SI
	MOVL 16(SP), BX

	// emitRepeat
	LEAL -1(SI), BX
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBetterBlockAsm64K
	LEAL -30(SI), BX
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBetterBlockAsm64K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBetterBlockAsm64K
	MOVB $0xfc, (CX)
	MOVL BX, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm64K

repeat_three_match_repeat_encodeBetterBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm64K

repeat_two_match_repeat_encodeBetterBlockAsm64K:
	MOVB $0xec, (CX)
	MOVB BL, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm64K

repeat_one_match_repeat_encodeBetterBlockAsm64K:
	XORL BX, BX
	LEAL -4(BX)(SI*8), BX
	MOVB BL, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBetterBlockAsm64K:
	MOVL AX, 12(SP)
	JMP  search_loop_encodeBetterBlockAsm64K
	PCALIGN $0x10

no_repeat_found_encodeBetterBlockAsm64K:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm64K
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm64K
	MOVL 20(SP), AX
	JMP  search_loop_encodeBetterBlockAsm64K

candidateS_match_encodeBetterBlockAsm64K:
	SHRQ    $0x08, DI
	MOVQ    DI, R10
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x31, R10
	MOVWLZX (BX)(R10*2), SI
	INCL    AX
	MOVW    AX, (BX)(R10*2)
	CMPL    (DX)(SI*1), DI
	JEQ     candidate_match_encodeBetterBlockAsm64K
	DECL    AX
	MOVL    R8, SI
	PCALIGN $0x10

candidate_match_encodeBetterBlockAsm64K:
	MOVL  12(SP), BX
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm64K

match_extend_back_loop_encodeBetterBlockAsm64K:
	CMPL AX, BX
	JBE  match_extend_back_end_encodeBetterBlockAsm64K
	MOVB -1(DX)(SI*1), DI
	MOVB -1(DX)(AX*1), R8
	CMPB DI, R8
	JNE  match_extend_back_end_encodeBetterBlockAsm64K
	LEAL -1(AX), AX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm64K
	JMP  match_extend_back_loop_encodeBetterBlockAsm64K

match_extend_back_end_encodeBetterBlockAsm64K:
	MOVL AX, BX
	SUBL 12(SP), BX
	LEAQ 4(CX)(BX*1), BX
	CMPQ BX, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm64K:
	MOVL AX, BX
	ADDL $0x04, AX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), R9

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm64K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm64K:
	MOVQ (R8)(R11*1), R10
	MOVQ 8(R8)(R11*1), R12
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm64K
	XORQ 8(R9)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm64K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm64K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm64K
	JMP  matchlen_match8_match_nolit_encodeBetterBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBetterBlockAsm64K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBetterBlockAsm64K

matchlen_match8_match_nolit_encodeBetterBlockAsm64K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm64K
	MOVQ (R8)(R11*1), R10
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm64K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm64K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm64K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBetterBlockAsm64K

matchlen_match4_match_nolit_encodeBetterBlockAsm64K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm64K
	MOVL (R8)(R11*1), R10
	CMPL (R9)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm64K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBetterBlockAsm64K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm64K
	JB   match_nolit_end_encodeBetterBlockAsm64K
	MOVW (R8)(R11*1), R10
	CMPW (R9)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm64K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBetterBlockAsm64K

matchlen_match1_match_nolit_encodeBetterBlockAsm64K:
	MOVB (R8)(R11*1), R10
	CMPB (R9)(R11*1), R10
	JNE  match_nolit_end_encodeBetterBlockAsm64K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBetterBlockAsm64K
	PCALIGN $0x10

match_nolit_end_encodeBetterBlockAsm64K:
	MOVL AX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)

	// Check if we can combine lit+copy
	MOVLQZX 12(SP), R8
	MOVL    BX, SI
	SUBL    R8, SI
	JZ      match_emit_nolits_encodeBetterBlockAsm64K
	CMPL    DI, $0x00000040
	JL      match_emit_lits_encodeBetterBlockAsm64K
	CMPL    SI, $0x04
	JA      match_emit_lits_encodeBetterBlockAsm64K
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, DI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    DI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, DI
	CMOVLLT R11, DI
	LEAL    -1(SI)(DI*4), DI
	MOVL    $0x00000003, R10
	LEAL    (R10)(DI*8), DI
	MOVB    DI, (CX)
	ADDQ    $0x03, CX
	MOVL    R8, (CX)
	ADDQ    SI, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBetterBlockAsm64K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm64K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm64K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm64K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm64K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm64K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

match_emit_lits_encodeBetterBlockAsm64K:
	LEAQ (DX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBetterBlockAsm64K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm64K
	JB   three_bytes_match_emit_encodeBetterBlockAsm64K
	MOVL R9, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R9, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm64K

three_bytes_match_emit_encodeBetterBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm64K

two_bytes_match_emit_encodeBetterBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBetterBlockAsm64K
	JMP  memmove_long_match_emit_encodeBetterBlockAsm64K

one_byte_match_emit_encodeBetterBlockAsm64K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ SI, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm64K_memmove_move_8
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm64K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm64K_memmove_move_8:
	MOVQ (R8), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm64K
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBetterBlockAsm64K_memmove_move_8through16:
	MOVQ (R8), R10
	MOVQ -8(R8)(SI*1), R8
	MOVQ R10, (CX)
	MOVQ R8, -8(CX)(SI*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm64K

emit_lit_memmove_match_emit_encodeBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm64K

emit_lit_memmove_match_emit_encodeBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm64K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm64K

memmove_midmatch_emit_encodeBetterBlockAsm64K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBetterBlockAsm64K

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_match_emit_encodeBetterBlockAsm64K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm64K

memmove_long_match_emit_encodeBetterBlockAsm64K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBetterBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm64Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R13*1), X4
	MOVOU -16(R8)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  SI, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

match_emit_nolits_encodeBetterBlockAsm64K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitCopy
	CMPL DI, $0x00000400
	JA   two_byte_match_nolit_encodeBetterBlockAsm64K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBetterBlockAsm64K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

emit_one_longer_match_nolit_encodeBetterBlockAsm64K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBetterBlockAsm64K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

emit_copy1_repeat_match_nolit_encodeBetterBlockAsm64K:
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm64K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm64K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm64K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm64K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm64K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

two_byte_match_nolit_encodeBetterBlockAsm64K:
	// emitCopy2
	LEAL -64(DI), DI
	LEAL -4(R11), R11
	MOVW DI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBetterBlockAsm64K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBetterBlockAsm64K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBetterBlockAsm64K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

emit_copy2_2_match_nolit_encodeBetterBlockAsm64K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

emit_copy2_1_match_nolit_encodeBetterBlockAsm64K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

emit_copy2_0_match_nolit_encodeBetterBlockAsm64K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm64K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm64K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm64K
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm64K
	MOVL SI, R9
	SHRL $0x10, R9
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R9, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm64K

three_bytes_match_emit_repeat_encodeBetterBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm64K

two_bytes_match_emit_repeat_encodeBetterBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midmatch_emit_repeat_encodeBetterBlockAsm64K
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm64K

one_byte_match_emit_repeat_encodeBetterBlockAsm64K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm64K
	PCALIGN $0x10

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm64K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm64K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm64K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm64K

memmove_midmatch_emit_repeat_encodeBetterBlockAsm64K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm64K

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm64K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm64K

memmove_long_match_emit_repeat_encodeBetterBlockAsm64K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R12
	SUBQ    R9, R12
	DECQ    R10
	JA      emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R9
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm64Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R9
	ADDQ  $0x20, R12
	DECQ  R10
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  DI, R12
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm64K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_match_nolit_repeat_encodeBetterBlockAsm64K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_match_nolit_repeat_encodeBetterBlockAsm64K
	CMPL R11, $0x0001001e
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm64K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

repeat_three_match_nolit_repeat_encodeBetterBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

repeat_two_match_nolit_repeat_encodeBetterBlockAsm64K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm64K

repeat_one_match_nolit_repeat_encodeBetterBlockAsm64K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm64K:
	CMPL AX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm64K
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm64K:
	MOVQ  tmp+48(FP), SI
	MOVQ  $0x0000cf1bbcdcbf9b, DI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(BX), BX
	LEAQ  -2(AX), R9
	MOVQ  (DX)(BX*1), R10
	MOVQ  1(DX)(BX*1), R11
	MOVQ  (DX)(R9*1), R12
	MOVQ  1(DX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ DI, R10
	SHRQ  $0x31, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x34, R11
	SHLQ  $0x10, R12
	IMULQ DI, R12
	SHRQ  $0x31, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x34, R13
	LEAQ  1(BX), R8
	LEAQ  1(R9), R14
	MOVW  BX, (SI)(R10*2)
	MOVW  R9, (SI)(R12*2)
	LEAQ  1(R9)(BX*1), R10
	SHRQ  $0x01, R10
	ADDQ  $0x01, BX
	SUBQ  $0x01, R9
	MOVW  R8, 65536(SI)(R11*2)
	MOVW  R14, 65536(SI)(R13*2)

index_loop_encodeBetterBlockAsm64K:
	CMPQ  R10, R9
	JAE   search_loop_encodeBetterBlockAsm64K
	MOVQ  (DX)(BX*1), R8
	MOVQ  (DX)(R10*1), R11
	SHLQ  $0x10, R8
	IMULQ DI, R8
	SHRQ  $0x31, R8
	SHLQ  $0x10, R11
	IMULQ DI, R11
	SHRQ  $0x31, R11
	MOVW  BX, (SI)(R8*2)
	MOVW  R9, (SI)(R11*2)
	ADDQ  $0x02, BX
	ADDQ  $0x02, R10
	JMP   index_loop_encodeBetterBlockAsm64K

emit_remainder_encodeBetterBlockAsm64K:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 4(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm64K:
	MOVQ src_len+32(FP), AX

	// emitLiteralsDstP
	MOVL 12(SP), BX
	CMPL BX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm64K
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (DX)(BX*1), AX
	SUBL BX, SI

	// emitLiteral
	LEAL -1(SI), DX
	CMPL DX, $0x1d
	JB   one_byte_emit_remainder_encodeBetterBlockAsm64K
	SUBL $0x1d, DX
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm64K
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm64K
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm64K

three_bytes_emit_remainder_encodeBetterBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm64K

two_bytes_emit_remainder_encodeBetterBlockAsm64K:
	MOVB $0xe8, (CX)
	MOVB DL, 1(CX)
	ADDL $0x1d, DX
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_midemit_remainder_encodeBetterBlockAsm64K
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm64K

one_byte_emit_remainder_encodeBetterBlockAsm64K:
	SHLB $0x03, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -1, min move: 1
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_3
	CMPQ BX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_4through8
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm64K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm64K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_4through8:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm64K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm64K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm64K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm64K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm64K

memmove_midemit_remainder_encodeBetterBlockAsm64K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -2, min move: 30
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm64K

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm64K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm64K

memmove_long_emit_remainder_encodeBetterBlockAsm64K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU   (AX), X0
	MOVOU   16(AX), X1
	MOVOU   -32(AX)(BX*1), X2
	MOVOU   -16(AX)(BX*1), X3
	MOVQ    BX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm64Klarge_forward_sse_loop_32
	LEAQ    -32(AX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm64Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm64K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm16K(dst []byte, src []byte, tmp *[36864]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBetterBlockAsm16K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000120, DX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm16K:
	MOVOU   X0, (AX)
	MOVOU   X0, 16(AX)
	MOVOU   X0, 32(AX)
	MOVOU   X0, 48(AX)
	MOVOU   X0, 64(AX)
	MOVOU   X0, 80(AX)
	MOVOU   X0, 96(AX)
	MOVOU   X0, 112(AX)
	ADDQ    $0x80, AX
	DECQ    DX
	JNZ     zero_loop_encodeBetterBlockAsm16K
	MOVL    $0x00000000, 12(SP)
	MOVQ    src_len+32(FP), AX
	LEAQ    -11(AX), DX
	LEAQ    -8(AX), BX
	MOVL    BX, 8(SP)
	SHRQ    $0x05, AX
	SUBL    AX, DX
	LEAQ    (CX)(DX*1), DX
	MOVQ    DX, (SP)
	MOVL    $0x00000001, AX
	MOVL    AX, 16(SP)
	MOVQ    src_base+24(FP), DX
	PCALIGN $0x10

search_loop_encodeBetterBlockAsm16K:
	MOVQ    tmp+48(FP), BX
	MOVL    AX, SI
	SUBL    12(SP), SI
	SHRL    $0x06, SI
	LEAL    1(AX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeBetterBlockAsm16K
	MOVQ    (DX)(AX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0x0000cf1bbcdcbf9b, R9
	MOVQ    $0x9e3779b1, SI
	MOVQ    DI, R10
	MOVQ    DI, R11
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x32, R10
	SHLQ    $0x20, R11
	IMULQ   SI, R11
	SHRQ    $0x35, R11
	MOVWLZX (BX)(R10*2), SI
	MOVWLZX 32768(BX)(R11*2), R8
	MOVW    AX, (BX)(R10*2)
	MOVW    AX, 32768(BX)(R11*2)
	MOVQ    (DX)(SI*1), R10
	CMPQ    R10, DI
	JEQ     candidate_match_encodeBetterBlockAsm16K
	MOVQ    (DX)(R8*1), R11
	CMPQ    R11, DI
	MOVL    AX, R12
	SUBL    16(SP), R12
	MOVQ    (DX)(R12*1), R12
	MOVQ    $0x000000ffffffff00, R13
	XORQ    DI, R12
	TESTQ   R13, R12
	JNE     no_repeat_found_encodeBetterBlockAsm16K
	LEAL    1(AX), BX
	MOVL    12(SP), SI
	MOVL    BX, DI
	SUBL    16(SP), DI
	JZ      repeat_extend_back_end_encodeBetterBlockAsm16K

repeat_extend_back_loop_encodeBetterBlockAsm16K:
	CMPL BX, SI
	JBE  repeat_extend_back_end_encodeBetterBlockAsm16K
	MOVB -1(DX)(DI*1), R8
	MOVB -1(DX)(BX*1), R9
	CMPB R8, R9
	JNE  repeat_extend_back_end_encodeBetterBlockAsm16K
	LEAL -1(BX), BX
	DECL DI
	JNZ  repeat_extend_back_loop_encodeBetterBlockAsm16K

repeat_extend_back_end_encodeBetterBlockAsm16K:
	MOVL BX, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBetterBlockAsm16K
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBetterBlockAsm16K:
	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_repeat_emit_encodeBetterBlockAsm16K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_repeat_emit_encodeBetterBlockAsm16K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBetterBlockAsm16K
	JB   three_bytes_repeat_emit_encodeBetterBlockAsm16K

three_bytes_repeat_emit_encodeBetterBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm16K

two_bytes_repeat_emit_encodeBetterBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midrepeat_emit_encodeBetterBlockAsm16K
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm16K

one_byte_repeat_emit_encodeBetterBlockAsm16K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm16K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm16K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm16K_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm16K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm16K
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm16K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm16K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_repeat_emit_encodeBetterBlockAsm16K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_repeat_emit_encodeBetterBlockAsm16K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm16K

memmove_midrepeat_emit_encodeBetterBlockAsm16K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm16K

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm16K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm16K

memmove_long_repeat_emit_encodeBetterBlockAsm16K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R11
	SUBQ    R9, R11
	DECQ    R10
	JA      emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R11*1), R9
	LEAQ    -32(CX)(R11*1), R12
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm16Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R12)
	MOVOA X5, 16(R12)
	ADDQ  $0x20, R12
	ADDQ  $0x20, R9
	ADDQ  $0x20, R11
	DECQ  R10
	JNA   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R11*1), X4
	MOVOU -16(R8)(R11*1), X5
	MOVOA X4, -32(CX)(R11*1)
	MOVOA X5, -16(CX)(R11*1)
	ADDQ  $0x20, R11
	CMPQ  DI, R11
	JAE   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBetterBlockAsm16K:
	ADDL $0x05, AX
	MOVL AX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), SI

	// matchLen
	XORL R10, R10
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm16K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm16K:
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm16K
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16repeat_extend_encodeBetterBlockAsm16K
	LEAL -16(DI), DI
	LEAL 16(R10), R10

matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm16K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm16K
	JMP  matchlen_match8_repeat_extend_encodeBetterBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBetterBlockAsm16K:
	TZCNTQ R11, R11
	SARQ   $0x03, R11
	LEAL   8(R10)(R11*1), R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm16K

matchlen_match8_repeat_extend_encodeBetterBlockAsm16K:
	CMPL DI, $0x08
	JB   matchlen_match4_repeat_extend_encodeBetterBlockAsm16K
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm16K
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_repeat_extend_encodeBetterBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm16K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm16K

matchlen_match4_repeat_extend_encodeBetterBlockAsm16K:
	CMPL DI, $0x04
	JB   matchlen_match2_repeat_extend_encodeBetterBlockAsm16K
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_repeat_extend_encodeBetterBlockAsm16K
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_repeat_extend_encodeBetterBlockAsm16K:
	CMPL DI, $0x01
	JE   matchlen_match1_repeat_extend_encodeBetterBlockAsm16K
	JB   repeat_extend_forward_end_encodeBetterBlockAsm16K
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_repeat_extend_encodeBetterBlockAsm16K
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   repeat_extend_forward_end_encodeBetterBlockAsm16K

matchlen_match1_repeat_extend_encodeBetterBlockAsm16K:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  repeat_extend_forward_end_encodeBetterBlockAsm16K
	LEAL 1(R10), R10
	JMP  repeat_extend_forward_end_encodeBetterBlockAsm16K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBetterBlockAsm16K:
	ADDL R10, AX
	MOVL AX, SI
	SUBL BX, SI
	MOVL 16(SP), BX

	// emitRepeat
	LEAL -1(SI), BX
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBetterBlockAsm16K
	LEAL -30(SI), BX
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBetterBlockAsm16K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBetterBlockAsm16K
	MOVB $0xfc, (CX)
	MOVL BX, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm16K

repeat_three_match_repeat_encodeBetterBlockAsm16K:
	MOVB $0xf4, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm16K

repeat_two_match_repeat_encodeBetterBlockAsm16K:
	MOVB $0xec, (CX)
	MOVB BL, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm16K

repeat_one_match_repeat_encodeBetterBlockAsm16K:
	XORL BX, BX
	LEAL -4(BX)(SI*8), BX
	MOVB BL, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBetterBlockAsm16K:
	MOVL AX, 12(SP)
	JMP  search_loop_encodeBetterBlockAsm16K
	PCALIGN $0x10

no_repeat_found_encodeBetterBlockAsm16K:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm16K
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm16K
	MOVL 20(SP), AX
	JMP  search_loop_encodeBetterBlockAsm16K

candidateS_match_encodeBetterBlockAsm16K:
	SHRQ    $0x08, DI
	MOVQ    DI, R10
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x32, R10
	MOVWLZX (BX)(R10*2), SI
	INCL    AX
	MOVW    AX, (BX)(R10*2)
	CMPL    (DX)(SI*1), DI
	JEQ     candidate_match_encodeBetterBlockAsm16K
	DECL    AX
	MOVL    R8, SI
	PCALIGN $0x10

candidate_match_encodeBetterBlockAsm16K:
	MOVL  12(SP), BX
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm16K

match_extend_back_loop_encodeBetterBlockAsm16K:
	CMPL AX, BX
	JBE  match_extend_back_end_encodeBetterBlockAsm16K
	MOVB -1(DX)(SI*1), DI
	MOVB -1(DX)(AX*1), R8
	CMPB DI, R8
	JNE  match_extend_back_end_encodeBetterBlockAsm16K
	LEAL -1(AX), AX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm16K
	JMP  match_extend_back_loop_encodeBetterBlockAsm16K

match_extend_back_end_encodeBetterBlockAsm16K:
	MOVL AX, BX
	SUBL 12(SP), BX
	LEAQ 3(CX)(BX*1), BX
	CMPQ BX, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm16K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm16K:
	MOVL AX, BX
	ADDL $0x04, AX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), R9

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm16K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm16K:
	MOVQ (R8)(R11*1), R10
	MOVQ 8(R8)(R11*1), R12
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm16K
	XORQ 8(R9)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm16K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm16K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm16K
	JMP  matchlen_match8_match_nolit_encodeBetterBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBetterBlockAsm16K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBetterBlockAsm16K

matchlen_match8_match_nolit_encodeBetterBlockAsm16K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm16K
	MOVQ (R8)(R11*1), R10
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm16K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm16K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm16K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBetterBlockAsm16K

matchlen_match4_match_nolit_encodeBetterBlockAsm16K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm16K
	MOVL (R8)(R11*1), R10
	CMPL (R9)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm16K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBetterBlockAsm16K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm16K
	JB   match_nolit_end_encodeBetterBlockAsm16K
	MOVW (R8)(R11*1), R10
	CMPW (R9)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm16K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBetterBlockAsm16K

matchlen_match1_match_nolit_encodeBetterBlockAsm16K:
	MOVB (R8)(R11*1), R10
	CMPB (R9)(R11*1), R10
	JNE  match_nolit_end_encodeBetterBlockAsm16K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBetterBlockAsm16K
	PCALIGN $0x10

match_nolit_end_encodeBetterBlockAsm16K:
	MOVL AX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)

	// Check if we can combine lit+copy
	MOVLQZX 12(SP), R8
	MOVL    BX, SI
	SUBL    R8, SI
	JZ      match_emit_nolits_encodeBetterBlockAsm16K
	CMPL    DI, $0x00000040
	JL      match_emit_lits_encodeBetterBlockAsm16K
	CMPL    SI, $0x04
	JA      match_emit_lits_encodeBetterBlockAsm16K
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, DI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    DI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, DI
	CMOVLLT R11, DI
	LEAL    -1(SI)(DI*4), DI
	MOVL    $0x00000003, R10
	LEAL    (R10)(DI*8), DI
	MOVB    DI, (CX)
	ADDQ    $0x03, CX
	MOVL    R8, (CX)
	ADDQ    SI, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBetterBlockAsm16K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm16K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm16K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm16K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm16K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm16K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm16K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

match_emit_lits_encodeBetterBlockAsm16K:
	LEAQ (DX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBetterBlockAsm16K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm16K
	JB   three_bytes_match_emit_encodeBetterBlockAsm16K

three_bytes_match_emit_encodeBetterBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm16K

two_bytes_match_emit_encodeBetterBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBetterBlockAsm16K
	JMP  memmove_long_match_emit_encodeBetterBlockAsm16K

one_byte_match_emit_encodeBetterBlockAsm16K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ SI, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm16K_memmove_move_8
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm16K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm16K_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm16K_memmove_move_8:
	MOVQ (R8), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm16K
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBetterBlockAsm16K_memmove_move_8through16:
	MOVQ (R8), R10
	MOVQ -8(R8)(SI*1), R8
	MOVQ R10, (CX)
	MOVQ R8, -8(CX)(SI*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm16K

emit_lit_memmove_match_emit_encodeBetterBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm16K

emit_lit_memmove_match_emit_encodeBetterBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm16K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm16K

memmove_midmatch_emit_encodeBetterBlockAsm16K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBetterBlockAsm16K

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_match_emit_encodeBetterBlockAsm16K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm16K

memmove_long_match_emit_encodeBetterBlockAsm16K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBetterBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm16Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R13*1), X4
	MOVOU -16(R8)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  SI, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

match_emit_nolits_encodeBetterBlockAsm16K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitCopy
	CMPL DI, $0x00000400
	JA   two_byte_match_nolit_encodeBetterBlockAsm16K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBetterBlockAsm16K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

emit_one_longer_match_nolit_encodeBetterBlockAsm16K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBetterBlockAsm16K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

emit_copy1_repeat_match_nolit_encodeBetterBlockAsm16K:
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm16K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm16K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm16K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm16K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm16K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm16K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

two_byte_match_nolit_encodeBetterBlockAsm16K:
	// emitCopy2
	LEAL -64(DI), DI
	LEAL -4(R11), R11
	MOVW DI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBetterBlockAsm16K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBetterBlockAsm16K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBetterBlockAsm16K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

emit_copy2_2_match_nolit_encodeBetterBlockAsm16K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

emit_copy2_1_match_nolit_encodeBetterBlockAsm16K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

emit_copy2_0_match_nolit_encodeBetterBlockAsm16K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm16K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm16K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm16K
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm16K

three_bytes_match_emit_repeat_encodeBetterBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm16K

two_bytes_match_emit_repeat_encodeBetterBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midmatch_emit_repeat_encodeBetterBlockAsm16K
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm16K

one_byte_match_emit_repeat_encodeBetterBlockAsm16K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm16K
	PCALIGN $0x10

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm16K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm16K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm16K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm16K

memmove_midmatch_emit_repeat_encodeBetterBlockAsm16K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm16K

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm16K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm16K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm16K

memmove_long_match_emit_repeat_encodeBetterBlockAsm16K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R12
	SUBQ    R9, R12
	DECQ    R10
	JA      emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R9
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm16Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R9
	ADDQ  $0x20, R12
	DECQ  R10
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  DI, R12
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm16K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_match_nolit_repeat_encodeBetterBlockAsm16K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_match_nolit_repeat_encodeBetterBlockAsm16K
	CMPL R11, $0x0001001e
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm16K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

repeat_three_match_nolit_repeat_encodeBetterBlockAsm16K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

repeat_two_match_nolit_repeat_encodeBetterBlockAsm16K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm16K

repeat_one_match_nolit_repeat_encodeBetterBlockAsm16K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm16K:
	CMPL AX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm16K
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm16K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm16K:
	MOVQ  tmp+48(FP), SI
	MOVQ  $0x0000cf1bbcdcbf9b, DI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(BX), BX
	LEAQ  -2(AX), R9
	MOVQ  (DX)(BX*1), R10
	MOVQ  1(DX)(BX*1), R11
	MOVQ  (DX)(R9*1), R12
	MOVQ  1(DX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ DI, R10
	SHRQ  $0x32, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x35, R11
	SHLQ  $0x10, R12
	IMULQ DI, R12
	SHRQ  $0x32, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x35, R13
	LEAQ  1(BX), R8
	LEAQ  1(R9), R14
	MOVW  BX, (SI)(R10*2)
	MOVW  R9, (SI)(R12*2)
	LEAQ  1(R9)(BX*1), R10
	SHRQ  $0x01, R10
	ADDQ  $0x01, BX
	SUBQ  $0x01, R9
	MOVW  R8, 32768(SI)(R11*2)
	MOVW  R14, 32768(SI)(R13*2)

index_loop_encodeBetterBlockAsm16K:
	CMPQ  R10, R9
	JAE   search_loop_encodeBetterBlockAsm16K
	MOVQ  (DX)(BX*1), R8
	MOVQ  (DX)(R10*1), R11
	SHLQ  $0x10, R8
	IMULQ DI, R8
	SHRQ  $0x32, R8
	SHLQ  $0x10, R11
	IMULQ DI, R11
	SHRQ  $0x32, R11
	MOVW  BX, (SI)(R8*2)
	MOVW  R9, (SI)(R11*2)
	ADDQ  $0x02, BX
	ADDQ  $0x02, R10
	JMP   index_loop_encodeBetterBlockAsm16K

emit_remainder_encodeBetterBlockAsm16K:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm16K
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm16K:
	MOVQ src_len+32(FP), AX

	// emitLiteralsDstP
	MOVL 12(SP), BX
	CMPL BX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm16K
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (DX)(BX*1), AX
	SUBL BX, SI

	// emitLiteral
	LEAL -1(SI), DX
	CMPL DX, $0x1d
	JB   one_byte_emit_remainder_encodeBetterBlockAsm16K
	SUBL $0x1d, DX
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm16K
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm16K

three_bytes_emit_remainder_encodeBetterBlockAsm16K:
	MOVB $0xf0, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm16K

two_bytes_emit_remainder_encodeBetterBlockAsm16K:
	MOVB $0xe8, (CX)
	MOVB DL, 1(CX)
	ADDL $0x1d, DX
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_midemit_remainder_encodeBetterBlockAsm16K
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm16K

one_byte_emit_remainder_encodeBetterBlockAsm16K:
	SHLB $0x03, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -1, min move: 1
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_3
	CMPQ BX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_4through8
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm16K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm16K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_4through8:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm16K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm16K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm16K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm16K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm16K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm16K

memmove_midemit_remainder_encodeBetterBlockAsm16K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -2, min move: 30
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm16K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm16K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm16K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm16K

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm16K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm16K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm16K

memmove_long_emit_remainder_encodeBetterBlockAsm16K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU   (AX), X0
	MOVOU   16(AX), X1
	MOVOU   -32(AX)(BX*1), X2
	MOVOU   -16(AX)(BX*1), X3
	MOVQ    BX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm16Klarge_forward_sse_loop_32
	LEAQ    -32(AX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm16Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm16Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm16Klarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm16Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm16K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm4K(dst []byte, src []byte, tmp *[10240]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBetterBlockAsm4K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000050, DX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm4K:
	MOVOU   X0, (AX)
	MOVOU   X0, 16(AX)
	MOVOU   X0, 32(AX)
	MOVOU   X0, 48(AX)
	MOVOU   X0, 64(AX)
	MOVOU   X0, 80(AX)
	MOVOU   X0, 96(AX)
	MOVOU   X0, 112(AX)
	ADDQ    $0x80, AX
	DECQ    DX
	JNZ     zero_loop_encodeBetterBlockAsm4K
	MOVL    $0x00000000, 12(SP)
	MOVQ    src_len+32(FP), AX
	LEAQ    -11(AX), DX
	LEAQ    -8(AX), BX
	MOVL    BX, 8(SP)
	SHRQ    $0x05, AX
	SUBL    AX, DX
	LEAQ    (CX)(DX*1), DX
	MOVQ    DX, (SP)
	MOVL    $0x00000001, AX
	MOVL    AX, 16(SP)
	MOVQ    src_base+24(FP), DX
	PCALIGN $0x10

search_loop_encodeBetterBlockAsm4K:
	MOVQ    tmp+48(FP), BX
	MOVL    AX, SI
	SUBL    12(SP), SI
	SHRL    $0x05, SI
	LEAL    1(AX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeBetterBlockAsm4K
	MOVQ    (DX)(AX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0x0000cf1bbcdcbf9b, R9
	MOVQ    $0x9e3779b1, SI
	MOVQ    DI, R10
	MOVQ    DI, R11
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x34, R10
	SHLQ    $0x20, R11
	IMULQ   SI, R11
	SHRQ    $0x36, R11
	MOVWLZX (BX)(R10*2), SI
	MOVWLZX 8192(BX)(R11*2), R8
	MOVW    AX, (BX)(R10*2)
	MOVW    AX, 8192(BX)(R11*2)
	MOVQ    (DX)(SI*1), R10
	CMPQ    R10, DI
	JEQ     candidate_match_encodeBetterBlockAsm4K
	MOVQ    (DX)(R8*1), R11
	CMPQ    R11, DI
	MOVL    AX, R12
	SUBL    16(SP), R12
	MOVQ    (DX)(R12*1), R12
	MOVQ    $0x000000ffffffff00, R13
	XORQ    DI, R12
	TESTQ   R13, R12
	JNE     no_repeat_found_encodeBetterBlockAsm4K
	LEAL    1(AX), BX
	MOVL    12(SP), SI
	MOVL    BX, DI
	SUBL    16(SP), DI
	JZ      repeat_extend_back_end_encodeBetterBlockAsm4K

repeat_extend_back_loop_encodeBetterBlockAsm4K:
	CMPL BX, SI
	JBE  repeat_extend_back_end_encodeBetterBlockAsm4K
	MOVB -1(DX)(DI*1), R8
	MOVB -1(DX)(BX*1), R9
	CMPB R8, R9
	JNE  repeat_extend_back_end_encodeBetterBlockAsm4K
	LEAL -1(BX), BX
	DECL DI
	JNZ  repeat_extend_back_loop_encodeBetterBlockAsm4K

repeat_extend_back_end_encodeBetterBlockAsm4K:
	MOVL BX, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBetterBlockAsm4K
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBetterBlockAsm4K:
	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_repeat_emit_encodeBetterBlockAsm4K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_repeat_emit_encodeBetterBlockAsm4K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBetterBlockAsm4K
	JB   three_bytes_repeat_emit_encodeBetterBlockAsm4K

three_bytes_repeat_emit_encodeBetterBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm4K

two_bytes_repeat_emit_encodeBetterBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midrepeat_emit_encodeBetterBlockAsm4K
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm4K

one_byte_repeat_emit_encodeBetterBlockAsm4K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm4K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm4K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm4K_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm4K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm4K
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm4K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm4K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_repeat_emit_encodeBetterBlockAsm4K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_repeat_emit_encodeBetterBlockAsm4K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm4K

memmove_midrepeat_emit_encodeBetterBlockAsm4K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm4K

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm4K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm4K

memmove_long_repeat_emit_encodeBetterBlockAsm4K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R11
	SUBQ    R9, R11
	DECQ    R10
	JA      emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R11*1), R9
	LEAQ    -32(CX)(R11*1), R12
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm4Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R12)
	MOVOA X5, 16(R12)
	ADDQ  $0x20, R12
	ADDQ  $0x20, R9
	ADDQ  $0x20, R11
	DECQ  R10
	JNA   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R11*1), X4
	MOVOU -16(R8)(R11*1), X5
	MOVOA X4, -32(CX)(R11*1)
	MOVOA X5, -16(CX)(R11*1)
	ADDQ  $0x20, R11
	CMPQ  DI, R11
	JAE   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBetterBlockAsm4K:
	ADDL $0x05, AX
	MOVL AX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), SI

	// matchLen
	XORL R10, R10
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm4K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm4K:
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm4K
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16repeat_extend_encodeBetterBlockAsm4K
	LEAL -16(DI), DI
	LEAL 16(R10), R10

matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm4K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm4K
	JMP  matchlen_match8_repeat_extend_encodeBetterBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBetterBlockAsm4K:
	TZCNTQ R11, R11
	SARQ   $0x03, R11
	LEAL   8(R10)(R11*1), R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm4K

matchlen_match8_repeat_extend_encodeBetterBlockAsm4K:
	CMPL DI, $0x08
	JB   matchlen_match4_repeat_extend_encodeBetterBlockAsm4K
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm4K
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_repeat_extend_encodeBetterBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm4K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm4K

matchlen_match4_repeat_extend_encodeBetterBlockAsm4K:
	CMPL DI, $0x04
	JB   matchlen_match2_repeat_extend_encodeBetterBlockAsm4K
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_repeat_extend_encodeBetterBlockAsm4K
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_repeat_extend_encodeBetterBlockAsm4K:
	CMPL DI, $0x01
	JE   matchlen_match1_repeat_extend_encodeBetterBlockAsm4K
	JB   repeat_extend_forward_end_encodeBetterBlockAsm4K
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_repeat_extend_encodeBetterBlockAsm4K
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   repeat_extend_forward_end_encodeBetterBlockAsm4K

matchlen_match1_repeat_extend_encodeBetterBlockAsm4K:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  repeat_extend_forward_end_encodeBetterBlockAsm4K
	LEAL 1(R10), R10
	JMP  repeat_extend_forward_end_encodeBetterBlockAsm4K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBetterBlockAsm4K:
	ADDL R10, AX
	MOVL AX, SI
	SUBL BX, SI
	MOVL 16(SP), BX

	// emitRepeat
	LEAL -1(SI), BX
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBetterBlockAsm4K
	LEAL -30(SI), BX
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBetterBlockAsm4K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBetterBlockAsm4K
	MOVB $0xfc, (CX)
	MOVL BX, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm4K

repeat_three_match_repeat_encodeBetterBlockAsm4K:
	MOVB $0xf4, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm4K

repeat_two_match_repeat_encodeBetterBlockAsm4K:
	MOVB $0xec, (CX)
	MOVB BL, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm4K

repeat_one_match_repeat_encodeBetterBlockAsm4K:
	XORL BX, BX
	LEAL -4(BX)(SI*8), BX
	MOVB BL, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBetterBlockAsm4K:
	MOVL AX, 12(SP)
	JMP  search_loop_encodeBetterBlockAsm4K
	PCALIGN $0x10

no_repeat_found_encodeBetterBlockAsm4K:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm4K
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm4K
	MOVL 20(SP), AX
	JMP  search_loop_encodeBetterBlockAsm4K

candidateS_match_encodeBetterBlockAsm4K:
	SHRQ    $0x08, DI
	MOVQ    DI, R10
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x34, R10
	MOVWLZX (BX)(R10*2), SI
	INCL    AX
	MOVW    AX, (BX)(R10*2)
	CMPL    (DX)(SI*1), DI
	JEQ     candidate_match_encodeBetterBlockAsm4K
	DECL    AX
	MOVL    R8, SI
	PCALIGN $0x10

candidate_match_encodeBetterBlockAsm4K:
	MOVL  12(SP), BX
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm4K

match_extend_back_loop_encodeBetterBlockAsm4K:
	CMPL AX, BX
	JBE  match_extend_back_end_encodeBetterBlockAsm4K
	MOVB -1(DX)(SI*1), DI
	MOVB -1(DX)(AX*1), R8
	CMPB DI, R8
	JNE  match_extend_back_end_encodeBetterBlockAsm4K
	LEAL -1(AX), AX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm4K
	JMP  match_extend_back_loop_encodeBetterBlockAsm4K

match_extend_back_end_encodeBetterBlockAsm4K:
	MOVL AX, BX
	SUBL 12(SP), BX
	LEAQ 3(CX)(BX*1), BX
	CMPQ BX, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm4K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm4K:
	MOVL AX, BX
	ADDL $0x04, AX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), R9

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm4K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4K:
	MOVQ (R8)(R11*1), R10
	MOVQ 8(R8)(R11*1), R12
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4K
	XORQ 8(R9)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm4K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm4K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4K
	JMP  matchlen_match8_match_nolit_encodeBetterBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBetterBlockAsm4K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBetterBlockAsm4K

matchlen_match8_match_nolit_encodeBetterBlockAsm4K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm4K
	MOVQ (R8)(R11*1), R10
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm4K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBetterBlockAsm4K

matchlen_match4_match_nolit_encodeBetterBlockAsm4K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm4K
	MOVL (R8)(R11*1), R10
	CMPL (R9)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm4K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBetterBlockAsm4K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm4K
	JB   match_nolit_end_encodeBetterBlockAsm4K
	MOVW (R8)(R11*1), R10
	CMPW (R9)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm4K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBetterBlockAsm4K

matchlen_match1_match_nolit_encodeBetterBlockAsm4K:
	MOVB (R8)(R11*1), R10
	CMPB (R9)(R11*1), R10
	JNE  match_nolit_end_encodeBetterBlockAsm4K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBetterBlockAsm4K
	PCALIGN $0x10

match_nolit_end_encodeBetterBlockAsm4K:
	MOVL AX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)

	// Check if we can combine lit+copy
	MOVLQZX 12(SP), R8
	MOVL    BX, SI
	SUBL    R8, SI
	JZ      match_emit_nolits_encodeBetterBlockAsm4K
	CMPL    DI, $0x00000040
	JL      match_emit_lits_encodeBetterBlockAsm4K
	CMPL    SI, $0x04
	JA      match_emit_lits_encodeBetterBlockAsm4K
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, DI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    DI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, DI
	CMOVLLT R11, DI
	LEAL    -1(SI)(DI*4), DI
	MOVL    $0x00000003, R10
	LEAL    (R10)(DI*8), DI
	MOVB    DI, (CX)
	ADDQ    $0x03, CX
	MOVL    R8, (CX)
	ADDQ    SI, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBetterBlockAsm4K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm4K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm4K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm4K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm4K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm4K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm4K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

match_emit_lits_encodeBetterBlockAsm4K:
	LEAQ (DX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBetterBlockAsm4K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm4K
	JB   three_bytes_match_emit_encodeBetterBlockAsm4K

three_bytes_match_emit_encodeBetterBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm4K

two_bytes_match_emit_encodeBetterBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBetterBlockAsm4K
	JMP  memmove_long_match_emit_encodeBetterBlockAsm4K

one_byte_match_emit_encodeBetterBlockAsm4K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ SI, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm4K_memmove_move_8
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm4K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm4K_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm4K_memmove_move_8:
	MOVQ (R8), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm4K
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBetterBlockAsm4K_memmove_move_8through16:
	MOVQ (R8), R10
	MOVQ -8(R8)(SI*1), R8
	MOVQ R10, (CX)
	MOVQ R8, -8(CX)(SI*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm4K

emit_lit_memmove_match_emit_encodeBetterBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm4K

emit_lit_memmove_match_emit_encodeBetterBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm4K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm4K

memmove_midmatch_emit_encodeBetterBlockAsm4K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBetterBlockAsm4K

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_match_emit_encodeBetterBlockAsm4K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm4K

memmove_long_match_emit_encodeBetterBlockAsm4K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R13*1), X4
	MOVOU -16(R8)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  SI, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

match_emit_nolits_encodeBetterBlockAsm4K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitCopy
	CMPL DI, $0x00000400
	JA   two_byte_match_nolit_encodeBetterBlockAsm4K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBetterBlockAsm4K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

emit_one_longer_match_nolit_encodeBetterBlockAsm4K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBetterBlockAsm4K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

emit_copy1_repeat_match_nolit_encodeBetterBlockAsm4K:
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm4K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm4K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm4K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm4K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm4K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm4K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

two_byte_match_nolit_encodeBetterBlockAsm4K:
	// emitCopy2
	LEAL -64(DI), DI
	LEAL -4(R11), R11
	MOVW DI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBetterBlockAsm4K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBetterBlockAsm4K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBetterBlockAsm4K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

emit_copy2_2_match_nolit_encodeBetterBlockAsm4K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

emit_copy2_1_match_nolit_encodeBetterBlockAsm4K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

emit_copy2_0_match_nolit_encodeBetterBlockAsm4K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm4K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm4K
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm4K

three_bytes_match_emit_repeat_encodeBetterBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm4K

two_bytes_match_emit_repeat_encodeBetterBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midmatch_emit_repeat_encodeBetterBlockAsm4K
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm4K

one_byte_match_emit_repeat_encodeBetterBlockAsm4K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4K
	PCALIGN $0x10

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4K

memmove_midmatch_emit_repeat_encodeBetterBlockAsm4K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm4K

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm4K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm4K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4K

memmove_long_match_emit_repeat_encodeBetterBlockAsm4K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R12
	SUBQ    R9, R12
	DECQ    R10
	JA      emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R9
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R9
	ADDQ  $0x20, R12
	DECQ  R10
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  DI, R12
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_match_nolit_repeat_encodeBetterBlockAsm4K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_match_nolit_repeat_encodeBetterBlockAsm4K
	CMPL R11, $0x0001001e
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm4K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

repeat_three_match_nolit_repeat_encodeBetterBlockAsm4K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

repeat_two_match_nolit_repeat_encodeBetterBlockAsm4K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4K

repeat_one_match_nolit_repeat_encodeBetterBlockAsm4K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm4K:
	CMPL AX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm4K
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm4K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm4K:
	MOVQ  tmp+48(FP), SI
	MOVQ  $0x0000cf1bbcdcbf9b, DI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(BX), BX
	LEAQ  -2(AX), R9
	MOVQ  (DX)(BX*1), R10
	MOVQ  1(DX)(BX*1), R11
	MOVQ  (DX)(R9*1), R12
	MOVQ  1(DX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ DI, R10
	SHRQ  $0x34, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x36, R11
	SHLQ  $0x10, R12
	IMULQ DI, R12
	SHRQ  $0x34, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x36, R13
	LEAQ  1(BX), R8
	LEAQ  1(R9), R14
	MOVW  BX, (SI)(R10*2)
	MOVW  R9, (SI)(R12*2)
	LEAQ  1(R9)(BX*1), R10
	SHRQ  $0x01, R10
	ADDQ  $0x01, BX
	SUBQ  $0x01, R9
	MOVW  R8, 8192(SI)(R11*2)
	MOVW  R14, 8192(SI)(R13*2)

index_loop_encodeBetterBlockAsm4K:
	CMPQ  R10, R9
	JAE   search_loop_encodeBetterBlockAsm4K
	MOVQ  (DX)(BX*1), R8
	MOVQ  (DX)(R10*1), R11
	SHLQ  $0x10, R8
	IMULQ DI, R8
	SHRQ  $0x34, R8
	SHLQ  $0x10, R11
	IMULQ DI, R11
	SHRQ  $0x34, R11
	MOVW  BX, (SI)(R8*2)
	MOVW  R9, (SI)(R11*2)
	ADDQ  $0x02, BX
	ADDQ  $0x02, R10
	JMP   index_loop_encodeBetterBlockAsm4K

emit_remainder_encodeBetterBlockAsm4K:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm4K
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm4K:
	MOVQ src_len+32(FP), AX

	// emitLiteralsDstP
	MOVL 12(SP), BX
	CMPL BX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm4K
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (DX)(BX*1), AX
	SUBL BX, SI

	// emitLiteral
	LEAL -1(SI), DX
	CMPL DX, $0x1d
	JB   one_byte_emit_remainder_encodeBetterBlockAsm4K
	SUBL $0x1d, DX
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm4K
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm4K

three_bytes_emit_remainder_encodeBetterBlockAsm4K:
	MOVB $0xf0, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm4K

two_bytes_emit_remainder_encodeBetterBlockAsm4K:
	MOVB $0xe8, (CX)
	MOVB DL, 1(CX)
	ADDL $0x1d, DX
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_midemit_remainder_encodeBetterBlockAsm4K
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm4K

one_byte_emit_remainder_encodeBetterBlockAsm4K:
	SHLB $0x03, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -1, min move: 1
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_3
	CMPQ BX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_4through8
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm4K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm4K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_4through8:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm4K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm4K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm4K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm4K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm4K

memmove_midemit_remainder_encodeBetterBlockAsm4K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -2, min move: 30
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm4K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm4K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm4K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm4K

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm4K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm4K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm4K

memmove_long_emit_remainder_encodeBetterBlockAsm4K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU   (AX), X0
	MOVOU   16(AX), X1
	MOVOU   -32(AX)(BX*1), X2
	MOVOU   -16(AX)(BX*1), X3
	MOVQ    BX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4Klarge_forward_sse_loop_32
	LEAQ    -32(AX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4Klarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm4K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm1K(dst []byte, src []byte, tmp *[4608]byte) int
// Requires: BMI, CMOV, SSE2
TEXT ·encodeBetterBlockAsm1K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000024, DX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm1K:
	MOVOU   X0, (AX)
	MOVOU   X0, 16(AX)
	MOVOU   X0, 32(AX)
	MOVOU   X0, 48(AX)
	MOVOU   X0, 64(AX)
	MOVOU   X0, 80(AX)
	MOVOU   X0, 96(AX)
	MOVOU   X0, 112(AX)
	ADDQ    $0x80, AX
	DECQ    DX
	JNZ     zero_loop_encodeBetterBlockAsm1K
	MOVL    $0x00000000, 12(SP)
	MOVQ    src_len+32(FP), AX
	LEAQ    -11(AX), DX
	LEAQ    -8(AX), BX
	MOVL    BX, 8(SP)
	SHRQ    $0x05, AX
	SUBL    AX, DX
	LEAQ    (CX)(DX*1), DX
	MOVQ    DX, (SP)
	MOVL    $0x00000001, AX
	MOVL    AX, 16(SP)
	MOVQ    src_base+24(FP), DX
	PCALIGN $0x10

search_loop_encodeBetterBlockAsm1K:
	MOVQ    tmp+48(FP), BX
	MOVL    AX, SI
	SUBL    12(SP), SI
	SHRL    $0x04, SI
	LEAL    1(AX)(SI*1), SI
	CMPL    SI, 8(SP)
	JAE     emit_remainder_encodeBetterBlockAsm1K
	MOVQ    (DX)(AX*1), DI
	MOVL    SI, 20(SP)
	MOVQ    $0x0000cf1bbcdcbf9b, R9
	MOVQ    $0x9e3779b1, SI
	MOVQ    DI, R10
	MOVQ    DI, R11
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x35, R10
	SHLQ    $0x20, R11
	IMULQ   SI, R11
	SHRQ    $0x38, R11
	MOVWLZX (BX)(R10*2), SI
	MOVWLZX 4096(BX)(R11*2), R8
	MOVW    AX, (BX)(R10*2)
	MOVW    AX, 4096(BX)(R11*2)
	MOVQ    (DX)(SI*1), R10
	CMPQ    R10, DI
	JEQ     candidate_match_encodeBetterBlockAsm1K
	MOVQ    (DX)(R8*1), R11
	CMPQ    R11, DI
	MOVL    AX, R12
	SUBL    16(SP), R12
	MOVQ    (DX)(R12*1), R12
	MOVQ    $0x000000ffffffff00, R13
	XORQ    DI, R12
	TESTQ   R13, R12
	JNE     no_repeat_found_encodeBetterBlockAsm1K
	LEAL    1(AX), BX
	MOVL    12(SP), SI
	MOVL    BX, DI
	SUBL    16(SP), DI
	JZ      repeat_extend_back_end_encodeBetterBlockAsm1K

repeat_extend_back_loop_encodeBetterBlockAsm1K:
	CMPL BX, SI
	JBE  repeat_extend_back_end_encodeBetterBlockAsm1K
	MOVB -1(DX)(DI*1), R8
	MOVB -1(DX)(BX*1), R9
	CMPB R8, R9
	JNE  repeat_extend_back_end_encodeBetterBlockAsm1K
	LEAL -1(BX), BX
	DECL DI
	JNZ  repeat_extend_back_loop_encodeBetterBlockAsm1K

repeat_extend_back_end_encodeBetterBlockAsm1K:
	MOVL BX, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBetterBlockAsm1K
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBetterBlockAsm1K:
	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_repeat_emit_encodeBetterBlockAsm1K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_repeat_emit_encodeBetterBlockAsm1K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBetterBlockAsm1K
	JB   three_bytes_repeat_emit_encodeBetterBlockAsm1K

three_bytes_repeat_emit_encodeBetterBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm1K

two_bytes_repeat_emit_encodeBetterBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midrepeat_emit_encodeBetterBlockAsm1K
	JMP  memmove_long_repeat_emit_encodeBetterBlockAsm1K

one_byte_repeat_emit_encodeBetterBlockAsm1K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm1K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm1K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBetterBlockAsm1K_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm1K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm1K
	PCALIGN $0x10

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm1K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_repeat_emit_encodeBetterBlockAsm1K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_repeat_emit_encodeBetterBlockAsm1K

emit_lit_memmove_repeat_emit_encodeBetterBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_repeat_emit_encodeBetterBlockAsm1K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm1K

memmove_midrepeat_emit_encodeBetterBlockAsm1K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm1K

emit_lit_memmove_mid_repeat_emit_encodeBetterBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_repeat_emit_encodeBetterBlockAsm1K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBetterBlockAsm1K

memmove_long_repeat_emit_encodeBetterBlockAsm1K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R11
	SUBQ    R9, R11
	DECQ    R10
	JA      emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R11*1), R9
	LEAQ    -32(CX)(R11*1), R12
	PCALIGN $0x10

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm1Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R12)
	MOVOA X5, 16(R12)
	ADDQ  $0x20, R12
	ADDQ  $0x20, R9
	ADDQ  $0x20, R11
	DECQ  R10
	JNA   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R11*1), X4
	MOVOU -16(R8)(R11*1), X5
	MOVOA X4, -32(CX)(R11*1)
	MOVOA X5, -16(CX)(R11*1)
	ADDQ  $0x20, R11
	CMPQ  DI, R11
	JAE   emit_lit_memmove_long_repeat_emit_encodeBetterBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBetterBlockAsm1K:
	ADDL $0x05, AX
	MOVL AX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), SI

	// matchLen
	XORL R10, R10
	JMP  matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm1K
	PCALIGN $0x10

matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm1K:
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm1K
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16repeat_extend_encodeBetterBlockAsm1K
	LEAL -16(DI), DI
	LEAL 16(R10), R10

matchlen_loop_16_entry_repeat_extend_encodeBetterBlockAsm1K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_repeat_extend_encodeBetterBlockAsm1K
	JMP  matchlen_match8_repeat_extend_encodeBetterBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_16repeat_extend_encodeBetterBlockAsm1K:
	TZCNTQ R11, R11
	SARQ   $0x03, R11
	LEAL   8(R10)(R11*1), R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm1K

matchlen_match8_repeat_extend_encodeBetterBlockAsm1K:
	CMPL DI, $0x08
	JB   matchlen_match4_repeat_extend_encodeBetterBlockAsm1K
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm1K
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_repeat_extend_encodeBetterBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_8_repeat_extend_encodeBetterBlockAsm1K:
	TZCNTQ R9, R9
	SARQ   $0x03, R9
	ADDL   R9, R10
	JMP    repeat_extend_forward_end_encodeBetterBlockAsm1K

matchlen_match4_repeat_extend_encodeBetterBlockAsm1K:
	CMPL DI, $0x04
	JB   matchlen_match2_repeat_extend_encodeBetterBlockAsm1K
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_repeat_extend_encodeBetterBlockAsm1K
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_repeat_extend_encodeBetterBlockAsm1K:
	CMPL DI, $0x01
	JE   matchlen_match1_repeat_extend_encodeBetterBlockAsm1K
	JB   repeat_extend_forward_end_encodeBetterBlockAsm1K
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_repeat_extend_encodeBetterBlockAsm1K
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   repeat_extend_forward_end_encodeBetterBlockAsm1K

matchlen_match1_repeat_extend_encodeBetterBlockAsm1K:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  repeat_extend_forward_end_encodeBetterBlockAsm1K
	LEAL 1(R10), R10
	JMP  repeat_extend_forward_end_encodeBetterBlockAsm1K
	PCALIGN $0x10

repeat_extend_forward_end_encodeBetterBlockAsm1K:
	ADDL R10, AX
	MOVL AX, SI
	SUBL BX, SI
	MOVL 16(SP), BX

	// emitRepeat
	LEAL -1(SI), BX
	CMPL SI, $0x1d
	JBE  repeat_one_match_repeat_encodeBetterBlockAsm1K
	LEAL -30(SI), BX
	CMPL SI, $0x0000011e
	JB   repeat_two_match_repeat_encodeBetterBlockAsm1K
	CMPL SI, $0x0001001e
	JB   repeat_three_match_repeat_encodeBetterBlockAsm1K
	MOVB $0xfc, (CX)
	MOVL BX, 1(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm1K

repeat_three_match_repeat_encodeBetterBlockAsm1K:
	MOVB $0xf4, (CX)
	MOVW BX, 1(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm1K

repeat_two_match_repeat_encodeBetterBlockAsm1K:
	MOVB $0xec, (CX)
	MOVB BL, 1(CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBetterBlockAsm1K

repeat_one_match_repeat_encodeBetterBlockAsm1K:
	XORL BX, BX
	LEAL -4(BX)(SI*8), BX
	MOVB BL, (CX)
	ADDQ $0x01, CX

repeat_end_emit_encodeBetterBlockAsm1K:
	MOVL AX, 12(SP)
	JMP  search_loop_encodeBetterBlockAsm1K
	PCALIGN $0x10

no_repeat_found_encodeBetterBlockAsm1K:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm1K
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm1K
	MOVL 20(SP), AX
	JMP  search_loop_encodeBetterBlockAsm1K

candidateS_match_encodeBetterBlockAsm1K:
	SHRQ    $0x08, DI
	MOVQ    DI, R10
	SHLQ    $0x10, R10
	IMULQ   R9, R10
	SHRQ    $0x35, R10
	MOVWLZX (BX)(R10*2), SI
	INCL    AX
	MOVW    AX, (BX)(R10*2)
	CMPL    (DX)(SI*1), DI
	JEQ     candidate_match_encodeBetterBlockAsm1K
	DECL    AX
	MOVL    R8, SI
	PCALIGN $0x10

candidate_match_encodeBetterBlockAsm1K:
	MOVL  12(SP), BX
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm1K

match_extend_back_loop_encodeBetterBlockAsm1K:
	CMPL AX, BX
	JBE  match_extend_back_end_encodeBetterBlockAsm1K
	MOVB -1(DX)(SI*1), DI
	MOVB -1(DX)(AX*1), R8
	CMPB DI, R8
	JNE  match_extend_back_end_encodeBetterBlockAsm1K
	LEAL -1(AX), AX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm1K
	JMP  match_extend_back_loop_encodeBetterBlockAsm1K

match_extend_back_end_encodeBetterBlockAsm1K:
	MOVL AX, BX
	SUBL 12(SP), BX
	LEAQ 3(CX)(BX*1), BX
	CMPQ BX, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm1K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm1K:
	MOVL AX, BX
	ADDL $0x04, AX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL AX, DI
	LEAQ (DX)(AX*1), R8
	LEAQ (DX)(SI*1), R9

	// matchLen
	XORL R11, R11
	JMP  matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm1K
	PCALIGN $0x10

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm1K:
	MOVQ (R8)(R11*1), R10
	MOVQ 8(R8)(R11*1), R12
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm1K
	XORQ 8(R9)(R11*1), R12
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm1K
	LEAL -16(DI), DI
	LEAL 16(R11), R11

matchlen_loop_16_entry_match_nolit_encodeBetterBlockAsm1K:
	CMPL DI, $0x10
	JAE  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm1K
	JMP  matchlen_match8_match_nolit_encodeBetterBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_16match_nolit_encodeBetterBlockAsm1K:
	TZCNTQ R12, R12
	SARQ   $0x03, R12
	LEAL   8(R11)(R12*1), R11
	JMP    match_nolit_end_encodeBetterBlockAsm1K

matchlen_match8_match_nolit_encodeBetterBlockAsm1K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm1K
	MOVQ (R8)(R11*1), R10
	XORQ (R9)(R11*1), R10
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm1K
	LEAL -8(DI), DI
	LEAL 8(R11), R11
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm1K
	PCALIGN $0x10

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm1K:
	TZCNTQ R10, R10
	SARQ   $0x03, R10
	ADDL   R10, R11
	JMP    match_nolit_end_encodeBetterBlockAsm1K

matchlen_match4_match_nolit_encodeBetterBlockAsm1K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm1K
	MOVL (R8)(R11*1), R10
	CMPL (R9)(R11*1), R10
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm1K
	LEAL -4(DI), DI
	LEAL 4(R11), R11

matchlen_match2_match_nolit_encodeBetterBlockAsm1K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm1K
	JB   match_nolit_end_encodeBetterBlockAsm1K
	MOVW (R8)(R11*1), R10
	CMPW (R9)(R11*1), R10
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm1K
	LEAL 2(R11), R11
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBetterBlockAsm1K

matchlen_match1_match_nolit_encodeBetterBlockAsm1K:
	MOVB (R8)(R11*1), R10
	CMPB (R9)(R11*1), R10
	JNE  match_nolit_end_encodeBetterBlockAsm1K
	LEAL 1(R11), R11
	JMP  match_nolit_end_encodeBetterBlockAsm1K
	PCALIGN $0x10

match_nolit_end_encodeBetterBlockAsm1K:
	MOVL AX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)

	// Check if we can combine lit+copy
	MOVLQZX 12(SP), R8
	MOVL    BX, SI
	SUBL    R8, SI
	JZ      match_emit_nolits_encodeBetterBlockAsm1K
	CMPL    DI, $0x00000040
	JL      match_emit_lits_encodeBetterBlockAsm1K
	CMPL    SI, $0x04
	JA      match_emit_lits_encodeBetterBlockAsm1K
	MOVL    (DX)(R8*1), R8
	ADDL    R11, AX
	ADDL    $0x04, R11
	MOVL    AX, 12(SP)

	// emitCopy2WithLits
	XORQ    R9, R9
	SUBL    $0x40, DI
	LEAL    -11(R11), R10
	LEAL    -4(R11), R11
	MOVW    DI, 1(CX)
	CMPL    R11, $0x07
	CMOVLGE R10, R9
	MOVQ    $0x00000007, DI
	CMOVLLT R11, DI
	LEAL    -1(SI)(DI*4), DI
	MOVL    $0x00000003, R10
	LEAL    (R10)(DI*8), DI
	MOVB    DI, (CX)
	ADDQ    $0x03, CX
	MOVL    R8, (CX)
	ADDQ    SI, CX
	TESTL   R9, R9
	JZ      match_nolit_emitcopy_end_encodeBetterBlockAsm1K

	// emitRepeat
	LEAL -1(R9), SI
	CMPL R9, $0x1d
	JBE  repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm1K
	LEAL -30(R9), SI
	CMPL R9, $0x0000011e
	JB   repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm1K
	CMPL R9, $0x0001001e
	JB   repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm1K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

repeat_three_match_emit_repeat_copy2_encodeBetterBlockAsm1K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

repeat_two_match_emit_repeat_copy2_encodeBetterBlockAsm1K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

repeat_one_match_emit_repeat_copy2_encodeBetterBlockAsm1K:
	XORL SI, SI
	LEAL -4(SI)(R9*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

match_emit_lits_encodeBetterBlockAsm1K:
	LEAQ (DX)(R8*1), R8

	// emitLiteral
	LEAL -1(SI), R9
	CMPL R9, $0x1d
	JB   one_byte_match_emit_encodeBetterBlockAsm1K
	SUBL $0x1d, R9
	CMPL R9, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm1K
	JB   three_bytes_match_emit_encodeBetterBlockAsm1K

three_bytes_match_emit_encodeBetterBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW R9, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, R9
	JMP  memmove_long_match_emit_encodeBetterBlockAsm1K

two_bytes_match_emit_encodeBetterBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB R9, 1(CX)
	ADDL $0x1d, R9
	ADDQ $0x02, CX
	CMPL R9, $0x40
	JB   memmove_midmatch_emit_encodeBetterBlockAsm1K
	JMP  memmove_long_match_emit_encodeBetterBlockAsm1K

one_byte_match_emit_encodeBetterBlockAsm1K:
	SHLB $0x03, R9
	MOVB R9, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ SI, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm1K_memmove_move_8
	CMPQ SI, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm1K_memmove_move_8through16
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm1K_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm1K_memmove_move_8:
	MOVQ (R8), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm1K
	PCALIGN $0x10

emit_lit_memmove_match_emit_encodeBetterBlockAsm1K_memmove_move_8through16:
	MOVQ (R8), R10
	MOVQ -8(R8)(SI*1), R8
	MOVQ R10, (CX)
	MOVQ R8, -8(CX)(SI*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm1K

emit_lit_memmove_match_emit_encodeBetterBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm1K

emit_lit_memmove_match_emit_encodeBetterBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm1K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm1K

memmove_midmatch_emit_encodeBetterBlockAsm1K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ SI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(SI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(SI*1)
	JMP   memmove_mid_end_copy_match_emit_encodeBetterBlockAsm1K

emit_lit_memmove_mid_match_emit_encodeBetterBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(SI*1), X2
	MOVOU -16(R8)(SI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)

memmove_mid_end_copy_match_emit_encodeBetterBlockAsm1K:
	MOVQ R9, CX
	JMP  match_emit_nolits_encodeBetterBlockAsm1K

memmove_long_match_emit_encodeBetterBlockAsm1K:
	LEAQ (CX)(SI*1), R9

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(SI*1), X2
	MOVOU   -16(R8)(SI*1), X3
	MOVQ    SI, R12
	SHRQ    $0x05, R12
	MOVQ    CX, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R12
	JA      emit_lit_memmove_long_match_emit_encodeBetterBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R13*1), R10
	LEAQ    -32(CX)(R13*1), R14
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm1Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R13*1), X4
	MOVOU -16(R8)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  SI, R13
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(SI*1)
	MOVOU X3, -16(CX)(SI*1)
	MOVQ  R9, CX

match_emit_nolits_encodeBetterBlockAsm1K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitCopy
	CMPL DI, $0x00000400
	JA   two_byte_match_nolit_encodeBetterBlockAsm1K
	CMPL R11, $0x00000013
	JAE  emit_one_longer_match_nolit_encodeBetterBlockAsm1K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL -15(SI)(R11*4), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

emit_one_longer_match_nolit_encodeBetterBlockAsm1K:
	CMPL R11, $0x00000112
	JAE  emit_copy1_repeat_match_nolit_encodeBetterBlockAsm1K
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 61(SI), SI
	MOVW SI, (CX)
	LEAL -18(R11), SI
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

emit_copy1_repeat_match_nolit_encodeBetterBlockAsm1K:
	LEAL -1(DI), SI
	SHLL $0x06, SI
	LEAL 57(SI), SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x12, R11

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm1K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm1K
	CMPL R11, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm1K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

repeat_three_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm1K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

repeat_two_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm1K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

repeat_one_emit_copy1_do_repeat_match_nolit_encodeBetterBlockAsm1K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

two_byte_match_nolit_encodeBetterBlockAsm1K:
	// emitCopy2
	LEAL -64(DI), DI
	LEAL -4(R11), R11
	MOVW DI, 1(CX)
	CMPL R11, $0x3c
	JBE  emit_copy2_0_match_nolit_encodeBetterBlockAsm1K_emit2
	LEAL -60(R11), SI
	CMPL R11, $0x0000013c
	JB   emit_copy2_1_match_nolit_encodeBetterBlockAsm1K_emit2
	CMPL R11, $0x0001003c
	JB   emit_copy2_2_match_nolit_encodeBetterBlockAsm1K_emit2
	MOVB $0xfe, (CX)
	MOVL SI, 3(CX)
	ADDQ $0x06, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

emit_copy2_2_match_nolit_encodeBetterBlockAsm1K_emit2:
	MOVB $0xfa, (CX)
	MOVW SI, 3(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

emit_copy2_1_match_nolit_encodeBetterBlockAsm1K_emit2:
	MOVB $0xf6, (CX)
	MOVB SI, 3(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

emit_copy2_0_match_nolit_encodeBetterBlockAsm1K_emit2:
	MOVL $0x00000002, SI
	LEAL (SI)(R11*4), SI
	MOVB SI, (CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

	// emitLiteralsDstP
	MOVL 12(SP), SI
	CMPL SI, BX
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm1K
	MOVL BX, DI
	MOVL BX, 12(SP)
	LEAQ (DX)(SI*1), R8
	SUBL SI, DI

	// emitLiteral
	LEAL -1(DI), SI
	CMPL SI, $0x1d
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm1K
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm1K
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm1K

three_bytes_match_emit_repeat_encodeBetterBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, SI
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm1K

two_bytes_match_emit_repeat_encodeBetterBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB SI, 1(CX)
	ADDL $0x1d, SI
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_midmatch_emit_repeat_encodeBetterBlockAsm1K
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm1K

one_byte_match_emit_repeat_encodeBetterBlockAsm1K:
	SHLB $0x03, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 1
	CMPQ DI, $0x08
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_8
	CMPQ DI, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_8through16
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_8:
	MOVQ (R8), R9
	MOVQ R9, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm1K
	PCALIGN $0x10

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_8through16:
	MOVQ (R8), R9
	MOVQ -8(R8)(DI*1), R8
	MOVQ R9, (CX)
	MOVQ R8, -8(CX)(DI*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm1K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm1K

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm1K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm1K

memmove_midmatch_emit_repeat_encodeBetterBlockAsm1K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveShort
	// margin: 8, min move: 30
	CMPQ DI, $0x20
	JBE  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DI*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(DI*1)
	JMP   memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm1K

emit_lit_memmove_mid_match_emit_repeat_encodeBetterBlockAsm1K_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DI*1), X2
	MOVOU -16(R8)(DI*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)

memmove_mid_end_copy_match_emit_repeat_encodeBetterBlockAsm1K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm1K

memmove_long_match_emit_repeat_encodeBetterBlockAsm1K:
	LEAQ (CX)(DI*1), SI

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DI*1), X2
	MOVOU   -16(R8)(DI*1), X3
	MOVQ    DI, R10
	SHRQ    $0x05, R10
	MOVQ    CX, R9
	ANDL    $0x0000001f, R9
	MOVQ    $0x00000040, R12
	SUBQ    R9, R12
	DECQ    R10
	JA      emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(R8)(R12*1), R9
	LEAQ    -32(CX)(R12*1), R13
	PCALIGN $0x10

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm1Klarge_big_loop_back:
	MOVOU (R9), X4
	MOVOU 16(R9), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R9
	ADDQ  $0x20, R12
	DECQ  R10
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(R8)(R12*1), X4
	MOVOU -16(R8)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  DI, R12
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(DI*1)
	MOVOU X3, -16(CX)(DI*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm1K:
	ADDL R11, AX
	ADDL $0x04, R11
	MOVL AX, 12(SP)

	// emitRepeat
	LEAL -1(R11), SI
	CMPL R11, $0x1d
	JBE  repeat_one_match_nolit_repeat_encodeBetterBlockAsm1K
	LEAL -30(R11), SI
	CMPL R11, $0x0000011e
	JB   repeat_two_match_nolit_repeat_encodeBetterBlockAsm1K
	CMPL R11, $0x0001001e
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm1K
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

repeat_three_match_nolit_repeat_encodeBetterBlockAsm1K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

repeat_two_match_nolit_repeat_encodeBetterBlockAsm1K:
	MOVB $0xec, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm1K

repeat_one_match_nolit_repeat_encodeBetterBlockAsm1K:
	XORL SI, SI
	LEAL -4(SI)(R11*8), SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm1K:
	CMPL AX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm1K
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm1K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm1K:
	MOVQ  tmp+48(FP), SI
	MOVQ  $0x0000cf1bbcdcbf9b, DI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(BX), BX
	LEAQ  -2(AX), R9
	MOVQ  (DX)(BX*1), R10
	MOVQ  1(DX)(BX*1), R11
	MOVQ  (DX)(R9*1), R12
	MOVQ  1(DX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ DI, R10
	SHRQ  $0x35, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x38, R11
	SHLQ  $0x10, R12
	IMULQ DI, R12
	SHRQ  $0x35, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x38, R13
	LEAQ  1(BX), R8
	LEAQ  1(R9), R14
	MOVW  BX, (SI)(R10*2)
	MOVW  R9, (SI)(R12*2)
	LEAQ  1(R9)(BX*1), R10
	SHRQ  $0x01, R10
	ADDQ  $0x01, BX
	SUBQ  $0x01, R9
	MOVW  R8, 4096(SI)(R11*2)
	MOVW  R14, 4096(SI)(R13*2)

index_loop_encodeBetterBlockAsm1K:
	CMPQ  R10, R9
	JAE   search_loop_encodeBetterBlockAsm1K
	MOVQ  (DX)(BX*1), R8
	MOVQ  (DX)(R10*1), R11
	SHLQ  $0x10, R8
	IMULQ DI, R8
	SHRQ  $0x35, R8
	SHLQ  $0x10, R11
	IMULQ DI, R11
	SHRQ  $0x35, R11
	MOVW  BX, (SI)(R8*2)
	MOVW  R9, (SI)(R11*2)
	ADDQ  $0x02, BX
	ADDQ  $0x02, R10
	JMP   index_loop_encodeBetterBlockAsm1K

emit_remainder_encodeBetterBlockAsm1K:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm1K
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm1K:
	MOVQ src_len+32(FP), AX

	// emitLiteralsDstP
	MOVL 12(SP), BX
	CMPL BX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm1K
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (DX)(BX*1), AX
	SUBL BX, SI

	// emitLiteral
	LEAL -1(SI), DX
	CMPL DX, $0x1d
	JB   one_byte_emit_remainder_encodeBetterBlockAsm1K
	SUBL $0x1d, DX
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm1K
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm1K

three_bytes_emit_remainder_encodeBetterBlockAsm1K:
	MOVB $0xf0, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	ADDL $0x1d, DX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm1K

two_bytes_emit_remainder_encodeBetterBlockAsm1K:
	MOVB $0xe8, (CX)
	MOVB DL, 1(CX)
	ADDL $0x1d, DX
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_midemit_remainder_encodeBetterBlockAsm1K
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm1K

one_byte_emit_remainder_encodeBetterBlockAsm1K:
	SHLB $0x03, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -1, min move: 1
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_3
	CMPQ BX, $0x08
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_4through8
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm1K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm1K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_4through8:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm1K
	PCALIGN $0x10

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm1K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm1K

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm1K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm1K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm1K

memmove_midemit_remainder_encodeBetterBlockAsm1K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	// margin: -2, min move: 30
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm1K_memmove_move_17through32
	JMP  emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm1K_memmove_move_33through64

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm1K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm1K

emit_lit_memmove_mid_emit_remainder_encodeBetterBlockAsm1K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_mid_end_copy_emit_remainder_encodeBetterBlockAsm1K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm1K

memmove_long_emit_remainder_encodeBetterBlockAsm1K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU   (AX), X0
	MOVOU   16(AX), X1
	MOVOU   -32(AX)(BX*1), X2
	MOVOU   -16(AX)(BX*1), X3
	MOVQ    BX, DI
	SHRQ    $0x05, DI
	MOVQ    CX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm1Klarge_forward_sse_loop_32
	LEAQ    -32(AX)(R8*1), SI
	LEAQ    -32(CX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm1Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm1Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm1Klarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm1Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm1K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func emitLiteral(dst []byte, lit []byte) int
// Requires: SSE2
TEXT ·emitLiteral(SB), NOSPLIT, $0-56
	MOVQ  lit_len+32(FP), DX
	MOVQ  dst_base+0(FP), AX
	MOVQ  lit_base+24(FP), CX
	TESTQ DX, DX
	JZ    emit_literal_end_standalone_skip

	// emitLiteral
	MOVL DX, BX
	LEAL -1(DX), SI
	CMPL SI, $0x1d
	JB   one_byte_standalone
	SUBL $0x1d, SI
	CMPL SI, $0x00000100
	JB   two_bytes_standalone
	CMPL SI, $0x00010000
	JB   three_bytes_standalone
	MOVL SI, DI
	SHRL $0x10, DI
	MOVB $0xf8, (AX)
	MOVW SI, 1(AX)
	MOVB DI, 3(AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	ADDL $0x1d, SI
	JMP  memmove_long_standalone

three_bytes_standalone:
	MOVB $0xf0, (AX)
	MOVW SI, 1(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX
	ADDL $0x1d, SI
	JMP  memmove_long_standalone

two_bytes_standalone:
	MOVB $0xe8, (AX)
	MOVB SI, 1(AX)
	ADDL $0x1d, SI
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	CMPL SI, $0x40
	JB   memmove_midstandalone
	JMP  memmove_long_standalone

one_byte_standalone:
	SHLB $0x03, SI
	MOVB SI, (AX)
	ADDQ $0x01, BX
	ADDQ $0x01, AX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ DX, $0x03
	JB   emit_lit_memmove_standalone_memmove_move_1or2
	JE   emit_lit_memmove_standalone_memmove_move_3
	CMPQ DX, $0x08
	JBE  emit_lit_memmove_standalone_memmove_move_4through8
	CMPQ DX, $0x10
	JBE  emit_lit_memmove_standalone_memmove_move_8through16
	CMPQ DX, $0x20
	JBE  emit_lit_memmove_standalone_memmove_move_17through32
	JMP  emit_lit_memmove_standalone_memmove_move_33through64

emit_lit_memmove_standalone_memmove_move_1or2:
	MOVB (CX), SI
	MOVB -1(CX)(DX*1), CL
	MOVB SI, (AX)
	MOVB CL, -1(AX)(DX*1)
	JMP  emit_literal_end_standalone

emit_lit_memmove_standalone_memmove_move_3:
	MOVW (CX), SI
	MOVB 2(CX), CL
	MOVW SI, (AX)
	MOVB CL, 2(AX)
	JMP  emit_literal_end_standalone

emit_lit_memmove_standalone_memmove_move_4through8:
	MOVL (CX), SI
	MOVL -4(CX)(DX*1), CX
	MOVL SI, (AX)
	MOVL CX, -4(AX)(DX*1)
	JMP  emit_literal_end_standalone
	PCALIGN $0x10

emit_lit_memmove_standalone_memmove_move_8through16:
	MOVQ (CX), SI
	MOVQ -8(CX)(DX*1), CX
	MOVQ SI, (AX)
	MOVQ CX, -8(AX)(DX*1)
	JMP  emit_literal_end_standalone

emit_lit_memmove_standalone_memmove_move_17through32:
	MOVOU (CX), X0
	MOVOU -16(CX)(DX*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(DX*1)
	JMP   emit_literal_end_standalone

emit_lit_memmove_standalone_memmove_move_33through64:
	MOVOU (CX), X0
	MOVOU 16(CX), X1
	MOVOU -32(CX)(DX*1), X2
	MOVOU -16(CX)(DX*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(DX*1)
	MOVOU X3, -16(AX)(DX*1)
	JMP   emit_literal_end_standalone
	JMP emit_literal_end_standalone

memmove_midstandalone:
	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ DX, $0x20
	JBE  emit_lit_memmove_mid_standalone_memmove_move_17through32
	JMP  emit_lit_memmove_mid_standalone_memmove_move_33through64

emit_lit_memmove_mid_standalone_memmove_move_17through32:
	MOVOU (CX), X0
	MOVOU -16(CX)(DX*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(DX*1)
	JMP   emit_literal_end_standalone

emit_lit_memmove_mid_standalone_memmove_move_33through64:
	MOVOU (CX), X0
	MOVOU 16(CX), X1
	MOVOU -32(CX)(DX*1), X2
	MOVOU -16(CX)(DX*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(DX*1)
	MOVOU X3, -16(AX)(DX*1)
	JMP   emit_literal_end_standalone
	JMP emit_literal_end_standalone

memmove_long_standalone:
	// genMemMoveLong
	MOVOU   (CX), X0
	MOVOU   16(CX), X1
	MOVOU   -32(CX)(DX*1), X2
	MOVOU   -16(CX)(DX*1), X3
	MOVQ    DX, DI
	SHRQ    $0x05, DI
	MOVQ    AX, SI
	ANDL    $0x0000001f, SI
	MOVQ    $0x00000040, R8
	SUBQ    SI, R8
	DECQ    DI
	JA      emit_lit_memmove_long_standalonelarge_forward_sse_loop_32
	LEAQ    -32(CX)(R8*1), SI
	LEAQ    -32(AX)(R8*1), R9
	PCALIGN $0x10

emit_lit_memmove_long_standalonelarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_standalonelarge_big_loop_back

emit_lit_memmove_long_standalonelarge_forward_sse_loop_32:
	MOVOU -32(CX)(R8*1), X4
	MOVOU -16(CX)(R8*1), X5
	MOVOA X4, -32(AX)(R8*1)
	MOVOA X5, -16(AX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  DX, R8
	JAE   emit_lit_memmove_long_standalonelarge_forward_sse_loop_32
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(DX*1)
	MOVOU X3, -16(AX)(DX*1)
	JMP   emit_literal_end_standalone
	JMP emit_literal_end_standalone

emit_literal_end_standalone_skip:
	XORQ BX, BX

emit_literal_end_standalone:
	MOVQ BX, ret+48(FP)
	RET

// func emitRepeat(dst []byte, length int) int
TEXT ·emitRepeat(SB), NOSPLIT, $0-40
	XORQ DX, DX
	MOVQ dst_base+0(FP), AX
	MOVQ length+24(FP), CX

	// emitRepeat
	LEAL -1(CX), BX
	CMPL CX, $0x1d
	JBE  repeat_one_standalone
	LEAL -30(CX), BX
	CMPL CX, $0x0000011e
	JB   repeat_two_standalone
	CMPL CX, $0x0001001e
	JB   repeat_three_standalone
	MOVB $0xfc, (AX)
	MOVL BX, 1(AX)
	ADDQ $0x04, DX
	ADDQ $0x04, AX
	JMP  gen_emit_repeat_end

repeat_three_standalone:
	MOVB $0xf4, (AX)
	MOVW BX, 1(AX)
	ADDQ $0x03, DX
	ADDQ $0x03, AX
	JMP  gen_emit_repeat_end

repeat_two_standalone:
	MOVB $0xec, (AX)
	MOVB BL, 1(AX)
	ADDQ $0x02, DX
	ADDQ $0x02, AX
	JMP  gen_emit_repeat_end

repeat_one_standalone:
	XORL BX, BX
	LEAL -4(BX)(CX*8), BX
	MOVB BL, (AX)
	ADDQ $0x01, DX
	ADDQ $0x01, AX

gen_emit_repeat_end:
	MOVQ DX, ret+32(FP)
	RET

// func emitCopy(dst []byte, offset int, length int) int
TEXT ·emitCopy(SB), NOSPLIT, $0-48
	XORQ BX, BX
	MOVQ dst_base+0(FP), AX
	MOVQ offset+24(FP), CX
	MOVQ length+32(FP), DX

	// emitCopy
	CMPL CX, $0x0001003f
	JBE  two_byte_offset_standalone

	// emitCopy3
	LEAL -4(DX), DX
	LEAL -65536(CX), CX
	SHLL $0x0b, CX
	ADDL $0x07, CX
	CMPL DX, $0x3c
	JBE  emit_copy3_0_standalone_emit3
	LEAL -60(DX), SI
	CMPL DX, $0x0000013c
	JB   emit_copy3_1_standalone_emit3
	CMPL DX, $0x0001003c
	JB   emit_copy3_2_standalone_emit3
	ADDL $0x000007e0, CX
	MOVL CX, (AX)
	MOVL SI, 4(AX)
	ADDQ $0x07, BX
	ADDQ $0x07, AX
	JMP  gen_emit_copy_end

emit_copy3_2_standalone_emit3:
	ADDL $0x000007c0, CX
	MOVL CX, (AX)
	MOVW SI, 4(AX)
	ADDQ $0x06, BX
	ADDQ $0x06, AX
	JMP  gen_emit_copy_end

emit_copy3_1_standalone_emit3:
	ADDL $0x000007a0, CX
	MOVL CX, (AX)
	MOVB SI, 4(AX)
	ADDQ $0x05, BX
	ADDQ $0x05, AX
	JMP  gen_emit_copy_end

emit_copy3_0_standalone_emit3:
	SHLL $0x05, DX
	ORL  DX, CX
	MOVL CX, (AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	JMP  gen_emit_copy_end

two_byte_offset_standalone:
	CMPL CX, $0x00000400
	JA   two_byte_standalone
	CMPL DX, $0x00000013
	JAE  emit_one_longer_standalone
	LEAL -1(CX), CX
	SHLL $0x06, CX
	LEAL -15(CX)(DX*4), CX
	MOVW CX, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end

emit_one_longer_standalone:
	CMPL DX, $0x00000112
	JAE  emit_copy1_repeat_standalone
	LEAL -1(CX), CX
	SHLL $0x06, CX
	LEAL 61(CX), CX
	MOVW CX, (AX)
	LEAL -18(DX), CX
	MOVB CL, 2(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX
	JMP  gen_emit_copy_end

emit_copy1_repeat_standalone:
	LEAL -1(CX), CX
	SHLL $0x06, CX
	LEAL 57(CX), CX
	MOVW CX, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	SUBL $0x12, DX

	// emitRepeat
	LEAL -1(DX), CX
	CMPL DX, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_standalone
	LEAL -30(DX), CX
	CMPL DX, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_standalone
	CMPL DX, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_standalone
	MOVB $0xfc, (AX)
	MOVL CX, 1(AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	JMP  gen_emit_copy_end

repeat_three_emit_copy1_do_repeat_standalone:
	MOVB $0xf4, (AX)
	MOVW CX, 1(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX
	JMP  gen_emit_copy_end

repeat_two_emit_copy1_do_repeat_standalone:
	MOVB $0xec, (AX)
	MOVB CL, 1(AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end

repeat_one_emit_copy1_do_repeat_standalone:
	XORL CX, CX
	LEAL -4(CX)(DX*8), CX
	MOVB CL, (AX)
	ADDQ $0x01, BX
	ADDQ $0x01, AX
	JMP  gen_emit_copy_end

two_byte_standalone:
	// emitCopy2
	LEAL -64(CX), CX
	LEAL -4(DX), DX
	MOVW CX, 1(AX)
	CMPL DX, $0x3c
	JBE  emit_copy2_0_standalone_emit2
	LEAL -60(DX), CX
	CMPL DX, $0x0000013c
	JB   emit_copy2_1_standalone_emit2
	CMPL DX, $0x0001003c
	JB   emit_copy2_2_standalone_emit2
	MOVB $0xfe, (AX)
	MOVL CX, 3(AX)
	ADDQ $0x06, BX
	ADDQ $0x06, AX
	JMP  gen_emit_copy_end

emit_copy2_2_standalone_emit2:
	MOVB $0xfa, (AX)
	MOVW CX, 3(AX)
	ADDQ $0x05, BX
	ADDQ $0x05, AX
	JMP  gen_emit_copy_end

emit_copy2_1_standalone_emit2:
	MOVB $0xf6, (AX)
	MOVB CL, 3(AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	JMP  gen_emit_copy_end

emit_copy2_0_standalone_emit2:
	MOVL $0x00000002, CX
	LEAL (CX)(DX*4), CX
	MOVB CL, (AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX

gen_emit_copy_end:
	MOVQ BX, ret+40(FP)
	RET

// func emitCopyLits2(dst []byte, lits []byte, offset int, length int) int
// Requires: CMOV
TEXT ·emitCopyLits2(SB), NOSPLIT, $0-72
	XORQ BX, BX
	MOVQ dst_base+0(FP), AX
	MOVQ lits_len+32(FP), SI
	MOVQ offset+48(FP), CX
	MOVQ length+56(FP), DX
	CMPL DX, $0x0b

	// emitCopy2WithLits
	XORQ    DI, DI
	SUBL    $0x40, CX
	LEAL    -11(DX), R8
	LEAL    -4(DX), DX
	MOVW    CX, 1(AX)
	CMPL    DX, $0x07
	CMOVLGE R8, DI
	MOVQ    $0x00000007, CX
	CMOVLLT DX, CX
	LEAL    -1(SI)(CX*4), CX
	MOVL    $0x00000003, DX
	LEAL    (DX)(CX*8), CX
	MOVB    CL, (AX)
	ADDQ    $0x03, BX
	ADDQ    $0x03, AX
	MOVQ    lits_base+24(FP), CX

	// genMemMoveVeryShort
	CMPQ SI, $0x03
	JE   standalone_emitcopy2_lits_move_3
	JA   standalone_emitcopy2_lits_move_4
	MOVB (CX), DL
	MOVB -1(CX)(SI*1), CL
	MOVB DL, (AX)
	MOVB CL, -1(AX)(SI*1)
	JMP  standalone_emitcopy2_lits_end

standalone_emitcopy2_lits_move_3:
	MOVW (CX), DX
	MOVB 2(CX), CL
	MOVW DX, (AX)
	MOVB CL, 2(AX)
	JMP  standalone_emitcopy2_lits_end

standalone_emitcopy2_lits_move_4:
	MOVL (CX), DX
	MOVL DX, (AX)

standalone_emitcopy2_lits_end:
	ADDQ  SI, BX
	ADDQ  SI, AX
	TESTL DI, DI
	JZ    standalone_emitcopy2_lits_done

	// emitRepeat
	LEAL -1(DI), CX
	CMPL DI, $0x1d
	JBE  repeat_one_standalone_emitcopy2_lits
	LEAL -30(DI), CX
	CMPL DI, $0x0000011e
	JB   repeat_two_standalone_emitcopy2_lits
	CMPL DI, $0x0001001e
	JB   repeat_three_standalone_emitcopy2_lits
	MOVB $0xfc, (AX)
	MOVL CX, 1(AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	JMP  standalone_emitcopy2_lits_done

repeat_three_standalone_emitcopy2_lits:
	MOVB $0xf4, (AX)
	MOVW CX, 1(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX
	JMP  standalone_emitcopy2_lits_done

repeat_two_standalone_emitcopy2_lits:
	MOVB $0xec, (AX)
	MOVB CL, 1(AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  standalone_emitcopy2_lits_done

repeat_one_standalone_emitcopy2_lits:
	XORL CX, CX
	LEAL -4(CX)(DI*8), CX
	MOVB CL, (AX)
	ADDQ $0x01, BX
	ADDQ $0x01, AX

standalone_emitcopy2_lits_done:
	MOVQ BX, ret+64(FP)
	RET

// func emitCopyLits3(dst []byte, lits []byte, offset int, length int) int
TEXT ·emitCopyLits3(SB), NOSPLIT, $0-72
	XORQ BX, BX
	MOVQ dst_base+0(FP), AX
	MOVQ lits_len+32(FP), SI
	MOVQ offset+48(FP), CX
	MOVQ length+56(FP), DX

	// emitCopy3
	LEAL -4(DX), DX
	LEAL -65536(CX), CX
	SHLL $0x0b, CX
	LEAL 7(CX)(SI*8), CX
	CMPL DX, $0x3c
	JBE  emit_copy3_0_standalone_lits
	LEAL -60(DX), DI
	CMPL DX, $0x0000013c
	JB   emit_copy3_1_standalone_lits
	CMPL DX, $0x0001003c
	JB   emit_copy3_2_standalone_lits
	ADDL $0x000007e0, CX
	MOVL CX, (AX)
	MOVL DI, 4(AX)
	ADDQ $0x07, BX
	ADDQ $0x07, AX
	JMP  gen_emit_copy_lits_copylits

emit_copy3_2_standalone_lits:
	ADDL $0x000007c0, CX
	MOVL CX, (AX)
	MOVW DI, 4(AX)
	ADDQ $0x06, BX
	ADDQ $0x06, AX
	JMP  gen_emit_copy_lits_copylits

emit_copy3_1_standalone_lits:
	ADDL $0x000007a0, CX
	MOVL CX, (AX)
	MOVB DI, 4(AX)
	ADDQ $0x05, BX
	ADDQ $0x05, AX
	JMP  gen_emit_copy_lits_copylits

emit_copy3_0_standalone_lits:
	SHLL $0x05, DX
	ORL  DX, CX
	MOVL CX, (AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX

gen_emit_copy_lits_copylits:
	MOVQ lits_base+24(FP), CX

	// genMemMoveVeryShort
	CMPQ SI, $0x03
	JE   standalone_emitcopy3_lits_move_3
	MOVB (CX), DL
	MOVB -1(CX)(SI*1), CL
	MOVB DL, (AX)
	MOVB CL, -1(AX)(SI*1)
	JMP  standalone_emitcopy3_lits_end

standalone_emitcopy3_lits_move_3:
	MOVW (CX), DX
	MOVB 2(CX), CL
	MOVW DX, (AX)
	MOVB CL, 2(AX)

standalone_emitcopy3_lits_end:
	ADDQ SI, BX
	MOVQ BX, ret+64(FP)
	RET

// func matchLen(a []byte, b []byte) int
// Requires: BMI
TEXT ·matchLen(SB), NOSPLIT, $0-56
	MOVQ a_base+0(FP), AX
	MOVQ b_base+24(FP), CX
	MOVQ a_len+8(FP), DX

	// matchLen
	XORL SI, SI
	JMP  matchlen_loop_16_entry_standalone
	PCALIGN $0x10

matchlen_loopback_16_standalone:
	MOVQ (AX)(SI*1), BX
	MOVQ 8(AX)(SI*1), DI
	XORQ (CX)(SI*1), BX
	JNZ  matchlen_bsf_8_standalone
	XORQ 8(CX)(SI*1), DI
	JNZ  matchlen_bsf_16standalone
	LEAL -16(DX), DX
	LEAL 16(SI), SI

matchlen_loop_16_entry_standalone:
	CMPL DX, $0x10
	JAE  matchlen_loopback_16_standalone
	JMP  matchlen_match8_standalone
	PCALIGN $0x10

matchlen_bsf_16standalone:
	TZCNTQ DI, DI
	SARQ   $0x03, DI
	LEAL   8(SI)(DI*1), SI
	JMP    gen_match_len_end

matchlen_match8_standalone:
	CMPL DX, $0x08
	JB   matchlen_match4_standalone
	MOVQ (AX)(SI*1), BX
	XORQ (CX)(SI*1), BX
	JNZ  matchlen_bsf_8_standalone
	LEAL -8(DX), DX
	LEAL 8(SI), SI
	JMP  matchlen_match4_standalone
	PCALIGN $0x10

matchlen_bsf_8_standalone:
	TZCNTQ BX, BX
	SARQ   $0x03, BX
	ADDL   BX, SI
	JMP    gen_match_len_end

matchlen_match4_standalone:
	CMPL DX, $0x04
	JB   matchlen_match2_standalone
	MOVL (AX)(SI*1), BX
	CMPL (CX)(SI*1), BX
	JNE  matchlen_match2_standalone
	LEAL -4(DX), DX
	LEAL 4(SI), SI

matchlen_match2_standalone:
	CMPL DX, $0x01
	JE   matchlen_match1_standalone
	JB   gen_match_len_end
	MOVW (AX)(SI*1), BX
	CMPW (CX)(SI*1), BX
	JNE  matchlen_match1_standalone
	LEAL 2(SI), SI
	SUBL $0x02, DX
	JZ   gen_match_len_end

matchlen_match1_standalone:
	MOVB (AX)(SI*1), BL
	CMPB (CX)(SI*1), BL
	JNE  gen_match_len_end
	LEAL 1(SI), SI

gen_match_len_end:
	MOVQ SI, ret+48(FP)
	RET

// func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
// Requires: CMOV, SSE2
TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $8-64
	XORQ SI, SI
	MOVQ dst_base+0(FP), AX
	MOVQ dst_len+8(FP), CX
	MOVQ src_base+24(FP), DX
	MOVQ src_len+32(FP), BX
	LEAQ (DX)(BX*1), BX
	LEAQ -12(AX)(CX*1), CX
	MOVL $0x00000001, (SP)

lz4_mz_loop:
	CMPQ    DX, BX
	JAE     lz4_mz_corrupt
	CMPQ    AX, CX
	JAE     lz4_mz_dstfull
	MOVBQZX (DX), DI
	MOVQ    DI, R8
	MOVQ    DI, R9
	ANDQ    $0x0f, R9
	XORQ    R10, R10
	SHRQ    $0x04, R8
	CMPQ    DI, $0x50
	CMOVQLT R8, R10
	JLT     lz4_mz_ll_end
	CMPQ    DI, $0xf0
	JB      lz4_mz_ll_end

lz4_mz_ll_loop:
	INCQ    DX
	CMPQ    DX, BX
	JAE     lz4_mz_corrupt
	MOVBQZX (DX), DI
	ADDQ    DI, R8
	CMPQ    DI, $0xff
	JEQ     lz4_mz_ll_loop

lz4_mz_ll_end:
	LEAQ  (DX)(R8*1), DI
	ADDQ  $0x04, R9
	CMPQ  DI, BX
	JAE   lz4_mz_corrupt
	INCQ  DX
	INCQ  DI
	TESTQ R8, R8
	JZ    lz4_mz_lits_done
	TESTQ R10, R10
	JNZ   lz4_mz_lits_done
	LEAQ  (AX)(R8*1), R11
	CMPQ  R11, CX
	JAE   lz4_mz_dstfull

	// emitLiteral
	LEAL -1(R8), R11
	CMPL R11, $0x1d
	JB   one_byte_lz4_mz
	SUBL $0x1d, R11
	CMPL R11, $0x00000100
	JB   two_bytes_lz4_mz
	CMPL R11, $0x00010000
	JB   three_bytes_lz4_mz
	MOVL R11, R12
	SHRL $0x10, R12
	MOVB $0xf8, (AX)
	MOVW R11, 1(AX)
	MOVB R12, 3(AX)
	ADDQ $0x04, AX
	ADDL $0x1d, R11
	JMP  memmove_long_lz4_mz

three_bytes_lz4_mz:
	MOVB $0xf0, (AX)
	MOVW R11, 1(AX)
	ADDQ $0x03, AX
	ADDL $0x1d, R11
	JMP  memmove_long_lz4_mz

two_bytes_lz4_mz:
	MOVB $0xe8, (AX)
	MOVB R11, 1(AX)
	ADDL $0x1d, R11
	ADDQ $0x02, AX
	CMPL R11, $0x40
	JB   memmove_midlz4_mz
	JMP  memmove_long_lz4_mz

one_byte_lz4_mz:
	SHLB $0x03, R11
	MOVB R11, (AX)
	ADDQ $0x01, AX
	LEAQ (AX)(R8*1), R11
	MOVL R8, R12

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ R12, $0x03
	JB   emit_lit_memmove_lz4_mz_memmove_move_1or2
	JE   emit_lit_memmove_lz4_mz_memmove_move_3
	CMPQ R12, $0x08
	JBE  emit_lit_memmove_lz4_mz_memmove_move_4through8
	CMPQ R12, $0x10
	JBE  emit_lit_memmove_lz4_mz_memmove_move_8through16
	CMPQ R12, $0x20
	JBE  emit_lit_memmove_lz4_mz_memmove_move_17through32
	JMP  emit_lit_memmove_lz4_mz_memmove_move_33through64

emit_lit_memmove_lz4_mz_memmove_move_1or2:
	MOVB (DX), R13
	MOVB -1(DX)(R12*1), R14
	MOVB R13, (AX)
	MOVB R14, -1(AX)(R12*1)
	JMP  memmove_end_copy_lz4_mz

emit_lit_memmove_lz4_mz_memmove_move_3:
	MOVW (DX), R13
	MOVB 2(DX), R14
	MOVW R13, (AX)
	MOVB R14, 2(AX)
	JMP  memmove_end_copy_lz4_mz

emit_lit_memmove_lz4_mz_memmove_move_4through8:
	MOVL (DX), R13
	MOVL -4(DX)(R12*1), R14
	MOVL R13, (AX)
	MOVL R14, -4(AX)(R12*1)
	JMP  memmove_end_copy_lz4_mz
	PCALIGN $0x10

emit_lit_memmove_lz4_mz_memmove_move_8through16:
	MOVQ (DX), R13
	MOVQ -8(DX)(R12*1), R14
	MOVQ R13, (AX)
	MOVQ R14, -8(AX)(R12*1)
	JMP  memmove_end_copy_lz4_mz

emit_lit_memmove_lz4_mz_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(R12*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(R12*1)
	JMP   memmove_end_copy_lz4_mz

emit_lit_memmove_lz4_mz_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R12*1), X2
	MOVOU -16(DX)(R12*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R12*1)
	MOVOU X3, -16(AX)(R12*1)

memmove_end_copy_lz4_mz:
	MOVQ R11, AX
	JMP  lz4_mz_lits_emit_done

memmove_midlz4_mz:
	LEAQ (AX)(R8*1), R11
	MOVL R8, R12

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ R12, $0x20
	JBE  emit_lit_memmove_mid_lz4_mz_memmove_move_17through32
	JMP  emit_lit_memmove_mid_lz4_mz_memmove_move_33through64

emit_lit_memmove_mid_lz4_mz_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(R12*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(R12*1)
	JMP   memmove_mid_end_copy_lz4_mz

emit_lit_memmove_mid_lz4_mz_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R12*1), X2
	MOVOU -16(DX)(R12*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R12*1)
	MOVOU X3, -16(AX)(R12*1)

memmove_mid_end_copy_lz4_mz:
	MOVQ R11, AX
	JMP  lz4_mz_lits_emit_done

memmove_long_lz4_mz:
	LEAQ (AX)(R8*1), R11
	MOVL R8, R12

	// genMemMoveLong
	MOVOU   (DX), X0
	MOVOU   16(DX), X1
	MOVOU   -32(DX)(R12*1), X2
	MOVOU   -16(DX)(R12*1), X3
	MOVQ    R12, R14
	SHRQ    $0x05, R14
	MOVQ    AX, R13
	ANDL    $0x0000001f, R13
	MOVQ    $0x00000040, R15
	SUBQ    R13, R15
	DECQ    R14
	JA      emit_lit_memmove_long_lz4_mzlarge_forward_sse_loop_32
	LEAQ    -32(DX)(R15*1), R13
	LEAQ    -32(AX)(R15*1), BP
	PCALIGN $0x10

emit_lit_memmove_long_lz4_mzlarge_big_loop_back:
	MOVOU (R13), X4
	MOVOU 16(R13), X5
	MOVOA X4, (BP)
	MOVOA X5, 16(BP)
	ADDQ  $0x20, BP
	ADDQ  $0x20, R13
	ADDQ  $0x20, R15
	DECQ  R14
	JNA   emit_lit_memmove_long_lz4_mzlarge_big_loop_back

emit_lit_memmove_long_lz4_mzlarge_forward_sse_loop_32:
	MOVOU -32(DX)(R15*1), X4
	MOVOU -16(DX)(R15*1), X5
	MOVOA X4, -32(AX)(R15*1)
	MOVOA X5, -16(AX)(R15*1)
	ADDQ  $0x20, R15
	CMPQ  R12, R15
	JAE   emit_lit_memmove_long_lz4_mzlarge_forward_sse_loop_32
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R12*1)
	MOVOU X3, -16(AX)(R12*1)
	MOVQ  R11, AX

lz4_mz_lits_emit_done:
lz4_mz_lits_done:
	ADDQ  R8, SI
	MOVQ  DI, R8
	MOVQ  DX, DI
	MOVQ  R8, DX
	CMPQ  DX, BX
	JNE   lz4_mz_match
	CMPQ  R9, $0x04
	JNE   lz4_mz_corrupt
	TESTQ R10, R10
	JNZ   lz4_mz_emit_final
	JMP   lz4_mz_done

lz4_mz_match:
	ADDQ    $0x02, DX
	CMPQ    DX, BX
	JAE     lz4_mz_corrupt
	MOVWQZX -2(DX), R8
	TESTQ   R8, R8
	JZ      lz4_mz_corrupt
	CMPQ    R8, SI
	JA      lz4_mz_corrupt
	CMPQ    R9, $0x13
	JNE     lz4_mz_ml_done

lz4_mz_ml_loop:
	MOVBQZX (DX), R11
	INCQ    DX
	ADDQ    R11, R9
	CMPQ    DX, BX
	JAE     lz4_mz_corrupt
	CMPQ    R11, $0xff
	JEQ     lz4_mz_ml_loop

lz4_mz_ml_done:
	ADDQ  R9, SI
	TESTQ R10, R10
	JNZ   lz4_mz_dofuse
	CMPQ  (SP), R8
	JNE   lz4_mz_docopy

	// emitRepeat
	LEAL -1(R9), DI
	CMPL R9, $0x1d
	JBE  repeat_one_lz4_mz
	LEAL -30(R9), DI
	CMPL R9, $0x0000011e
	JB   repeat_two_lz4_mz
	CMPL R9, $0x0001001e
	JB   repeat_three_lz4_mz
	MOVB $0xfc, (AX)
	MOVL DI, 1(AX)
	ADDQ $0x04, AX
	JMP  lz4_mz_loop

repeat_three_lz4_mz:
	MOVB $0xf4, (AX)
	MOVW DI, 1(AX)
	ADDQ $0x03, AX
	JMP  lz4_mz_loop

repeat_two_lz4_mz:
	MOVB $0xec, (AX)
	MOVB DI, 1(AX)
	ADDQ $0x02, AX
	JMP  lz4_mz_loop

repeat_one_lz4_mz:
	XORL DI, DI
	LEAL -4(DI)(R9*8), DI
	MOVB DI, (AX)
	ADDQ $0x01, AX
	JMP  lz4_mz_loop

lz4_mz_dofuse:
	MOVQ R8, (SP)
	CMPQ R8, $0x40
	JB   lz4_mz_doemitcopy

	// emitCopy2WithLits
	XORQ    R11, R11
	SUBL    $0x40, R8
	LEAL    -11(R9), R12
	LEAL    -4(R9), R9
	MOVW    R8, 1(AX)
	CMPL    R9, $0x07
	CMOVLGE R12, R11
	MOVQ    $0x00000007, R8
	CMOVLLT R9, R8
	LEAL    -1(R10)(R8*4), R8
	MOVL    $0x00000003, R9
	LEAL    (R9)(R8*8), R8
	MOVB    R8, (AX)
	ADDQ    $0x03, AX
	MOVL    (DI), DI
	MOVL    DI, (AX)
	ADDQ    R10, AX
	TESTL   R11, R11
	JZ      lz4_mz_loop

	// emitRepeat
	LEAL -1(R11), DI
	CMPL R11, $0x1d
	JBE  repeat_one_fused_emitrep_lz4_mz_
	LEAL -30(R11), DI
	CMPL R11, $0x0000011e
	JB   repeat_two_fused_emitrep_lz4_mz_
	CMPL R11, $0x0001001e
	JB   repeat_three_fused_emitrep_lz4_mz_
	MOVB $0xfc, (AX)
	MOVL DI, 1(AX)
	ADDQ $0x04, AX
	JMP  lz4_mz_loop

repeat_three_fused_emitrep_lz4_mz_:
	MOVB $0xf4, (AX)
	MOVW DI, 1(AX)
	ADDQ $0x03, AX
	JMP  lz4_mz_loop

repeat_two_fused_emitrep_lz4_mz_:
	MOVB $0xec, (AX)
	MOVB DI, 1(AX)
	ADDQ $0x02, AX
	JMP  lz4_mz_loop

repeat_one_fused_emitrep_lz4_mz_:
	XORL DI, DI
	LEAL -4(DI)(R11*8), DI
	MOVB DI, (AX)
	ADDQ $0x01, AX
	JMP  lz4_mz_loop

lz4_mz_doemitcopy:
	// emitLiteral
	LEAL -1(R10), R11
	CMPL R11, $0x1d
	JB   one_byte_lz4_mz_emitcopy
	SUBL $0x1d, R11
	CMPL R11, $0x00000100
	JB   two_bytes_lz4_mz_emitcopy
	CMPL R11, $0x00010000
	JB   three_bytes_lz4_mz_emitcopy
	MOVL R11, R12
	SHRL $0x10, R12
	MOVB $0xf8, (AX)
	MOVW R11, 1(AX)
	MOVB R12, 3(AX)
	ADDQ $0x04, AX
	ADDL $0x1d, R11
	JMP  memmove_long_lz4_mz_emitcopy

three_bytes_lz4_mz_emitcopy:
	MOVB $0xf0, (AX)
	MOVW R11, 1(AX)
	ADDQ $0x03, AX
	ADDL $0x1d, R11
	JMP  memmove_long_lz4_mz_emitcopy

two_bytes_lz4_mz_emitcopy:
	MOVB $0xe8, (AX)
	MOVB R11, 1(AX)
	ADDL $0x1d, R11
	ADDQ $0x02, AX
	CMPL R11, $0x40
	JB   memmove_midlz4_mz_emitcopy
	JMP  memmove_long_lz4_mz_emitcopy

one_byte_lz4_mz_emitcopy:
	SHLB $0x03, R11
	MOVB R11, (AX)
	ADDQ $0x01, AX
	LEAQ (AX)(R10*1), R11

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ R10, $0x03
	JB   emit_lit_memmove_lz4_mz_emitcopy_memmove_move_1or2
	JE   emit_lit_memmove_lz4_mz_emitcopy_memmove_move_3
	CMPQ R10, $0x08
	JBE  emit_lit_memmove_lz4_mz_emitcopy_memmove_move_4through8
	CMPQ R10, $0x10
	JBE  emit_lit_memmove_lz4_mz_emitcopy_memmove_move_8through16
	CMPQ R10, $0x20
	JBE  emit_lit_memmove_lz4_mz_emitcopy_memmove_move_17through32
	JMP  emit_lit_memmove_lz4_mz_emitcopy_memmove_move_33through64

emit_lit_memmove_lz4_mz_emitcopy_memmove_move_1or2:
	MOVB (DI), R12
	MOVB -1(DI)(R10*1), DI
	MOVB R12, (AX)
	MOVB DI, -1(AX)(R10*1)
	JMP  memmove_end_copy_lz4_mz_emitcopy

emit_lit_memmove_lz4_mz_emitcopy_memmove_move_3:
	MOVW (DI), R12
	MOVB 2(DI), DI
	MOVW R12, (AX)
	MOVB DI, 2(AX)
	JMP  memmove_end_copy_lz4_mz_emitcopy

emit_lit_memmove_lz4_mz_emitcopy_memmove_move_4through8:
	MOVL (DI), R12
	MOVL -4(DI)(R10*1), DI
	MOVL R12, (AX)
	MOVL DI, -4(AX)(R10*1)
	JMP  memmove_end_copy_lz4_mz_emitcopy
	PCALIGN $0x10

emit_lit_memmove_lz4_mz_emitcopy_memmove_move_8through16:
	MOVQ (DI), R12
	MOVQ -8(DI)(R10*1), DI
	MOVQ R12, (AX)
	MOVQ DI, -8(AX)(R10*1)
	JMP  memmove_end_copy_lz4_mz_emitcopy

emit_lit_memmove_lz4_mz_emitcopy_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R10*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(R10*1)
	JMP   memmove_end_copy_lz4_mz_emitcopy

emit_lit_memmove_lz4_mz_emitcopy_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R10*1), X2
	MOVOU -16(DI)(R10*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R10*1)
	MOVOU X3, -16(AX)(R10*1)

memmove_end_copy_lz4_mz_emitcopy:
	MOVQ R11, AX
	JMP  lz4_mz__emit_done

memmove_midlz4_mz_emitcopy:
	LEAQ (AX)(R10*1), R11

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ R10, $0x20
	JBE  emit_lit_memmove_mid_lz4_mz_emitcopy_memmove_move_17through32
	JMP  emit_lit_memmove_mid_lz4_mz_emitcopy_memmove_move_33through64

emit_lit_memmove_mid_lz4_mz_emitcopy_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R10*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(R10*1)
	JMP   memmove_mid_end_copy_lz4_mz_emitcopy

emit_lit_memmove_mid_lz4_mz_emitcopy_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R10*1), X2
	MOVOU -16(DI)(R10*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R10*1)
	MOVOU X3, -16(AX)(R10*1)

memmove_mid_end_copy_lz4_mz_emitcopy:
	MOVQ R11, AX
	JMP  lz4_mz__emit_done

memmove_long_lz4_mz_emitcopy:
	LEAQ (AX)(R10*1), R11

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(R10*1), X2
	MOVOU   -16(DI)(R10*1), X3
	MOVQ    R10, R13
	SHRQ    $0x05, R13
	MOVQ    AX, R12
	ANDL    $0x0000001f, R12
	MOVQ    $0x00000040, R14
	SUBQ    R12, R14
	DECQ    R13
	JA      emit_lit_memmove_long_lz4_mz_emitcopylarge_forward_sse_loop_32
	LEAQ    -32(DI)(R14*1), R12
	LEAQ    -32(AX)(R14*1), R15
	PCALIGN $0x10

emit_lit_memmove_long_lz4_mz_emitcopylarge_big_loop_back:
	MOVOU (R12), X4
	MOVOU 16(R12), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R12
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_lz4_mz_emitcopylarge_big_loop_back

emit_lit_memmove_long_lz4_mz_emitcopylarge_forward_sse_loop_32:
	MOVOU -32(DI)(R14*1), X4
	MOVOU -16(DI)(R14*1), X5
	MOVOA X4, -32(AX)(R14*1)
	MOVOA X5, -16(AX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R10, R14
	JAE   emit_lit_memmove_long_lz4_mz_emitcopylarge_forward_sse_loop_32
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R10*1)
	MOVOU X3, -16(AX)(R10*1)
	MOVQ  R11, AX

lz4_mz__emit_done:
	// emitCopy
	CMPL R8, $0x00000400
	JA   two_byte_lz4_mz__lz4_mz_short_
	CMPL R9, $0x00000013
	JAE  emit_one_longer_lz4_mz__lz4_mz_short_
	LEAL -1(R8), DI
	SHLL $0x06, DI
	LEAL -15(DI)(R9*4), DI
	MOVW DI, (AX)
	ADDQ $0x02, AX
	JMP  lz4_mz_loop

emit_one_longer_lz4_mz__lz4_mz_short_:
	CMPL R9, $0x00000112
	JAE  emit_copy1_repeat_lz4_mz__lz4_mz_short_
	LEAL -1(R8), DI
	SHLL $0x06, DI
	LEAL 61(DI), DI
	MOVW DI, (AX)
	LEAL -18(R9), DI
	MOVB DI, 2(AX)
	ADDQ $0x03, AX
	JMP  lz4_mz_loop

emit_copy1_repeat_lz4_mz__lz4_mz_short_:
	LEAL -1(R8), DI
	SHLL $0x06, DI
	LEAL 57(DI), DI
	MOVW DI, (AX)
	ADDQ $0x02, AX
	SUBL $0x12, R9

	// emitRepeat
	LEAL -1(R9), DI
	CMPL R9, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_lz4_mz__lz4_mz_short_
	LEAL -30(R9), DI
	CMPL R9, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_lz4_mz__lz4_mz_short_
	CMPL R9, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_lz4_mz__lz4_mz_short_
	MOVB $0xfc, (AX)
	MOVL DI, 1(AX)
	ADDQ $0x04, AX
	JMP  lz4_mz_loop

repeat_three_emit_copy1_do_repeat_lz4_mz__lz4_mz_short_:
	MOVB $0xf4, (AX)
	MOVW DI, 1(AX)
	ADDQ $0x03, AX
	JMP  lz4_mz_loop

repeat_two_emit_copy1_do_repeat_lz4_mz__lz4_mz_short_:
	MOVB $0xec, (AX)
	MOVB DI, 1(AX)
	ADDQ $0x02, AX
	JMP  lz4_mz_loop

repeat_one_emit_copy1_do_repeat_lz4_mz__lz4_mz_short_:
	XORL DI, DI
	LEAL -4(DI)(R9*8), DI
	MOVB DI, (AX)
	ADDQ $0x01, AX
	JMP  lz4_mz_loop

two_byte_lz4_mz__lz4_mz_short_:
	// emitCopy2
	LEAL -64(R8), R8
	LEAL -4(R9), R9
	MOVW R8, 1(AX)
	CMPL R9, $0x3c
	JBE  emit_copy2_0_lz4_mz__lz4_mz_short__emit2
	LEAL -60(R9), DI
	CMPL R9, $0x0000013c
	JB   emit_copy2_1_lz4_mz__lz4_mz_short__emit2
	CMPL R9, $0x0001003c
	JB   emit_copy2_2_lz4_mz__lz4_mz_short__emit2
	MOVB $0xfe, (AX)
	MOVL DI, 3(AX)
	ADDQ $0x06, AX
	JMP  lz4_mz_loop

emit_copy2_2_lz4_mz__lz4_mz_short__emit2:
	MOVB $0xfa, (AX)
	MOVW DI, 3(AX)
	ADDQ $0x05, AX
	JMP  lz4_mz_loop

emit_copy2_1_lz4_mz__lz4_mz_short__emit2:
	MOVB $0xf6, (AX)
	MOVB DI, 3(AX)
	ADDQ $0x04, AX
	JMP  lz4_mz_loop

emit_copy2_0_lz4_mz__lz4_mz_short__emit2:
	MOVL $0x00000002, DI
	LEAL (DI)(R9*4), DI
	MOVB DI, (AX)
	ADDQ $0x03, AX
	JMP  lz4_mz_loop

lz4_mz_docopy:
	MOVQ R8, (SP)

	// emitCopy
	CMPL R8, $0x00000400
	JA   two_byte_lz4_mz__lz4_mz
	CMPL R9, $0x00000013
	JAE  emit_one_longer_lz4_mz__lz4_mz
	LEAL -1(R8), DI
	SHLL $0x06, DI
	LEAL -15(DI)(R9*4), DI
	MOVW DI, (AX)
	ADDQ $0x02, AX
	JMP  lz4_mz_loop

emit_one_longer_lz4_mz__lz4_mz:
	CMPL R9, $0x00000112
	JAE  emit_copy1_repeat_lz4_mz__lz4_mz
	LEAL -1(R8), DI
	SHLL $0x06, DI
	LEAL 61(DI), DI
	MOVW DI, (AX)
	LEAL -18(R9), DI
	MOVB DI, 2(AX)
	ADDQ $0x03, AX
	JMP  lz4_mz_loop

emit_copy1_repeat_lz4_mz__lz4_mz:
	LEAL -1(R8), DI
	SHLL $0x06, DI
	LEAL 57(DI), DI
	MOVW DI, (AX)
	ADDQ $0x02, AX
	SUBL $0x12, R9

	// emitRepeat
	LEAL -1(R9), DI
	CMPL R9, $0x1d
	JBE  repeat_one_emit_copy1_do_repeat_lz4_mz__lz4_mz
	LEAL -30(R9), DI
	CMPL R9, $0x0000011e
	JB   repeat_two_emit_copy1_do_repeat_lz4_mz__lz4_mz
	CMPL R9, $0x0001001e
	JB   repeat_three_emit_copy1_do_repeat_lz4_mz__lz4_mz
	MOVB $0xfc, (AX)
	MOVL DI, 1(AX)
	ADDQ $0x04, AX
	JMP  lz4_mz_loop

repeat_three_emit_copy1_do_repeat_lz4_mz__lz4_mz:
	MOVB $0xf4, (AX)
	MOVW DI, 1(AX)
	ADDQ $0x03, AX
	JMP  lz4_mz_loop

repeat_two_emit_copy1_do_repeat_lz4_mz__lz4_mz:
	MOVB $0xec, (AX)
	MOVB DI, 1(AX)
	ADDQ $0x02, AX
	JMP  lz4_mz_loop

repeat_one_emit_copy1_do_repeat_lz4_mz__lz4_mz:
	XORL DI, DI
	LEAL -4(DI)(R9*8), DI
	MOVB DI, (AX)
	ADDQ $0x01, AX
	JMP  lz4_mz_loop

two_byte_lz4_mz__lz4_mz:
	// emitCopy2
	LEAL -64(R8), R8
	LEAL -4(R9), R9
	MOVW R8, 1(AX)
	CMPL R9, $0x3c
	JBE  emit_copy2_0_lz4_mz__lz4_mz_emit2
	LEAL -60(R9), DI
	CMPL R9, $0x0000013c
	JB   emit_copy2_1_lz4_mz__lz4_mz_emit2
	CMPL R9, $0x0001003c
	JB   emit_copy2_2_lz4_mz__lz4_mz_emit2
	MOVB $0xfe, (AX)
	MOVL DI, 3(AX)
	ADDQ $0x06, AX
	JMP  lz4_mz_loop

emit_copy2_2_lz4_mz__lz4_mz_emit2:
	MOVB $0xfa, (AX)
	MOVW DI, 3(AX)
	ADDQ $0x05, AX
	JMP  lz4_mz_loop

emit_copy2_1_lz4_mz__lz4_mz_emit2:
	MOVB $0xf6, (AX)
	MOVB DI, 3(AX)
	ADDQ $0x04, AX
	JMP  lz4_mz_loop

emit_copy2_0_lz4_mz__lz4_mz_emit2:
	MOVL $0x00000002, DI
	LEAL (DI)(R9*4), DI
	MOVB DI, (AX)
	ADDQ $0x03, AX
	JMP  lz4_mz_loop

lz4_mz_emit_final:
	// emitLiteral
	LEAL -1(R10), CX
	CMPL CX, $0x1d
	JB   one_byte_lz4_mz_emit_final
	SUBL $0x1d, CX
	CMPL CX, $0x00000100
	JB   two_bytes_lz4_mz_emit_final
	CMPL CX, $0x00010000
	JB   three_bytes_lz4_mz_emit_final
	MOVL CX, DX
	SHRL $0x10, DX
	MOVB $0xf8, (AX)
	MOVW CX, 1(AX)
	MOVB DL, 3(AX)
	ADDQ $0x04, AX
	ADDL $0x1d, CX
	JMP  memmove_long_lz4_mz_emit_final

three_bytes_lz4_mz_emit_final:
	MOVB $0xf0, (AX)
	MOVW CX, 1(AX)
	ADDQ $0x03, AX
	ADDL $0x1d, CX
	JMP  memmove_long_lz4_mz_emit_final

two_bytes_lz4_mz_emit_final:
	MOVB $0xe8, (AX)
	MOVB CL, 1(AX)
	ADDL $0x1d, CX
	ADDQ $0x02, AX
	CMPL CX, $0x40
	JB   memmove_midlz4_mz_emit_final
	JMP  memmove_long_lz4_mz_emit_final

one_byte_lz4_mz_emit_final:
	SHLB $0x03, CL
	MOVB CL, (AX)
	ADDQ $0x01, AX
	LEAQ (AX)(R10*1), CX
	MOVL R10, DX

	// genMemMoveShort
	// margin: 0, min move: 1
	CMPQ DX, $0x03
	JB   emit_lit_memmove_lz4_mz_emit_final_memmove_move_1or2
	JE   emit_lit_memmove_lz4_mz_emit_final_memmove_move_3
	CMPQ DX, $0x08
	JBE  emit_lit_memmove_lz4_mz_emit_final_memmove_move_4through8
	CMPQ DX, $0x10
	JBE  emit_lit_memmove_lz4_mz_emit_final_memmove_move_8through16
	CMPQ DX, $0x20
	JBE  emit_lit_memmove_lz4_mz_emit_final_memmove_move_17through32
	JMP  emit_lit_memmove_lz4_mz_emit_final_memmove_move_33through64

emit_lit_memmove_lz4_mz_emit_final_memmove_move_1or2:
	MOVB (DI), BL
	MOVB -1(DI)(DX*1), DI
	MOVB BL, (AX)
	MOVB DI, -1(AX)(DX*1)
	JMP  memmove_end_copy_lz4_mz_emit_final

emit_lit_memmove_lz4_mz_emit_final_memmove_move_3:
	MOVW (DI), BX
	MOVB 2(DI), DI
	MOVW BX, (AX)
	MOVB DI, 2(AX)
	JMP  memmove_end_copy_lz4_mz_emit_final

emit_lit_memmove_lz4_mz_emit_final_memmove_move_4through8:
	MOVL (DI), BX
	MOVL -4(DI)(DX*1), DI
	MOVL BX, (AX)
	MOVL DI, -4(AX)(DX*1)
	JMP  memmove_end_copy_lz4_mz_emit_final
	PCALIGN $0x10

emit_lit_memmove_lz4_mz_emit_final_memmove_move_8through16:
	MOVQ (DI), BX
	MOVQ -8(DI)(DX*1), DI
	MOVQ BX, (AX)
	MOVQ DI, -8(AX)(DX*1)
	JMP  memmove_end_copy_lz4_mz_emit_final

emit_lit_memmove_lz4_mz_emit_final_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(DX*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(DX*1)
	JMP   memmove_end_copy_lz4_mz_emit_final

emit_lit_memmove_lz4_mz_emit_final_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(DX*1), X2
	MOVOU -16(DI)(DX*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(DX*1)
	MOVOU X3, -16(AX)(DX*1)

memmove_end_copy_lz4_mz_emit_final:
	MOVQ CX, AX
	JMP  lz4_mz_done

memmove_midlz4_mz_emit_final:
	LEAQ (AX)(R10*1), CX
	MOVL R10, DX

	// genMemMoveShort
	// margin: 0, min move: 30
	CMPQ DX, $0x20
	JBE  emit_lit_memmove_mid_lz4_mz_emit_final_memmove_move_17through32
	JMP  emit_lit_memmove_mid_lz4_mz_emit_final_memmove_move_33through64

emit_lit_memmove_mid_lz4_mz_emit_final_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(DX*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(DX*1)
	JMP   memmove_mid_end_copy_lz4_mz_emit_final

emit_lit_memmove_mid_lz4_mz_emit_final_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(DX*1), X2
	MOVOU -16(DI)(DX*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(DX*1)
	MOVOU X3, -16(AX)(DX*1)

memmove_mid_end_copy_lz4_mz_emit_final:
	MOVQ CX, AX
	JMP  lz4_mz_done

memmove_long_lz4_mz_emit_final:
	LEAQ (AX)(R10*1), CX
	MOVL R10, DX

	// genMemMoveLong
	MOVOU   (DI), X0
	MOVOU   16(DI), X1
	MOVOU   -32(DI)(DX*1), X2
	MOVOU   -16(DI)(DX*1), X3
	MOVQ    DX, R8
	SHRQ    $0x05, R8
	MOVQ    AX, BX
	ANDL    $0x0000001f, BX
	MOVQ    $0x00000040, R9
	SUBQ    BX, R9
	DECQ    R8
	JA      emit_lit_memmove_long_lz4_mz_emit_finallarge_forward_sse_loop_32
	LEAQ    -32(DI)(R9*1), BX
	LEAQ    -32(AX)(R9*1), R10
	PCALIGN $0x10

emit_lit_memmove_long_lz4_mz_emit_finallarge_big_loop_back:
	MOVOU (BX), X4
	MOVOU 16(BX), X5
	MOVOA X4, (R10)
	MOVOA X5, 16(R10)
	ADDQ  $0x20, R10
	ADDQ  $0x20, BX
	ADDQ  $0x20, R9
	DECQ  R8
	JNA   emit_lit_memmove_long_lz4_mz_emit_finallarge_big_loop_back

emit_lit_memmove_long_lz4_mz_emit_finallarge_forward_sse_loop_32:
	MOVOU -32(DI)(R9*1), X4
	MOVOU -16(DI)(R9*1), X5
	MOVOA X4, -32(AX)(R9*1)
	MOVOA X5, -16(AX)(R9*1)
	ADDQ  $0x20, R9
	CMPQ  DX, R9
	JAE   emit_lit_memmove_long_lz4_mz_emit_finallarge_forward_sse_loop_32
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(DX*1)
	MOVOU X3, -16(AX)(DX*1)
	MOVQ  CX, AX

lz4_mz_done:
	MOVQ dst_base+0(FP), CX
	SUBQ CX, AX
	MOVQ SI, uncompressed+48(FP)
	MOVQ AX, dstUsed+56(FP)
	RET

lz4_mz_corrupt:
	XORQ AX, AX
	LEAQ -1(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET

lz4_mz_dstfull:
	XORQ AX, AX
	LEAQ -2(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET

// func decodeBlockAsm(dst []byte, src []byte) int
// Requires: CMOV, SSE2
TEXT ·decodeBlockAsm(SB), $8-56
	MOVQ    dst_base+0(FP), AX
	MOVQ    dst_len+8(FP), CX
	MOVQ    src_base+24(FP), DX
	MOVQ    src_len+32(FP), BX
	MOVQ    AX, SI
	XORQ    DI, DI
	MOVQ    DX, R8
	MOVQ    $0x00000001, R9
	LEAQ    (AX)(CX*1), AX
	LEAQ    (DX)(BX*1), CX
	LEAQ    -20(CX), DX
	LEAQ    -20(AX), BX
	CMPQ    R8, DX
	JAE     decodeBlockAsm_fast_end_copy
	MOVBQZX (R8), R10
	MOVQ    R10, R11
	SHRQ    $0x02, R11
	PCALIGN $0x10

decodeBlockAsm_fast_loop_nofetch:
	CMPQ    SI, BX
	JAE     decodeBlockAsm_fast_end_copy
	ANDQ    $0x03, R10
	JNZ     decodeBlockAsm_fast_copy
	PCALIGN $0x10

decodeBlockAsm_fast_lits:
	MOVL    R11, R12
	SHRL    $0x01, R12
	CMPL    R11, $0x3a
	JB      decodeBlockAsm_fast_lit_0
	CMPL    R12, $0x1e
	MOVBQZX 1(R8), R12
	MOVWQZX 1(R8), R13
	JA      decodeBlockAsm_fast_lit_3
	MOVQ    $0x00000002, R14
	MOVQ    $0x00000003, R10
	CMOVQEQ R10, R14
	CMOVLEQ R13, R12
	ADDQ    R14, R8
	JMP     decodeBlockAsm_fast_litcopy_long
	PCALIGN $0x10

decodeBlockAsm_fast_lit_0:
	INCQ R8
	INCL R12
	BTL  $0x00, R11
	JC   decodeBlockAsm_fast_copy_exec_short
	LEAQ (SI)(R12*1), R10
	CMPQ R10, AX
	JA   corrupt
	LEAQ (R8)(R12*1), R10
	CMPQ R10, CX
	JA   corrupt

	// genMemMoveShort
	// margin: 19, min move: 1
	CMPQ R12, $0x10
	JBE  decodeBlockAsm_fast_lit_0_copy_memmove_move_8through16
	CMPQ R12, $0x20
	JBE  decodeBlockAsm_fast_lit_0_copy_memmove_move_17through32
	JMP  decodeBlockAsm_fast_lit_0_copy_memmove_move_33through64
	PCALIGN $0x10

decodeBlockAsm_fast_lit_0_copy_memmove_move_8through16:
	MOVOU (R8), X0
	MOVOU X0, (SI)
	JMP   decodeBlockAsm_fast_litcopy_done

decodeBlockAsm_fast_lit_0_copy_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(R12*1), X1
	MOVOU X0, (SI)
	MOVOU X1, -16(SI)(R12*1)
	JMP   decodeBlockAsm_fast_litcopy_done

decodeBlockAsm_fast_lit_0_copy_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(R12*1), X2
	MOVOU -16(R8)(R12*1), X3
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(R12*1)
	MOVOU X3, -16(SI)(R12*1)
	JMP   decodeBlockAsm_fast_litcopy_done

decodeBlockAsm_fast_lit_3:
	MOVL (R8), R12
	ADDQ $0x04, R8
	SHRL $0x08, R12

decodeBlockAsm_fast_litcopy_long:
	LEAQ 30(R12), R12
	BTL  $0x00, R11
	JC   decodeBlockAsm_fast_copy_exec
	LEAQ (SI)(R12*1), R10
	CMPQ R10, AX
	JA   corrupt
	LEAQ (R8)(R12*1), R10
	CMPQ R10, CX
	JA   corrupt
	CMPL R12, $0x40
	JBE  decodeBlockAsm_fast_litcopy_short_reduced

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(R12*1), X2
	MOVOU   -16(R8)(R12*1), X3
	MOVQ    R12, R11
	SHRQ    $0x05, R11
	MOVQ    SI, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R13
	SUBQ    R10, R13
	DECQ    R11
	JA      decodeBlockAsm_fast_litcopy_longlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R13*1), R10
	LEAQ    -32(SI)(R13*1), R14
	PCALIGN $0x10

decodeBlockAsm_fast_litcopy_longlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R11
	JNA   decodeBlockAsm_fast_litcopy_longlarge_big_loop_back

decodeBlockAsm_fast_litcopy_longlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R13*1), X4
	MOVOU -16(R8)(R13*1), X5
	MOVOA X4, -32(SI)(R13*1)
	MOVOA X5, -16(SI)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R12, R13
	JAE   decodeBlockAsm_fast_litcopy_longlarge_forward_sse_loop_32
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(R12*1)
	MOVOU X3, -16(SI)(R12*1)
	JMP   decodeBlockAsm_fast_litcopy_done

decodeBlockAsm_fast_litcopy_short_reduced:
	// genMemMoveShort
	// margin: 16, min move: 30
	CMPQ R12, $0x20
	JBE  decodeBlockAsm_fast_lit_longer_copy_memmove_move_17through32
	JMP  decodeBlockAsm_fast_lit_longer_copy_memmove_move_33through64

decodeBlockAsm_fast_lit_longer_copy_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(R12*1), X1
	MOVOU X0, (SI)
	MOVOU X1, -16(SI)(R12*1)
	JMP   decodeBlockAsm_fast_litcopy_done

decodeBlockAsm_fast_lit_longer_copy_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(R12*1), X2
	MOVOU -16(R8)(R12*1), X3
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(R12*1)
	MOVOU X3, -16(SI)(R12*1)

decodeBlockAsm_fast_litcopy_done:
	ADDQ    R12, R8
	ADDQ    R12, SI
	ADDQ    R12, DI
	CMPQ    R8, DX
	JAE     decodeBlockAsm_fast_end_done
	MOVBQZX (R8), R10
	MOVQ    R10, R11
	SHRQ    $0x02, R11
	CMPQ    SI, BX
	JAE     decodeBlockAsm_fast_end_done
	ANDQ    $0x03, R10
	JZ      decodeBlockAsm_fast_lits

decodeBlockAsm_fast_copy:
	MOVL (R8), R13
	CMPL R10, $0x02
	JB   decodeBlockAsm_fast_copy_1
	JEQ  decodeBlockAsm_fast_copy_2
	JMP  decodeBlockAsm_fast_copy_3
	PCALIGN $0x10

decodeBlockAsm_fast_copy_1:
	MOVWQZX R13, R9
	ADDQ    $0x02, R8
	MOVQ    R11, R12
	ANDL    $0x0f, R12
	SHRL    $0x06, R9
	INCL    R9
	SHRL    $0x10, R13
	LEAQ    1(R8), R10
	MOVBLZX R13, R11
	ADDL    $0x04, R12
	LEAL    18(R11), R11
	CMPL    R12, $0x13
	CMOVLEQ R11, R12
	CMOVQEQ R10, R8
	JMP     decodeBlockAsm_fast_copy_exec
	PCALIGN $0x10

decodeBlockAsm_fast_copy_2:
	MOVQ    R11, R12
	CMPL    R11, $0x3d
	JB      decodeBlockAsm_fast_copy_2_0_extra
	JEQ     decodeBlockAsm_fast_copy_2_1_extra
	CMPL    R12, $0x3f
	JB      decodeBlockAsm_fast_copy_2_2_extra
	MOVWQZX 1(R8), R9
	MOVL    2(R8), R12
	ADDQ    $0x06, R8
	SHRL    $0x08, R12
	LEAL    64(R12), R12
	ADDQ    $0x40, R9
	JMP     decodeBlockAsm_fast_copy_exec_long_long

decodeBlockAsm_fast_copy_2_2_extra:
	MOVWQZX 1(R8), R9
	MOVWLZX 3(R8), R12
	ADDQ    $0x05, R8
	LEAL    64(R12), R12
	ADDQ    $0x40, R9
	JMP     decodeBlockAsm_fast_copy_exec_long_long

decodeBlockAsm_fast_copy_2_1_extra:
	MOVL    R13, R12
	SHRL    $0x08, R13
	SHRL    $0x18, R12
	MOVWQZX R13, R9
	ADDQ    $0x04, R8
	LEAL    64(R12), R12
	ADDQ    $0x40, R9
	JMP     decodeBlockAsm_fast_copy_exec_long_long

decodeBlockAsm_fast_copy_2_0_extra:
	SHRL    $0x08, R13
	MOVWQZX R13, R9
	LEAQ    3(R8), R8
	LEAL    4(R12), R12
	ADDQ    $0x40, R9
	JMP     decodeBlockAsm_fast_copy_short_no_ol
	PCALIGN $0x10

decodeBlockAsm_fast_copy_3:
	MOVL    R13, R9
	ADDQ    $0x04, R8
	MOVQ    R11, R10
	SHRQ    $0x01, R10
	ANDQ    $0x03, R10
	BTL     $0x00, R11
	JC      decodeBlockAsm_fast_copy3_read
	SHRL    $0x03, R11
	ANDL    $0x07, R11
	LEAL    4(R11), R12
	SHRL    $0x08, R13
	MOVWQZX R13, R9
	DECQ    R8
	INCQ    R10
	MOVL    (R8), R11
	MOVL    R11, (SI)
	ADDQ    $0x40, R9
	ADDQ    R10, R8
	ADDQ    R10, SI
	ADDQ    R10, DI
	JMP     decodeBlockAsm_fast_copy_short_no_ol

decodeBlockAsm_fast_copy3_read:
	MOVL R9, R12
	SHRL $0x05, R12
	ANDL $0x3f, R12
	SHRL $0x0b, R9
	ADDL $0x00010000, R9
	CMPL R12, $0x3d
	JB   decodeBlockAsm_fast_copy_3_0_extra
	JEQ  decodeBlockAsm_fast_copy_3_1_extra
	CMPL R12, $0x3e
	JEQ  decodeBlockAsm_fast_copy_3_2_extra
	MOVL -1(R8), R12
	ADDQ $0x03, R8
	SHRL $0x08, R12
	LEAL 64(R12), R12
	JMP  decodeBlockAsm_fast_copy_fused_long

decodeBlockAsm_fast_copy_3_2_extra:
	MOVWLZX (R8), R12
	ADDQ    $0x02, R8
	LEAL    64(R12), R12
	JMP     decodeBlockAsm_fast_copy_fused_long

decodeBlockAsm_fast_copy_3_1_extra:
	MOVBLZX (R8), R12
	ADDQ    $0x01, R8
	LEAL    64(R12), R12
	JMP     decodeBlockAsm_fast_copy_fused_long

decodeBlockAsm_fast_copy_3_0_extra:
	LEAL 4(R12), R12
	MOVL (R8), R11
	MOVL R11, (SI)
	ADDQ R10, R8
	ADDQ R10, SI
	ADDQ R10, DI
	JMP  decodeBlockAsm_fast_copy_short_no_ol

decodeBlockAsm_fast_copy_fused_long:
	MOVL (R8), R11
	MOVL R11, (SI)
	ADDQ R10, R8
	ADDQ R10, SI
	ADDQ R10, DI
	JMP  decodeBlockAsm_fast_copy_exec_long_long
	PCALIGN $0x10

decodeBlockAsm_fast_copy_exec_short:
	CMPL R9, DI
	JA   corrupt
	LEAQ (SI)(R12*1), R10
	CMPQ R10, AX
	JA   corrupt

	// Prefetch next tag
	MOVBQZX (R8), R10
	MOVQ    SI, R11
	SUBQ    R9, R11
	CMPL    R9, R12
	JB      decodeBlockAsm_fast_copy_overlap
	JMP     decodeBlockAsm_fast_copy_short
	PCALIGN $0x10

decodeBlockAsm_fast_copy_exec_long_long:
	MOVQ SI, R11
	SUBQ R9, R11
	CMPL R9, DI
	JA   corrupt
	LEAQ (SI)(R12*1), R10
	CMPQ R10, AX
	JA   corrupt

	// Prefetch next tag
	MOVBQZX (R8), R10

	// genMemMoveLong
	MOVQ    R12, R13
	SHRQ    $0x06, R13
	MOVQ    SI, R14
	MOVQ    R12, R15
	PCALIGN $0x10

decodeBlockAsm_fast_copy_long_longlarge_big_loop_back:
	MOVOU (R11), X0
	MOVOU 16(R11), X1
	MOVOU 32(R11), X2
	MOVOU 48(R11), X3
	MOVOU X0, (R14)
	MOVOU X1, 16(R14)
	MOVOU X2, 32(R14)
	MOVOU X3, 48(R14)
	ADDQ  $0x40, R14
	ADDQ  $0x40, R11
	SUBQ  $0x40, R15
	JZ    decodeBlockAsm_fast_copy_done
	DECQ  R13
	JNZ   decodeBlockAsm_fast_copy_long_longlarge_big_loop_back
	TESTQ R15, R15
	JZ    decodeBlockAsm_fast_copy_done
	MOVOU -64(R11)(R15*1), X0
	MOVOU -48(R11)(R15*1), X1
	MOVOU -32(R11)(R15*1), X2
	MOVOU -16(R11)(R15*1), X3
	MOVOU X0, -64(R14)(R15*1)
	MOVOU X1, -48(R14)(R15*1)
	MOVOU X2, -32(R14)(R15*1)
	MOVOU X3, -16(R14)(R15*1)
	JMP   decodeBlockAsm_fast_copy_done
	PCALIGN $0x10

decodeBlockAsm_fast_copy_short_no_ol:
	MOVQ SI, R11
	SUBQ R9, R11
	CMPL R9, DI
	JA   corrupt
	LEAQ (SI)(R12*1), R10
	CMPQ R10, AX
	JA   corrupt

	// Prefetch next tag
	MOVBQZX (R8), R10

	// genMemMoveShort
	// margin: 16, min move: 4
	CMPQ R12, $0x10
	JBE  decodeBlockAsm_fast_copy_short_no_ol_memmove_move_8through16
	CMPQ R12, $0x20
	JBE  decodeBlockAsm_fast_copy_short_no_ol_memmove_move_17through32
	JMP  decodeBlockAsm_fast_copy_short_no_ol_memmove_move_33through64
	PCALIGN $0x10

decodeBlockAsm_fast_copy_short_no_ol_memmove_move_8through16:
	MOVOU (R11), X0
	MOVOU X0, (SI)
	JMP   decodeBlockAsm_fast_copy_done

decodeBlockAsm_fast_copy_short_no_ol_memmove_move_17through32:
	MOVOU (R11), X0
	MOVOU -16(R11)(R12*1), X1
	MOVOU X0, (SI)
	MOVOU X1, -16(SI)(R12*1)
	JMP   decodeBlockAsm_fast_copy_done

decodeBlockAsm_fast_copy_short_no_ol_memmove_move_33through64:
	MOVOU (R11), X0
	MOVOU 16(R11), X1
	MOVOU -32(R11)(R12*1), X2
	MOVOU -16(R11)(R12*1), X3
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(R12*1)
	MOVOU X3, -16(SI)(R12*1)
	JMP   decodeBlockAsm_fast_copy_done
	PCALIGN $0x10

decodeBlockAsm_fast_copy_exec:
	CMPL R9, DI
	JA   corrupt
	LEAQ (SI)(R12*1), R10
	CMPQ R10, AX
	JA   corrupt
	MOVQ SI, R11
	SUBQ R9, R11

	// Prefetch next tag
	MOVBQZX (R8), R10
	CMPL    R9, R12
	JB      decodeBlockAsm_fast_copy_overlap
	CMPL    R12, $0x40
	JAE     decodeBlockAsm_fast_copy_long

decodeBlockAsm_fast_copy_short:
	// genMemMoveShort
	// margin: 16, min move: 1
	CMPQ R12, $0x10
	JBE  decodeBlockAsm_fast_copy_short_memmove_move_8through16
	CMPQ R12, $0x20
	JBE  decodeBlockAsm_fast_copy_short_memmove_move_17through32
	JMP  decodeBlockAsm_fast_copy_short_memmove_move_33through64
	PCALIGN $0x10

decodeBlockAsm_fast_copy_short_memmove_move_8through16:
	MOVOU (R11), X0
	MOVOU X0, (SI)
	JMP   decodeBlockAsm_fast_copy_done

decodeBlockAsm_fast_copy_short_memmove_move_17through32:
	MOVOU (R11), X0
	MOVOU -16(R11)(R12*1), X1
	MOVOU X0, (SI)
	MOVOU X1, -16(SI)(R12*1)
	JMP   decodeBlockAsm_fast_copy_done

decodeBlockAsm_fast_copy_short_memmove_move_33through64:
	MOVOU (R11), X0
	MOVOU 16(R11), X1
	MOVOU -32(R11)(R12*1), X2
	MOVOU -16(R11)(R12*1), X3
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(R12*1)
	MOVOU X3, -16(SI)(R12*1)
	JMP   decodeBlockAsm_fast_copy_done

decodeBlockAsm_fast_copy_long:
	// genMemMoveLong
	MOVOU   (R11), X0
	MOVOU   16(R11), X1
	MOVOU   -32(R11)(R12*1), X2
	MOVOU   -16(R11)(R12*1), X3
	MOVQ    R12, R14
	SHRQ    $0x05, R14
	MOVQ    SI, R13
	ANDL    $0x0000001f, R13
	MOVQ    $0x00000040, R15
	SUBQ    R13, R15
	DECQ    R14
	JA      decodeBlockAsm_fast_copy_longlarge_forward_sse_loop_32
	LEAQ    -32(R11)(R15*1), R13
	LEAQ    -32(SI)(R15*1), BP
	PCALIGN $0x10

decodeBlockAsm_fast_copy_longlarge_big_loop_back:
	MOVOU (R13), X4
	MOVOU 16(R13), X5
	MOVOA X4, (BP)
	MOVOA X5, 16(BP)
	ADDQ  $0x20, BP
	ADDQ  $0x20, R13
	ADDQ  $0x20, R15
	DECQ  R14
	JNA   decodeBlockAsm_fast_copy_longlarge_big_loop_back

decodeBlockAsm_fast_copy_longlarge_forward_sse_loop_32:
	MOVOU -32(R11)(R15*1), X4
	MOVOU -16(R11)(R15*1), X5
	MOVOA X4, -32(SI)(R15*1)
	MOVOA X5, -16(SI)(R15*1)
	ADDQ  $0x20, R15
	CMPQ  R12, R15
	JAE   decodeBlockAsm_fast_copy_longlarge_forward_sse_loop_32
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(R12*1)
	MOVOU X3, -16(SI)(R12*1)

decodeBlockAsm_fast_copy_done:
	ADDQ R12, SI
	ADDQ R12, DI
	MOVQ R10, R11
	SHRQ $0x02, R11
	CMPQ R8, DX
	JB   decodeBlockAsm_fast_loop_nofetch
	JMP  decodeBlockAsm_fast_end_copy

decodeBlockAsm_fast_copy_overlap:
	CMPL R9, $0x03
	JA   decodeBlockAsm_fast_copy_overlap_4
	JE   decodeBlockAsm_fast_copy_overlap_3
	CMPL R9, $0x02
	JE   decodeBlockAsm_fast_copy_overlap_2
	MOVB (R11), R11
	ADDQ R12, DI

decodeBlockAsm_fast_loop_overlap_1:
	MOVB R11, (SI)
	INCQ SI
	DECQ R12
	JNZ  decodeBlockAsm_fast_loop_overlap_1
	MOVQ R10, R11
	SHRQ $0x02, R11
	CMPQ R8, DX
	JB   decodeBlockAsm_fast_loop_nofetch
	JMP  decodeBlockAsm_fast_end_copy

decodeBlockAsm_fast_copy_overlap_2:
	MOVW (R11), R13
	ADDQ R12, DI
	BTL  $0x00, R12
	JNC  decodeBlockAsm_fast_loop_overlap_2
	MOVB R13, (SI)
	MOVW 1(R11), R13
	INCQ SI
	DECQ R12

decodeBlockAsm_fast_loop_overlap_2:
	MOVW R13, (SI)
	ADDQ $0x02, SI
	SUBQ $0x02, R12
	JNZ  decodeBlockAsm_fast_loop_overlap_2
	MOVQ R10, R11
	SHRQ $0x02, R11
	CMPQ R8, DX
	JB   decodeBlockAsm_fast_loop_nofetch
	JMP  decodeBlockAsm_fast_end_copy

decodeBlockAsm_fast_copy_overlap_3:
	MOVL (R11), R13
	ADDQ R12, DI
	SUBQ $0x03, R12

decodeBlockAsm_fast_loop_overlap_3:
	MOVL R13, (SI)
	ADDQ $0x03, SI
	SUBQ $0x03, R12
	JA   decodeBlockAsm_fast_loop_overlap_3
	MOVW 3(R11)(R12*1), R13
	MOVW R13, (SI)(R12*1)
	MOVB 5(R11)(R12*1), R13
	MOVB R13, 2(SI)(R12*1)
	LEAQ 3(SI)(R12*1), SI
	MOVQ R10, R11
	SHRQ $0x02, R11
	CMPQ R8, DX
	JB   decodeBlockAsm_fast_loop_nofetch
	JMP  decodeBlockAsm_fast_end_copy

decodeBlockAsm_fast_copy_overlap_4:
	ADDQ R12, DI
	SUBQ $0x04, R12

decodeBlockAsm_fast_loop_overlap_4:
	MOVL (R11), R13
	ADDQ $0x04, R11
	MOVL R13, (SI)
	ADDQ $0x04, SI
	SUBQ $0x04, R12
	JA   decodeBlockAsm_fast_loop_overlap_4
	MOVL (R11)(R12*1), R13
	MOVL R13, (SI)(R12*1)
	LEAQ 4(SI)(R12*1), SI
	MOVQ R10, R11
	SHRQ $0x02, R11
	CMPQ R8, DX
	JB   decodeBlockAsm_fast_loop_nofetch

decodeBlockAsm_fast_end_copy:
decodeBlockAsm_fast_end_done:
decodeBlockAsm_remain_loop:
	CMPQ    R8, CX
	JAE     decodeBlockAsm_remain_end_copy
	MOVBQZX (R8), DX
	MOVQ    DX, BX
	SHRQ    $0x02, BX
	CMPQ    SI, AX
	JAE     decodeBlockAsm_remain_end_copy
	ANDQ    $0x03, DX
	JNZ     decodeBlockAsm_remain_copy
	PCALIGN $0x10

decodeBlockAsm_remain_lits:
	MOVL    BX, DX
	SHRL    $0x01, DX
	CMPL    BX, $0x3a
	JB      decodeBlockAsm_remain_lit_0
	CMPL    DX, $0x1e
	JEQ     decodeBlockAsm_remain_lit_2
	JA      decodeBlockAsm_remain_lit_3
	ADDQ    $0x02, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVBQZX -1(R8), DX
	JMP     decodeBlockAsm_remain_litcopy_long

decodeBlockAsm_remain_lit_2:
	ADDQ    $0x03, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVWQZX -2(R8), DX
	JMP     decodeBlockAsm_remain_litcopy_long
	PCALIGN $0x10

decodeBlockAsm_remain_lit_0:
	INCQ R8
	INCL DX
	BTL  $0x00, BX
	JC   decodeBlockAsm_remain_copy_exec_short
	LEAQ (SI)(DX*1), BX
	CMPQ BX, AX
	JA   corrupt
	LEAQ (R8)(DX*1), BX
	CMPQ BX, CX
	JA   corrupt

	// genMemMoveShort
	// margin: -1, min move: 1
	CMPQ DX, $0x03
	JB   decodeBlockAsm_remain_lit_0_copy_memmove_move_1or2
	JE   decodeBlockAsm_remain_lit_0_copy_memmove_move_3
	CMPQ DX, $0x08
	JBE  decodeBlockAsm_remain_lit_0_copy_memmove_move_4through8
	CMPQ DX, $0x10
	JBE  decodeBlockAsm_remain_lit_0_copy_memmove_move_8through16
	CMPQ DX, $0x20
	JBE  decodeBlockAsm_remain_lit_0_copy_memmove_move_17through32
	JMP  decodeBlockAsm_remain_lit_0_copy_memmove_move_33through64

decodeBlockAsm_remain_lit_0_copy_memmove_move_1or2:
	MOVB (R8), BL
	MOVB -1(R8)(DX*1), R10
	MOVB BL, (SI)
	MOVB R10, -1(SI)(DX*1)
	JMP  decodeBlockAsm_remain_litcopy_done

decodeBlockAsm_remain_lit_0_copy_memmove_move_3:
	MOVW (R8), BX
	MOVB 2(R8), R10
	MOVW BX, (SI)
	MOVB R10, 2(SI)
	JMP  decodeBlockAsm_remain_litcopy_done

decodeBlockAsm_remain_lit_0_copy_memmove_move_4through8:
	MOVL (R8), BX
	MOVL -4(R8)(DX*1), R10
	MOVL BX, (SI)
	MOVL R10, -4(SI)(DX*1)
	JMP  decodeBlockAsm_remain_litcopy_done
	PCALIGN $0x10

decodeBlockAsm_remain_lit_0_copy_memmove_move_8through16:
	MOVQ (R8), BX
	MOVQ -8(R8)(DX*1), R10
	MOVQ BX, (SI)
	MOVQ R10, -8(SI)(DX*1)
	JMP  decodeBlockAsm_remain_litcopy_done

decodeBlockAsm_remain_lit_0_copy_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DX*1), X1
	MOVOU X0, (SI)
	MOVOU X1, -16(SI)(DX*1)
	JMP   decodeBlockAsm_remain_litcopy_done

decodeBlockAsm_remain_lit_0_copy_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DX*1), X2
	MOVOU -16(R8)(DX*1), X3
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(DX*1)
	MOVOU X3, -16(SI)(DX*1)
	JMP   decodeBlockAsm_remain_litcopy_done

decodeBlockAsm_remain_lit_3:
	ADDQ $0x04, R8
	CMPQ R8, CX
	JA   corrupt
	MOVL -4(R8), DX
	SHRL $0x08, DX

decodeBlockAsm_remain_litcopy_long:
	LEAQ 30(DX), DX
	BTL  $0x00, BX
	JC   decodeBlockAsm_remain_copy_exec
	LEAQ (SI)(DX*1), BX
	CMPQ BX, AX
	JA   corrupt
	LEAQ (R8)(DX*1), BX
	CMPQ BX, CX
	JA   corrupt
	CMPL DX, $0x40
	JBE  decodeBlockAsm_remain_litcopy_short_reduced

	// genMemMoveLong
	MOVOU   (R8), X0
	MOVOU   16(R8), X1
	MOVOU   -32(R8)(DX*1), X2
	MOVOU   -16(R8)(DX*1), X3
	MOVQ    DX, R10
	SHRQ    $0x05, R10
	MOVQ    SI, BX
	ANDL    $0x0000001f, BX
	MOVQ    $0x00000040, R11
	SUBQ    BX, R11
	DECQ    R10
	JA      decodeBlockAsm_remain_litcopy_longlarge_forward_sse_loop_32
	LEAQ    -32(R8)(R11*1), BX
	LEAQ    -32(SI)(R11*1), R12
	PCALIGN $0x10

decodeBlockAsm_remain_litcopy_longlarge_big_loop_back:
	MOVOU (BX), X4
	MOVOU 16(BX), X5
	MOVOA X4, (R12)
	MOVOA X5, 16(R12)
	ADDQ  $0x20, R12
	ADDQ  $0x20, BX
	ADDQ  $0x20, R11
	DECQ  R10
	JNA   decodeBlockAsm_remain_litcopy_longlarge_big_loop_back

decodeBlockAsm_remain_litcopy_longlarge_forward_sse_loop_32:
	MOVOU -32(R8)(R11*1), X4
	MOVOU -16(R8)(R11*1), X5
	MOVOA X4, -32(SI)(R11*1)
	MOVOA X5, -16(SI)(R11*1)
	ADDQ  $0x20, R11
	CMPQ  DX, R11
	JAE   decodeBlockAsm_remain_litcopy_longlarge_forward_sse_loop_32
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(DX*1)
	MOVOU X3, -16(SI)(DX*1)
	JMP   decodeBlockAsm_remain_litcopy_done

decodeBlockAsm_remain_litcopy_short_reduced:
	// genMemMoveShort
	// margin: -4, min move: 30
	CMPQ DX, $0x20
	JBE  decodeBlockAsm_remain_lit_longer_copy_memmove_move_17through32
	JMP  decodeBlockAsm_remain_lit_longer_copy_memmove_move_33through64

decodeBlockAsm_remain_lit_longer_copy_memmove_move_17through32:
	MOVOU (R8), X0
	MOVOU -16(R8)(DX*1), X1
	MOVOU X0, (SI)
	MOVOU X1, -16(SI)(DX*1)
	JMP   decodeBlockAsm_remain_litcopy_done

decodeBlockAsm_remain_lit_longer_copy_memmove_move_33through64:
	MOVOU (R8), X0
	MOVOU 16(R8), X1
	MOVOU -32(R8)(DX*1), X2
	MOVOU -16(R8)(DX*1), X3
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(DX*1)
	MOVOU X3, -16(SI)(DX*1)

decodeBlockAsm_remain_litcopy_done:
	ADDQ    DX, R8
	ADDQ    DX, SI
	ADDQ    DX, DI
	CMPQ    R8, CX
	JAE     decodeBlockAsm_remain_end_done
	MOVBQZX (R8), DX
	MOVQ    DX, BX
	SHRQ    $0x02, BX
	CMPQ    SI, AX
	JAE     decodeBlockAsm_remain_end_done
	ANDQ    $0x03, DX
	JZ      decodeBlockAsm_remain_lits

decodeBlockAsm_remain_copy:
	CMPL DX, $0x02
	JB   decodeBlockAsm_remain_copy_1
	JEQ  decodeBlockAsm_remain_copy_2
	JMP  decodeBlockAsm_remain_copy_3
	PCALIGN $0x10

decodeBlockAsm_remain_copy_1:
	ADDQ    $0x02, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVWQZX -2(R8), R9
	MOVQ    BX, DX
	ANDL    $0x0f, DX
	SHRL    $0x06, R9
	INCL    R9
	CMPL    DX, $0x0f
	JNE     decodeBlockAsm_remain_copy_1_short
	ADDQ    $0x01, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVBLZX -1(R8), DX
	LEAL    18(DX), DX
	JMP     decodeBlockAsm_remain_copy_exec

decodeBlockAsm_remain_copy_1_short:
	LEAL 4(DX), DX
	JMP  decodeBlockAsm_remain_copy_exec_short
	PCALIGN $0x10

decodeBlockAsm_remain_copy_2:
	MOVQ    BX, DX
	CMPL    BX, $0x3d
	JB      decodeBlockAsm_remain_copy_2_0_extra
	JEQ     decodeBlockAsm_remain_copy_2_1_extra
	CMPL    DX, $0x3f
	JB      decodeBlockAsm_remain_copy_2_2_extra
	ADDQ    $0x06, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVWQZX -5(R8), R9
	MOVL    -4(R8), DX
	SHRL    $0x08, DX
	LEAL    64(DX), DX
	ADDQ    $0x40, R9
	JMP     decodeBlockAsm_remain_copy_exec_long_long

decodeBlockAsm_remain_copy_2_2_extra:
	ADDQ    $0x05, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVWQZX -4(R8), R9
	MOVWLZX -2(R8), DX
	LEAL    64(DX), DX
	ADDQ    $0x40, R9
	JMP     decodeBlockAsm_remain_copy_exec_long_long

decodeBlockAsm_remain_copy_2_1_extra:
	ADDQ    $0x04, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVWQZX -3(R8), R9
	MOVBLZX -1(R8), DX
	LEAL    64(DX), DX
	ADDQ    $0x40, R9
	JMP     decodeBlockAsm_remain_copy_exec_long_long

decodeBlockAsm_remain_copy_2_0_extra:
	LEAQ    3(R8), R8
	CMPQ    R8, CX
	JA      corrupt
	MOVWQZX -2(R8), R9
	LEAL    4(DX), DX
	ADDQ    $0x40, R9
	JMP     decodeBlockAsm_remain_copy_short_no_ol
	PCALIGN $0x10

decodeBlockAsm_remain_copy_3:
	ADDQ    $0x04, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVL    -4(R8), R9
	MOVQ    BX, R10
	SHRQ    $0x01, R10
	ANDQ    $0x03, R10
	BTL     $0x00, BX
	JC      decodeBlockAsm_remain_copy3_read
	SHRL    $0x03, BX
	ANDL    $0x07, BX
	LEAL    4(BX), DX
	MOVWQZX -3(R8), R9
	DECQ    R8
	INCQ    R10
	LEAQ    (R8)(R10*1), BX
	LEAQ    (SI)(R10*1), R11
	CMPQ    BX, CX
	JA      corrupt
	CMPQ    R11, AX
	JA      corrupt

	// genMemMoveVeryShort
	CMPQ R10, $0x03
	JE   decodeBlockAsm_remain_copy2_fused_lits_move_3
	JA   decodeBlockAsm_remain_copy2_fused_lits_move_4
	MOVB (R8), BL
	MOVB -1(R8)(R10*1), R11
	MOVB BL, (SI)
	MOVB R11, -1(SI)(R10*1)
	JMP  decodeBlockAsm_remain_copy2_fused_lits_done

decodeBlockAsm_remain_copy2_fused_lits_move_3:
	MOVW (R8), BX
	MOVB 2(R8), R11
	MOVW BX, (SI)
	MOVB R11, 2(SI)
	JMP  decodeBlockAsm_remain_copy2_fused_lits_done

decodeBlockAsm_remain_copy2_fused_lits_move_4:
	MOVL (R8), BX
	MOVL BX, (SI)

decodeBlockAsm_remain_copy2_fused_lits_done:
	ADDQ $0x40, R9
	ADDQ R10, R8
	ADDQ R10, SI
	ADDQ R10, DI
	JMP  decodeBlockAsm_remain_copy_short_no_ol

decodeBlockAsm_remain_copy3_read:
	MOVL R9, DX
	SHRL $0x05, DX
	ANDL $0x3f, DX
	SHRL $0x0b, R9
	ADDL $0x00010000, R9
	CMPL DX, $0x3d
	JB   decodeBlockAsm_remain_copy_3_0_extra
	JEQ  decodeBlockAsm_remain_copy_3_1_extra
	CMPL DX, $0x3e
	JEQ  decodeBlockAsm_remain_copy_3_2_extra
	ADDQ $0x03, R8
	CMPQ R8, CX
	JA   corrupt
	MOVL -4(R8), DX
	SHRL $0x08, DX
	LEAL 64(DX), DX
	JMP  decodeBlockAsm_remain_copy_fused_long

decodeBlockAsm_remain_copy_3_2_extra:
	ADDQ    $0x02, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVWLZX -2(R8), DX
	LEAL    64(DX), DX
	JMP     decodeBlockAsm_remain_copy_fused_long

decodeBlockAsm_remain_copy_3_1_extra:
	ADDQ    $0x01, R8
	CMPQ    R8, CX
	JA      corrupt
	MOVBLZX -1(R8), DX
	LEAL    64(DX), DX
	JMP     decodeBlockAsm_remain_copy_fused_long

decodeBlockAsm_remain_copy_3_0_extra:
	LEAL  4(DX), DX
	TESTL R10, R10
	JZ    decodeBlockAsm_remain_copy_short_no_ol
	LEAQ  (R8)(R10*1), BX
	LEAQ  (SI)(R10*1), R11
	CMPQ  BX, CX
	JA    corrupt
	CMPQ  R11, AX
	JA    corrupt

	// genMemMoveVeryShort
	CMPQ R10, $0x03
	JE   decodeBlockAsm_remain_copy3s_fused_lits_move_3
	JA   decodeBlockAsm_remain_copy3s_fused_lits_move_4
	MOVB (R8), BL
	MOVB -1(R8)(R10*1), R11
	MOVB BL, (SI)
	MOVB R11, -1(SI)(R10*1)
	JMP  decodeBlockAsm_remain_copy3s_fused_lits_done

decodeBlockAsm_remain_copy3s_fused_lits_move_3:
	MOVW (R8), BX
	MOVB 2(R8), R11
	MOVW BX, (SI)
	MOVB R11, 2(SI)
	JMP  decodeBlockAsm_remain_copy3s_fused_lits_done

decodeBlockAsm_remain_copy3s_fused_lits_move_4:
	MOVL (R8), BX
	MOVL BX, (SI)

decodeBlockAsm_remain_copy3s_fused_lits_done:
	ADDQ R10, R8
	ADDQ R10, SI
	ADDQ R10, DI
	JMP  decodeBlockAsm_remain_copy_short_no_ol

decodeBlockAsm_remain_copy_fused_long:
	TESTL R10, R10
	JZ    decodeBlockAsm_remain_copy_exec_long_long
	LEAQ  (R8)(R10*1), BX
	LEAQ  (SI)(R10*1), R11
	CMPQ  BX, CX
	JA    corrupt
	CMPQ  R11, AX
	JA    corrupt

	// genMemMoveVeryShort
	CMPQ R10, $0x03
	JE   decodeBlockAsm_remain_copy3_fused_lits_move_3
	JA   decodeBlockAsm_remain_copy3_fused_lits_move_4
	MOVB (R8), BL
	MOVB -1(R8)(R10*1), R11
	MOVB BL, (SI)
	MOVB R11, -1(SI)(R10*1)
	JMP  decodeBlockAsm_remain_copy3_fused_lits_done

decodeBlockAsm_remain_copy3_fused_lits_move_3:
	MOVW (R8), BX
	MOVB 2(R8), R11
	MOVW BX, (SI)
	MOVB R11, 2(SI)
	JMP  decodeBlockAsm_remain_copy3_fused_lits_done

decodeBlockAsm_remain_copy3_fused_lits_move_4:
	MOVL (R8), BX
	MOVL BX, (SI)

decodeBlockAsm_remain_copy3_fused_lits_done:
	ADDQ R10, R8
	ADDQ R10, SI
	ADDQ R10, DI
	JMP  decodeBlockAsm_remain_copy_exec_long_long
	PCALIGN $0x10

decodeBlockAsm_remain_copy_exec_short:
	CMPL R9, DI
	JA   corrupt
	LEAQ (SI)(DX*1), BX
	CMPQ BX, AX
	JA   corrupt
	MOVQ SI, BX
	SUBQ R9, BX
	CMPL R9, DX
	JB   decodeBlockAsm_remain_copy_overlap
	JMP  decodeBlockAsm_remain_copy_short
	PCALIGN $0x10

decodeBlockAsm_remain_copy_exec_long_long:
	MOVQ SI, BX
	SUBQ R9, BX
	CMPL R9, DI
	JA   corrupt
	LEAQ (SI)(DX*1), R10
	CMPQ R10, AX
	JA   corrupt

	// genMemMoveLong
	MOVQ    DX, R10
	SHRQ    $0x06, R10
	MOVQ    SI, R11
	MOVQ    DX, R12
	PCALIGN $0x10

decodeBlockAsm_remain_copy_long_longlarge_big_loop_back:
	MOVOU (BX), X0
	MOVOU 16(BX), X1
	MOVOU 32(BX), X2
	MOVOU 48(BX), X3
	MOVOU X0, (R11)
	MOVOU X1, 16(R11)
	MOVOU X2, 32(R11)
	MOVOU X3, 48(R11)
	ADDQ  $0x40, R11
	ADDQ  $0x40, BX
	SUBQ  $0x40, R12
	JZ    decodeBlockAsm_remain_copy_done
	DECQ  R10
	JNZ   decodeBlockAsm_remain_copy_long_longlarge_big_loop_back
	TESTQ R12, R12
	JZ    decodeBlockAsm_remain_copy_done
	MOVOU -64(BX)(R12*1), X0
	MOVOU -48(BX)(R12*1), X1
	MOVOU -32(BX)(R12*1), X2
	MOVOU -16(BX)(R12*1), X3
	MOVOU X0, -64(R11)(R12*1)
	MOVOU X1, -48(R11)(R12*1)
	MOVOU X2, -32(R11)(R12*1)
	MOVOU X3, -16(R11)(R12*1)
	JMP   decodeBlockAsm_remain_copy_done
	PCALIGN $0x10

decodeBlockAsm_remain_copy_short_no_ol:
	MOVQ SI, BX
	SUBQ R9, BX
	CMPL R9, DI
	JA   corrupt
	LEAQ (SI)(DX*1), R10
	CMPQ R10, AX
	JA   corrupt

	// genMemMoveShort
	// margin: -4, min move: 4
	CMPQ DX, $0x08
	JBE  decodeBlockAsm_remain_copy_short_no_ol_memmove_move_4through8
	CMPQ DX, $0x10
	JBE  decodeBlockAsm_remain_copy_short_no_ol_memmove_move_8through16
	CMPQ DX, $0x20
	JBE  decodeBlockAsm_remain_copy_short_no_ol_memmove_move_17through32
	JMP  decodeBlockAsm_remain_copy_short_no_ol_memmove_move_33through64

decodeBlockAsm_remain_copy_short_no_ol_memmove_move_4through8:
	MOVL (BX), R10
	MOVL -4(BX)(DX*1), BX
	MOVL R10, (SI)
	MOVL BX, -4(SI)(DX*1)
	JMP  decodeBlockAsm_remain_copy_done
	PCALIGN $0x10

decodeBlockAsm_remain_copy_short_no_ol_memmove_move_8through16:
	MOVQ (BX), R10
	MOVQ -8(BX)(DX*1), BX
	MOVQ R10, (SI)
	MOVQ BX, -8(SI)(DX*1)
	JMP  decodeBlockAsm_remain_copy_done

decodeBlockAsm_remain_copy_short_no_ol_memmove_move_17through32:
	MOVOU (BX), X0
	MOVOU -16(BX)(DX*1), X1
	MOVOU X0, (SI)
	MOVOU X1, -16(SI)(DX*1)
	JMP   decodeBlockAsm_remain_copy_done

decodeBlockAsm_remain_copy_short_no_ol_memmove_move_33through64:
	MOVOU (BX), X0
	MOVOU 16(BX), X1
	MOVOU -32(BX)(DX*1), X2
	MOVOU -16(BX)(DX*1), X3
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(DX*1)
	MOVOU X3, -16(SI)(DX*1)
	JMP   decodeBlockAsm_remain_copy_done
	PCALIGN $0x10

decodeBlockAsm_remain_copy_exec:
	CMPL R9, DI
	JA   corrupt
	LEAQ (SI)(DX*1), BX
	CMPQ BX, AX
	JA   corrupt
	MOVQ SI, BX
	SUBQ R9, BX
	CMPL R9, DX
	JB   decodeBlockAsm_remain_copy_overlap
	CMPL DX, $0x40
	JAE  decodeBlockAsm_remain_copy_long

decodeBlockAsm_remain_copy_short:
	// genMemMoveShort
	// margin: -4, min move: 1
	CMPQ DX, $0x03
	JB   decodeBlockAsm_remain_copy_short_memmove_move_1or2
	JE   decodeBlockAsm_remain_copy_short_memmove_move_3
	CMPQ DX, $0x08
	JBE  decodeBlockAsm_remain_copy_short_memmove_move_4through8
	CMPQ DX, $0x10
	JBE  decodeBlockAsm_remain_copy_short_memmove_move_8through16
	CMPQ DX, $0x20
	JBE  decodeBlockAsm_remain_copy_short_memmove_move_17through32
	JMP  decodeBlockAsm_remain_copy_short_memmove_move_33through64

decodeBlockAsm_remain_copy_short_memmove_move_1or2:
	MOVB (BX), R10
	MOVB -1(BX)(DX*1), BL
	MOVB R10, (SI)
	MOVB BL, -1(SI)(DX*1)
	JMP  decodeBlockAsm_remain_copy_done

decodeBlockAsm_remain_copy_short_memmove_move_3:
	MOVW (BX), R10
	MOVB 2(BX), BL
	MOVW R10, (SI)
	MOVB BL, 2(SI)
	JMP  decodeBlockAsm_remain_copy_done

decodeBlockAsm_remain_copy_short_memmove_move_4through8:
	MOVL (BX), R10
	MOVL -4(BX)(DX*1), BX
	MOVL R10, (SI)
	MOVL BX, -4(SI)(DX*1)
	JMP  decodeBlockAsm_remain_copy_done
	PCALIGN $0x10

decodeBlockAsm_remain_copy_short_memmove_move_8through16:
	MOVQ (BX), R10
	MOVQ -8(BX)(DX*1), BX
	MOVQ R10, (SI)
	MOVQ BX, -8(SI)(DX*1)
	JMP  decodeBlockAsm_remain_copy_done

decodeBlockAsm_remain_copy_short_memmove_move_17through32:
	MOVOU (BX), X0
	MOVOU -16(BX)(DX*1), X1
	MOVOU X0, (SI)
	MOVOU X1, -16(SI)(DX*1)
	JMP   decodeBlockAsm_remain_copy_done

decodeBlockAsm_remain_copy_short_memmove_move_33through64:
	MOVOU (BX), X0
	MOVOU 16(BX), X1
	MOVOU -32(BX)(DX*1), X2
	MOVOU -16(BX)(DX*1), X3
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(DX*1)
	MOVOU X3, -16(SI)(DX*1)
	JMP   decodeBlockAsm_remain_copy_done

decodeBlockAsm_remain_copy_long:
	// genMemMoveLong
	MOVOU   (BX), X0
	MOVOU   16(BX), X1
	MOVOU   -32(BX)(DX*1), X2
	MOVOU   -16(BX)(DX*1), X3
	MOVQ    DX, R11
	SHRQ    $0x05, R11
	MOVQ    SI, R10
	ANDL    $0x0000001f, R10
	MOVQ    $0x00000040, R12
	SUBQ    R10, R12
	DECQ    R11
	JA      decodeBlockAsm_remain_copy_longlarge_forward_sse_loop_32
	LEAQ    -32(BX)(R12*1), R10
	LEAQ    -32(SI)(R12*1), R13
	PCALIGN $0x10

decodeBlockAsm_remain_copy_longlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   decodeBlockAsm_remain_copy_longlarge_big_loop_back

decodeBlockAsm_remain_copy_longlarge_forward_sse_loop_32:
	MOVOU -32(BX)(R12*1), X4
	MOVOU -16(BX)(R12*1), X5
	MOVOA X4, -32(SI)(R12*1)
	MOVOA X5, -16(SI)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  DX, R12
	JAE   decodeBlockAsm_remain_copy_longlarge_forward_sse_loop_32
	MOVOU X0, (SI)
	MOVOU X1, 16(SI)
	MOVOU X2, -32(SI)(DX*1)
	MOVOU X3, -16(SI)(DX*1)

decodeBlockAsm_remain_copy_done:
	ADDQ DX, SI
	ADDQ DX, DI
	JMP  decodeBlockAsm_remain_loop

decodeBlockAsm_remain_copy_overlap:
	ADDQ DX, DI

decodeBlockAsm_remain_copy_overlap_simple:
	MOVB (BX), R10
	MOVB R10, (SI)
	INCQ BX
	INCQ SI
	DECQ DX
	JNZ  decodeBlockAsm_remain_copy_overlap_simple
	JMP  decodeBlockAsm_remain_loop

decodeBlockAsm_remain_end_copy:
decodeBlockAsm_remain_end_done:
	MOVQ src_base+24(FP), AX
	MOVQ src_len+32(FP), CX
	MOVQ dst_base+0(FP), DX
	MOVQ dst_len+8(FP), BX
	LEAQ (DX)(BX*1), DX
	LEAQ (AX)(CX*1), AX
	CMPQ SI, DX
	JNE  corrupt
	CMPQ R8, AX
	JNE  corrupt
	MOVQ $0x00000000, ret+48(FP)
	RET

corrupt:
	MOVQ $0x00000001, ret+48(FP)
	RET
