Current File : //usr/local/go/src/runtime/memmove_arm64.s
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// See memmove Go doc for important implementation constraints.
// Register map
//
// dstin R0
// src R1
// count R2
// dst R3 (same as R0, but gets modified in unaligned cases)
// srcend R4
// dstend R5
// data R6-R17
// tmp1 R14
// Copies are split into 3 main cases: small copies of up to 32 bytes, medium
// copies of up to 128 bytes, and large copies. The overhead of the overlap
// check is negligible since it is only required for large copies.
//
// Large copies use a software pipelined loop processing 64 bytes per iteration.
// The destination pointer is 16-byte aligned to minimize unaligned accesses.
// The loop tail is handled by always copying 64 bytes from the end.
// func memmove(to, from unsafe.Pointer, n uintptr)
TEXT runtime·memmove(SB), NOSPLIT|NOFRAME, $0-24
MOVD to+0(FP), R0
MOVD from+8(FP), R1
MOVD n+16(FP), R2
CBZ R2, copy0
// Small copies: 1..16 bytes
CMP $16, R2
BLE copy16
// Large copies
CMP $128, R2
BHI copy_long
CMP $32, R2
BHI copy32_128
// Small copies: 17..32 bytes.
LDP (R1), (R6, R7)
ADD R1, R2, R4 // R4 points just past the last source byte
LDP -16(R4), (R12, R13)
STP (R6, R7), (R0)
ADD R0, R2, R5 // R5 points just past the last destination byte
STP (R12, R13), -16(R5)
RET
// Small copies: 1..16 bytes.
copy16:
ADD R1, R2, R4 // R4 points just past the last source byte
ADD R0, R2, R5 // R5 points just past the last destination byte
CMP $8, R2
BLT copy7
MOVD (R1), R6
MOVD -8(R4), R7
MOVD R6, (R0)
MOVD R7, -8(R5)
RET
copy7:
TBZ $2, R2, copy3
MOVWU (R1), R6
MOVWU -4(R4), R7
MOVW R6, (R0)
MOVW R7, -4(R5)
RET
copy3:
TBZ $1, R2, copy1
MOVHU (R1), R6
MOVHU -2(R4), R7
MOVH R6, (R0)
MOVH R7, -2(R5)
RET
copy1:
MOVBU (R1), R6
MOVB R6, (R0)
copy0:
RET
// Medium copies: 33..128 bytes.
copy32_128:
ADD R1, R2, R4 // R4 points just past the last source byte
ADD R0, R2, R5 // R5 points just past the last destination byte
LDP (R1), (R6, R7)
LDP 16(R1), (R8, R9)
LDP -32(R4), (R10, R11)
LDP -16(R4), (R12, R13)
CMP $64, R2
BHI copy128
STP (R6, R7), (R0)
STP (R8, R9), 16(R0)
STP (R10, R11), -32(R5)
STP (R12, R13), -16(R5)
RET
// Copy 65..128 bytes.
copy128:
LDP 32(R1), (R14, R15)
LDP 48(R1), (R16, R17)
CMP $96, R2
BLS copy96
LDP -64(R4), (R2, R3)
LDP -48(R4), (R1, R4)
STP (R2, R3), -64(R5)
STP (R1, R4), -48(R5)
copy96:
STP (R6, R7), (R0)
STP (R8, R9), 16(R0)
STP (R14, R15), 32(R0)
STP (R16, R17), 48(R0)
STP (R10, R11), -32(R5)
STP (R12, R13), -16(R5)
RET
// Copy more than 128 bytes.
copy_long:
ADD R1, R2, R4 // R4 points just past the last source byte
ADD R0, R2, R5 // R5 points just past the last destination byte
MOVD ZR, R7
MOVD ZR, R8
CMP $1024, R2
BLT backward_check
// feature detect to decide how to align
MOVBU runtime·arm64UseAlignedLoads(SB), R6
CBNZ R6, use_aligned_loads
MOVD R0, R7
MOVD R5, R8
B backward_check
use_aligned_loads:
MOVD R1, R7
MOVD R4, R8
// R7 and R8 are used here for the realignment calculation. In
// the use_aligned_loads case, R7 is the src pointer and R8 is
// srcend pointer, which is used in the backward copy case.
// When doing aligned stores, R7 is the dst pointer and R8 is
// the dstend pointer.
backward_check:
// Use backward copy if there is an overlap.
SUB R1, R0, R14
CBZ R14, copy0
CMP R2, R14
BCC copy_long_backward
// Copy 16 bytes and then align src (R1) or dst (R0) to 16-byte alignment.
LDP (R1), (R12, R13) // Load A
AND $15, R7, R14 // Calculate the realignment offset
SUB R14, R1, R1
SUB R14, R0, R3 // move dst back same amount as src
ADD R14, R2, R2
LDP 16(R1), (R6, R7) // Load B
STP (R12, R13), (R0) // Store A
LDP 32(R1), (R8, R9) // Load C
LDP 48(R1), (R10, R11) // Load D
LDP.W 64(R1), (R12, R13) // Load E
// 80 bytes have been loaded; if less than 80+64 bytes remain, copy from the end
SUBS $144, R2, R2
BLS copy64_from_end
loop64:
STP (R6, R7), 16(R3) // Store B
LDP 16(R1), (R6, R7) // Load B (next iteration)
STP (R8, R9), 32(R3) // Store C
LDP 32(R1), (R8, R9) // Load C
STP (R10, R11), 48(R3) // Store D
LDP 48(R1), (R10, R11) // Load D
STP.W (R12, R13), 64(R3) // Store E
LDP.W 64(R1), (R12, R13) // Load E
SUBS $64, R2, R2
BHI loop64
// Write the last iteration and copy 64 bytes from the end.
copy64_from_end:
LDP -64(R4), (R14, R15) // Load F
STP (R6, R7), 16(R3) // Store B
LDP -48(R4), (R6, R7) // Load G
STP (R8, R9), 32(R3) // Store C
LDP -32(R4), (R8, R9) // Load H
STP (R10, R11), 48(R3) // Store D
LDP -16(R4), (R10, R11) // Load I
STP (R12, R13), 64(R3) // Store E
STP (R14, R15), -64(R5) // Store F
STP (R6, R7), -48(R5) // Store G
STP (R8, R9), -32(R5) // Store H
STP (R10, R11), -16(R5) // Store I
RET
// Large backward copy for overlapping copies.
// Copy 16 bytes and then align srcend (R4) or dstend (R5) to 16-byte alignment.
copy_long_backward:
LDP -16(R4), (R12, R13)
AND $15, R8, R14
SUB R14, R4, R4
SUB R14, R2, R2
LDP -16(R4), (R6, R7)
STP (R12, R13), -16(R5)
LDP -32(R4), (R8, R9)
LDP -48(R4), (R10, R11)
LDP.W -64(R4), (R12, R13)
SUB R14, R5, R5
SUBS $128, R2, R2
BLS copy64_from_start
loop64_backward:
STP (R6, R7), -16(R5)
LDP -16(R4), (R6, R7)
STP (R8, R9), -32(R5)
LDP -32(R4), (R8, R9)
STP (R10, R11), -48(R5)
LDP -48(R4), (R10, R11)
STP.W (R12, R13), -64(R5)
LDP.W -64(R4), (R12, R13)
SUBS $64, R2, R2
BHI loop64_backward
// Write the last iteration and copy 64 bytes from the start.
copy64_from_start:
LDP 48(R1), (R2, R3)
STP (R6, R7), -16(R5)
LDP 32(R1), (R6, R7)
STP (R8, R9), -32(R5)
LDP 16(R1), (R8, R9)
STP (R10, R11), -48(R5)
LDP (R1), (R10, R11)
STP (R12, R13), -64(R5)
STP (R2, R3), 48(R0)
STP (R6, R7), 32(R0)
STP (R8, R9), 16(R0)
STP (R10, R11), (R0)
RET
Mini Shell