[gcc(refs/users/acsawdey/heads/memcpy-unk-lxvl)] Inline small memcpy/memmove code with lxvl/stxvl
Aaron Sawdey
acsawdey@gcc.gnu.org
Tue Nov 10 16:52:12 GMT 2020
https://gcc.gnu.org/g:098a460262a7895753b04773ac5683783c4230cd
commit 098a460262a7895753b04773ac5683783c4230cd
Author: Aaron Sawdey <acsawdey@linux.ibm.com>
Date: Tue Nov 3 11:20:59 2020 -0600
Inline small memcpy/memmove code with lxvl/stxvl
Diff:
---
gcc/config/rs6000/rs6000-string.c | 91 ++++++++++++++++++++++++++++++++++++++-
gcc/config/rs6000/rs6000.c | 10 +++++
gcc/config/rs6000/rs6000.md | 42 +++++++++++++++++-
gcc/config/rs6000/rs6000.opt | 4 ++
4 files changed, 144 insertions(+), 3 deletions(-)
diff --git a/gcc/config/rs6000/rs6000-string.c b/gcc/config/rs6000/rs6000-string.c
index 82cc24ecdda..6a3658a593b 100644
--- a/gcc/config/rs6000/rs6000-string.c
+++ b/gcc/config/rs6000/rs6000-string.c
@@ -2734,6 +2734,88 @@ gen_lxvl_stxvl_move (rtx dest, rtx src, int length)
return gen_lxvl (dest, addr, len);
}
+/* Expand a block move operation of unknown length, and return 1 if successful.
+ Return 0 if we should let the compiler generate normal code.
+
+ This emits a test for length <= 16 and code to do the move/copy with
+ lxvl/stxvl if true and a call to the library function if false.
+
+ operands[0] is the destination
+ operands[1] is the source
+ operands[2] is the length
+ operands[3] is the alignment */
+
+int
+expand_block_move_unk_lxvl (rtx operands[], bool might_overlap)
+{
+ rtx orig_dest = operands[0];
+ rtx orig_src = operands[1];
+ rtx bytes_rtx = operands[2];
+ rtx dest_addr = force_reg (Pmode, XEXP (orig_dest, 0));
+ rtx src_addr = force_reg (Pmode, XEXP (orig_src, 0));
+
+ /*
+ cmpli bytes,16
+ ble Linline
+ bl library_func
+ b Lfinish
+ .Linline
+ lxvl
+ stxvl
+ .Lfinish
+ */
+
+ rtx len_rtx = gen_reg_rtx (Pmode);
+ emit_move_insn (len_rtx, bytes_rtx);
+
+ rtx inline_label = gen_label_rtx ();
+ rtx finish_label = gen_label_rtx ();
+ rtx inline_ref = gen_rtx_LABEL_REF (VOIDmode, inline_label);
+ rtx cond = gen_reg_rtx (CCmode);
+ emit_move_insn (cond, gen_rtx_COMPARE (CCmode, len_rtx,
+ GEN_INT (16)));
+
+ rtx cmp_rtx = gen_rtx_LE (VOIDmode, cond, const0_rtx);
+
+ rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx,
+ inline_ref, pc_rtx);
+ rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
+ add_reg_br_prob_note (j, profile_probability::even ());
+ JUMP_LABEL (j) = inline_label;
+ LABEL_NUSES (inline_label) += 1;
+
+ if (might_overlap)
+ {
+ tree fun = builtin_decl_explicit (BUILT_IN_MEMMOVE);
+ emit_library_call (XEXP (DECL_RTL (fun), 0), LCT_NORMAL,
+ Pmode, dest_addr, Pmode,
+ src_addr, Pmode, len_rtx, Pmode);
+ }
+ else
+ {
+ tree fun = builtin_decl_explicit (BUILT_IN_MEMCPY);
+ emit_library_call (XEXP (DECL_RTL (fun), 0),
+ LCT_NORMAL, Pmode,
+ dest_addr, Pmode, src_addr, Pmode, len_rtx, Pmode);
+ }
+
+ rtx finish_ref = gen_rtx_LABEL_REF (VOIDmode, finish_label);
+ j = emit_jump_insn (gen_rtx_SET (pc_rtx, finish_ref));
+ JUMP_LABEL (j) = finish_label;
+ LABEL_NUSES (finish_label) += 1;
+ emit_barrier ();
+
+ emit_label (inline_label);
+
+ rtx data = gen_reg_rtx (V16QImode);
+ emit_insn (gen_lxvl (data, src_addr, len_rtx));
+ emit_insn (gen_stxvl (data, dest_addr, len_rtx));
+
+ emit_label (finish_label);
+
+ return 1;
+}
+
/* Expand a block move operation, and return 1 if successful. Return 0
if we should let the compiler generate normal code.
@@ -2762,7 +2844,14 @@ expand_block_move (rtx operands[], bool might_overlap)
/* If this is not a fixed size move, just call memcpy */
if (! constp)
- return 0;
+ {
+ if (!TARGET_BLOCK_OPS_UNKNOWN_LEN
+ || !TARGET_BLOCK_OPS_UNALIGNED_VSX
+ || !TARGET_POWER10)
+ return 0;
+ else
+ return expand_block_move_unk_lxvl (operands, might_overlap);
+ }
/* This must be a fixed size alignment */
gcc_assert (CONST_INT_P (align_rtx));
diff --git a/gcc/config/rs6000/rs6000.c b/gcc/config/rs6000/rs6000.c
index 1e506b83762..04f9ddeccc1 100644
--- a/gcc/config/rs6000/rs6000.c
+++ b/gcc/config/rs6000/rs6000.c
@@ -4129,6 +4129,14 @@ rs6000_option_override_internal (bool global_init_p)
rs6000_isa_flags &= ~OPTION_MASK_BLOCK_OPS_VECTOR_PAIR;
}
+ if (!(rs6000_isa_flags_explicit & OPTION_MASK_BLOCK_OPS_UNKNOWN_LEN))
+ {
+ if (TARGET_POWER10)
+ rs6000_isa_flags |= OPTION_MASK_BLOCK_OPS_UNKNOWN_LEN;
+ else
+ rs6000_isa_flags &= ~OPTION_MASK_BLOCK_OPS_UNKNOWN_LEN;
+ }
+
/* Use long double size to select the appropriate long double. We use
TYPE_PRECISION to differentiate the 3 different long double types. We map
128 into the precision used for TFmode. */
@@ -23371,6 +23379,8 @@ static struct rs6000_opt_mask const rs6000_opt_masks[] =
false, true },
{ "block-ops-vector-pair", OPTION_MASK_BLOCK_OPS_VECTOR_PAIR,
false, true },
+ { "block-ops-unknown-len", OPTION_MASK_BLOCK_OPS_UNKNOWN_LEN,
+ false, true },
{ "cmpb", OPTION_MASK_CMPB, false, true },
{ "crypto", OPTION_MASK_CRYPTO, false, true },
{ "direct-move", OPTION_MASK_DIRECT_MOVE, false, true },
diff --git a/gcc/config/rs6000/rs6000.md b/gcc/config/rs6000/rs6000.md
index dc060143104..1eeb0edf5cf 100644
--- a/gcc/config/rs6000/rs6000.md
+++ b/gcc/config/rs6000/rs6000.md
@@ -9338,7 +9338,7 @@
(match_operand:BLK 1 ""))
(use (match_operand:SI 2 ""))
(use (match_operand:SI 3 ""))])]
- ""
+ "!TARGET_POWERPC64"
{
if (expand_block_move (operands, false))
DONE;
@@ -9357,7 +9357,45 @@
(match_operand:BLK 1 ""))
(use (match_operand:SI 2 ""))
(use (match_operand:SI 3 ""))])]
- ""
+ "!TARGET_POWERPC64"
+{
+ if (expand_block_move (operands, true))
+ DONE;
+ else
+ FAIL;
+})
+
+;; String/block copy insn (source and destination must not overlap).
+;; Argument 0 is the destination
+;; Argument 1 is the source
+;; Argument 2 is the length
+;; Argument 3 is the alignment
+
+(define_expand "cpymemdi"
+ [(parallel [(set (match_operand:BLK 0 "")
+ (match_operand:BLK 1 ""))
+ (use (match_operand:DI 2 ""))
+ (use (match_operand:DI 3 ""))])]
+ "TARGET_POWERPC64"
+{
+ if (expand_block_move (operands, false))
+ DONE;
+ else
+ FAIL;
+})
+
+;; String/block move insn (source and destination may overlap).
+;; Argument 0 is the destination
+;; Argument 1 is the source
+;; Argument 2 is the length
+;; Argument 3 is the alignment
+
+(define_expand "movmemdi"
+ [(parallel [(set (match_operand:BLK 0 "")
+ (match_operand:BLK 1 ""))
+ (use (match_operand:DI 2 ""))
+ (use (match_operand:DI 3 ""))])]
+ "TARGET_POWERPC64"
{
if (expand_block_move (operands, true))
DONE;
diff --git a/gcc/config/rs6000/rs6000.opt b/gcc/config/rs6000/rs6000.opt
index b2a70e88ca8..3386e550674 100644
--- a/gcc/config/rs6000/rs6000.opt
+++ b/gcc/config/rs6000/rs6000.opt
@@ -324,6 +324,10 @@ mblock-move-inline-limit=
Target Report Var(rs6000_block_move_inline_limit) Init(0) RejectNegative Joined UInteger Save
Max number of bytes to move inline.
+mblock-ops-unknown-len
+Target Report Mask(BLOCK_OPS_UNKNOWN_LEN) Var(rs6000_isa_flags)
+Generate code to do memcpy/memmove of unknown length inline with lxvl/stxvl if length <= 16.
+
mblock-ops-unaligned-vsx
Target Report Mask(BLOCK_OPS_UNALIGNED_VSX) Var(rs6000_isa_flags)
Generate unaligned VSX load/store for inline expansion of memcpy/memmove.
More information about the Gcc-cvs
mailing list