[gcc.git] / gcc / config / ia64 / ia64.md

;; IA-64 Machine description template
;; Copyright (C) 1999, 2000 Free Software Foundation, Inc.
;; Contributed by James E. Wilson <wilson@cygnus.com> and
;;                David Mosberger <davidm@hpl.hp.com>.

;; This file is part of GNU CC.

;; GNU CC is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 2, or (at your option)
;; any later version.

;; GNU CC is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
;; along with GNU CC; see the file COPYING.  If not, write to
;; the Free Software Foundation, 59 Temple Place - Suite 330,
;; Boston, MA 02111-1307, USA.

;;- See file "rtl.def" for documentation on define_insn, match_*, et. al.

;; ??? Add support for long double XFmode patterns.

;; ??? register_operand accepts (subreg:DI (mem:SI X)) which forces later
;; reload.  This will be fixed once scheduling support is turned on.

;; ??? Optimize for post-increment addressing modes.

;; ??? fselect is not supported, because there is no integer register
;; equivalent.

;; ??? fp abs/min/max instructions may also work for integer values.

;; ??? Would a predicate_reg_operand predicate be useful?  The HP one is buggy,
;; it assumes the operand is a register and takes REGNO of it without checking.

;; ??? Would a branch_reg_operand predicate be useful?  The HP one is buggy,
;; it assumes the operand is a register and takes REGNO of it without checking.

;; ??? Go through list of documented named patterns and look for more to
;; implement.

;; ??? Go through instruction manual and look for more instructions that
;; can be emitted.

;; ??? Add function unit scheduling info for Itanium (TM) processor.

;; ??? The explicit stop in the flushrs pattern is not ideal.  It
;; would be better if rtx_needs_barrier took care of this, but this is
;; something that can be fixed later.

;; Unspec usage:
;;
;; unspec:
;;      1       gr_spill
;;      2       gr_restore
;;      3       fr_spill
;;      4       fr_restore
;;      5       pr_spill
;;      8       popcnt
;;      9       unat_spill
;;      10      unat_restore
;;      13      cmpxchg_acq
;;      14      val_compare_and_swap
;;      16      lock_test_and_set
;;      17      op_and_fetch
;;      18      fetch_and_op
;;      19      fetchadd_acq
;;      20      bsp_value
;;      21      flushrs
;;
;; unspec_volatile:
;;      0       alloc
;;      1       blockage
;;      2       insn_group_barrier
;;      3       flush_cache
;;      4       pfs_restore
;;      5       set_bsp
;;      6       pr_restore
;;      7       pred.rel.mutex
\f
;; ::::::::::::::::::::
;; ::
;; :: Attributes
;; ::
;; ::::::::::::::::::::

;; Instruction type.  This primarily determines how instructions can be
;; packed in bundles, and secondarily affects scheduling to function units.

;; A alu, can go in I or M syllable of a bundle
;; I integer
;; M memory
;; F floating-point
;; B branch
;; L long immediate, takes two syllables
;; S stop bit

;; ??? Should not have any pattern with type unknown.  Perhaps add code to
;; check this in md_reorg?  Currently use unknown for patterns which emit
;; multiple instructions, patterns which emit 0 instructions, and patterns
;; which emit instruction that can go in any slot (e.g. nop).

(define_attr "type" "unknown,A,I,M,F,B,L,S" (const_string "unknown"))

;; Predication.  True iff this instruction can be predicated.

(define_attr "predicable" "no,yes" (const_string "yes"))

\f
;; ::::::::::::::::::::
;; ::
;; :: Function Units
;; ::
;; ::::::::::::::::::::

;; Each usage of a function units by a class of insns is specified with a
;; `define_function_unit' expression, which looks like this:
;; (define_function_unit NAME MULTIPLICITY SIMULTANEITY TEST READY-DELAY
;;   ISSUE-DELAY [CONFLICT-LIST])

;; This default scheduling info seeks to pack instructions into bundles
;; efficiently to reduce code size, so we just list how many of each
;; instruction type can go in a bundle.  ISSUE_RATE is set to 3.

;; ??? Add scheduler ready-list hook (MD_SCHED_REORDER) that orders
;; instructions, so that the next instruction can fill the next bundle slot.
;; This really needs to know where the stop bits are though.

;; ??? Use MD_SCHED_REORDER to put alloc first instead of using an unspec
;; volatile.  Use ADJUST_PRIORITY to set the priority of alloc very high to
;; make it schedule first.

;; ??? Modify the md_reorg code that emits stop bits so that instead of putting
;; them in the last possible place, we put them in places where bundles allow
;; them.  This should reduce code size, but may decrease performance if we end
;; up with more stop bits than the minimum we need.

;; Alu instructions can execute on either the integer or memory function
;; unit.  We indicate this by defining an alu function unit, and then marking
;; it as busy everytime we issue a integer or memory type instruction.

(define_function_unit "alu" 3 1 (eq_attr "type" "A,I,M") 1 0)

(define_function_unit "integer" 2 1 (eq_attr "type" "I") 1 0)

(define_function_unit "memory" 3 1 (eq_attr "type" "M") 1 0)

(define_function_unit "floating_point" 1 1 (eq_attr "type" "F") 1 0)

(define_function_unit "branch" 3 1 (eq_attr "type" "B") 1 0)

;; ??? This isn't quite right, because we can only fit two insns in a bundle
;; when using an L type instruction.  That isn't modeled currently.

(define_function_unit "long_immediate" 1 1 (eq_attr "type" "L") 1 0)

\f
;; ::::::::::::::::::::
;; ::
;; :: Moves
;; ::
;; ::::::::::::::::::::

(define_expand "movqi"
  [(set (match_operand:QI 0 "general_operand" "")
        (match_operand:QI 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && GET_CODE (operands[0]) == MEM
      && GET_CODE (operands[1]) == MEM)
    operands[1] = copy_to_mode_reg (QImode, operands[1]);
}")

(define_insn "*movqi_internal"
  [(set (match_operand:QI 0 "nonimmediate_operand" "=r,r,r, m, r,*f,*f")
        (match_operand:QI 1 "move_operand"         "rO,J,m,rO,*f,rO,*f"))]
  "! memory_operand (operands[0], QImode)
   || ! memory_operand (operands[1], QImode)"
  "@
   mov %0 = %r1
   addl %0 = %1, r0
   ld1%O1 %0 = %1%P1
   st1%Q0 %0 = %r1%P0
   getf.sig %0 = %1
   setf.sig %0 = %r1
   mov %0 = %1"
  [(set_attr "type" "A,A,M,M,M,M,F")])

(define_expand "movhi"
  [(set (match_operand:HI 0 "general_operand" "")
        (match_operand:HI 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && GET_CODE (operands[0]) == MEM
      && GET_CODE (operands[1]) == MEM)
    operands[1] = copy_to_mode_reg (HImode, operands[1]);
}")

(define_insn "*movhi_internal"
  [(set (match_operand:HI 0 "nonimmediate_operand" "=r,r,r, m, r,*f,*f")
        (match_operand:HI 1 "move_operand"         "rO,J,m,rO,*f,rO,*f"))]
  "! memory_operand (operands[0], HImode)
   || !memory_operand (operands[1], HImode)"
  "@
   mov %0 = %r1
   addl %0 = %1, r0
   ld2%O1 %0 = %1%P1
   st2%Q0 %0 = %r1%P0
   getf.sig %0 = %1
   setf.sig %0 = %r1
   mov %0 = %1"
  [(set_attr "type" "A,A,M,M,M,M,F")])

(define_expand "movsi"
  [(set (match_operand:SI 0 "general_operand" "")
        (match_operand:SI 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && GET_CODE (operands[0]) == MEM
      && GET_CODE (operands[1]) == MEM)
    operands[1] = copy_to_mode_reg (SImode, operands[1]);
}")

(define_insn "*movsi_internal"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=r,r,r,r, m, r,*f,*f")
        (match_operand:SI 1 "move_operand"         "rO,J,i,m,rO,*f,rO,*f"))]
  "! memory_operand (operands[0], SImode)
   || ! memory_operand (operands[1], SImode)"
  "@
  mov %0 = %r1
  addl %0 = %1, r0
  movl %0 = %1
  ld4%O1 %0 = %1%P1
  st4%Q0 %0 = %r1%P0
  getf.sig %0 = %1
  setf.sig %0 = %r1
  mov %0 = %1"
  [(set_attr "type" "A,A,L,M,M,M,M,F")])

(define_expand "movdi"
  [(set (match_operand:DI 0 "general_operand" "")
        (match_operand:DI 1 "general_operand" ""))]
  ""
  "
{
  /* ??? Should generalize this, so that we can also support 32 bit
     pointers.  */
  if (! TARGET_NO_PIC && symbolic_operand (operands[1], DImode))
    {
      rtx temp;

      /* Operand[0] could be a MEM, which isn't a valid destination for the
         PIC load address patterns.  */
      if (! register_operand (operands[0], DImode))
        temp = gen_reg_rtx (DImode);
      else
        temp = operands[0];

      if (TARGET_AUTO_PIC)
        emit_insn (gen_load_gprel64 (temp, operands[1]));
      else if (GET_CODE (operands[1]) == SYMBOL_REF
               && SYMBOL_REF_FLAG (operands[1]))
        emit_insn (gen_load_fptr (temp, operands[1]));
      else if (sdata_symbolic_operand (operands[1], DImode))
        emit_insn (gen_load_gprel (temp, operands[1]));
      else
        emit_insn (gen_load_symptr (temp, operands[1]));

      if (temp == operands[0])
        DONE;

      operands[1] = temp;
    }

  if (! reload_in_progress && ! reload_completed
      && GET_CODE (operands[0]) == MEM
      && GET_CODE (operands[1]) == MEM)
    operands[1] = copy_to_mode_reg (DImode, operands[1]);
}")

(define_insn "*movdi_internal"
  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,r,r,r, m,r,*f,*f,*f,Q, r,*b")
        (match_operand:DI 1 "move_operand"         "rO,J,i,m,rO,*f,rO,*f,Q,*f,*b,rO"))]
  "! memory_operand (operands[0], DImode)
   || ! memory_operand (operands[1], DImode)"
  "@
  mov %0 = %r1
  addl %0 = %1, r0
  movl %0 = %1
  ld8%O1 %0 = %1%P1
  st8%Q0 %0 = %r1%P0
  getf.sig %0 = %1
  setf.sig %0 = %r1
  mov %0 = %1
  ldf8 %0 = %1%P1
  stf8 %0 = %1%P0
  mov %0 = %1
  mov %0 = %r1"
  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M,I,I")])

(define_expand "load_fptr"
  [(set (match_dup 2)
        (plus:DI (reg:DI 1) (match_operand:DI 1 "function_operand" "")))
   (set (match_operand:DI 0 "register_operand" "") (mem:DI (match_dup 2)))]
  ""
  "
{
  if (reload_in_progress)
    operands[2] = operands[0];
  else
    operands[2] = gen_reg_rtx (DImode);
}")

(define_insn "*load_fptr_internal1"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (plus:DI (reg:DI 1) (match_operand:DI 1 "function_operand" "s")))]
  ""
  "addl %0 = @ltoff(@fptr(%1)), gp"
  [(set_attr "type" "A")])

(define_insn "load_gprel"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (plus:DI (reg:DI 1) (match_operand:DI 1 "sdata_symbolic_operand" "s")))]
  ""
  "addl %0 = @gprel(%1), gp"
  [(set_attr "type" "A")])

(define_insn "gprel64_offset"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (minus:DI (match_operand:DI 1 "symbolic_operand" "") (reg:DI 1)))]
  ""
  "movl %0 = @gprel(%1)"
  [(set_attr "type" "L")])

(define_expand "load_gprel64"
  [(set (match_dup 2)
        (minus:DI (match_operand:DI 1 "symbolic_operand" "") (reg:DI 1)))
   (set (match_operand:DI 0 "register_operand" "")
        (plus:DI (reg:DI 1) (match_dup 2)))]
  ""
  "{
  if (reload_in_progress)
    operands[2] = operands[0];
  else
    operands[2] = gen_reg_rtx (DImode);
}")

(define_expand "load_symptr"
  [(set (match_dup 2)
        (plus:DI (reg:DI 1) (match_operand:DI 1 "symbolic_operand" "")))
   (set (match_operand:DI 0 "register_operand" "") (mem:DI (match_dup 2)))]
  ""
  "
{
  if (reload_in_progress)
    operands[2] = operands[0];
  else
    operands[2] = gen_reg_rtx (DImode);
}")

(define_insn "*load_symptr_internal1"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (plus:DI (reg:DI 1) (match_operand:DI 1 "symbolic_operand" "s")))]
  ""
  "addl %0 = @ltoff(%1), gp"
  [(set_attr "type" "A")])

;; Floating Point Moves
;;
;; Note - Patterns for SF mode moves are compulsory, but
;; patterns for DF are optional, as GCC can synthesise them.

(define_expand "movsf"
  [(set (match_operand:SF 0 "general_operand" "")
        (match_operand:SF 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && GET_CODE (operands[0]) == MEM
      && GET_CODE (operands[1]) == MEM)
    operands[1] = copy_to_mode_reg (SFmode, operands[1]);
}")

(define_insn "*movsf_internal"
  [(set (match_operand:SF 0 "nonimmediate_operand" "=f,f, Q,*r, f,*r,*r, m")
        (match_operand:SF 1 "general_operand"      "fG,Q,fG,fG,*r,*r, m,*r"))]
  "! memory_operand (operands[0], SFmode)
   || ! memory_operand (operands[1], SFmode)"
  "@
  mov %0 = %F1
  ldfs %0 = %1%P1
  stfs %0 = %F1%P0
  getf.s %0 = %F1
  setf.s %0 = %1
  mov %0 = %1
  ld4%O1 %0 = %1%P1
  st4%Q0 %0 = %1%P0"
  [(set_attr "type" "F,M,M,M,M,A,M,M")])

(define_expand "movdf"
  [(set (match_operand:DF 0 "general_operand" "")
        (match_operand:DF 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && GET_CODE (operands[0]) == MEM
      && GET_CODE (operands[1]) == MEM)
    operands[1] = copy_to_mode_reg (DFmode, operands[1]);
}")

(define_insn "*movdf_internal"
  [(set (match_operand:DF 0 "nonimmediate_operand" "=f,f, Q,*r, f,*r,*r, m")
        (match_operand:DF 1 "general_operand"      "fG,Q,fG,fG,*r,*r, m,*r"))]
  "! memory_operand (operands[0], DFmode)
   || ! memory_operand (operands[1], DFmode)"
  "@
  mov %0 = %F1
  ldfd %0 = %1%P1
  stfd %0 = %F1%P0
  getf.d %0 = %F1
  setf.d %0 = %1
  mov %0 = %1
  ld8%O1 %0 = %1%P1
  st8%Q0 %0 = %1%P0"
  [(set_attr "type" "F,M,M,M,M,A,M,M")])

(define_expand "movxf"
  [(set (match_operand:XF 0 "general_operand" "")
        (match_operand:XF 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && GET_CODE (operands[0]) == MEM
      && GET_CODE (operands[1]) == MEM)
    operands[1] = copy_to_mode_reg (XFmode, operands[1]);
}")

;; ??? There's no easy way to mind volatile acquire/release semantics.
(define_insn "*movxf_internal"
  [(set (match_operand:XF 0 "nonimmediate_operand" "=f,f, m")
        (match_operand:XF 1 "general_operand"      "fG,m,fG"))]
  "! memory_operand (operands[0], XFmode)
   || ! memory_operand (operands[1], XFmode)"
  "@
  mov %0 = %F1
  ldfe %0 = %1%P1
  stfe %0 = %F1%P0"
  [(set_attr "type" "F,M,M")])

\f
;; ::::::::::::::::::::
;; ::
;; :: Conversions
;; ::
;; ::::::::::::::::::::

;; Signed conversions from a smaller integer to a larger integer

(define_insn "extendqidi2"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (sign_extend:DI (match_operand:QI 1 "register_operand" "r")))]
  ""
  "sxt1 %0 = %1"
  [(set_attr "type" "I")])

(define_insn "extendhidi2"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (sign_extend:DI (match_operand:HI 1 "register_operand" "r")))]
  ""
  "sxt2 %0 = %1"
  [(set_attr "type" "I")])

(define_insn "extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "=r,*f")
        (sign_extend:DI (match_operand:SI 1 "register_operand" "r,*f")))]
  ""
  "@
   sxt4 %0 = %1
   fsxt.r %0 = %1, %1%B0"
  [(set_attr "type" "I,F")])

;; Unsigned conversions from a smaller integer to a larger integer

(define_insn "zero_extendqidi2"
  [(set (match_operand:DI 0 "register_operand" "=r,r")
        (zero_extend:DI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
  ""
  "@
   zxt1 %0 = %1
   ld1%O1 %0 = %1%P1"
  [(set_attr "type" "I,M")])

(define_insn "zero_extendhidi2"
  [(set (match_operand:DI 0 "register_operand" "=r,r")
        (zero_extend:DI (match_operand:HI 1 "nonimmediate_operand" "r,m")))]
  ""
  "@
   zxt2 %0 = %1
   ld2%O1 %0 = %1%P1"
  [(set_attr "type" "I,M")])

(define_insn "zero_extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "=r,r,*f")
        (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,*f")))]
  ""
  "@
   zxt4 %0 = %1
   ld4%O1 %0 = %1%P1
   fsxt.r %0 = f1, %1%B0"
  [(set_attr "type" "I,M,F")])

;; Convert between floating point types of different sizes.

;; ??? Optimization opportunity here.  Get rid of the insn altogether
;; when we can.  Should probably use a scheme like has been proposed
;; for ia32 in dealing with operands that match unary operators.  This
;; would let combine merge the thing into adjacent insns.

(define_insn_and_split "extendsfdf2"
  [(set (match_operand:DF 0 "register_operand" "=f,f")
        (float_extend:DF (match_operand:SF 1 "register_operand" "0,f")))]
  ""
  "mov %0 = %1"
  "reload_completed"
  [(set (match_dup 0) (float_extend:DF (match_dup 1)))]
  "if (true_regnum (operands[0]) == true_regnum (operands[1])) DONE;"
  [(set_attr "type" "F")])

(define_insn "truncdfsf2"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (float_truncate:SF (match_operand:DF 1 "register_operand" "f")))]
  ""
  "fnorm.s %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "truncxfsf2"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (float_truncate:SF (match_operand:XF 1 "register_operand" "f")))]
  ""
  "fnorm.s %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "truncxfdf2"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (float_truncate:DF (match_operand:XF 1 "register_operand" "f")))]
  ""
  "fnorm.d %0 = %1%B0"
  [(set_attr "type" "F")])

;; Convert between signed integer types and floating point.

(define_insn "floatdixf2"
  [(set (match_operand:XF 0 "register_operand" "=f")
        (float:XF (match_operand:DI 1 "register_operand" "f")))]
  ""
  "fcvt.xf %0 = %1"
  [(set_attr "type" "F")])

(define_insn "fix_truncsfdi2"
  [(set (match_operand:DI 0 "register_operand" "=f")
        (fix:DI (match_operand:SF 1 "register_operand" "f")))]
  ""
  "fcvt.fx.trunc %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "fix_truncdfdi2"
  [(set (match_operand:DI 0 "register_operand" "=f")
        (fix:DI (match_operand:DF 1 "register_operand" "f")))]
  ""
  "fcvt.fx.trunc %0 = %1%B0"
  [(set_attr "type" "F")])

;; Convert between unsigned integer types and floating point.

(define_insn "floatunsdisf2"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (unsigned_float:SF (match_operand:DI 1 "register_operand" "f")))]
  ""
  "fcvt.xuf.s %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "floatunsdidf2"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (unsigned_float:DF (match_operand:DI 1 "register_operand" "f")))]
  ""
  "fcvt.xuf.d %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "fixuns_truncsfdi2"
  [(set (match_operand:DI 0 "register_operand" "=f")
        (unsigned_fix:DI (match_operand:SF 1 "register_operand" "f")))]
  ""
  "fcvt.fxu.trunc %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "fixuns_truncdfdi2"
  [(set (match_operand:DI 0 "register_operand" "=f")
        (unsigned_fix:DI (match_operand:DF 1 "register_operand" "f")))]
  ""
  "fcvt.fxu.trunc %0 = %1%B0"
  [(set_attr "type" "F")])

\f
;; ::::::::::::::::::::
;; ::
;; :: Bit field extraction
;; ::
;; ::::::::::::::::::::

;; ??? It would be useful to have SImode versions of the extract and insert
;; patterns.

(define_insn "extv"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (sign_extract:DI (match_operand:DI 1 "register_operand" "r")
                         (match_operand:DI 2 "const_int_operand" "n")
                         (match_operand:DI 3 "const_int_operand" "n")))]
  ""
  "extr %0 = %1, %3, %2"
  [(set_attr "type" "I")])

(define_insn "extzv"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (zero_extract:DI (match_operand:DI 1 "register_operand" "r")
                         (match_operand:DI 2 "const_int_operand" "n")
                         (match_operand:DI 3 "const_int_operand" "n")))]
  ""
  "extr.u %0 = %1, %3, %2"
  [(set_attr "type" "I")])

;; Insert a bit field.
;; Can have 3 operands, source1 (inserter), source2 (insertee), dest.
;; Source1 can be 0 or -1.
;; Source2 can be 0.

;; ??? Actual dep instruction is more powerful than what these insv
;; patterns support.  Unfortunately, combine is unable to create patterns
;; where source2 != dest.

(define_expand "insv"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
                         (match_operand:DI 1 "const_int_operand" "")
                         (match_operand:DI 2 "const_int_operand" ""))
        (match_operand:DI 3 "nonmemory_operand" ""))]
  ""
  "
{
  int width = INTVAL (operands[1]);
  int shift = INTVAL (operands[2]);

  /* If operand[3] is a constant, and isn't 0 or -1, then load it into a
     pseudo.  */
  if (! register_operand (operands[3], DImode)
      && operands[3] != const0_rtx && operands[3] != constm1_rtx)
    operands[3] = force_reg (DImode, operands[3]);

  /* If this is a single dep instruction, we have nothing to do.  */
  if (! ((register_operand (operands[3], DImode) && width <= 16)
         || operands[3] == const0_rtx || operands[3] == constm1_rtx))
    {
      /* Check for cases that can be implemented with a mix instruction.  */
      if (width == 32 && shift == 0)
        {
          /* Directly generating the mix4left instruction confuses
             optimize_bit_field in function.c.  Since this is performing
             a useful optimization, we defer generation of the complicated
             mix4left RTL to the first splitting phase.  */
          rtx tmp = gen_reg_rtx (DImode);
          emit_insn (gen_shift_mix4left (operands[0], operands[3], tmp));
          DONE;
        }
      else if (width == 32 && shift == 32)
        {
          emit_insn (gen_mix4right (operands[0], operands[3]));
          DONE;
        }

      /* We could handle remaining cases by emitting multiple dep
         instructions.

         If we need more than two dep instructions then we lose.  A 6
         insn sequence mov mask1,mov mask2,shl;;and,and;;or is better than
         mov;;dep,shr;;dep,shr;;dep.  The former can be executed in 3 cycles,
         the latter is 6 cycles on an Itanium (TM) processor, because there is
         only one function unit that can execute dep and shr immed.

         If we only need two dep instruction, then we still lose.
         mov;;dep,shr;;dep is still 4 cycles.  Even if we optimize away
         the unnecessary mov, this is still undesirable because it will be
         hard to optimize, and it creates unnecessary pressure on the I0
         function unit.  */

      FAIL;

#if 0
      /* This code may be useful for other IA-64 processors, so we leave it in
         for now.  */
      while (width > 16)
        {
          rtx tmp;

          emit_insn (gen_insv (operands[0], GEN_INT (16), GEN_INT (shift),
                               operands[3]));
          shift += 16;
          width -= 16;
          tmp = gen_reg_rtx (DImode);
          emit_insn (gen_lshrdi3 (tmp, operands[3], GEN_INT (16)));
          operands[3] = tmp;
        }
      operands[1] = GEN_INT (width);
      operands[2] = GEN_INT (shift);
#endif
    }
}")

(define_insn "*insv_internal"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
                         (match_operand:DI 1 "const_int_operand" "n")
                         (match_operand:DI 2 "const_int_operand" "n"))
        (match_operand:DI 3 "nonmemory_operand" "rP"))]
  "(register_operand (operands[3], DImode) && INTVAL (operands[1]) <= 16)
   || operands[3] == const0_rtx || operands[3] == constm1_rtx"
  "dep %0 = %3, %0, %2, %1"
  [(set_attr "type" "I")])

(define_insn "shift_mix4left"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
                         (const_int 32) (const_int 0))
        (match_operand:DI 1 "register_operand" "r"))
   (clobber (match_operand:DI 2 "register_operand" "=r"))]
  ""
  "#"
  [(set_attr "type" "unknown")])

;; ??? Need to emit an instruction group barrier here because this gets split
;; after md_reorg.

(define_split
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
                         (const_int 32) (const_int 0))
        (match_operand:DI 1 "register_operand" ""))
   (clobber (match_operand:DI 2 "register_operand" ""))]
  "reload_completed"
  [(set (match_dup 3) (ashift:DI (match_dup 1) (const_int 32)))
   (unspec_volatile [(const_int 0)] 2)
   (set (zero_extract:DI (match_dup 0) (const_int 32) (const_int 0))
        (lshiftrt:DI (match_dup 3) (const_int 32)))]
  "operands[3] = operands[2];")

(define_split
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
                         (const_int 32) (const_int 0))
        (match_operand:DI 1 "register_operand" ""))
   (clobber (match_operand:DI 2 "register_operand" ""))]
  "! reload_completed"
  [(set (match_dup 3) (ashift:DI (match_dup 1) (const_int 32)))
   (set (zero_extract:DI (match_dup 0) (const_int 32) (const_int 0))
        (lshiftrt:DI (match_dup 3) (const_int 32)))]
  "operands[3] = operands[2];")

(define_insn "*mix4left"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
                         (const_int 32) (const_int 0))
        (lshiftrt:DI (match_operand:DI 1 "register_operand" "r")
                     (const_int 32)))]
  ""
  "mix4.l %0 = %0, %r1"
  [(set_attr "type" "I")])

(define_insn "mix4right"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
                         (const_int 32) (const_int 32))
        (match_operand:DI 1 "reg_or_0_operand" "rO"))]
  ""
  "mix4.r %0 = %r1, %0"
  [(set_attr "type" "I")])

;; This is used by the rotrsi3 pattern.

(define_insn "*mix4right_3op"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (ior:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "r"))
                (ashift:DI (zero_extend:DI (match_operand:SI 2 "register_operand" "r"))
                           (const_int 32))))]
  ""
  "mix4.r %0 = %2, %1"
  [(set_attr "type" "I")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 32 bit Integer arithmetic
;; ::
;; ::::::::::::::::::::

;; We handle 32-bit arithmetic just like the alpha port does.

(define_expand "addsi3"
  [(set (match_operand:SI 0 "register_operand" "")
        (plus:SI (match_operand:SI 1 "register_operand" "")
                 (match_operand:SI 2 "reg_or_22bit_operand" "")))]
  ""
  "
{
  if (optimize)
    {
      rtx op1 = gen_lowpart (DImode, operands[1]);
      rtx op2 = gen_lowpart (DImode, operands[2]);

      if (! cse_not_expected)
        {
          rtx tmp = gen_reg_rtx (DImode);
          emit_insn (gen_adddi3 (tmp, op1, op2));
          emit_move_insn (operands[0], gen_lowpart (SImode, tmp));
        }
      else
        emit_insn (gen_adddi3 (gen_lowpart (DImode, operands[0]), op1, op2));
      DONE;
    }
}")

(define_insn "*addsi3_internal"
  [(set (match_operand:SI 0 "register_operand" "=r,r,r")
        (plus:SI (match_operand:SI 1 "register_operand" "%r,r,a")
                 (match_operand:SI 2 "reg_or_22bit_operand" "r,I,J")))]
  ""
  "@
  add %0 = %1, %2
  adds %0 = %2, %1
  addl %0 = %2, %1"
  [(set_attr "type" "A")])

(define_insn "*addsi3_plus1"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (plus:SI (plus:SI (match_operand:SI 1 "register_operand" "r")
                          (match_operand:SI 2 "register_operand" "r"))
                 (const_int 1)))]
  ""
  "add %0 = %1, %2, 1"
  [(set_attr "type" "A")])

(define_expand "subsi3"
  [(set (match_operand:SI 0 "register_operand" "")
        (minus:SI (match_operand:SI 1 "reg_or_8bit_operand" "")
                  (match_operand:SI 2 "register_operand" "")))]
  ""
  "
{
  if (optimize)
    {
      rtx op1 = gen_lowpart (DImode, operands[1]);
      rtx op2 = gen_lowpart (DImode, operands[2]);

      if (! cse_not_expected)
        {
          rtx tmp = gen_reg_rtx (DImode);
          emit_insn (gen_subdi3 (tmp, op1, op2));
          emit_move_insn (operands[0], gen_lowpart (SImode, tmp));
        }
      else
        emit_insn (gen_subdi3 (gen_lowpart (DImode, operands[0]), op1, op2));
      DONE;
    }
}")

(define_insn "*subsi3_internal"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (minus:SI (match_operand:SI 1 "reg_or_8bit_operand" "rK")
                  (match_operand:SI 2 "register_operand" "r")))]
  ""
  "sub %0 = %1, %2"
  [(set_attr "type" "A")])

(define_insn "*subsi3_minus1"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (plus:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
                 (match_operand:SI 2 "register_operand" "r")))]
  ""
  "sub %0 = %2, %1, 1"
  [(set_attr "type" "A")])

(define_expand "mulsi3"
  [(set (match_operand:SI 0 "register_operand" "")
        (mult:SI (match_operand:SI 1 "register_operand" "")
                 (match_operand:SI 2 "register_operand" "")))]
  ""
  "
{
  if (optimize)
    {
      rtx op1 = gen_lowpart (DImode, operands[1]);
      rtx op2 = gen_lowpart (DImode, operands[2]);

      if (! cse_not_expected)
        {
          rtx tmp = gen_reg_rtx (DImode);
          emit_insn (gen_muldi3 (tmp, op1, op2));
          emit_move_insn (operands[0], gen_lowpart (SImode, tmp));
        }
      else
        emit_insn (gen_muldi3 (gen_lowpart (DImode, operands[0]), op1, op2));
      DONE;
    }
}")

;; ??? Could add maddsi3 patterns patterned after the madddi3 patterns.

(define_insn "*mulsi3_internal"
  [(set (match_operand:SI 0 "register_operand" "=f")
        (mult:SI (match_operand:SI 1 "register_operand" "f")
                 (match_operand:SI 2 "nonmemory_operand" "f")))]
  ""
  "xma.l %0 = %1, %2, f0%B0"
  [(set_attr "type" "F")])

(define_expand "negsi2"
  [(set (match_operand:SI 0 "register_operand" "")
        (neg:SI (match_operand:SI 1 "register_operand" "")))]
  ""
  "
{
  if (optimize)
    {
      rtx op1 = gen_lowpart (DImode, operands[1]);

      if (! cse_not_expected)
        {
          rtx tmp = gen_reg_rtx (DImode);
          emit_insn (gen_negdi2 (tmp, op1));
          emit_move_insn (operands[0], gen_lowpart (SImode, tmp));
        }
      else
        emit_insn (gen_negdi2 (gen_lowpart (DImode, operands[0]), op1));
      DONE;
    }
}")

(define_insn "*negsi2_internal"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (neg:SI (match_operand:SI 1 "register_operand" "r")))]
  ""
  "sub %0 = r0, %1"
  [(set_attr "type" "A")])

(define_expand "abssi2"
  [(set (match_dup 2)
        (ge:CC (match_operand:SI 1 "register_operand" "") (const_int 0)))
   (set (match_operand:SI 0 "register_operand" "")
        (if_then_else:SI (eq:CC (match_dup 2) (const_int 0))
                         (neg:SI (match_dup 1))
                         (match_dup 1)))]
  ""
  "
{
  operands[2] = gen_reg_rtx (CCmode);
}")

(define_expand "sminsi3"
  [(set (match_dup 3)
        (ge:CC (match_operand:SI 1 "register_operand" "")
               (match_operand:SI 2 "register_operand" "")))
   (set (match_operand:SI 0 "register_operand" "")
        (if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
                         (match_dup 2) (match_dup 1)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "smaxsi3"
  [(set (match_dup 3)
        (ge:CC (match_operand:SI 1 "register_operand" "")
               (match_operand:SI 2 "register_operand" "")))
   (set (match_operand:SI 0 "register_operand" "")
        (if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
                         (match_dup 1) (match_dup 2)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "uminsi3"
  [(set (match_dup 3)
        (geu:CC (match_operand:SI 1 "register_operand" "")
                (match_operand:SI 2 "register_operand" "")))
   (set (match_operand:SI 0 "register_operand" "")
        (if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
                         (match_dup 2) (match_dup 1)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "umaxsi3"
  [(set (match_dup 3)
        (geu:CC (match_operand:SI 1 "register_operand" "")
                (match_operand:SI 2 "register_operand" "")))
   (set (match_operand:SI 0 "register_operand" "")
        (if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
                         (match_dup 1) (match_dup 2)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

\f
;; ::::::::::::::::::::
;; ::
;; :: 64 bit Integer arithmetic
;; ::
;; ::::::::::::::::::::

(define_insn "adddi3"
  [(set (match_operand:DI 0 "register_operand" "=r,r,r")
        (plus:DI (match_operand:DI 1 "register_operand" "%r,r,a")
                 (match_operand:DI 2 "reg_or_22bit_operand" "r,I,J")))]
  ""
  "@
  add %0 = %1, %2
  adds %0 = %2, %1
  addl %0 = %2, %1"
  [(set_attr "type" "A")])

(define_insn "*adddi3_plus1"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (plus:DI (plus:DI (match_operand:DI 1 "register_operand" "r")
                          (match_operand:DI 2 "register_operand" "r"))
                 (const_int 1)))]
  ""
  "add %0 = %1, %2, 1"
  [(set_attr "type" "A")])

(define_insn "subdi3"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (minus:DI (match_operand:DI 1 "reg_or_8bit_operand" "rK")
                  (match_operand:DI 2 "register_operand" "r")))]
  ""
  "sub %0 = %1, %2"
  [(set_attr "type" "A")])

(define_insn "*subdi3_minus1"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (plus:DI (not:DI (match_operand:DI 1 "register_operand" "r"))
                 (match_operand:DI 2 "register_operand" "r")))]
  ""
  "sub %0 = %2, %1, 1"
  [(set_attr "type" "A")])

(define_insn "muldi3"
  [(set (match_operand:DI 0 "register_operand" "=f")
        (mult:DI (match_operand:DI 1 "register_operand" "f")
                 (match_operand:DI 2 "register_operand" "f")))]
  ""
  "xma.l %0 = %1, %2, f0%B0"
  [(set_attr "type" "F")])

;; ??? If operand 3 is an eliminable reg, then register elimination causes the
;; same problem that we have with shladd below.  Unfortunately, this case is
;; much harder to fix because the multiply puts the result in an FP register,
;; but the add needs inputs from a general register.  We add a spurious clobber
;; here so that it will be present just in case register elimination gives us
;; the funny result.

;; ??? Maybe validate_changes should try adding match_scratch clobbers?

;; ??? Maybe we should change how adds are canonicalized.

(define_insn "*madddi3"
  [(set (match_operand:DI 0 "register_operand" "=f")
        (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "f")
                          (match_operand:DI 2 "register_operand" "f"))
                 (match_operand:DI 3 "register_operand" "f")))
   (clobber (match_scratch:DI 4 "=X"))]
  ""
  "xma.l %0 = %1, %2, %3%B0"
  [(set_attr "type" "F")])

;; This can be created by register elimination if operand3 of shladd is an
;; eliminable register or has reg_equiv_constant set.

;; We have to use nonmemory_operand for operand 4, to ensure that the
;; validate_changes call inside eliminate_regs will always succeed.  If it
;; doesn't succeed, then this remain a madddi3 pattern, and will be reloaded
;; incorrectly.

(define_insn "*madddi3_elim"
  [(set (match_operand:DI 0 "register_operand" "=&r")
        (plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "f")
                                   (match_operand:DI 2 "register_operand" "f"))
                          (match_operand:DI 3 "register_operand" "f"))
                 (match_operand:DI 4 "nonmemory_operand" "rI")))
   (clobber (match_scratch:DI 5 "=f"))]
  "reload_in_progress"
  "#"
  [(set_attr "type" "unknown")])

;; ??? Need to emit an instruction group barrier here because this gets split
;; after md_reorg.

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "")
                                   (match_operand:DI 2 "register_operand" ""))
                          (match_operand:DI 3 "register_operand" ""))
                 (match_operand:DI 4 "reg_or_14bit_operand" "")))
   (clobber (match_scratch:DI 5 ""))]
  "reload_completed"
  [(parallel [(set (match_dup 5) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
                                          (match_dup 3)))
              (clobber (match_dup 0))])
   (unspec_volatile [(const_int 0)] 2)
   (set (match_dup 0) (match_dup 5))
   (unspec_volatile [(const_int 0)] 2)
   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 4)))]
  "")

;; ??? There are highpart multiply and add instructions, but we have no way
;; to generate them.

(define_insn "smuldi3_highpart"
  [(set (match_operand:DI 0 "register_operand" "=f")
        (truncate:DI
         (lshiftrt:TI
          (mult:TI (sign_extend:TI (match_operand:DI 1 "register_operand" "f"))
                   (sign_extend:TI (match_operand:DI 2 "register_operand" "f")))
          (const_int 64))))]
  ""
  "xma.h %0 = %1, %2, f0%B0"
  [(set_attr "type" "F")])

(define_insn "umuldi3_highpart"
  [(set (match_operand:DI 0 "register_operand" "=f")
        (truncate:DI
         (lshiftrt:TI
          (mult:TI (zero_extend:TI (match_operand:DI 1 "register_operand" "f"))
                   (zero_extend:TI (match_operand:DI 2 "register_operand" "f")))
          (const_int 64))))]
  ""
  "xma.hu %0 = %1, %2, f0%B0"
  [(set_attr "type" "F")])

(define_insn "negdi2"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (neg:DI (match_operand:DI 1 "register_operand" "r")))]
  ""
  "sub %0 = r0, %1"
  [(set_attr "type" "A")])

(define_expand "absdi2"
  [(set (match_dup 2)
        (ge:CC (match_operand:DI 1 "register_operand" "") (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI (eq:CC (match_dup 2) (const_int 0))
                         (neg:DI (match_dup 1))
                         (match_dup 1)))]
  ""
  "
{
  operands[2] = gen_reg_rtx (CCmode);
}")

(define_expand "smindi3"
  [(set (match_dup 3)
        (ge:CC (match_operand:DI 1 "register_operand" "")
               (match_operand:DI 2 "register_operand" "")))
   (set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
                         (match_dup 2) (match_dup 1)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "smaxdi3"
  [(set (match_dup 3)
        (ge:CC (match_operand:DI 1 "register_operand" "")
               (match_operand:DI 2 "register_operand" "")))
   (set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
                         (match_dup 1) (match_dup 2)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "umindi3"
  [(set (match_dup 3)
        (geu:CC (match_operand:DI 1 "register_operand" "")
                (match_operand:DI 2 "register_operand" "")))
   (set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
                         (match_dup 2) (match_dup 1)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "umaxdi3"
  [(set (match_dup 3)
        (geu:CC (match_operand:DI 1 "register_operand" "")
                (match_operand:DI 2 "register_operand" "")))
   (set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
                         (match_dup 1) (match_dup 2)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "ffsdi2"
  [(set (match_dup 6)
        (eq:CC (match_operand:DI 1 "register_operand" "") (const_int 0)))
   (set (match_dup 2) (plus:DI (match_dup 1) (const_int -1)))
   (set (match_dup 5) (const_int 0))
   (set (match_dup 3) (xor:DI (match_dup 1) (match_dup 2)))
   (set (match_dup 4) (unspec:DI [(match_dup 3)] 8))
   (set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI (ne:CC (match_dup 6) (const_int 0))
                         (match_dup 5) (match_dup 4)))]
  ""
  "
{
  operands[2] = gen_reg_rtx (DImode);
  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_reg_rtx (DImode);
  operands[5] = gen_reg_rtx (DImode);
  operands[6] = gen_reg_rtx (CCmode);
}")

(define_insn "*popcnt"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec:DI [(match_operand:DI 1 "register_operand" "r")] 8))]
  ""
  "popcnt %0 = %1"
  [(set_attr "type" "I")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 32 bit floating point arithmetic
;; ::
;; ::::::::::::::::::::

(define_insn "addsf3"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (plus:SF (match_operand:SF 1 "register_operand" "%f")
                 (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
  ""
  "fadd.s %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "subsf3"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (minus:SF (match_operand:SF 1 "reg_or_fp01_operand" "fG")
                  (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
  ""
  "fsub.s %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "mulsf3"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (mult:SF (match_operand:SF 1 "register_operand" "%f")
                 (match_operand:SF 2 "register_operand" "f")))]
  ""
  "fmpy.s %0 = %1, %2%B0"
  [(set_attr "type" "F")])

(define_insn "abssf2"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (abs:SF (match_operand:SF 1 "register_operand" "f")))]
  ""
  "fabs %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "negsf2"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (neg:SF (match_operand:SF 1 "register_operand" "f")))]
  ""
  "fneg %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "*nabssf2"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (neg:SF (abs:SF (match_operand:SF 1 "register_operand" "f"))))]
  ""
  "fnegabs %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "minsf3"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (smin:SF (match_operand:SF 1 "register_operand" "f")
                 (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
  ""
  "fmin %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "maxsf3"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (smax:SF (match_operand:SF 1 "register_operand" "f")
                 (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
  ""
  "fmax %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "*maddsf3"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (plus:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
                          (match_operand:SF 2 "register_operand" "f"))
                 (match_operand:SF 3 "reg_or_fp01_operand" "fG")))]
  ""
  "fma.s %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*msubsf3"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (minus:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
                           (match_operand:SF 2 "register_operand" "f"))
                  (match_operand:SF 3 "reg_or_fp01_operand" "fG")))]
  ""
  "fms.s %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*nmulsf3"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (neg:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
                         (match_operand:SF 2 "register_operand" "f"))))]
  ""
  "fnmpy.s %0 = %1, %2%B0"
  [(set_attr "type" "F")])

;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?

(define_insn "*nmaddsf3"
  [(set (match_operand:SF 0 "register_operand" "=f")
        (plus:SF (neg:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
                                  (match_operand:SF 2 "register_operand" "f")))
                 (match_operand:SF 3 "reg_or_fp01_operand" "fG")))]
  ""
  "fnma.s %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 64 bit floating point arithmetic
;; ::
;; ::::::::::::::::::::

(define_insn "adddf3"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (plus:DF (match_operand:DF 1 "register_operand" "%f")
                 (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
  ""
  "fadd.d %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "subdf3"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (minus:DF (match_operand:DF 1 "reg_or_fp01_operand" "fG")
                  (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
  ""
  "fsub.d %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "muldf3"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (mult:DF (match_operand:DF 1 "register_operand" "f")
                 (match_operand:DF 2 "register_operand" "f")))]
  ""
  "fmpy.d %0 = %1, %2%B0"
  [(set_attr "type" "F")])

(define_insn "absdf2"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (abs:DF (match_operand:DF 1 "register_operand" "f")))]
  ""
  "fabs %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "negdf2"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (neg:DF (match_operand:DF 1 "register_operand" "f")))]
  ""
  "fneg %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "*nabsdf2"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (neg:DF (abs:DF (match_operand:DF 1 "register_operand" "f"))))]
  ""
  "fnegabs %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "mindf3"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (smin:DF (match_operand:DF 1 "register_operand" "f")
                 (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
  ""
  "fmin %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "maxdf3"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (smax:DF (match_operand:DF 1 "register_operand" "f")
                 (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
  ""
  "fmax %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "*madddf3"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (plus:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
                          (match_operand:DF 2 "register_operand" "f"))
                 (match_operand:DF 3 "reg_or_fp01_operand" "fG")))]
  ""
  "fma.d %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*msubdf3"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (minus:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
                           (match_operand:DF 2 "register_operand" "f"))
                  (match_operand:DF 3 "reg_or_fp01_operand" "fG")))]
  ""
  "fms.d %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*nmuldf3"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (neg:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
                         (match_operand:DF 2 "register_operand" "f"))))]
  ""
  "fnmpy.d %0 = %1, %2%B0"
  [(set_attr "type" "F")])

;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?

(define_insn "*nmadddf3"
  [(set (match_operand:DF 0 "register_operand" "=f")
        (plus:DF (neg:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
                                  (match_operand:DF 2 "register_operand" "f")))
                 (match_operand:DF 3 "reg_or_fp01_operand" "fG")))]
  ""
  "fnma.d %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 32 bit Integer Shifts and Rotates
;; ::
;; ::::::::::::::::::::

;; There is no sign-extend form of dep, so we only get 32 bits of valid result
;; instead of 64 like the patterns below.

;; Using a predicate that accepts only constants doesn't work, because optabs
;; will load the operand into a register and call the pattern if the predicate
;; did not accept it on the first try.  So we use nonmemory_operand and then
;; verify that we have an appropriate constant in the expander.

(define_expand "ashlsi3"
  [(set (match_operand:SI 0 "register_operand" "")
        (ashift:SI (match_operand:SI 1 "register_operand" "")
                   (match_operand:SI 2 "nonmemory_operand" "")))]
  ""
  "
{
  if (! shift_32bit_count_operand (operands[2], SImode))
    FAIL;
}")

(define_insn "*ashlsi3_internal"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (ashift:SI (match_operand:SI 1 "register_operand" "r")
                   (match_operand:SI 2 "shift_32bit_count_operand" "n")))]
  ""
  "dep.z %0 = %1, %2, %E2"
  [(set_attr "type" "I")])

;; This is really an extract, but this is how combine canonicalizes the
;; operation.

(define_expand "ashrsi3"
  [(set (match_dup 3)
        (ashiftrt:DI (sign_extend:DI
                      (match_operand:SI 1 "register_operand" ""))
                     (match_operand:DI 2 "nonmemory_operand" "")))
   (set (match_operand:SI 0 "register_operand" "") (match_dup 4))]
  ""
  "
{
  if (! shift_32bit_count_operand (operands[2], SImode))
    FAIL;

  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_lowpart (SImode, operands[3]);
}")

(define_insn "*ashrsi3_internal"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (ashiftrt:DI (sign_extend:DI
                      (match_operand:SI 1 "register_operand" "r"))
                     (match_operand:DI 2 "shift_32bit_count_operand" "n")))]
  ""
  "extr %0 = %1, %2, %E2"
  [(set_attr "type" "I")])

;; This is really an extract, but this is how combine canonicalizes the
;; operation.

(define_expand "lshrsi3"
  [(set (match_dup 3)
        (lshiftrt:DI (zero_extend:DI
                      (match_operand:SI 1 "register_operand" ""))
                     (match_operand:DI 2 "nonmemory_operand" "")))
   (set (match_operand:SI 0 "register_operand" "") (match_dup 4))]
  ""
  "
{
  if (! shift_32bit_count_operand (operands[2], SImode))
    FAIL;

  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_lowpart (SImode, operands[3]);
}")

(define_insn "*lshrsi3_internal"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (lshiftrt:DI (zero_extend:DI
                      (match_operand:SI 1 "register_operand" "r"))
                     (match_operand:DI 2 "shift_32bit_count_operand" "n")))]
  ""
  "extr.u %0 = %1, %2, %E2"
  [(set_attr "type" "I")])

;; Use mix4.r/shr to implement rotrsi3.  We only get 32 bits of valid result
;; here, instead of 64 like the patterns above.

(define_expand "rotrsi3"
  [(set (match_dup 3)
        (ior:DI (zero_extend:DI (match_operand:SI 1 "register_operand" ""))
                (ashift:DI (zero_extend:DI (match_dup 1)) (const_int 32))))
   (set (match_dup 3)
        (lshiftrt:DI (match_dup 3)
                     (match_operand:DI 2 "nonmemory_operand" "")))
   (set (match_operand:SI 0 "register_operand" "") (match_dup 4))]
  ""
  "
{
  if (! shift_32bit_count_operand (operands[2], SImode))
    FAIL;

  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_lowpart (SImode, operands[3]);
}")

\f
;; ::::::::::::::::::::
;; ::
;; :: 64 bit Integer Shifts and Rotates
;; ::
;; ::::::::::::::::::::

(define_insn "ashldi3"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (ashift:DI (match_operand:DI 1 "register_operand" "r")
                   (match_operand:DI 2 "reg_or_6bit_operand" "rM")))]
  ""
  "shl %0 = %1, %2"
  [(set_attr "type" "I")])

;; ??? Maybe combine this with the multiply and add instruction?

(define_insn "*shladd"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
                          (match_operand:DI 2 "shladd_operand" "n"))
                 (match_operand:DI 3 "register_operand" "r")))]
  ""
  "shladd %0 = %1, %S2, %3"
  [(set_attr "type" "A")])

;; This can be created by register elimination if operand3 of shladd is an
;; eliminable register or has reg_equiv_constant set.

;; We have to use nonmemory_operand for operand 4, to ensure that the
;; validate_changes call inside eliminate_regs will always succeed.  If it
;; doesn't succeed, then this remain a shladd pattern, and will be reloaded
;; incorrectly.

(define_insn "*shladd_elim"
  [(set (match_operand:DI 0 "register_operand" "=&r")
        (plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
                                   (match_operand:DI 2 "shladd_operand" "n"))
                          (match_operand:DI 3 "register_operand" "r"))
                 (match_operand:DI 4 "nonmemory_operand" "rI")))]
  "reload_in_progress"
  "#"
  [(set_attr "type" "unknown")])

;; ??? Need to emit an instruction group barrier here because this gets split
;; after md_reorg.

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "")
                                   (match_operand:DI 2 "shladd_operand" ""))
                          (match_operand:DI 3 "register_operand" ""))
                 (match_operand:DI 4 "reg_or_14bit_operand" "")))]
  "reload_completed"
  [(set (match_dup 0) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
                               (match_dup 3)))
   (unspec_volatile [(const_int 0)] 2)
   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 4)))]
  "")

(define_insn "ashrdi3"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (ashiftrt:DI (match_operand:DI 1 "register_operand" "r")
                     (match_operand:DI 2 "reg_or_6bit_operand" "rM")))]
  ""
  "shr %0 = %1, %2"
  [(set_attr "type" "I")])

(define_insn "lshrdi3"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (lshiftrt:DI (match_operand:DI 1 "register_operand" "r")
                     (match_operand:DI 2 "reg_or_6bit_operand" "rM")))]
  ""
  "shr.u %0 = %1, %2"
  [(set_attr "type" "I")])

;; Using a predicate that accepts only constants doesn't work, because optabs
;; will load the operand into a register and call the pattern if the predicate
;; did not accept it on the first try.  So we use nonmemory_operand and then
;; verify that we have an appropriate constant in the expander.

(define_expand "rotrdi3"
  [(set (match_operand:DI 0 "register_operand" "")
        (rotatert:DI (match_operand:DI 1 "register_operand" "")
                     (match_operand:DI 2 "nonmemory_operand" "")))]
  ""
  "
{
  if (! shift_count_operand (operands[2], DImode))
    FAIL;
}")

(define_insn "*rotrdi3_internal"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (rotatert:DI (match_operand:DI 1 "register_operand" "r")
                     (match_operand:DI 2 "shift_count_operand" "M")))]
  ""
  "shrp %0 = %1, %1, %2"
  [(set_attr "type" "I")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 32 Bit Integer Logical operations
;; ::
;; ::::::::::::::::::::

;; We don't seem to need any other 32-bit logical operations, because gcc
;; generates zero-extend;zero-extend;DImode-op, which combine optimizes to
;; DImode-op;zero-extend, and then we can optimize away the zero-extend.
;; This doesn't work for unary logical operations, because we don't call
;; apply_distributive_law for them.

;; ??? Likewise, this doesn't work for andnot, which isn't handled by
;; apply_distributive_law.  We get inefficient code for
;; int sub4 (int i, int j) { return i & ~j; }
;; We could convert (and (not (sign_extend A)) (sign_extend B)) to
;; (zero_extend (and (not A) B)) in combine.
;; Or maybe fix this by adding andsi3/iorsi3/xorsi3 patterns like the
;; one_cmplsi2 pattern.

(define_expand "one_cmplsi2"
  [(set (match_operand:SI 0 "register_operand" "")
        (not:SI (match_operand:SI 1 "register_operand" "")))]
  ""
  "
{
  if (optimize)
    {
      rtx op1 = gen_lowpart (DImode, operands[1]);

      if (! cse_not_expected)
        {
          rtx tmp = gen_reg_rtx (DImode);
          emit_insn (gen_one_cmpldi2 (tmp, op1));
          emit_move_insn (operands[0], gen_lowpart (SImode, tmp));
        }
      else
        emit_insn (gen_one_cmpldi2 (gen_lowpart (DImode, operands[0]), op1));
      DONE;
    }
}")

(define_insn "*one_cmplsi2_internal"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (not:SI (match_operand:SI 1 "register_operand" "r")))]
  ""
  "andcm %0 = -1, %1"
  [(set_attr "type" "A")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 64 Bit Integer Logical operations
;; ::
;; ::::::::::::::::::::

(define_insn "anddi3"
  [(set (match_operand:DI 0 "register_operand" "=r,*f")
        (and:DI (match_operand:DI 1 "register_operand" "%r,*f")
                (match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
  ""
  "@
   and %0 = %2, %1
   fand %0 = %2, %1%B0"
  [(set_attr "type" "A,F")])

(define_insn "*andnot"
  [(set (match_operand:DI 0 "register_operand" "=r,*f")
        (and:DI (not:DI (match_operand:DI 1 "register_operand" "r,*f"))
                (match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
  ""
  "@
   andcm %0 = %2, %1
   fandcm %0 = %2, %1%B0"
  [(set_attr "type" "A,F")])

(define_insn "iordi3"
  [(set (match_operand:DI 0 "register_operand" "=r,*f")
        (ior:DI (match_operand:DI 1 "register_operand" "%r,*f")
                (match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
  ""
  "@
   or %0 = %2, %1
   for %0 = %2, %1%B0"
  [(set_attr "type" "A,F")])

(define_insn "xordi3"
  [(set (match_operand:DI 0 "register_operand" "=r,*f")
        (xor:DI (match_operand:DI 1 "register_operand" "%r,*f")
                (match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
  ""
  "@
   xor %0 = %2, %1
   fxor %0 = %2, %1%B0"
  [(set_attr "type" "A,F")])

(define_insn "one_cmpldi2"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (not:DI (match_operand:DI 1 "register_operand" "r")))]
  ""
  "andcm %0 = -1, %1"
  [(set_attr "type" "A")])
\f
;; ::::::::::::::::::::
;; ::
;; :: Comparisons
;; ::
;; ::::::::::::::::::::

(define_expand "cmpsi"
  [(set (cc0)
        (compare (match_operand:SI 0 "register_operand" "")
                 (match_operand:SI 1 "reg_or_8bit_and_adjusted_operand" "")))]
  ""
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

(define_expand "cmpdi"
  [(set (cc0)
        (compare (match_operand:DI 0 "register_operand" "")
                 (match_operand:DI 1 "reg_or_8bit_and_adjusted_operand" "")))]
  ""
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

(define_expand "cmpsf"
  [(set (cc0)
        (compare (match_operand:SF 0 "reg_or_fp01_operand" "")
                 (match_operand:SF 1 "reg_or_fp01_operand" "")))]
  ""
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

(define_expand "cmpdf"
  [(set (cc0)
        (compare (match_operand:DF 0 "reg_or_fp01_operand" "")
                 (match_operand:DF 1 "reg_or_fp01_operand" "")))]
  ""
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

;; ??? Enable this for XFmode support.

(define_expand "cmpxf"
  [(set (cc0)
        (compare (match_operand:XF 0 "reg_or_fp01_operand" "")
                 (match_operand:XF 1 "reg_or_fp01_operand" "")))]
  "0"
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

(define_insn "*cmpsi_normal"
  [(set (match_operand:CC 0 "register_operand" "=c")
        (match_operator:CC 1 "normal_comparison_operator"
                           [(match_operand:SI 2 "register_operand" "r")
                            (match_operand:SI 3 "reg_or_8bit_operand" "rK")]))]
  ""
  "cmp4.%C1 %0, %I0 = %3, %2"
  [(set_attr "type" "A")])

(define_insn "*cmpsi_adjusted"
  [(set (match_operand:CC 0 "register_operand" "=c")
        (match_operator:CC 1 "adjusted_comparison_operator"
                           [(match_operand:SI 2 "register_operand" "r")
                            (match_operand:SI 3 "reg_or_8bit_adjusted_operand"
                                              "rL")]))]
  ""
  "cmp4.%C1 %0, %I0 = %3, %2"
  [(set_attr "type" "A")])

(define_insn "*cmpdi_normal"
  [(set (match_operand:CC 0 "register_operand" "=c")
        (match_operator:CC 1 "normal_comparison_operator"
                           [(match_operand:DI 2 "register_operand" "r")
                            (match_operand:DI 3 "reg_or_8bit_operand" "rK")]))]
  ""
  "cmp.%C1 %0, %I0 = %3, %2"
  [(set_attr "type" "A")])

(define_insn "*cmpdi_adjusted"
  [(set (match_operand:CC 0 "register_operand" "=c")
        (match_operator:CC 1 "adjusted_comparison_operator"
                           [(match_operand:DI 2 "register_operand" "r")
                            (match_operand:DI 3 "reg_or_8bit_adjusted_operand"
                                              "rL")]))]
  ""
  "cmp.%C1 %0, %I0 = %3, %2"
  [(set_attr "type" "A")])

(define_insn "*cmpsf_internal"
  [(set (match_operand:CC 0 "register_operand" "=c")
        (match_operator:CC 1 "comparison_operator"
                           [(match_operand:SF 2 "reg_or_fp01_operand" "fG")
                            (match_operand:SF 3 "reg_or_fp01_operand" "fG")]))]
  ""
  "fcmp.%D1 %0, %I0 = %F2, %F3"
  [(set_attr "type" "F")])

(define_insn "*cmpdf_internal"
  [(set (match_operand:CC 0 "register_operand" "=c")
        (match_operator:CC 1 "comparison_operator"
                           [(match_operand:DF 2 "reg_or_fp01_operand" "fG")
                            (match_operand:DF 3 "reg_or_fp01_operand" "fG")]))]
  ""
  "fcmp.%D1 %0, %I0 = %F2, %F3"
  [(set_attr "type" "F")])

;; ??? Can this pattern be generated?

(define_insn "*bit_zero"
  [(set (match_operand:CC 0 "register_operand" "=c")
        (eq:CC (zero_extract:DI (match_operand:DI 1 "register_operand" "r")
                                (const_int 1)
                                (match_operand:DI 2 "immediate_operand" "n"))
               (const_int 0)))]
  ""
  "tbit.z %0, %I0 = %1, %2"
  [(set_attr "type" "I")])

(define_insn "*bit_one"
  [(set (match_operand:CC 0 "register_operand" "=c")
        (ne:CC (zero_extract:DI (match_operand:DI 1 "register_operand" "r")
                                (const_int 1)
                                (match_operand:DI 2 "immediate_operand" "n"))
               (const_int 0)))]
  ""
  "tbit.nz %0, %I0 = %1, %2"
  [(set_attr "type" "I")])

;; ??? We also need this if we run out of PR regs and need to spill some.

;; ??? We need this if a CCmode value does not get allocated to a hard
;; register.  This happens if we cse/gcse a CCmode value across a call, and the
;; function has a nonlocal goto.  This is because global does not allocate
;; call crossing pseudos to hard registers when current_function_has_
;; nonlocal_goto is true.  This is relatively common for C++ programs that
;; use exceptions.  See ia64_secondary_reload_class.

;; We use a define_expand here so that cse/gcse/combine can't accidentally
;; create movcc insns.  If this was a named define_insn, we would not be able
;; to make it conditional on reload.

(define_expand "movcc"
  [(set (match_operand:CC 0 "nonimmediate_operand" "")
        (match_operand:CC 1 "move_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed)
    FAIL;
}")

(define_insn "*movcc_internal"
  [(set (match_operand:CC 0 "nonimmediate_operand" "=r,c,r,m")
        (match_operand:CC 1 "move_operand" "c,r,m,r"))]
  "reload_in_progress || reload_completed"
  "@
   #
   cmp4.ne %0, %I0 = %1, r0
   ld4%O1 %0 = %1%P1
   st4%Q0 %0 = %1%P0"
  [(set_attr "type" "unknown,A,M,M")])

(define_split
  [(set (match_operand:CC 0 "register_operand" "")
        (match_operand:CC 1 "register_operand" ""))]
  "reload_completed
   && GET_CODE (operands[0]) == REG && GR_REGNO_P (REGNO (operands[0]))
   && GET_CODE (operands[1]) == REG && PR_REGNO_P (REGNO (operands[1]))"
  [(set (match_dup 2)
        (if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
                         (const_int 1)
                         (match_dup 2)))
   (set (match_dup 2)
        (if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
                         (match_dup 2)
                         (const_int 0)))]
  "operands[2] = gen_rtx_SUBREG (DImode, operands[0], 0);")

\f
;; ::::::::::::::::::::
;; ::
;; :: Branches
;; ::
;; ::::::::::::::::::::

(define_expand "beq"
  [(set (match_dup 1)
        (eq:CC (match_dup 2)
               (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bne"
  [(set (match_dup 1)
        (ne:CC (match_dup 2)
               (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "blt"
  [(set (match_dup 1)
        (lt:CC (match_dup 2)
               (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "ble"
  [(set (match_dup 1)
        (le:CC (match_dup 2)
               (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bgt"
  [(set (match_dup 1)
        (gt:CC (match_dup 2)
               (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bge"
  [(set (match_dup 1)
        (ge:CC (match_dup 2)
               (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bltu"
  [(set (match_dup 1)
        (ltu:CC (match_dup 2)
                (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bleu"
  [(set (match_dup 1)
        (leu:CC (match_dup 2)
                (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bgtu"
  [(set (match_dup 1)
        (gtu:CC (match_dup 2)
                (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bgeu"
  [(set (match_dup 1)
        (geu:CC (match_dup 2)
                (match_dup 3)))
   (set (pc)
        (if_then_else (ne:CC (match_dup 1)
                             (const_int 0))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_insn "*br_true"
  [(set (pc)
        (if_then_else (match_operator 0 "predicate_operator"
                        [(match_operand:CC 1 "register_operand" "c")
                         (const_int 0)])
                      (label_ref (match_operand 2 "" ""))
                      (pc)))]
  ""
  "(%J0) br.cond%+ %l2"
  [(set_attr "type" "B")
   (set_attr "predicable" "no")])

(define_insn "*br_false"
  [(set (pc)
        (if_then_else (match_operator 0 "predicate_operator"
                        [(match_operand:CC 1 "register_operand" "c")
                         (const_int 0)])
                      (pc)
                      (label_ref (match_operand 2 "" ""))))]
  ""
  "(%j0) br.cond%+ %l2"
  [(set_attr "type" "B")
   (set_attr "predicable" "no")])
\f
;; ::::::::::::::::::::
;; ::
;; :: Set flag operations
;; ::
;; ::::::::::::::::::::

(define_expand "seq"
  [(set (match_dup 1)
        (eq:CC (match_dup 2)
               (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sne"
  [(set (match_dup 1)
        (ne:CC (match_dup 2)
               (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "slt"
  [(set (match_dup 1)
        (lt:CC (match_dup 2)
               (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sle"
  [(set (match_dup 1)
        (le:CC (match_dup 2)
               (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sgt"
  [(set (match_dup 1)
        (gt:CC (match_dup 2)
               (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sge"
  [(set (match_dup 1)
        (ge:CC (match_dup 2)
               (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sltu"
  [(set (match_dup 1)
        (ltu:CC (match_dup 2)
                (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sleu"
  [(set (match_dup 1)
        (leu:CC (match_dup 2)
                (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sgtu"
  [(set (match_dup 1)
        (gtu:CC (match_dup 2)
                (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sgeu"
  [(set (match_dup 1)
        (geu:CC (match_dup 2)
                (match_dup 3)))
   (set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

;; Don't allow memory as destination here, because cmov/cmov/st is more
;; efficient than mov/mov/cst/cst.

(define_insn "*sne_internal"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (ne:DI (match_operand:CC 1 "register_operand" "c")
               (const_int 0)))]
  ""
  "#"
  [(set_attr "type" "unknown")])

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (ne:DI (match_operand:CC 1 "register_operand" "")
               (const_int 0)))]
  "reload_completed"
  [(set (match_dup 0)
        (if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
                         (const_int 1)
                         (match_dup 0)))
   (set (match_dup 0)
        (if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
                         (match_dup 0)
                         (const_int 0)))]
  "")

;; ??? Unknown if this can be matched.

(define_insn "*seq_internal"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (eq:DI (match_operand:CC 1 "register_operand" "c")
               (const_int 0)))]
  ""
  "#"
  [(set_attr "type" "unknown")])

;; ??? Unknown if this can be matched.

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (eq:DI (match_operand:CC 1 "register_operand" "")
               (const_int 0)))]
  "reload_completed"
  [(set (match_dup 0)
        (if_then_else:DI (eq:CC (match_dup 1) (const_int 0))
                         (const_int 1)
                         (match_dup 0)))
   (set (match_dup 0)
        (if_then_else:DI (eq:CC (match_dup 1) (const_int 0))
                         (match_dup 0)
                         (const_int 0)))]
  "")

\f
;; ::::::::::::::::::::
;; ::
;; :: Conditional move instructions.
;; ::
;; ::::::::::::::::::::

;; ??? Add movXXcc patterns?

;;
;; DImode if_then_else patterns.
;;

(define_insn_and_split "*cmovdi_internal"
  [(set (match_operand:DI 0 "nonimmediate_operand"
                          "=r,m,*f,Q,*b,r,m,*f,Q,*b,r,m,*f,Q,*b")
        (if_then_else:DI
          (match_operator:CC 4 "predicate_operator"
            [(match_operand:CC 1 "register_operand"
                               "c,c,c,c,c,c,c,c,c,c,c,c,c,c,c")
             (const_int 0)])
          (match_operand:DI 2 "general_operand"
                    "0,0,0,0,0,rim*f*b,rO,rOQ,*f,r,rim*f*b,rO,rOQ,*f,r")
          (match_operand:DI 3 "general_operand"
                    "rim*f*b,rO,rOQ,*f,r,0,0,0,0,0,rim*f*b,rO,rOQ,*f,r")))]
  ""
  "#"
  "reload_completed"
  [(const_int 0)]
  "
{
  rtx tmp;
  if (! rtx_equal_p (operands[0], operands[2]))
    {
      tmp = gen_rtx_SET (VOIDmode, operands[0], operands[2]);
      tmp = gen_rtx_COND_EXEC (VOIDmode, operands[4], tmp);
      emit_insn (tmp);
    }
  if (! rtx_equal_p (operands[0], operands[3]))
    {
      tmp = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
                            CCmode, operands[1], const0_rtx);
      tmp = gen_rtx_COND_EXEC (VOIDmode, tmp,
                               gen_rtx_SET (VOIDmode, operands[0],
                                            operands[3]));
      emit_insn (tmp);
    }
  DONE;
}"
  [(set_attr "predicable" "no")])

;; Absolute value pattern.

(define_insn "*absdi2_internal"
  [(set (match_operand:DI 0 "register_operand" "=r,r")
        (if_then_else:DI
          (match_operator:CC 4 "predicate_operator"
            [(match_operand:CC 1 "register_operand" "c,c")
             (const_int 0)])
          (neg:DI (match_operand:DI 2 "reg_or_22bit_operand" "rI,rI"))
          (match_operand:DI 3 "reg_or_22bit_operand" "0,rI")))]
  ""
  "#"
  [(set_attr "type" "A,unknown")
   (set_attr "predicable" "no")])

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI
          (match_operator:CC 4 "predicate_operator"
            [(match_operand:CC 1 "register_operand" "c,c")
             (const_int 0)])
          (neg:DI (match_operand:DI 2 "reg_or_22bit_operand" ""))
          (match_operand:DI 3 "reg_or_22bit_operand" "")))]
  "reload_completed && rtx_equal_p (operands[0], operands[3])"
  [(cond_exec
     (match_dup 4)
     (set (match_dup 0)
          (neg:DI (match_dup 2))))]
  "")

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI
          (match_operator:CC 4 "predicate_operator"
            [(match_operand:CC 1 "register_operand" "c,c")
             (const_int 0)])
          (neg:DI (match_operand:DI 2 "reg_or_22bit_operand" ""))
          (match_operand:DI 3 "reg_or_22bit_operand" "")))]
  "reload_completed"
  [(cond_exec
     (match_dup 4)
     (set (match_dup 0) (neg:DI (match_dup 2))))
   (cond_exec
     (match_dup 5)
     (set (match_dup 0) (match_dup 3)))]
  "
{
  operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
                                CCmode, operands[1], const0_rtx);
}")

;;
;; SImode if_then_else patterns.
;;

(define_insn_and_split "*cmovsi_internal"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=r,m,*f,r,m,*f,r,m,*f")
        (if_then_else:SI
          (match_operator:CC 4 "predicate_operator"
            [(match_operand:CC 1 "register_operand" "c,c,c,c,c,c,c,c,c")
             (const_int 0)])
          (match_operand:SI 2 "general_operand"
                    "0,0,0,rim*f,rO,rO,rim*f,rO,rO")
          (match_operand:SI 3 "general_operand"
                    "rim*f,rO,rO,0,0,0,rim*f,rO,rO")))]
  ""
  "#"
  "reload_completed"
  [(const_int 0)]
  "
{
  rtx tmp;
  if (! rtx_equal_p (operands[0], operands[2]))
    {
      tmp = gen_rtx_SET (VOIDmode, operands[0], operands[2]);
      tmp = gen_rtx_COND_EXEC (VOIDmode, operands[4], tmp);
      emit_insn (tmp);
    }
  if (! rtx_equal_p (operands[0], operands[3]))
    {
      tmp = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
                            CCmode, operands[1], const0_rtx);
      tmp = gen_rtx_COND_EXEC (VOIDmode, tmp,
                               gen_rtx_SET (VOIDmode, operands[0],
                                            operands[3]));
      emit_insn (tmp);
    }
  DONE;
}"
  [(set_attr "predicable" "no")])

(define_insn "*abssi2_internal"
  [(set (match_operand:SI 0 "register_operand" "=r,r")
        (if_then_else:SI
          (match_operator:CC 4 "predicate_operator"
            [(match_operand:CC 1 "register_operand" "c,c")
             (const_int 0)])
          (neg:SI (match_operand:SI 3 "reg_or_22bit_operand" "rI,rI"))
          (match_operand:SI 2 "reg_or_22bit_operand" "0,rI")))]
  ""
  "#"
  [(set_attr "type" "A,unknown")
   (set_attr "predicable" "no")])

(define_split
  [(set (match_operand:SI 0 "register_operand" "")
        (if_then_else:SI
          (match_operator:CC 4 "predicate_operator"
            [(match_operand:CC 1 "register_operand" "c,c")
             (const_int 0)])
          (neg:SI (match_operand:SI 2 "reg_or_22bit_operand" ""))
          (match_operand:SI 3 "reg_or_22bit_operand" "")))]
  "reload_completed && rtx_equal_p (operands[0], operands[3])"
  [(cond_exec
     (match_dup 4)
     (set (match_dup 0)
          (neg:SI (match_dup 2))))]
  "")

(define_split
  [(set (match_operand:SI 0 "register_operand" "")
        (if_then_else:SI
          (match_operator:CC 4 "predicate_operator"
            [(match_operand:CC 1 "register_operand" "c,c")
             (const_int 0)])
          (neg:SI (match_operand:SI 2 "reg_or_22bit_operand" ""))
          (match_operand:SI 3 "reg_or_22bit_operand" "")))]
  "reload_completed"
  [(cond_exec
     (match_dup 4)
     (set (match_dup 0) (neg:SI (match_dup 2))))
   (cond_exec
     (match_dup 5)
     (set (match_dup 0) (match_dup 3)))]
  "
{
  operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
                                CCmode, operands[1], const0_rtx);
}")

\f
;; ::::::::::::::::::::
;; ::
;; :: Call and branch instructions
;; ::
;; ::::::::::::::::::::

;; Subroutine call instruction returning no value.  Operand 0 is the function
;; to call; operand 1 is the number of bytes of arguments pushed (in mode
;; `SImode', except it is normally a `const_int'); operand 2 is the number of
;; registers used as operands.

;; On most machines, operand 2 is not actually stored into the RTL pattern.  It
;; is supplied for the sake of some RISC machines which need to put this
;; information into the assembler code; they can put it in the RTL instead of
;; operand 1.

(define_expand "call"
  [(use (match_operand:DI 0 "" ""))
   (use (match_operand 1 "" ""))
   (use (match_operand 2 "" ""))
   (use (match_operand 3 "" ""))]
  ""
  "
{
  /* ??? Stripping off the MEM isn't correct.  Will lose alias info.  */
  rtx addr = XEXP (operands[0], 0);
  enum machine_mode mode = GET_MODE (addr);

  if (TARGET_NO_PIC || TARGET_AUTO_PIC)
    emit_call_insn (gen_call_internal (addr, operands[1],
                                       gen_rtx_REG (DImode, R_BR (0))));

  /* If this is an indirect call, then we have the address of a descriptor.  */
  else if (! symbolic_operand (addr, mode))
    emit_insn (gen_indirect_call_pic (addr, operands[1]));
  else if (TARGET_CONST_GP)
    emit_call_insn (gen_call_internal (addr, operands[1],
                                       gen_rtx_REG (DImode, R_BR (0))));
  /* ??? This is an unsatisfying solution.  Should rethink.  */
  else if (setjmp_operand (addr, mode))
    emit_insn (gen_setjmp_call_pic (addr, operands[1]));
  else
    emit_insn (gen_call_pic (addr, operands[1]));

  DONE;
}")

(define_expand "indirect_call_pic"
  [(set (match_dup 2) (reg:DI 1))
   (set (match_dup 3) (mem:DI (match_operand 0 "" "")))
   (set (match_dup 4) (plus:DI (match_dup 0) (const_int 8)))
   (set (reg:DI 1) (mem:DI (match_dup 4)))
   (parallel [(call (mem:DI (match_dup 3)) (match_operand 1 "" ""))
              (use (reg:DI 1))
              (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 2))]
  ""
  "
{
  operands[2] = gen_reg_rtx (DImode);
  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_reg_rtx (DImode);
}")

;; We can't save GP in a pseudo if we are calling setjmp, because pseudos
;; won't be restored by longjmp.  For now, we save it in r4.

;; ??? It would be more efficient to save this directly into a stack slot.
;; Unfortunately, the stack slot address gets cse'd across the setjmp call
;; because the NOTE_INSN_SETJMP note is in the wrong place.

;; ??? This is an unsatisfying solution.  Should rethink.

(define_expand "setjmp_call_pic"
  [(set (match_dup 2) (reg:DI 1))
   (parallel [(call (mem:DI (match_operand 0 "" "")) (match_operand 1 "" ""))
              (use (reg:DI 1))
              (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 2))]
  ""
  "
{
  operands[2] = gen_rtx_REG (DImode, GR_REG (4));
}")

;; ??? Saving/restoring the GP register is not needed if we are calling
;; a function in the same module.

(define_expand "call_pic"
  [(set (match_dup 2) (reg:DI 1))
   (parallel [(call (mem:DI (match_operand 0 "" "")) (match_operand 1 "" ""))
              (use (reg:DI 1))
              (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 2))]
  ""
  "
{
  operands[2] = gen_reg_rtx (DImode);
}")

(define_insn "call_internal"
  [(call (mem:DI (match_operand:DI 0 "call_operand" "bi"))
         (match_operand 1 "" ""))
   (clobber (match_operand:DI 2 "register_operand" "=b"))]
  ""
  "br.call%+.many %2 = %0"
  [(set_attr "type" "B")])

(define_insn "*call_internal1"
  [(call (mem:DI (match_operand:DI 0 "call_operand" "bi"))
         (match_operand 1 "" ""))
   (use (reg:DI 1))
   (clobber (match_operand:DI 2 "register_operand" "=b"))]
  ""
  "br.call%+.many %2 = %0"
  [(set_attr "type" "B")])

;; Subroutine call instruction returning a value.  Operand 0 is the hard
;; register in which the value is returned.  There are three more operands, the
;; same as the three operands of the `call' instruction (but with numbers
;; increased by one).

;; Subroutines that return `BLKmode' objects use the `call' insn.

(define_expand "call_value"
  [(use (match_operand 0 "" ""))
   (use (match_operand:DI 1 "" ""))
   (use (match_operand 2 "" ""))
   (use (match_operand 3 "" ""))
   (use (match_operand 4 "" ""))]
  ""
  "
{
  /* ??? Stripping off the MEM isn't correct.  Will lose alias info.  */
  rtx addr = XEXP (operands[1], 0);
  enum machine_mode mode = GET_MODE (addr);

  if (TARGET_NO_PIC || TARGET_AUTO_PIC)
    emit_call_insn (gen_call_value_internal (operands[0], addr, operands[2],
                                             gen_rtx_REG (DImode, R_BR (0))));

  /* If this is an indirect call, then we have the address of a descriptor.  */
  else if (! symbolic_operand (addr, mode))
    {
      /* This is for HFA returns.  */
      if (GET_CODE (operands[0]) == PARALLEL)
        emit_insn (gen_indirect_call_multiple_values_pic (operands[0], addr,
                                                          operands[2]));
      else
        emit_insn (gen_indirect_call_value_pic (operands[0], addr,
                                                operands[2]));
    }
  else if (TARGET_CONST_GP)
    emit_call_insn (gen_call_value_internal (operands[0], addr, operands[2],
                                             gen_rtx_REG (DImode, R_BR (0))));
  /* ??? This is an unsatisfying solution.  Should rethink.  */
  else if (setjmp_operand (addr, mode))
    emit_insn (gen_setjmp_call_value_pic (operands[0], addr, operands[2]));
  /* This is for HFA returns.  */
  else if (GET_CODE (operands[0]) == PARALLEL)
    emit_insn (gen_call_multiple_values_pic (operands[0], addr, operands[2]));
  else
    emit_insn (gen_call_value_pic (operands[0], addr, operands[2]));

  DONE;
}")

(define_expand "indirect_call_value_pic"
  [(set (match_dup 3) (reg:DI 1))
   (set (match_dup 4) (mem:DI (match_operand 1 "" "")))
   (set (match_dup 5) (plus:DI (match_dup 1) (const_int 8)))
   (set (reg:DI 1) (mem:DI (match_dup 5)))
   (parallel [(set (match_operand 0 "" "")
                   (call (mem:DI (match_dup 4)) (match_operand 2 "" "")))
              (use (reg:DI 1))
              (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 3))]
  ""
  "
{
  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_reg_rtx (DImode);
  operands[5] = gen_reg_rtx (DImode);
}")

(define_expand "indirect_call_multiple_values_pic"
  [(set (match_dup 3) (reg:DI 1))
   (set (match_dup 4) (mem:DI (match_operand 1 "" "")))
   (set (match_dup 5) (plus:DI (match_dup 1) (const_int 8)))
   (set (reg:DI 1) (mem:DI (match_dup 5)))
   (match_par_dup 6 [(set (match_operand 0 "" "")
                          (call (mem:DI (match_dup 4))
                                (match_operand 2 "" "")))
                     (use (reg:DI 1))
                     (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 3))]
  ""
  "
{
  int count;
  int i;
  rtx call;

  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_reg_rtx (DImode);
  operands[5] = gen_reg_rtx (DImode);

  /* This code is the same as the code in call_multiple_values_pic, except
     that op3 was replaced with op6 and op1 was replaced with op4.  */
  call = gen_rtx_CALL (VOIDmode, gen_rtx_MEM (DImode, operands[4]),
                       operands[2]);

  count = XVECLEN (operands[0], 0);
  operands[6] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count + 2));
  
  XVECEXP (operands[6], 0, 0)
    = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, 0), 0), call);

  XVECEXP (operands[6], 0, 1)
    = gen_rtx_USE (DImode, gen_rtx_REG (DImode, GR_REG (1)));
  XVECEXP (operands[6], 0, 2)
    = gen_rtx_CLOBBER (DImode, gen_rtx_REG (DImode, BR_REG (0)));

  for (i = 1; i < count; i++)
    XVECEXP (operands[6], 0, i + 2)
      = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, i), 0), call);

}")

;; We can't save GP in a pseudo if we are calling setjmp, because pseudos
;; won't be restored by longjmp.  For now, we save it in r4.

;; ??? It would be more efficient to save this directly into a stack slot.
;; Unfortunately, the stack slot address gets cse'd across the setjmp call
;; because the NOTE_INSN_SETJMP note is in the wrong place.

;; ??? This is an unsatisfying solution.  Should rethink.

(define_expand "setjmp_call_value_pic"
  [(set (match_dup 3) (reg:DI 1))
   (parallel [(set (match_operand 0 "" "")
                   (call (mem:DI (match_operand 1 "" ""))
                         (match_operand 2 "" "")))
              (use (reg:DI 1))
              (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 3))]
  ""
  "
{
  operands[3] = gen_rtx_REG (DImode, GR_REG (4));
}")

;; ??? Saving/restoring the GP register is not needed if we are calling
;; a function in the same module.

(define_expand "call_value_pic"
  [(set (match_dup 3) (reg:DI 1))
   (parallel [(set (match_operand 0 "" "")
                   (call (mem:DI (match_operand 1 "" ""))
                         (match_operand 2 "" "")))
              (use (reg:DI 1))
              (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 3))]
  ""
  "
{
  operands[3] = gen_reg_rtx (DImode);
}")

;; ??? Saving/restoring the GP register is not needed if we are calling
;; a function in the same module.

(define_expand "call_multiple_values_pic"
  [(set (match_dup 4) (reg:DI 1))
   (match_par_dup 3 [(set (match_operand 0 "" "")
                          (call (mem:DI (match_operand 1 "" ""))
                                (match_operand 2 "" "")))
                     (use (reg:DI 1))
                     (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 4))]
  ""
  "
{
  int count;
  int i;
  rtx call;

  operands[4] = gen_reg_rtx (DImode);

  call = gen_rtx_CALL (VOIDmode, gen_rtx_MEM (DImode, operands[1]),
                       operands[2]);

  count = XVECLEN (operands[0], 0);
  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count + 2));
  
  XVECEXP (operands[3], 0, 0)
    = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, 0), 0), call);

  XVECEXP (operands[3], 0, 1)
    = gen_rtx_USE (DImode, gen_rtx_REG (DImode, GR_REG (1)));
  XVECEXP (operands[3], 0, 2)
    = gen_rtx_CLOBBER (DImode, gen_rtx_REG (DImode, BR_REG (0)));

  for (i = 1; i < count; i++)
    XVECEXP (operands[3], 0, i + 2)
      = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, i), 0), call);
}")

(define_insn "call_value_internal"
  [(set (match_operand 0 "register_operand" "=rf")
        (call (mem:DI (match_operand:DI 1 "call_operand" "bi"))
              (match_operand 2 "" "")))
   (clobber (match_operand:DI 3 "register_operand" "=b"))]
  ""
  "br.call%+.many %3 = %1"
  [(set_attr "type" "B")])

(define_insn "*call_value_internal1"
  [(set (match_operand 0 "register_operand" "=rf")
        (call (mem:DI (match_operand:DI 1 "call_operand" "bi"))
              (match_operand 2 "" "")))
   (use (reg:DI 1))
   (clobber (match_operand:DI 3 "register_operand" "=b"))]
  ""
  "br.call%+.many %3 = %1"
  [(set_attr "type" "B")])

(define_insn "*call_multiple_values_internal1"
  [(match_parallel 0 "call_multiple_values_operation"
                   [(set (match_operand 1 "register_operand" "=rf")
                         (call (mem:DI (match_operand:DI 2 "call_operand" "bi"))
                               (match_operand 3 "" "")))
                    (use (reg:DI 1))
                    (clobber (match_operand:DI 4 "register_operand" "=b"))])]
  ""
  "br.call%+.many %4 = %2"
  [(set_attr "type" "B")])

;; Call subroutine returning any type.

(define_expand "untyped_call"
  [(parallel [(call (match_operand 0 "" "")
                    (const_int 0))
              (match_operand 1 "" "")
              (match_operand 2 "" "")])]
  ""
  "
{
  int i;

  emit_call_insn (gen_call (operands[0], const0_rtx, NULL, const0_rtx));

  for (i = 0; i < XVECLEN (operands[2], 0); i++)
    {
      rtx set = XVECEXP (operands[2], 0, i);
      emit_move_insn (SET_DEST (set), SET_SRC (set));
    }

  /* The optimizer does not know that the call sets the function value
     registers we stored in the result block.  We avoid problems by
     claiming that all hard registers are used and clobbered at this
     point.  */
  emit_insn (gen_blockage ());

  DONE;
}")

(define_insn "return_internal"
  [(return)
   (use (match_operand:DI 0 "register_operand" "b"))]
  ""
  "br.ret.sptk.many %0"
  [(set_attr "type" "B")])

(define_insn "return"
  [(return)]
  "ia64_direct_return ()"
  "br.ret.sptk.many rp"
  [(set_attr "type" "B")])

(define_insn "*return_true"
  [(set (pc)
        (if_then_else (match_operator 0 "predicate_operator"
                        [(match_operand:CC 1 "register_operand" "c")
                         (const_int 0)])
                      (return)
                      (pc)))]
  "ia64_direct_return ()"
  "(%J0) br.ret%+.many rp"
  [(set_attr "type" "B")
   (set_attr "predicable" "no")])

(define_insn "*return_false"
  [(set (pc)
        (if_then_else (match_operator 0 "predicate_operator"
                        [(match_operand:CC 1 "register_operand" "c")
                         (const_int 0)])
                      (pc)
                      (return)))]
  "ia64_direct_return ()"
  "(%j0) br.ret%+.many rp"
  [(set_attr "type" "B")
   (set_attr "predicable" "no")])

(define_insn "jump"
  [(set (pc) (label_ref (match_operand 0 "" "")))]
  ""
  "br %l0"
  [(set_attr "type" "B")])

(define_insn "indirect_jump"
  [(set (pc) (match_operand:DI 0 "register_operand" "b"))]
  ""
  "br %0"
  [(set_attr "type" "B")])

(define_expand "tablejump"
  [(match_operand:DI 0 "register_operand" "")
   (match_operand 1 "" "")]
  ""
  "
{
  rtx tmp1 = gen_reg_rtx (DImode);
  rtx tmp2 = gen_reg_rtx (DImode);

  emit_move_insn (tmp1, gen_rtx_LABEL_REF (Pmode, operands[1]));
  emit_insn (gen_adddi3 (tmp2, operands[0], tmp1));
  emit_jump_insn (gen_tablejump_internal (tmp2, operands[1]));
  DONE;
}")

(define_insn "tablejump_internal"
  [(set (pc) (match_operand:DI 0 "register_operand" "b"))
   (use (label_ref (match_operand 1 "" "")))]
  ""
  "br %0"
  [(set_attr "type" "B")])

\f
;; ::::::::::::::::::::
;; ::
;; :: Prologue and Epilogue instructions
;; ::
;; ::::::::::::::::::::

(define_expand "prologue"
  [(const_int 1)]
  ""
  "
{
  ia64_expand_prologue ();
  DONE;
}")

(define_expand "epilogue"
  [(const_int 2)]
  ""
  "
{
  ia64_expand_epilogue ();
  DONE;
}")

;; This prevents the scheduler from moving the SP decrement past FP-relative
;; stack accesses.  This is the same as adddi3 plus the extra set.

(define_insn "prologue_allocate_stack"
  [(set (match_operand:DI 0 "register_operand" "=r,r,r")
        (plus:DI (match_operand:DI 1 "register_operand" "%r,r,a")
                 (match_operand:DI 2 "reg_or_22bit_operand" "r,I,J")))
   (set (match_operand:DI 3 "register_operand" "=r,r,r")
        (match_dup 3))]
  ""
  "@
  add %0 = %1, %2
  adds %0 = %2, %1
  addl %0 = %2, %1"
  [(set_attr "type" "A")])

;; This prevents the scheduler from moving the SP restore past FP-relative
;; stack accesses.  This is similar to movdi plus the extra set.

(define_insn "epilogue_deallocate_stack"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (match_operand:DI 1 "register_operand" "+r"))
   (set (match_dup 1) (match_dup 1))]
  ""
  "mov %0 = %1"
  [(set_attr "type" "A")])

;; Allocate a new register frame.

(define_insn "alloc"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec_volatile:DI [(const_int 0)] 0))
   (use (match_operand:DI 1 "const_int_operand" "i"))
   (use (match_operand:DI 2 "const_int_operand" "i"))
   (use (match_operand:DI 3 "const_int_operand" "i"))
   (use (match_operand:DI 4 "const_int_operand" "i"))]
  ""
  "alloc %0 = ar.pfs, %1, %2, %3, %4"
  [(set_attr "type" "M")
   (set_attr "predicable" "no")])

(define_insn "gr_spill"
  [(set (match_operand:DI 0 "memory_operand" "=m")
        (unspec:DI [(match_operand:DI 1 "register_operand" "r")] 1))]
  ""
  "st8.spill %0 = %1%P0"
  [(set_attr "type" "M")])

(define_insn "gr_restore"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")] 2))]
  ""
  "ld8.fill %0 = %1%P1"
  [(set_attr "type" "M")])

(define_insn "fr_spill"
  [(set (match_operand:XF 0 "memory_operand" "=m")
        (unspec:XF [(match_operand:XF 1 "register_operand" "f")] 3))]
  ""
  "stf.spill %0 = %1%P0"
  [(set_attr "type" "M")])

(define_insn "fr_restore"
  [(set (match_operand:XF 0 "register_operand" "=f")
        (unspec:XF [(match_operand:XF 1 "memory_operand" "m")] 4))]
  ""
  "ldf.fill %0 = %1%P1"
  [(set_attr "type" "M")])

(define_insn "pr_spill"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec:DI [(const_int 0)] 5))]
  ""
  "mov %0 = pr"
  [(set_attr "type" "I")])

;; ??? This is volatile to prevent it from being moved before a conditional
;; expression that calculates the return value.

(define_insn "pr_restore"
  [(unspec_volatile [(const_int 0)] 6)
   (use (match_operand:DI 0 "register_operand" "r"))]
  ""
  "mov pr = %0, -1"
  [(set_attr "type" "I")])

;; ??? This is volatile to prevent it from being moved before a call.
;; Should instead add a ar.pfs hard register which is call clobbered.

(define_insn "pfs_restore"
  [(unspec_volatile [(const_int 0)] 4)
   (use (match_operand:DI 0 "register_operand" "r"))]
  ""
  "mov ar.pfs = %0"
  [(set_attr "type" "I")])

(define_insn "unat_spill"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec:DI [(const_int 0)] 9))]
  ""
  "mov %0 = ar.unat"
  [(set_attr "type" "M")])

(define_insn "unat_restore"
  [(unspec [(const_int 0)] 10)
   (use (match_operand:DI 0 "register_operand" "r"))]
  ""
  "mov ar.unat = %0"
  [(set_attr "type" "M")])

(define_insn "bsp_value"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec:DI [(const_int 0)] 20))]
  ""
  "mov %0 = ar.bsp"
  [(set_attr "type" "I")])

(define_insn "set_bsp"
  [(unspec_volatile [(const_int 0)] 5)
   (use (match_operand:DI 0 "register_operand" "r"))]
  ""
  "flushrs\;mov r19=ar.rsc\;;;\;and r19=0x1c,r19\;;;\;mov ar.rsc=r19\;;;\;mov ar.bspstore=%0\;;;\;or r19=0x3,r19\;;;\;loadrs\;invala\;;;\;mov ar.rsc=r19"
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])

(define_insn "flushrs"
  [(unspec [(const_int 0)] 21)]
  ""
  ";;\;flushrs"
  [(set_attr "type" "M")])
\f
;; ::::::::::::::::::::
;; ::
;; :: Miscellaneous instructions
;; ::
;; ::::::::::::::::::::

;; ??? Emiting a NOP instruction isn't very useful.  This should probably
;; be emitting ";;" to force a break in the instruction packing.

;; No operation, needed in case the user uses -g but not -O.
(define_insn "nop"
  [(const_int 0)]
  ""
  "nop 0"
  [(set_attr "type" "unknown")])

;; Pseudo instruction that prevents the scheduler from moving code above this
;; point.
(define_insn "blockage"
  [(unspec_volatile [(const_int 0)] 1)]
  ""
  ""
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])

(define_insn "insn_group_barrier"
  [(unspec_volatile [(const_int 0)] 2)]
  ""
  ";;"
  [(set_attr "type" "S")
   (set_attr "predicable" "no")])

\f
;; Non-local goto support.

(define_expand "save_stack_nonlocal"
  [(use (match_operand:OI 0 "memory_operand" ""))
   (use (match_operand:DI 1 "register_operand" ""))]
  ""
  "
{
  emit_library_call (gen_rtx_SYMBOL_REF (Pmode,
                                         \"__ia64_save_stack_nonlocal\"),
                     0, VOIDmode, 2, XEXP (operands[0], 0), Pmode,
                     operands[1], Pmode);
  DONE;
}")

(define_expand "nonlocal_goto"
  [(use (match_operand 0 "general_operand" ""))
   (use (match_operand 1 "general_operand" ""))
   (use (match_operand 2 "general_operand" ""))
   (use (match_operand 3 "general_operand" ""))]
  ""
  "
{
  if (GET_CODE (operands[0]) != REG)
    operands[0] = force_reg (Pmode, operands[0]);
  emit_move_insn (virtual_stack_vars_rtx, operands[0]);
  emit_insn (gen_rtx_USE (VOIDmode, frame_pointer_rtx));
  emit_insn (gen_rtx_USE (VOIDmode, stack_pointer_rtx));
  emit_insn (gen_rtx_USE (VOIDmode, static_chain_rtx));
  emit_library_call (gen_rtx_SYMBOL_REF (Pmode, \"__ia64_nonlocal_goto\"),
                     0, VOIDmode, 4,
                     operands[0], Pmode, operands[1], Pmode,
                     copy_to_reg (XEXP (operands[2], 0)), Pmode,
                     operands[3], Pmode);
  emit_barrier ();
  DONE;
}")

;; ??? We need this because the function __ia64_nonlocal_goto can't easily
;; access the FP which is currently stored in a local register.  Maybe move
;; the FP to a global register to avoid this problem?

(define_expand "nonlocal_goto_receiver"
  [(use (const_int 0))]
  ""
  "
{
  emit_move_insn (frame_pointer_rtx, gen_rtx_REG (DImode, GR_REG (7)));
  DONE;
}")

(define_expand "eh_epilogue"
  [(use (match_operand:DI 0 "register_operand" "r"))
   (use (match_operand:DI 1 "register_operand" "r"))
   (use (match_operand:DI 2 "register_operand" "r"))]
  ""
  "
{
  rtx bsp = gen_rtx_REG (Pmode, 10);
  rtx sp = gen_rtx_REG (Pmode, 9);

  if (GET_CODE (operands[0]) != REG || REGNO (operands[0]) != 10)
    {
      emit_move_insn (bsp, operands[0]);
      operands[0] = bsp;
    }
  if (GET_CODE (operands[2]) != REG || REGNO (operands[2]) != 9)
    {
      emit_move_insn (sp, operands[2]);
      operands[2] = sp;
    }
  emit_insn (gen_rtx_USE (VOIDmode, sp));
  emit_insn (gen_rtx_USE (VOIDmode, bsp));

  cfun->machine->ia64_eh_epilogue_sp = sp;
  cfun->machine->ia64_eh_epilogue_bsp = bsp;

}")

;; This flushes at least 64 bytes starting from the address pointed
;; to by operand[0].

;; ??? This should be a define expand.

(define_insn "flush_cache"
  [(unspec_volatile [(match_operand:DI 0 "register_operand" "=&r")] 3)]
  ""
  "fc %0\;;;\;adds %0=31,%0\;;;\;fc %0\;;;\;sync.i\;srlz.i"
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])
\f
;; Builtin apply support.

(define_expand "restore_stack_nonlocal"
  [(use (match_operand:DI 0 "register_operand" ""))
   (use (match_operand:OI 1 "memory_operand" ""))]
  ""
  "
{
  emit_library_call (gen_rtx_SYMBOL_REF (Pmode,
                                         \"__ia64_restore_stack_nonlocal\"),
                     0, VOIDmode, 1,
                     copy_to_reg (XEXP (operands[1], 0)), Pmode);
  DONE;
}")

\f
;;; Intrinsics support.

(define_insn "ccv_restore_si"
  [(unspec [(const_int 0)] 11)
   (use (match_operand:SI 0 "register_operand" "r"))]
  ""
  "mov ar.ccv = %0"
  [(set_attr "type" "M")])

(define_insn "ccv_restore_di"
  [(unspec [(const_int 0)] 11)
   (use (match_operand:DI 0 "register_operand" "r"))]
  ""
  "mov ar.ccv = %0"
  [(set_attr "type" "M")])

(define_insn "mf"
  [(unspec [(match_operand:BLK 0 "memory_operand" "m")] 12)]
  ""
  "mf"
  [(set_attr "type" "M")])

(define_insn "fetchadd_acq_si"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "fetchadd_operand" "n")] 19))]
  ""
  "fetchadd4.acq %0 = %1, %2"
  [(set_attr "type" "M")])

(define_insn "fetchadd_acq_di"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "fetchadd_operand" "n")] 19))]
  ""
  "fetchadd8.acq %0 = %1, %2"
  [(set_attr "type" "M")])

(define_insn "cmpxchg_acq_si"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 13))]
  ""
  "cmpxchg4.acq %0 = %1, %2, ar.ccv"
  [(set_attr "type" "M")])

(define_insn "cmpxchg_acq_di"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 13))]
  ""
  "cmpxchg8.acq %0 = %1, %2, ar.ccv"
  [(set_attr "type" "M")])

(define_expand "val_compare_and_swap_si"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")
                    (match_operand:SI 3 "register_operand" "r")] 14))]
  ""
  "
{
  rtx tmp_reg = gen_rtx_REG (DImode, GR_REG(0));
  rtx target = gen_rtx_MEM (BLKmode, tmp_reg);
  RTX_UNCHANGING_P (target) = 1;
  emit_insn (gen_ccv_restore_si (operands[2]));
  emit_insn (gen_mf (target));
  emit_insn (gen_cmpxchg_acq_si (operands[0], operands[1], operands[3]));
  DONE;
}")

(define_expand "val_compare_and_swap_di"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")
                    (match_operand:DI 3 "register_operand" "r")] 14))]
  ""
  "
{
  rtx tmp_reg = gen_rtx_REG (DImode, GR_REG(0));
  rtx target = gen_rtx_MEM (BLKmode, tmp_reg);
  RTX_UNCHANGING_P (target) = 1;
  emit_insn (gen_ccv_restore_di (operands[2]));
  emit_insn (gen_mf (target));
  emit_insn (gen_cmpxchg_acq_di (operands[0], operands[1], operands[3]));
  DONE;
}")

(define_insn "xchgsi"
  [(set (match_operand:SI 0 "register_operand" "=r")
        (match_operand:SI 1 "memory_operand" "+m"))
   (set (match_dup 1)
        (match_operand:SI 2 "register_operand" "r"))]
  ""
  "xchg4 %0 = %1, %2"
  [(set_attr "type" "M")])

(define_insn "xchgdi"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (match_operand:DI 1 "memory_operand" "+m"))
   (set (match_dup 1)
        (match_operand:DI 2 "register_operand" "r"))]
  ""
  "xchg8 %0 = %1, %2"
  [(set_attr "type" "M")])

(define_expand "lock_test_and_set_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 16))]
  ""
  "
{
  emit_insn (gen_xchgsi (operands[0], operands[1], operands[2]));
  DONE;
}")

(define_expand "lock_test_and_set_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 16))]
  ""
  "
{
  emit_insn (gen_xchgdi (operands[0], operands[1], operands[2]));
  DONE;
}")

(define_expand "fetch_and_add_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "nonmemory_operand" "")] 18))]
  ""
  "
{
  int x;

  if (GET_CODE (operands[2]) == CONST_INT)
    {
      x = INTVAL(operands[2]);
      if (x == -16 || x == -8 || x == -4 || x == -1 ||
          x ==  16 || x ==  8 || x ==  4 || x ==  1)
        {
          emit_insn (gen_fetchadd_acq_si (operands[0], operands[1], operands[2]));
          DONE;
        }
    }

  ia64_expand_fetch_and_op (IA64_ADD_OP, SImode, operands);
  DONE;
}")

(define_expand "fetch_and_sub_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_SUB_OP, SImode, operands);
  DONE;
}")

(define_expand "fetch_and_or_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_OR_OP, SImode, operands);
  DONE;
}")

(define_expand "fetch_and_and_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_AND_OP, SImode, operands);
  DONE;
}")

(define_expand "fetch_and_xor_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_XOR_OP, SImode, operands);
  DONE;
}")

(define_expand "fetch_and_nand_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_NAND_OP, SImode, operands);
  DONE;
}")

(define_expand "fetch_and_add_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "nonmemory_operand" "")] 18))]
  ""
  "
{
  int x;

  if (GET_CODE (operands[2]) == CONST_INT)
    {
      x = INTVAL(operands[2]);
      if (x == -16 || x == -8 || x == -4 || x == -1 ||
          x ==  16 || x ==  8 || x ==  4 || x ==  1)
        {
          emit_insn (gen_fetchadd_acq_di (operands[0], operands[1], operands[2]));
          DONE;
        }
    }

  ia64_expand_fetch_and_op (IA64_ADD_OP, DImode, operands);
  DONE;
}")

(define_expand "fetch_and_sub_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_SUB_OP, DImode, operands);
  DONE;
}")

(define_expand "fetch_and_or_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_OR_OP, DImode, operands);
  DONE;
}")

(define_expand "fetch_and_and_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_AND_OP, DImode, operands);
  DONE;
}")

(define_expand "fetch_and_xor_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_XOR_OP, DImode, operands);
  DONE;
}")

(define_expand "fetch_and_nand_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 18))]
  ""
  "
{
  ia64_expand_fetch_and_op (IA64_NAND_OP, DImode, operands);
  DONE;
}")

(define_expand "add_and_fetch_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_ADD_OP, DImode, operands);
  DONE;
}")

(define_expand "sub_and_fetch_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_SUB_OP, DImode, operands);
  DONE;
}")

(define_expand "or_and_fetch_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_OR_OP, DImode, operands);
  DONE;
}")

(define_expand "and_and_fetch_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_AND_OP, DImode, operands);
  DONE;
}")

(define_expand "xor_and_fetch_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_XOR_OP, DImode, operands);
  DONE;
}")

(define_expand "nand_and_fetch_di"
  [(set (match_operand:DI 0 "register_operand" "r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
                    (match_operand:DI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_NAND_OP, DImode, operands);
  DONE;
}")

(define_expand "add_and_fetch_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_ADD_OP, SImode, operands);
  DONE;
}")

(define_expand "sub_and_fetch_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_SUB_OP, SImode, operands);
  DONE;
}")

(define_expand "or_and_fetch_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_OR_OP, SImode, operands);
  DONE;
}")

(define_expand "and_and_fetch_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_AND_OP, SImode, operands);
  DONE;
}")

(define_expand "xor_and_fetch_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_XOR_OP, SImode, operands);
  DONE;
}")

(define_expand "nand_and_fetch_si"
  [(set (match_operand:SI 0 "register_operand" "r")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
                    (match_operand:SI 2 "register_operand" "r")] 17))]
  ""
  "
{
  ia64_expand_op_and_fetch (IA64_NAND_OP, SImode, operands);
  DONE;
}")
\f
;; Predication.

(define_cond_exec
  [(match_operator 0 "predicate_operator"
     [(match_operand:CC 1 "register_operand" "c")
      (const_int 0)])]
  ""
  "(%J0)")

(define_insn "pred_rel_mutex"
  [(unspec_volatile [(match_operand:CC 0 "register_operand" "c")] 7)]
  ""
  ".pred.rel.mutex %0, %I0"
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])