;; Machine description for RISC-V 'V' Extension for GNU compiler.
;; Copyright (C) 2022-2023 Free Software Foundation, Inc.
;; Contributed by Juzhe Zhong (juzhe.zhong@rivai.ai), RiVAI Technologies Ltd.
;; This file is part of GCC.
;; GCC is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3, or (at your option)
;; any later version.
;; GCC is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with GCC; see the file COPYING3. If not see
;; .
;; This file describes the RISC-V 'V' Extension, Version 1.0.
;;
;; This file include :
;;
;; - Intrinsics (https://github.com/riscv/rvv-intrinsic-doc)
;; - Auto-vectorization (autovec.md)
;; - Combine optimization (TBD)
(include "vector-iterators.md")
(define_constants [
(INVALID_ATTRIBUTE 255)
(X0_REGNUM 0)
])
;; True if the type is RVV instructions that include VTYPE
;; global status register in the use op list.
;; We known VTYPE has 4 fields: SEW, LMUL, TA, MA.
;; The instruction need any of VTYPE field is set as true
;; in this attribute.
(define_attr "has_vtype_op" "false,true"
(cond [(eq_attr "type" "vlde,vste,vldm,vstm,vlds,vsts,\
vldux,vldox,vstux,vstox,vldff,\
vialu,viwalu,vext,vicalu,vshift,vnshift,vicmp,viminmax,\
vimul,vidiv,viwmul,vimuladd,viwmuladd,vimerge,vimov,\
vsalu,vaalu,vsmul,vsshift,vnclip,\
vfalu,vfwalu,vfmul,vfdiv,vfwmul,vfmuladd,vfwmuladd,vfsqrt,vfrecp,\
vfcmp,vfminmax,vfsgnj,vfclass,vfmerge,vfmov,\
vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,\
vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,\
vired,viwred,vfredu,vfredo,vfwredu,vfwredo,\
vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,vfmovvf,vfmovfv,\
vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,\
vgather,vcompress,vlsegde,vssegte,vlsegds,vssegts,vlsegdux,vlsegdox,\
vssegtux,vssegtox,vlsegdff")
(const_string "true")]
(const_string "false")))
;; True if the type is RVV instructions that include VL
;; global status register in the use op list.
;; The instruction need vector length to be specified is set
;; in this attribute.
(define_attr "has_vl_op" "false,true"
(cond [(eq_attr "type" "vlde,vste,vldm,vstm,vlds,vsts,\
vldux,vldox,vstux,vstox,vldff,\
vialu,viwalu,vext,vicalu,vshift,vnshift,vicmp,viminmax,\
vimul,vidiv,viwmul,vimuladd,viwmuladd,vimerge,vimov,\
vsalu,vaalu,vsmul,vsshift,vnclip,\
vfalu,vfwalu,vfmul,vfdiv,vfwmul,vfmuladd,vfwmuladd,vfsqrt,vfrecp,\
vfcmp,vfminmax,vfsgnj,vfclass,vfmerge,vfmov,\
vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,\
vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,\
vired,viwred,vfredu,vfredo,vfwredu,vfwredo,\
vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovxv,vfmovfv,\
vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,\
vgather,vcompress,vlsegde,vssegte,vlsegds,vssegts,vlsegdux,vlsegdox,\
vssegtux,vssegtox,vlsegdff")
(const_string "true")]
(const_string "false")))
;; The default SEW of RVV instruction. This attribute doesn't mean the instruction
;; is necessary to require SEW check for example vlm.v which require ratio to
;; check. However, we need default value of SEW for vsetvl instruction since there
;; is no field for ratio in the vsetvl instruction encoding.
(define_attr "sew" ""
(cond [(eq_attr "mode" "VNx1QI,VNx2QI,VNx4QI,VNx8QI,VNx16QI,VNx32QI,VNx64QI,\
VNx1BI,VNx2BI,VNx4BI,VNx8BI,VNx16BI,VNx32BI,VNx64BI,\
VNx128QI,VNx128BI,VNx2x64QI,VNx2x32QI,VNx3x32QI,VNx4x32QI,\
VNx2x16QI,VNx3x16QI,VNx4x16QI,VNx5x16QI,VNx6x16QI,VNx7x16QI,VNx8x16QI,\
VNx2x8QI,VNx3x8QI,VNx4x8QI,VNx5x8QI,VNx6x8QI,VNx7x8QI,VNx8x8QI,\
VNx2x4QI,VNx3x4QI,VNx4x4QI,VNx5x4QI,VNx6x4QI,VNx7x4QI,VNx8x4QI,\
VNx2x2QI,VNx3x2QI,VNx4x2QI,VNx5x2QI,VNx6x2QI,VNx7x2QI,VNx8x2QI,\
VNx2x1QI,VNx3x1QI,VNx4x1QI,VNx5x1QI,VNx6x1QI,VNx7x1QI,VNx8x1QI")
(const_int 8)
(eq_attr "mode" "VNx1HI,VNx2HI,VNx4HI,VNx8HI,VNx16HI,VNx32HI,VNx64HI,\
VNx2x32HI,VNx2x16HI,VNx3x16HI,VNx4x16HI,\
VNx2x8HI,VNx3x8HI,VNx4x8HI,VNx5x8HI,VNx6x8HI,VNx7x8HI,VNx8x8HI,\
VNx2x4HI,VNx3x4HI,VNx4x4HI,VNx5x4HI,VNx6x4HI,VNx7x4HI,VNx8x4HI,\
VNx2x2HI,VNx3x2HI,VNx4x2HI,VNx5x2HI,VNx6x2HI,VNx7x2HI,VNx8x2HI,\
VNx2x1HI,VNx3x1HI,VNx4x1HI,VNx5x1HI,VNx6x1HI,VNx7x1HI,VNx8x1HI")
(const_int 16)
(eq_attr "mode" "VNx1SI,VNx2SI,VNx4SI,VNx8SI,VNx16SI,VNx32SI,\
VNx1SF,VNx2SF,VNx4SF,VNx8SF,VNx16SF,VNx32SF,\
VNx2x16SI,VNx2x8SI,VNx3x8SI,VNx4x8SI,\
VNx2x4SI,VNx3x4SI,VNx4x4SI,VNx5x4SI,VNx6x4SI,VNx7x4SI,VNx8x4SI,\
VNx2x2SI,VNx3x2SI,VNx4x2SI,VNx5x2SI,VNx6x2SI,VNx7x2SI,VNx8x2SI,\
VNx2x1SI,VNx3x1SI,VNx4x1SI,VNx5x1SI,VNx6x1SI,VNx7x1SI,VNx8x1SI,\
VNx2x16SF,VNx2x8SF,VNx3x8SF,VNx4x8SF,\
VNx2x4SF,VNx3x4SF,VNx4x4SF,VNx5x4SF,VNx6x4SF,VNx7x4SF,VNx8x4SF,\
VNx2x2SF,VNx3x2SF,VNx4x2SF,VNx5x2SF,VNx6x2SF,VNx7x2SF,VNx8x2SF,\
VNx2x1SF,VNx3x1SF,VNx4x1SF,VNx5x1SF,VNx6x1SF,VNx7x1SF,VNx8x1SF")
(const_int 32)
(eq_attr "mode" "VNx1DI,VNx2DI,VNx4DI,VNx8DI,VNx16DI,\
VNx1DF,VNx2DF,VNx4DF,VNx8DF,VNx16DF,\
VNx2x8DI,VNx2x4DI,VNx3x4DI,VNx4x4DI,\
VNx2x2DI,VNx3x2DI,VNx4x2DI,VNx5x2DI,VNx6x2DI,VNx7x2DI,VNx8x2DI,\
VNx2x1DI,VNx3x1DI,VNx4x1DI,VNx5x1DI,VNx6x1DI,VNx7x1DI,VNx8x1DI,\
VNx2x8DF,VNx2x4DF,VNx3x4DF,VNx4x4DF,\
VNx2x2DF,VNx3x2DF,VNx4x2DF,VNx5x2DF,VNx6x2DF,VNx7x2DF,VNx8x2DF,\
VNx2x1DF,VNx3x1DF,VNx4x1DF,VNx5x1DF,VNx6x1DF,VNx7x1DF,VNx8x1DF")
(const_int 64)]
(const_int INVALID_ATTRIBUTE)))
;; Ditto to LMUL.
(define_attr "vlmul" ""
(cond [(eq_attr "mode" "VNx1QI,VNx1BI,VNx2x1QI,VNx3x1QI,VNx4x1QI,VNx5x1QI,VNx6x1QI,VNx7x1QI,VNx8x1QI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx1QImode)")
(eq_attr "mode" "VNx2QI,VNx2BI,VNx2x2QI,VNx3x2QI,VNx4x2QI,VNx5x2QI,VNx6x2QI,VNx7x2QI,VNx8x2QI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx2QImode)")
(eq_attr "mode" "VNx4QI,VNx4BI,VNx2x4QI,VNx3x4QI,VNx4x4QI,VNx5x4QI,VNx6x4QI,VNx7x4QI,VNx8x4QI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx4QImode)")
(eq_attr "mode" "VNx8QI,VNx8BI,VNx2x8QI,VNx3x8QI,VNx4x8QI,VNx5x8QI,VNx6x8QI,VNx7x8QI,VNx8x8QI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx8QImode)")
(eq_attr "mode" "VNx16QI,VNx16BI,VNx2x16QI,VNx3x16QI,VNx4x16QI,VNx5x16QI,VNx6x16QI,VNx7x16QI,VNx8x16QI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx16QImode)")
(eq_attr "mode" "VNx32QI,VNx32BI,VNx2x32QI,VNx3x32QI,VNx4x32QI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx32QImode)")
(eq_attr "mode" "VNx64QI,VNx64BI,VNx2x64QI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx64QImode)")
(eq_attr "mode" "VNx128QI,VNx128BI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx128QImode)")
(eq_attr "mode" "VNx1HI,VNx2x1HI,VNx3x1HI,VNx4x1HI,VNx5x1HI,VNx6x1HI,VNx7x1HI,VNx8x1HI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx1HImode)")
(eq_attr "mode" "VNx2HI,VNx2x2HI,VNx3x2HI,VNx4x2HI,VNx5x2HI,VNx6x2HI,VNx7x2HI,VNx8x2HI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx2HImode)")
(eq_attr "mode" "VNx4HI,VNx2x4HI,VNx3x4HI,VNx4x4HI,VNx5x4HI,VNx6x4HI,VNx7x4HI,VNx8x4HI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx4HImode)")
(eq_attr "mode" "VNx8HI,VNx2x8HI,VNx3x8HI,VNx4x8HI,VNx5x8HI,VNx6x8HI,VNx7x8HI,VNx8x8HI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx8HImode)")
(eq_attr "mode" "VNx16HI,VNx2x16HI,VNx3x16HI,VNx4x16HI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx16HImode)")
(eq_attr "mode" "VNx32HI,VNx2x32HI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx32HImode)")
(eq_attr "mode" "VNx64HI")
(symbol_ref "riscv_vector::get_vlmul(E_VNx64HImode)")
(eq_attr "mode" "VNx1SI,VNx1SF,VNx2x1SI,VNx3x1SI,VNx4x1SI,VNx5x1SI,VNx6x1SI,VNx7x1SI,VNx8x1SI,\
VNx2x1SF,VNx3x1SF,VNx4x1SF,VNx5x1SF,VNx6x1SF,VNx7x1SF,VNx8x1SF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx1SImode)")
(eq_attr "mode" "VNx2SI,VNx2SF,VNx2x2SI,VNx3x2SI,VNx4x2SI,VNx5x2SI,VNx6x2SI,VNx7x2SI,VNx8x2SI,\
VNx2x2SF,VNx3x2SF,VNx4x2SF,VNx5x2SF,VNx6x2SF,VNx7x2SF,VNx8x2SF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx2SImode)")
(eq_attr "mode" "VNx4SI,VNx4SF,VNx2x4SI,VNx3x4SI,VNx4x4SI,VNx5x4SI,VNx6x4SI,VNx7x4SI,VNx8x4SI,\
VNx2x4SF,VNx3x4SF,VNx4x4SF,VNx5x4SF,VNx6x4SF,VNx7x4SF,VNx8x4SF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx4SImode)")
(eq_attr "mode" "VNx8SI,VNx8SF,VNx2x8SI,VNx3x8SI,VNx4x8SI,VNx2x8SF,VNx3x8SF,VNx4x8SF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx8SImode)")
(eq_attr "mode" "VNx16SI,VNx16SF,VNx2x16SI,VNx2x16SF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx16SImode)")
(eq_attr "mode" "VNx32SI,VNx32SF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx32SImode)")
(eq_attr "mode" "VNx1DI,VNx1DF,VNx2x1DI,VNx3x1DI,VNx4x1DI,VNx5x1DI,VNx6x1DI,VNx7x1DI,VNx8x1DI,\
VNx2x1DF,VNx3x1DF,VNx4x1DF,VNx5x1DF,VNx6x1DF,VNx7x1DF,VNx8x1DF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx1DImode)")
(eq_attr "mode" "VNx2DI,VNx2DF,VNx2x2DI,VNx3x2DI,VNx4x2DI,VNx5x2DI,VNx6x2DI,VNx7x2DI,VNx8x2DI,\
VNx2x2DF,VNx3x2DF,VNx4x2DF,VNx5x2DF,VNx6x2DF,VNx7x2DF,VNx8x2DF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx2DImode)")
(eq_attr "mode" "VNx4DI,VNx4DF,VNx2x4DI,VNx3x4DI,VNx4x4DI,VNx2x4DF,VNx3x4DF,VNx4x4DF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx4DImode)")
(eq_attr "mode" "VNx8DI,VNx8DF,VNx2x8DI,VNx2x8DF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx8DImode)")
(eq_attr "mode" "VNx16DI,VNx16DF")
(symbol_ref "riscv_vector::get_vlmul(E_VNx16DImode)")]
(const_int INVALID_ATTRIBUTE)))
;; It is valid for instruction that require sew/lmul ratio.
(define_attr "ratio" ""
(cond [(eq_attr "type" "vimov,vfmov,vldux,vldox,vstux,vstox,\
vialu,vshift,vicmp,vimul,vidiv,vsalu,\
vext,viwalu,viwmul,vicalu,vnshift,\
vimuladd,vimerge,vaalu,vsmul,vsshift,\
vnclip,viminmax,viwmuladd,vmpop,vmffs,vmsfs,\
vmiota,vmidx,vfalu,vfmul,vfminmax,vfdiv,\
vfwalu,vfwmul,vfsqrt,vfrecp,vfsgnj,vfcmp,\
vfmerge,vfcvtitof,vfcvtftoi,vfwcvtitof,\
vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,\
vfncvtftof,vfmuladd,vfwmuladd,vfclass,vired,\
viwred,vfredu,vfredo,vfwredu,vfwredo,vimovvx,\
vimovxv,vfmovvf,vfmovfv,vslideup,vslidedown,\
vislide1up,vislide1down,vfslide1up,vfslide1down,\
vgather,vcompress,vlsegdux,vlsegdox,vssegtux,vssegtox")
(const_int INVALID_ATTRIBUTE)
(eq_attr "mode" "VNx1QI,VNx1BI,VNx2x1QI,VNx3x1QI,VNx4x1QI,VNx5x1QI,VNx6x1QI,VNx7x1QI,VNx8x1QI")
(symbol_ref "riscv_vector::get_ratio(E_VNx1QImode)")
(eq_attr "mode" "VNx2QI,VNx2BI,VNx2x2QI,VNx3x2QI,VNx4x2QI,VNx5x2QI,VNx6x2QI,VNx7x2QI,VNx8x2QI")
(symbol_ref "riscv_vector::get_ratio(E_VNx2QImode)")
(eq_attr "mode" "VNx4QI,VNx4BI,VNx2x4QI,VNx3x4QI,VNx4x4QI,VNx5x4QI,VNx6x4QI,VNx7x4QI,VNx8x4QI")
(symbol_ref "riscv_vector::get_ratio(E_VNx4QImode)")
(eq_attr "mode" "VNx8QI,VNx8BI,VNx2x8QI,VNx3x8QI,VNx4x8QI,VNx5x8QI,VNx6x8QI,VNx7x8QI,VNx8x8QI")
(symbol_ref "riscv_vector::get_ratio(E_VNx8QImode)")
(eq_attr "mode" "VNx16QI,VNx16BI,VNx2x16QI,VNx3x16QI,VNx4x16QI,VNx5x16QI,VNx6x16QI,VNx7x16QI,VNx8x16QI")
(symbol_ref "riscv_vector::get_ratio(E_VNx16QImode)")
(eq_attr "mode" "VNx32QI,VNx32BI,VNx2x32QI,VNx3x32QI,VNx4x32QI")
(symbol_ref "riscv_vector::get_ratio(E_VNx32QImode)")
(eq_attr "mode" "VNx64QI,VNx64BI,VNx2x64QI")
(symbol_ref "riscv_vector::get_ratio(E_VNx64QImode)")
(eq_attr "mode" "VNx128QI,VNx128BI")
(symbol_ref "riscv_vector::get_ratio(E_VNx128QImode)")
(eq_attr "mode" "VNx1HI,VNx2x1HI,VNx3x1HI,VNx4x1HI,VNx5x1HI,VNx6x1HI,VNx7x1HI,VNx8x1HI")
(symbol_ref "riscv_vector::get_ratio(E_VNx1HImode)")
(eq_attr "mode" "VNx2HI,VNx2x2HI,VNx3x2HI,VNx4x2HI,VNx5x2HI,VNx6x2HI,VNx7x2HI,VNx8x2HI")
(symbol_ref "riscv_vector::get_ratio(E_VNx2HImode)")
(eq_attr "mode" "VNx4HI,VNx2x4HI,VNx3x4HI,VNx4x4HI,VNx5x4HI,VNx6x4HI,VNx7x4HI,VNx8x4HI")
(symbol_ref "riscv_vector::get_ratio(E_VNx4HImode)")
(eq_attr "mode" "VNx8HI,VNx2x8HI,VNx3x8HI,VNx4x8HI,VNx5x8HI,VNx6x8HI,VNx7x8HI,VNx8x8HI")
(symbol_ref "riscv_vector::get_ratio(E_VNx8HImode)")
(eq_attr "mode" "VNx16HI,VNx2x16HI,VNx3x16HI,VNx4x16HI")
(symbol_ref "riscv_vector::get_ratio(E_VNx16HImode)")
(eq_attr "mode" "VNx32HI,VNx2x32HI")
(symbol_ref "riscv_vector::get_ratio(E_VNx32HImode)")
(eq_attr "mode" "VNx64HI")
(symbol_ref "riscv_vector::get_ratio(E_VNx64HImode)")
(eq_attr "mode" "VNx1SI,VNx1SF,VNx2x1SI,VNx3x1SI,VNx4x1SI,VNx5x1SI,VNx6x1SI,VNx7x1SI,VNx8x1SI,\
VNx2x1SF,VNx3x1SF,VNx4x1SF,VNx5x1SF,VNx6x1SF,VNx7x1SF,VNx8x1SF")
(symbol_ref "riscv_vector::get_ratio(E_VNx1SImode)")
(eq_attr "mode" "VNx2SI,VNx2SF,VNx2x2SI,VNx3x2SI,VNx4x2SI,VNx5x2SI,VNx6x2SI,VNx7x2SI,VNx8x2SI,\
VNx2x2SF,VNx3x2SF,VNx4x2SF,VNx5x2SF,VNx6x2SF,VNx7x2SF,VNx8x2SF")
(symbol_ref "riscv_vector::get_ratio(E_VNx2SImode)")
(eq_attr "mode" "VNx4SI,VNx4SF,VNx2x4SI,VNx3x4SI,VNx4x4SI,VNx5x4SI,VNx6x4SI,VNx7x4SI,VNx8x4SI,\
VNx2x4SF,VNx3x4SF,VNx4x4SF,VNx5x4SF,VNx6x4SF,VNx7x4SF,VNx8x4SF")
(symbol_ref "riscv_vector::get_ratio(E_VNx4SImode)")
(eq_attr "mode" "VNx8SI,VNx8SF,VNx2x8SI,VNx3x8SI,VNx4x8SI,VNx2x8SF,VNx3x8SF,VNx4x8SF")
(symbol_ref "riscv_vector::get_ratio(E_VNx8SImode)")
(eq_attr "mode" "VNx16SI,VNx16SF,VNx2x16SI,VNx2x16SF")
(symbol_ref "riscv_vector::get_ratio(E_VNx16SImode)")
(eq_attr "mode" "VNx32SI,VNx32SF")
(symbol_ref "riscv_vector::get_ratio(E_VNx32SImode)")
(eq_attr "mode" "VNx1DI,VNx1DF,VNx2x1DI,VNx3x1DI,VNx4x1DI,VNx5x1DI,VNx6x1DI,VNx7x1DI,VNx8x1DI,\
VNx2x1DF,VNx3x1DF,VNx4x1DF,VNx5x1DF,VNx6x1DF,VNx7x1DF,VNx8x1DF")
(symbol_ref "riscv_vector::get_ratio(E_VNx1DImode)")
(eq_attr "mode" "VNx2DI,VNx2DF,VNx2x2DI,VNx3x2DI,VNx4x2DI,VNx5x2DI,VNx6x2DI,VNx7x2DI,VNx8x2DI,\
VNx2x2DF,VNx3x2DF,VNx4x2DF,VNx5x2DF,VNx6x2DF,VNx7x2DF,VNx8x2DF")
(symbol_ref "riscv_vector::get_ratio(E_VNx2DImode)")
(eq_attr "mode" "VNx4DI,VNx4DF,VNx2x4DI,VNx3x4DI,VNx4x4DI,VNx2x4DF,VNx3x4DF,VNx4x4DF")
(symbol_ref "riscv_vector::get_ratio(E_VNx4DImode)")
(eq_attr "mode" "VNx8DI,VNx8DF,VNx2x8DI,VNx2x8DF")
(symbol_ref "riscv_vector::get_ratio(E_VNx8DImode)")
(eq_attr "mode" "VNx16DI,VNx16DF")
(symbol_ref "riscv_vector::get_ratio(E_VNx16DImode)")]
(const_int INVALID_ATTRIBUTE)))
;; The index of operand[] to get the merge op.
(define_attr "merge_op_idx" ""
(cond [(eq_attr "type" "vlde,vimov,vfmov,vldm,vlds,vmalu,vldux,vldox,vicmp,\
vialu,vshift,viminmax,vimul,vidiv,vsalu,vext,viwalu,\
viwmul,vnshift,vaalu,vsmul,vsshift,vnclip,vmsfs,\
vmiota,vmidx,vfalu,vfmul,vfminmax,vfdiv,vfwalu,vfwmul,\
vfsqrt,vfrecp,vfsgnj,vfcmp,vfcvtitof,vfcvtftoi,vfwcvtitof,\
vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,vfclass,\
vired,viwred,vfredu,vfredo,vfwredu,vfwredo,vimovxv,vfmovfv,\
vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,\
vgather,vldff,viwmuladd,vfwmuladd,vlsegde,vlsegds,vlsegdux,vlsegdox,vlsegdff")
(const_int 2)
(eq_attr "type" "vimerge,vfmerge,vcompress")
(const_int 1)
(eq_attr "type" "vimuladd,vfmuladd")
(const_int 5)]
(const_int INVALID_ATTRIBUTE)))
;; The index of operand[] to get the avl op.
(define_attr "vl_op_idx" ""
(cond [(eq_attr "type" "vlde,vste,vimov,vfmov,vldm,vstm,vmalu,vsts,vstux,\
vstox,vext,vmsfs,vmiota,vfsqrt,vfrecp,vfcvtitof,vldff,\
vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,\
vfncvtftoi,vfncvtftof,vfclass,vimovxv,vfmovfv,vcompress,\
vlsegde,vssegts,vssegtux,vssegtox,vlsegdff")
(const_int 4)
;; If operands[3] of "vlds" is not vector mode, it is pred_broadcast.
;; wheras it is pred_strided_load if operands[3] is vector mode.
(eq_attr "type" "vlds")
(if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
(const_int 5)
(const_int 4))
(eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
viwalu,viwmul,vnshift,vimerge,vaalu,vsmul,\
vsshift,vnclip,vfalu,vfmul,vfminmax,vfdiv,vfwalu,vfwmul,\
vfsgnj,vfmerge,vired,viwred,vfredu,vfredo,vfwredu,vfwredo,\
vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,\
vgather,viwmuladd,vfwmuladd,vlsegds,vlsegdux,vlsegdox")
(const_int 5)
(eq_attr "type" "vicmp,vimuladd,vfcmp,vfmuladd")
(const_int 6)
(eq_attr "type" "vmpop,vmffs,vmidx,vssegte")
(const_int 3)]
(const_int INVALID_ATTRIBUTE)))
;; The tail policy op value.
(define_attr "ta" ""
(cond [(eq_attr "type" "vlde,vimov,vfmov,vext,vmiota,vfsqrt,vfrecp,\
vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,\
vfncvtitof,vfncvtftoi,vfncvtftof,vfclass,vimovxv,vfmovfv,\
vcompress,vldff,vlsegde,vlsegdff")
(symbol_ref "riscv_vector::get_ta(operands[5])")
;; If operands[3] of "vlds" is not vector mode, it is pred_broadcast.
;; wheras it is pred_strided_load if operands[3] is vector mode.
(eq_attr "type" "vlds")
(if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
(symbol_ref "riscv_vector::get_ta(operands[6])")
(symbol_ref "riscv_vector::get_ta(operands[5])"))
(eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
viwalu,viwmul,vnshift,vimerge,vaalu,vsmul,\
vsshift,vnclip,vfalu,vfmul,vfminmax,vfdiv,\
vfwalu,vfwmul,vfsgnj,vfmerge,vired,viwred,vfredu,\
vfredo,vfwredu,vfwredo,vslideup,vslidedown,vislide1up,\
vislide1down,vfslide1up,vfslide1down,vgather,viwmuladd,vfwmuladd,\
vlsegds,vlsegdux,vlsegdox")
(symbol_ref "riscv_vector::get_ta(operands[6])")
(eq_attr "type" "vimuladd,vfmuladd")
(symbol_ref "riscv_vector::get_ta(operands[7])")
(eq_attr "type" "vmidx")
(symbol_ref "riscv_vector::get_ta(operands[4])")]
(const_int INVALID_ATTRIBUTE)))
;; The mask policy op value.
(define_attr "ma" ""
(cond [(eq_attr "type" "vlde,vext,vmiota,vfsqrt,vfrecp,vfcvtitof,vfcvtftoi,\
vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,\
vfncvtftof,vfclass,vldff,vlsegde,vlsegdff")
(symbol_ref "riscv_vector::get_ma(operands[6])")
;; If operands[3] of "vlds" is not vector mode, it is pred_broadcast.
;; wheras it is pred_strided_load if operands[3] is vector mode.
(eq_attr "type" "vlds")
(if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
(symbol_ref "riscv_vector::get_ma(operands[7])")
(symbol_ref "riscv_vector::get_ma(operands[6])"))
(eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
viwalu,viwmul,vnshift,vaalu,vsmul,vsshift,\
vnclip,vicmp,vfalu,vfmul,vfminmax,vfdiv,\
vfwalu,vfwmul,vfsgnj,vfcmp,vslideup,vslidedown,\
vislide1up,vislide1down,vfslide1up,vfslide1down,vgather,\
viwmuladd,vfwmuladd,vlsegds,vlsegdux,vlsegdox")
(symbol_ref "riscv_vector::get_ma(operands[7])")
(eq_attr "type" "vimuladd,vfmuladd")
(symbol_ref "riscv_vector::get_ma(operands[8])")
(eq_attr "type" "vmsfs,vmidx")
(symbol_ref "riscv_vector::get_ma(operands[5])")]
(const_int INVALID_ATTRIBUTE)))
;; The avl type value.
(define_attr "avl_type" ""
(cond [(eq_attr "type" "vlde,vldff,vste,vimov,vimov,vimov,vfmov,vext,vimerge,\
vfsqrt,vfrecp,vfmerge,vfcvtitof,vfcvtftoi,vfwcvtitof,\
vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,\
vfclass,vired,viwred,vfredu,vfredo,vfwredu,vfwredo,\
vimovxv,vfmovfv,vlsegde,vlsegdff")
(symbol_ref "INTVAL (operands[7])")
(eq_attr "type" "vldm,vstm,vimov,vmalu,vmalu")
(symbol_ref "INTVAL (operands[5])")
;; If operands[3] of "vlds" is not vector mode, it is pred_broadcast.
;; wheras it is pred_strided_load if operands[3] is vector mode.
(eq_attr "type" "vlds")
(if_then_else (match_test "VECTOR_MODE_P (GET_MODE (operands[3]))")
(const_int INVALID_ATTRIBUTE)
(symbol_ref "INTVAL (operands[7])"))
(eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\
viwalu,viwmul,vnshift,vimuladd,vaalu,vsmul,vsshift,\
vnclip,vicmp,vfalu,vfmul,vfminmax,vfdiv,vfwalu,vfwmul,\
vfsgnj,vfcmp,vfmuladd,vslideup,vslidedown,vislide1up,\
vislide1down,vfslide1up,vfslide1down,vgather,viwmuladd,vfwmuladd,\
vlsegds,vlsegdux,vlsegdox")
(symbol_ref "INTVAL (operands[8])")
(eq_attr "type" "vstux,vstox,vssegts,vssegtux,vssegtox")
(symbol_ref "INTVAL (operands[5])")
(eq_attr "type" "vimuladd")
(symbol_ref "INTVAL (operands[9])")
(eq_attr "type" "vmsfs,vmidx,vcompress")
(symbol_ref "INTVAL (operands[6])")
(eq_attr "type" "vmpop,vmffs,vssegte")
(symbol_ref "INTVAL (operands[4])")]
(const_int INVALID_ATTRIBUTE)))
;; -----------------------------------------------------------------
;; ---- Miscellaneous Operations
;; -----------------------------------------------------------------
(define_insn "@vundefined"
[(set (match_operand:V 0 "register_operand" "=vr")
(unspec:V [(reg:SI X0_REGNUM)] UNSPEC_VUNDEF))]
"TARGET_VECTOR"
"")
(define_insn "@vundefined"
[(set (match_operand:VB 0 "register_operand" "=vr")
(unspec:VB [(reg:SI X0_REGNUM)] UNSPEC_VUNDEF))]
"TARGET_VECTOR"
"")
(define_expand "@vreinterpret"
[(set (match_operand:V 0 "register_operand")
(match_operand 1 "vector_any_register_operand"))]
"TARGET_VECTOR"
{
emit_move_insn (operands[0], gen_lowpart (mode, operands[1]));
DONE;
}
)
(define_expand "@vreinterpret"
[(set (match_operand:VB 0 "register_operand")
(match_operand 1 "vector_any_register_operand"))]
"TARGET_VECTOR"
{
emit_move_insn (operands[0], gen_lowpart (mode, operands[1]));
DONE;
}
)
(define_expand "@vlmul_extx2"
[(set (match_operand: 0 "register_operand")
(subreg:
(match_operand:VLMULEXT2 1 "register_operand") 0))]
"TARGET_VECTOR"
{})
(define_expand "@vlmul_extx4"
[(set (match_operand: 0 "register_operand")
(subreg:
(match_operand:VLMULEXT4 1 "register_operand") 0))]
"TARGET_VECTOR"
{})
(define_expand "@vlmul_extx8"
[(set (match_operand: 0 "register_operand")
(subreg:
(match_operand:VLMULEXT8 1 "register_operand") 0))]
"TARGET_VECTOR"
{})
(define_expand "@vlmul_extx16"
[(set (match_operand: 0 "register_operand")
(subreg:
(match_operand:VLMULEXT16 1 "register_operand") 0))]
"TARGET_VECTOR"
{})
(define_expand "@vlmul_extx32"
[(set (match_operand: 0 "register_operand")
(subreg:
(match_operand:VLMULEXT32 1 "register_operand") 0))]
"TARGET_VECTOR"
{})
(define_expand "@vlmul_extx64"
[(set (match_operand: 0 "register_operand")
(subreg:
(match_operand:VLMULEXT64 1 "register_operand") 0))]
"TARGET_VECTOR"
{})
(define_insn_and_split "*vlmul_extx2"
[(set (match_operand: 0 "register_operand" "=vr, ?&vr")
(subreg:
(match_operand:VLMULEXT2 1 "register_operand" " 0, vr") 0))]
"TARGET_VECTOR"
"#"
"&& reload_completed"
[(const_int 0)]
{
emit_insn (gen_rtx_SET (gen_lowpart (mode, operands[0]), operands[1]));
DONE;
})
(define_insn_and_split "*vlmul_extx4"
[(set (match_operand: 0 "register_operand" "=vr, ?&vr")
(subreg:
(match_operand:VLMULEXT4 1 "register_operand" " 0, vr") 0))]
"TARGET_VECTOR"
"#"
"&& reload_completed"
[(const_int 0)]
{
emit_insn (gen_rtx_SET (gen_lowpart (mode, operands[0]), operands[1]));
DONE;
})
(define_insn_and_split "*vlmul_extx8"
[(set (match_operand: 0 "register_operand" "=vr, ?&vr")
(subreg:
(match_operand:VLMULEXT8 1 "register_operand" " 0, vr") 0))]
"TARGET_VECTOR"
"#"
"&& reload_completed"
[(const_int 0)]
{
emit_insn (gen_rtx_SET (gen_lowpart (mode, operands[0]), operands[1]));
DONE;
})
(define_insn_and_split "*vlmul_extx16"
[(set (match_operand: 0 "register_operand" "=vr, ?&vr")
(subreg:
(match_operand:VLMULEXT16 1 "register_operand" " 0, vr") 0))]
"TARGET_VECTOR"
"#"
"&& reload_completed"
[(const_int 0)]
{
emit_insn (gen_rtx_SET (gen_lowpart (mode, operands[0]), operands[1]));
DONE;
})
(define_insn_and_split "*vlmul_extx32"
[(set (match_operand: 0 "register_operand" "=vr, ?&vr")
(subreg:
(match_operand:VLMULEXT32 1 "register_operand" " 0, vr") 0))]
"TARGET_VECTOR"
"#"
"&& reload_completed"
[(const_int 0)]
{
emit_insn (gen_rtx_SET (gen_lowpart (mode, operands[0]), operands[1]));
DONE;
})
(define_insn_and_split "*vlmul_extx64"
[(set (match_operand: 0 "register_operand" "=vr, ?&vr")
(subreg:
(match_operand:VLMULEXT64 1 "register_operand" " 0, vr") 0))]
"TARGET_VECTOR"
"#"
"&& reload_completed"
[(const_int 0)]
{
emit_insn (gen_rtx_SET (gen_lowpart (mode, operands[0]), operands[1]));
DONE;
})
;; This pattern is used to hold the AVL operand for
;; RVV instructions that implicity use VLMAX AVL.
;; RVV instruction implicitly use GPR that is ultimately
;; defined by this pattern is safe for VSETVL pass emit
;; a vsetvl instruction modify this register after RA.
;; Case 1:
;; vlmax_avl a5
;; ... (across many blocks)
;; vadd (implicit use a5) ====> emit: vsetvl a5,zero
;; Case 2:
;; vlmax_avl a5
;; ... (across many blocks)
;; mv a6,a5
;; ... (across many blocks)
;; vadd (implicit use a6) ====> emit: vsetvl a6,zero
;; Case 3:
;; vlmax_avl a5
;; ... (across many blocks)
;; store mem,a5 (spill)
;; ... (across many blocks)
;; load a7,mem (spill)
;; ... (across many blocks)
;; vadd (implicit use a7) ====> emit: vsetvl a7,zero
;; Such cases are all safe for VSETVL PASS to emit a vsetvl
;; instruction that modifies the AVL operand.
(define_insn "@vlmax_avl"
[(set (match_operand:P 0 "register_operand" "=r")
(unspec:P [(match_operand:P 1 "const_int_operand" "i")] UNSPEC_VLMAX))]
"TARGET_VECTOR"
"")
;; -----------------------------------------------------------------
;; ---- Moves Operations
;; -----------------------------------------------------------------
(define_expand "mov"
[(set (match_operand:V 0 "reg_or_mem_operand")
(match_operand:V 1 "general_operand"))]
"TARGET_VECTOR"
{
/* For whole register move, we transform the pattern into the format
that excludes the clobber of scratch register.
We include clobber of a scalar scratch register which is going to be
used for emit of vsetvl instruction after reload_completed since we
need vsetvl instruction to set VL/VTYPE global status for fractional
vector load/store.
For example:
[(set (match_operand:VNx1QI v24)
(match_operand:VNx1QI (mem: a4)))
(clobber (scratch:SI a5))]
====>> vsetvl a5,zero,e8,mf8
====>> vle8.v v24,(a4)
Philosophy:
- Clobber a scalar scratch register for each mov.
- Classify the machine_mode mode = mode into 2 class:
Whole register move and fractional register move.
- Transform and remove scratch clobber register for whole
register move so that we can avoid occupying the scalar
registers.
- We can not leave it to TARGET_SECONDARY_RELOAD since it happens
before spilling. The clobber scratch is used by spilling fractional
registers in IRA/LRA so it's too early. */
if (riscv_vector::legitimize_move (operands[0], operands[1], mode))
DONE;
})
;; This pattern is used for code-gen for whole register load/stores.
;; Also applicable for all register moves.
;; Fractional vector modes load/store are not allowed to match this pattern.
;; Mask modes load/store are not allowed to match this pattern.
;; We seperate "*mov" into "*mov_whole" and "*mov_fract" because
;; we don't want to include fractional load/store in "*mov" which will
;; create unexpected patterns in LRA.
;; For example:
;; ira rtl:
;; (insn 20 19 9 2 (set (reg/v:VNx2QI 97 v1 [ v1 ])
;; (reg:VNx2QI 134 [ _1 ])) "rvv.c":9:22 571 {*movvnx2qi_fract}
;; (nil))
;; When the value of pseudo register 134 of the insn above is discovered already
;; spilled in the memory during LRA.
;; LRA will reload this pattern into a memory load instruction pattern.
;; Because VNx2QI is a fractional vector, we want LRA reload this pattern into
;; (insn 20 19 9 2 (parallel [
;; (set (reg:VNx2QI 98 v2 [orig:134 _1 ] [134])
;; (mem/c:VNx2QI (reg:SI 13 a3 [155]) [1 %sfp+[-2, -2] S[2, 2] A8]))
;; (clobber (reg:SI 14 a4 [149]))])
;; So that we could be able to emit vsetvl instruction using clobber sratch a4.
;; To let LRA generate the expected pattern, we should exclude fractional vector
;; load/store in "*mov_whole". Otherwise, it will reload this pattern into:
;; (insn 20 19 9 2 (set (reg:VNx2QI 98 v2 [orig:134 _1 ] [134])
;; (mem/c:VNx2QI (reg:SI 13 a3 [155]) [1 %sfp+[-2, -2] S[2, 2] A8])))
;; which is not the pattern we want.
;; According the facts above, we make "*mov_whole" includes load/store/move for whole
;; vector modes according to '-march' and "*mov_fract" only include fractional vector modes.
(define_insn "*mov_whole"
[(set (match_operand:V_WHOLE 0 "reg_or_mem_operand" "=vr, m,vr")
(match_operand:V_WHOLE 1 "reg_or_mem_operand" " m,vr,vr"))]
"TARGET_VECTOR"
"@
vl%m1re.v\t%0,%1
vs%m1r.v\t%1,%0
vmv%m1r.v\t%0,%1"
[(set_attr "type" "vldr,vstr,vmov")
(set_attr "mode" "")])
(define_insn "*mov_fract"
[(set (match_operand:V_FRACT 0 "register_operand" "=vr")
(match_operand:V_FRACT 1 "register_operand" " vr"))]
"TARGET_VECTOR"
"vmv1r.v\t%0,%1"
[(set_attr "type" "vmov")
(set_attr "mode" "")])
(define_expand "mov"
[(set (match_operand:VB 0 "reg_or_mem_operand")
(match_operand:VB 1 "general_operand"))]
"TARGET_VECTOR"
{
if (riscv_vector::legitimize_move (operands[0], operands[1], mode))
DONE;
})
(define_insn "*mov"
[(set (match_operand:VB 0 "register_operand" "=vr")
(match_operand:VB 1 "register_operand" " vr"))]
"TARGET_VECTOR"
"vmv1r.v\t%0,%1"
[(set_attr "type" "vmov")
(set_attr "mode" "")])
(define_expand "@mov_lra"
[(parallel
[(set (match_operand:V_FRACT 0 "reg_or_mem_operand")
(match_operand:V_FRACT 1 "reg_or_mem_operand"))
(clobber (match_scratch:P 2))])]
"TARGET_VECTOR && (lra_in_progress || reload_completed)"
{})
(define_expand "@mov_lra"
[(parallel
[(set (match_operand:VB 0 "reg_or_mem_operand")
(match_operand:VB 1 "reg_or_mem_operand"))
(clobber (match_scratch:P 2))])]
"TARGET_VECTOR && (lra_in_progress || reload_completed)"
{})
(define_insn_and_split "*mov_lra"
[(set (match_operand:V_FRACT 0 "reg_or_mem_operand" "=vr, m,vr")
(match_operand:V_FRACT 1 "reg_or_mem_operand" " m,vr,vr"))
(clobber (match_scratch:P 2 "=&r,&r,X"))]
"TARGET_VECTOR && (lra_in_progress || reload_completed)"
"#"
"&& reload_completed"
[(const_int 0)]
{
if (REG_P (operands[0]) && REG_P (operands[1]))
emit_insn (gen_rtx_SET (operands[0], operands[1]));
else
{
riscv_vector::emit_vlmax_vsetvl (mode, operands[2]);
riscv_vector::emit_vlmax_reg_op (code_for_pred_mov (mode),
operands[0], operands[1], operands[2],
mode);
}
DONE;
})
(define_insn_and_split "*mov_lra"
[(set (match_operand:VB 0 "reg_or_mem_operand" "=vr, m,vr")
(match_operand:VB 1 "reg_or_mem_operand" " m,vr,vr"))
(clobber (match_scratch:P 2 "=&r,&r,X"))]
"TARGET_VECTOR && (lra_in_progress || reload_completed)"
"#"
"&& reload_completed"
[(const_int 0)]
{
if (REG_P (operands[0]) && REG_P (operands[1]))
emit_insn (gen_rtx_SET (operands[0], operands[1]));
else
{
riscv_vector::emit_vlmax_vsetvl (mode, operands[2]);
riscv_vector::emit_vlmax_reg_op (code_for_pred_mov (mode),
operands[0], operands[1], operands[2],
mode);
}
DONE;
})
;; Define tuple modes data movement.
;; operands[2] is used to save the offset of each subpart.
;; operands[3] is used to calculate the address for each subpart.
;; operands[4] is VL of vsevli instruction.
(define_expand "mov"
[(parallel [(set (match_operand:VT 0 "reg_or_mem_operand")
(match_operand:VT 1 "general_operand"))
(clobber (match_dup 2))
(clobber (match_dup 3))
(clobber (match_dup 4))])]
"TARGET_VECTOR"
{
/* Need to force register if mem <- !reg. */
if (MEM_P (operands[0]) && !REG_P (operands[1]))
operands[1] = force_reg (mode, operands[1]);
if (GET_CODE (operands[1]) == CONST_VECTOR)
{
riscv_vector::expand_tuple_move (mode, operands);
DONE;
}
operands[2] = gen_rtx_SCRATCH (Pmode);
operands[3] = gen_rtx_SCRATCH (Pmode);
operands[4] = gen_rtx_SCRATCH (Pmode);
})
(define_insn_and_split "*mov_"
[(set (match_operand:VT 0 "reg_or_mem_operand" "=vr,vr, m")
(match_operand:VT 1 "reg_or_mem_operand" " vr, m,vr"))
(clobber (match_scratch:P 2 "=X,&r,&r"))
(clobber (match_scratch:P 3 "=X,&r,&r"))
(clobber (match_scratch:P 4 "=X,&r,&r"))]
"TARGET_VECTOR"
"#"
"&& reload_completed"
[(const_int 0)]
{
riscv_vector::expand_tuple_move (mode, operands);
DONE;
}
[(set_attr "type" "vmov,vlde,vste")
(set_attr "mode" "")])
;; -----------------------------------------------------------------
;; ---- Duplicate Operations
;; -----------------------------------------------------------------
;; According to GCC internal:
;; This pattern only handles duplicates of non-constant inputs.
;; Constant vectors go through the movm pattern instead.
;; So "direct_broadcast_operand" can only be mem or reg, no CONSTANT.
(define_expand "vec_duplicate"
[(set (match_operand:V 0 "register_operand")
(vec_duplicate:V
(match_operand: 1 "direct_broadcast_operand")))]
"TARGET_VECTOR"
{
riscv_vector::emit_vlmax_op (code_for_pred_broadcast (mode),
operands[0], operands[1], mode);
DONE;
}
)
;; -----------------------------------------------------------------
;; ---- 6. Configuration-Setting Instructions
;; -----------------------------------------------------------------
;; Includes:
;; - 6.1 vsetvli/vsetivl/vsetvl instructions
;; -----------------------------------------------------------------
;; we dont't define vsetvli as unspec_volatile which has side effects.
;; This instruction can be scheduled by the instruction scheduler.
;; This means these instructions will be deleted when
;; there is no instructions using vl or vtype in the following.
;; rd | rs1 | AVL value | Effect on vl
;; - | !x0 | x[rs1] | Normal stripmining
;; !x0 | x0 | ~0 | Set vl to VLMAX
;; operands[0]: VL.
;; operands[1]: AVL.
;; operands[2]: SEW
;; operands[3]: LMUL
;; operands[4]: Tail policy 0 or 1 (undisturbed/agnostic)
;; operands[5]: Mask policy 0 or 1 (undisturbed/agnostic)
;; We define 2 types of "vsetvl*" instruction patterns:
;; - "@vsetvl" is a parallel format which has side effects.
;; - "@vsetvl_no_side_effects" has no side effects.
;; - "@vsetvl" is used by "vsetvl" intrinsics and "insert-vsetvl" PASS.
;; - "@vsetvl_no_side_effects" is used by GCC standard patterns.
;; - "@vsetvl" includes VL/VTYPE global registers status (define set)
;; and each RVV instruction includes VL/VTYPE global registers status (use)
;; so that we can guarantee each RVV instruction can execute with correct
;; VL/VTYPE global registers status after "insert-vsetvl" PASS.
;; - "@vsetvl_no_side_effects" has no side effects and excludes VL/VTYPE
;; global registers status (define set). It's only used by GCC standard pattern
;; expansion. For example: "mov" pattern for fractional vector modes which
;; need to set VL/VTYPE. Then we could manually call this pattern to gain benefits
;; from the optimization of each GCC internal PASS.
;; 1. void foo (float *in, float *out)
;; {
;; vfloat32mf2_t v = *(vfloat32mf2_t*)in;
;; *(vfloat32mf2_t*)out = v;
;; }
;; We could eliminate the second "vsetvl" by calling "@vsetvl_no_side_effects".
;;
;; "@vsetvl": ;; "@vsetvl_no_side_effects":
;; vsetvli a4,zero,e32,mf2,ta,ma ;; vsetvli a4,zero,e32,mf2,ta,ma
;; vle32.v v24,(a0) ;; vle32.v v24,(a0)
;; vsetvli a4,zero,e32,mf2,ta,ma ;; --
;; vse32.v v24,(a1) ;; vse32.v v24,(a1)
;; ret ;; ret
;; 2. void foo (int8_t *in, int8_t *out, int M)
;; {
;; for (int i = 0; i < M; i++){
;; vint8mf2_t v = *(vint8mf2_t*)(in + i);
;; *(vint8mf2_t*)(out + i) = v;
;; }
;; }
;;
;; Hoist "vsetvl" instruction in LICM:
;; "@vsetvl": ;; "@vsetvl_no_side_effects":
;; - ;; vsetvli a4,zero,e32,mf2,ta,ma
;; LOOP: ;; LOOP:
;; vsetvli a4,zero,e32,mf2,ta,ma ;; -
;; vle32.v v24,(a0) ;; vle32.v v24,(a0)
;; vsetvli a4,zero,e32,mf2,ta,ma ;; -
;; vse32.v v24,(a1) ;; vse32.v v24,(a1)
;; However, it may produce wrong codegen if we exclude VL/VTYPE in "vsevl".
;; 3. void foo (int8_t *in, int8_t *out, int32_t *in2, int32_t *out2, int M)
;; {
;; for (int i = 0; i < M; i++){
;; vint8mf2_t v = *(vint8mf2_t*)(in + i);
;; vint32mf2_t v2 = *(vint32mf2_t*)(in + i + i);
;; *(vint8mf2_t*)(out + i) = v;
;; *(vint32mf2_t*)(out + i + i) = v2;
;; }
;; }
;;
;; vsetvli a6,zero,e8,mf2,ta,ma
;; vsetvli a2,zero,e32,mf2,ta,ma
;; LOOP:
;; vle8.v v25,(a0)
;; vle32.v v24,(a5)
;; addi a0,a0,1
;; vse8.v v25,(a1)
;; vse32.v v24,(a3)
;;
;; Both vle8.v and vle32.v are using the wrong VL/VTYPE status.
;; We leave it to "insert-vsetvl" PASS to correct this situation.
;; The "insert-vsetvl" PASS mechanism:
;; 1. Before "insert-vsetvl" PASS, only RVV instructions are generated
;; by GCC standard pattern expansion has the corresponding "vsetvl".
;; We exploit each GCC internal optimization pass to optimize the "vsetvl".
;; 2. Correct the VL/VTYPE status for each GCC standard pattern RVV instructions.
;; Insert vsetvl for each RVV instructions that has no VL/VTYPE status if necessary.
;; For example: RVV intrinsics.
;; 3. Optimize "vsetvl" instructions.
(define_insn "@vsetvl"
[(set (match_operand:P 0 "register_operand" "=r")
(unspec:P [(match_operand:P 1 "csr_operand" "rK")
(match_operand 2 "const_int_operand" "i")
(match_operand 3 "const_int_operand" "i")
(match_operand 4 "const_int_operand" "i")
(match_operand 5 "const_int_operand" "i")] UNSPEC_VSETVL))
(set (reg:SI VL_REGNUM)
(unspec:SI [(match_dup 1)
(match_dup 2)
(match_dup 3)] UNSPEC_VSETVL))
(set (reg:SI VTYPE_REGNUM)
(unspec:SI [(match_dup 2)
(match_dup 3)
(match_dup 4)
(match_dup 5)] UNSPEC_VSETVL))]
"TARGET_VECTOR"
"vset%i1vli\t%0,%1,e%2,%m3,t%p4,m%p5"
[(set_attr "type" "vsetvl")
(set_attr "mode" "")
(set (attr "sew") (symbol_ref "INTVAL (operands[2])"))
(set (attr "vlmul") (symbol_ref "INTVAL (operands[3])"))
(set (attr "ta") (symbol_ref "INTVAL (operands[4])"))
(set (attr "ma") (symbol_ref "INTVAL (operands[5])"))])
;; vsetvl zero,zero,vtype instruction.
;; This pattern has no side effects and does not set X0 register.
(define_insn "vsetvl_vtype_change_only"
[(set (reg:SI VTYPE_REGNUM)
(unspec:SI
[(match_operand 0 "const_int_operand" "i")
(match_operand 1 "const_int_operand" "i")
(match_operand 2 "const_int_operand" "i")
(match_operand 3 "const_int_operand" "i")] UNSPEC_VSETVL))]
"TARGET_VECTOR"
"vsetvli\tzero,zero,e%0,%m1,t%p2,m%p3"
[(set_attr "type" "vsetvl")
(set_attr "mode" "SI")])
;; vsetvl zero,rs1,vtype instruction.
;; The reason we need this pattern since we should avoid setting X0 register
;; in vsetvl instruction pattern.
(define_insn "@vsetvl_discard_result"
[(set (reg:SI VL_REGNUM)
(unspec:SI [(match_operand:P 0 "csr_operand" "rK")
(match_operand 1 "const_int_operand" "i")
(match_operand 2 "const_int_operand" "i")] UNSPEC_VSETVL))
(set (reg:SI VTYPE_REGNUM)
(unspec:SI [(match_dup 1)
(match_dup 2)
(match_operand 3 "const_int_operand" "i")
(match_operand 4 "const_int_operand" "i")] UNSPEC_VSETVL))]
"TARGET_VECTOR"
"vset%i0vli\tzero,%0,e%1,%m2,t%p3,m%p4"
[(set_attr "type" "vsetvl")
(set_attr "mode" "")
(set (attr "sew") (symbol_ref "INTVAL (operands[1])"))
(set (attr "vlmul") (symbol_ref "INTVAL (operands[2])"))
(set (attr "ta") (symbol_ref "INTVAL (operands[3])"))
(set (attr "ma") (symbol_ref "INTVAL (operands[4])"))])
;; It's emit by vsetvl/vsetvlmax intrinsics with no side effects.
;; Since we have many optmization passes from "expand" to "reload_completed",
;; such pattern can allow us gain benefits of these optimizations.
(define_insn_and_split "@vsetvl_no_side_effects"
[(set (match_operand:P 0 "register_operand" "=r")
(unspec:P [(match_operand:P 1 "csr_operand" "rK")
(match_operand 2 "const_int_operand" "i")
(match_operand 3 "const_int_operand" "i")
(match_operand 4 "const_int_operand" "i")
(match_operand 5 "const_int_operand" "i")] UNSPEC_VSETVL))]
"TARGET_VECTOR"
"#"
"&& epilogue_completed"
[(parallel
[(set (match_dup 0)
(unspec:P [(match_dup 1) (match_dup 2) (match_dup 3)
(match_dup 4) (match_dup 5)] UNSPEC_VSETVL))
(set (reg:SI VL_REGNUM)
(unspec:SI [(match_dup 1) (match_dup 2) (match_dup 3)] UNSPEC_VSETVL))
(set (reg:SI VTYPE_REGNUM)
(unspec:SI [(match_dup 2) (match_dup 3) (match_dup 4)
(match_dup 5)] UNSPEC_VSETVL))])]
""
[(set_attr "type" "vsetvl")
(set_attr "mode" "SI")])
;; RVV machine description matching format
;; (define_insn ""
;; [(set (match_operand:MODE 0)
;; (if_then_else:MODE
;; (unspec:
;; [(match_operand: 1 "vector_mask_operand")
;; (match_operand N + 4 "vector_length_operand")
;; (match_operand N + 5 "const_int_operand")
;; (match_operand N + 6 "const_int_operand")
;; (reg:SI VL_REGNUM)
;; (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
;; (instruction operation:MODE
;; (match_operand 3
;; (match_operand 4
;; (match_operand 5
;; ................
;; (match_operand N + 3)
;; (match_operand:MODE 2 "vector_reg_or_const0_operand")))]
;;
;; (unspec:[........] UNSPEC_VPREDICATE) is a predicate wrapper.
;; Include mask predicate && length predicate && vector policy.
;; -------------------------------------------------------------------------------
;; ---- Predicated Mov
;; -------------------------------------------------------------------------------
;; Includes:
;; - 7.4. Vector Unit-Stride Instructions
;; - 11.15 Vector Integer Merge Instructions
;; - 11.16 Vector Integer Move Instructions
;; - 13.16 Vector Floating-Point Move Instruction
;; - 15.1 Vector Mask-Register Logical Instructions
;; -------------------------------------------------------------------------------
;; vle.v/vse.v/vmv.v.v/vmv.v.x/vmv.v.i/vfmv.v.f.
;; For vle.v/vmv.v.v/vmv.v.x/vmv.v.i/vfmv.v.f, we may need merge and mask operand.
;; For vse.v, we don't need merge operand, so it should always match "vu".
;; constraint alternative 0 ~ 1 match vle.v.
;; constraint alternative 2 match vse.v.
;; constraint alternative 3 match vmv.v.v.
;; constraint alternative 4 match vmv.v.i.
;; For vmv.v.i, we allow 2 following cases:
;; 1. (const_vector:VNx1QI repeat [
;; (const_int:QI N)]), -15 <= N < 16.
;; 2. (const_vector:VNx1SF repeat [
;; (const_double:SF 0.0 [0x0.0p+0])]).
;; We add "MEM_P (operands[0]) || MEM_P (operands[3]) || CONST_VECTOR_P (operands[1])" here to
;; make sure we don't want CSE to generate the following pattern:
;; (insn 17 8 19 2 (set (reg:VNx1HI 134 [ _1 ])
;; (if_then_else:VNx1HI (unspec:VNx1BI [
;; (reg/v:VNx1BI 137 [ mask ])
;; (reg:DI 151)
;; (const_int 0 [0]) repeated x3
;; (reg:SI 66 vl)
;; (reg:SI 67 vtype)
;; ] UNSPEC_VPREDICATE)
;; (const_vector:VNx1HI repeat [
;; (const_int 0 [0])
;; ])
;; (reg/v:VNx1HI 140 [ merge ]))) "rvv.c":8:12 608 {pred_movvnx1hi}
;; (expr_list:REG_DEAD (reg:DI 151)
;; (expr_list:REG_DEAD (reg/v:VNx1HI 140 [ merge ])
;; (expr_list:REG_DEAD (reg/v:VNx1BI 137 [ mask ])
;; (nil)))))
;; Since both vmv.v.v and vmv.v.i doesn't have mask operand.
(define_insn_and_split "@pred_mov"
[(set (match_operand:V 0 "nonimmediate_operand" "=vr, vr, vd, m, vr, vr, vr, vr")
(if_then_else:V
(unspec:
[(match_operand: 1 "vector_mask_operand" "vmWc1, Wc1, vm, vmWc1, Wc1, Wc1, Wc1, Wc1")
(match_operand 4 "vector_length_operand" " rK, rK, rK, rK, rK, rK, rK, rK")
(match_operand 5 "const_int_operand" " i, i, i, i, i, i, i, i")
(match_operand 6 "const_int_operand" " i, i, i, i, i, i, i, i")
(match_operand 7 "const_int_operand" " i, i, i, i, i, i, i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(match_operand:V 3 "vector_move_operand" " m, m, m, vr, vr, vr, viWc0, viWc0")
(match_operand:V 2 "vector_merge_operand" " 0, vu, vu, vu, vu, 0, vu, 0")))]
"TARGET_VECTOR && (MEM_P (operands[0]) || MEM_P (operands[3])
|| CONST_VECTOR_P (operands[1]))"
"@
vle.v\t%0,%3%p1
vle.v\t%0,%3
vle.v\t%0,%3,%1.t
vse.v\t%3,%0%p1
vmv.v.v\t%0,%3
vmv.v.v\t%0,%3
vmv.v.i\t%0,%v3
vmv.v.i\t%0,%v3"
"&& register_operand (operands[0], mode)
&& register_operand (operands[3], mode)
&& satisfies_constraint_vu (operands[2])
&& INTVAL (operands[7]) == riscv_vector::VLMAX"
[(set (match_dup 0) (match_dup 3))]
""
[(set_attr "type" "vlde,vlde,vlde,vste,vimov,vimov,vimov,vimov")
(set_attr "mode" "")])
;; Dedicated pattern for vse.v instruction since we can't reuse pred_mov pattern to include
;; memory operand as input which will produce inferior codegen.
(define_insn "@pred_store"
[(set (match_operand:V 0 "memory_operand" "+m")
(if_then_else:V
(unspec:
[(match_operand: 1 "vector_mask_operand" "vmWc1")
(match_operand 3 "vector_length_operand" " rK")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(match_operand:V 2 "register_operand" " vr")
(match_dup 0)))]
"TARGET_VECTOR"
"vse.v\t%2,%0%p1"
[(set_attr "type" "vste")
(set_attr "mode" "")
(set (attr "avl_type") (symbol_ref "riscv_vector::NONVLMAX"))
(set_attr "vl_op_idx" "3")])
;; vlm.v/vsm.v/vmclr.m/vmset.m.
;; constraint alternative 0 match vlm.v.
;; constraint alternative 1 match vsm.v.
;; constraint alternative 3 match vmclr.m.
;; constraint alternative 4 match vmset.m.
(define_insn_and_split "@pred_mov"
[(set (match_operand:VB 0 "nonimmediate_operand" "=vr, m, vr, vr, vr")
(if_then_else:VB
(unspec:VB
[(match_operand:VB 1 "vector_all_trues_mask_operand" "Wc1, Wc1, Wc1, Wc1, Wc1")
(match_operand 4 "vector_length_operand" " rK, rK, rK, rK, rK")
(match_operand 5 "const_int_operand" " i, i, i, i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(match_operand:VB 3 "vector_move_operand" " m, vr, vr, Wc0, Wc1")
(match_operand:VB 2 "vector_undef_operand" " vu, vu, vu, vu, vu")))]
"TARGET_VECTOR"
"@
vlm.v\t%0,%3
vsm.v\t%3,%0
vmmv.m\t%0,%3
vmclr.m\t%0
vmset.m\t%0"
"&& register_operand (operands[0], mode)
&& register_operand (operands[3], mode)
&& INTVAL (operands[5]) == riscv_vector::VLMAX"
[(set (match_dup 0) (match_dup 3))]
""
[(set_attr "type" "vldm,vstm,vmalu,vmalu,vmalu")
(set_attr "mode" "")])
;; Dedicated pattern for vsm.v instruction since we can't reuse pred_mov pattern to include
;; memory operand as input which will produce inferior codegen.
(define_insn "@pred_store"
[(set (match_operand:VB 0 "memory_operand" "+m")
(if_then_else:VB
(unspec:VB
[(match_operand:VB 1 "vector_all_trues_mask_operand" "Wc1")
(match_operand 3 "vector_length_operand" " rK")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(match_operand:VB 2 "register_operand" " vr")
(match_dup 0)))]
"TARGET_VECTOR"
"vsm.v\t%2,%0"
[(set_attr "type" "vstm")
(set_attr "mode" "")
(set (attr "avl_type") (symbol_ref "riscv_vector::NONVLMAX"))
(set_attr "vl_op_idx" "3")])
(define_insn "@pred_merge"
[(set (match_operand:V 0 "register_operand" "=vd,vd,vd,vd")
(if_then_else:V
(unspec:
[(match_operand 5 "vector_length_operand" " rK,rK,rK,rK")
(match_operand 6 "const_int_operand" " i, i, i, i")
(match_operand 7 "const_int_operand" " i, i, i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_merge:V
(match_operand:V 3 "vector_arith_operand" " vr,vr,vi,vi")
(match_operand:V 2 "register_operand" " vr,vr,vr,vr")
(match_operand: 4 "register_operand" " vm,vm,vm,vm"))
(match_operand:V 1 "vector_merge_operand" " vu, 0,vu, 0")))]
"TARGET_VECTOR"
"vmerge.v%o3m\t%0,%2,%v3,%4"
[(set_attr "type" "vimerge")
(set_attr "mode" "")])
(define_insn "@pred_merge_scalar"
[(set (match_operand:VI_QHS 0 "register_operand" "=vd,vd")
(if_then_else:VI_QHS
(unspec:
[(match_operand 5 "vector_length_operand" " rK,rK")
(match_operand 6 "const_int_operand" " i, i")
(match_operand 7 "const_int_operand" " i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_merge:VI_QHS
(vec_duplicate:VI_QHS
(match_operand: 3 "register_operand" " r, r"))
(match_operand:VI_QHS 2 "register_operand" " vr,vr")
(match_operand: 4 "register_operand" " vm,vm"))
(match_operand:VI_QHS 1 "vector_merge_operand" " vu, 0")))]
"TARGET_VECTOR"
"vmerge.vxm\t%0,%2,%3,%4"
[(set_attr "type" "vimerge")
(set_attr "mode" "")])
(define_expand "@pred_merge_scalar"
[(set (match_operand:VI_D 0 "register_operand")
(if_then_else:VI_D
(unspec:
[(match_operand 5 "vector_length_operand")
(match_operand 6 "const_int_operand")
(match_operand 7 "const_int_operand")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_merge:VI_D
(vec_duplicate:VI_D
(match_operand: 3 "reg_or_int_operand"))
(match_operand:VI_D 2 "register_operand")
(match_operand: 4 "register_operand"))
(match_operand:VI_D 1 "vector_merge_operand")))]
"TARGET_VECTOR"
{
if (riscv_vector::sew64_scalar_helper (
operands,
/* scalar op */&operands[3],
/* vl */operands[5],
mode,
mode,
riscv_vector::simm5_p (operands[3]),
[] (rtx *operands, rtx boardcast_scalar) {
emit_insn (gen_pred_merge (operands[0], operands[1],
operands[2], boardcast_scalar, operands[4], operands[5],
operands[6], operands[7]));
}))
DONE;
})
(define_insn "*pred_merge_scalar"
[(set (match_operand:VI_D 0 "register_operand" "=vd,vd")
(if_then_else:VI_D
(unspec:
[(match_operand 5 "vector_length_operand" " rK,rK")
(match_operand 6 "const_int_operand" " i, i")
(match_operand 7 "const_int_operand" " i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_merge:VI_D
(vec_duplicate:VI_D
(match_operand: 3 "register_operand" " r, r"))
(match_operand:VI_D 2 "register_operand" " vr,vr")
(match_operand: 4 "register_operand" " vm,vm"))
(match_operand:VI_D 1 "vector_merge_operand" " vu, 0")))]
"TARGET_VECTOR"
"vmerge.vxm\t%0,%2,%3,%4"
[(set_attr "type" "vimerge")
(set_attr "mode" "")])
(define_insn "*pred_merge_extended_scalar"
[(set (match_operand:VI_D 0 "register_operand" "=vd,vd")
(if_then_else:VI_D
(unspec:
[(match_operand 5 "vector_length_operand" " rK,rK")
(match_operand 6 "const_int_operand" " i, i")
(match_operand 7 "const_int_operand" " i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_merge:VI_D
(vec_duplicate:VI_D
(sign_extend:
(match_operand: 3 "register_operand" " r, r")))
(match_operand:VI_D 2 "register_operand" " vr,vr")
(match_operand: 4 "register_operand" " vm,vm"))
(match_operand:VI_D 1 "vector_merge_operand" " vu, 0")))]
"TARGET_VECTOR"
"vmerge.vxm\t%0,%2,%3,%4"
[(set_attr "type" "vimerge")
(set_attr "mode" "")])
;; -------------------------------------------------------------------------------
;; ---- Predicated Broadcast
;; -------------------------------------------------------------------------------
;; Includes:
;; - 7.5. Vector Strided Instructions (zero stride)
;; - 11.16 Vector Integer Move Instructions (vmv.v.x)
;; - 13.16 Vector Floating-Point Move Instruction (vfmv.v.f)
;; - 16.1 Integer Scalar Move Instructions (vmv.s.x)
;; - 16.2 Floating-Point Scalar Move Instructions (vfmv.s.f)
;; -------------------------------------------------------------------------------
;; According to RVV ISA, vector-scalar instruction doesn't support
;; operand fetched from 2 consecutive registers, so we should use
;; vlse.v which is a memory access to broadcast a DImode scalar into a vector.
;;
;; Since the optimization flow in GCC is as follows:
;; expand --> LICM (Loop invariant) --> split.
;; To use LICM optimization, we postpone generation of vlse.v to split stage since
;; a memory access instruction can not be optimized by LICM (Loop invariant).
(define_expand "@pred_broadcast"
[(set (match_operand:V 0 "register_operand")
(if_then_else:V
(unspec:
[(match_operand: 1 "vector_broadcast_mask_operand")
(match_operand 4 "vector_length_operand")
(match_operand 5 "const_int_operand")
(match_operand 6 "const_int_operand")
(match_operand 7 "const_int_operand")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_duplicate:V
(match_operand: 3 "direct_broadcast_operand"))
(match_operand:V 2 "vector_merge_operand")))]
"TARGET_VECTOR"
{
/* Handle vmv.s.x instruction which has memory scalar. */
if (satisfies_constraint_Wdm (operands[3]) || riscv_vector::simm5_p (operands[3])
|| rtx_equal_p (operands[3], CONST0_RTX (mode)))
{
if (satisfies_constraint_Wb1 (operands[1]))
{
// Case 1: vmv.s.x (TA) ==> vlse.v (TA)
if (satisfies_constraint_vu (operands[2]))
operands[1] = CONSTM1_RTX (mode);
else if (GET_MODE_BITSIZE (mode) > GET_MODE_BITSIZE (Pmode))
{
// Case 2: vmv.s.x (TU) ==> andi vl + vlse.v (TU) in RV32 system.
operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
operands[1] = CONSTM1_RTX (mode);
}
else
operands[3] = force_reg (mode, operands[3]);
}
}
else if (GET_MODE_BITSIZE (mode) > GET_MODE_BITSIZE (Pmode)
&& immediate_operand (operands[3], Pmode))
operands[3] = gen_rtx_SIGN_EXTEND (mode, force_reg (Pmode, operands[3]));
else
operands[3] = force_reg (mode, operands[3]);
})
(define_insn_and_split "*pred_broadcast"
[(set (match_operand:VI 0 "register_operand" "=vr, vr, vd, vd, vr, vr, vr, vr")
(if_then_else:VI
(unspec:
[(match_operand: 1 "vector_broadcast_mask_operand" "Wc1,Wc1, vm, vm,Wc1,Wc1,Wb1,Wb1")
(match_operand 4 "vector_length_operand" " rK, rK, rK, rK, rK, rK, rK, rK")
(match_operand 5 "const_int_operand" " i, i, i, i, i, i, i, i")
(match_operand 6 "const_int_operand" " i, i, i, i, i, i, i, i")
(match_operand 7 "const_int_operand" " i, i, i, i, i, i, i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_duplicate:VI
(match_operand: 3 "direct_broadcast_operand" " r, r,Wdm,Wdm,Wdm,Wdm, r, r"))
(match_operand:VI 2 "vector_merge_operand" "vu, 0, vu, 0, vu, 0, vu, 0")))]
"TARGET_VECTOR"
"@
vmv.v.x\t%0,%3
vmv.v.x\t%0,%3
vlse.v\t%0,%3,zero,%1.t
vlse.v\t%0,%3,zero,%1.t
vlse.v\t%0,%3,zero
vlse.v\t%0,%3,zero
vmv.s.x\t%0,%3
vmv.s.x\t%0,%3"
"register_operand (operands[3], mode)
&& GET_MODE_BITSIZE (mode) > GET_MODE_BITSIZE (Pmode)"
[(set (match_dup 0)
(if_then_else:VI (unspec: [(match_dup 1) (match_dup 4)
(match_dup 5) (match_dup 6) (match_dup 7)
(reg:SI VL_REGNUM) (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_duplicate:VI (match_dup 3))
(match_dup 2)))]
{
gcc_assert (can_create_pseudo_p ());
rtx m = assign_stack_local (mode, GET_MODE_SIZE (mode),
GET_MODE_ALIGNMENT (mode));
m = validize_mem (m);
emit_move_insn (m, operands[3]);
m = gen_rtx_MEM (mode, force_reg (Pmode, XEXP (m, 0)));
operands[3] = m;
/* For SEW = 64 in RV32 system, we expand vmv.s.x:
andi a2,a2,1
vsetvl zero,a2,e64
vlse64.v */
if (satisfies_constraint_Wb1 (operands[1]))
{
operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
operands[1] = CONSTM1_RTX (mode);
}
}
[(set_attr "type" "vimov,vimov,vlds,vlds,vlds,vlds,vimovxv,vimovxv")
(set_attr "mode" "")])
(define_insn "*pred_broadcast"
[(set (match_operand:VF 0 "register_operand" "=vr, vr, vr, vr, vr, vr, vr, vr")
(if_then_else:VF
(unspec:
[(match_operand: 1 "vector_broadcast_mask_operand" "Wc1,Wc1, vm, vm,Wc1,Wc1,Wb1,Wb1")
(match_operand 4 "vector_length_operand" " rK, rK, rK, rK, rK, rK, rK, rK")
(match_operand 5 "const_int_operand" " i, i, i, i, i, i, i, i")
(match_operand 6 "const_int_operand" " i, i, i, i, i, i, i, i")
(match_operand 7 "const_int_operand" " i, i, i, i, i, i, i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_duplicate:VF
(match_operand: 3 "direct_broadcast_operand" " f, f,Wdm,Wdm,Wdm,Wdm, f, f"))
(match_operand:VF 2 "vector_merge_operand" "vu, 0, vu, 0, vu, 0, vu, 0")))]
"TARGET_VECTOR"
"@
vfmv.v.f\t%0,%3
vfmv.v.f\t%0,%3
vlse.v\t%0,%3,zero,%1.t
vlse.v\t%0,%3,zero,%1.t
vlse.v\t%0,%3,zero
vlse.v\t%0,%3,zero
vfmv.s.f\t%0,%3
vfmv.s.f\t%0,%3"
[(set_attr "type" "vfmov,vfmov,vlds,vlds,vlds,vlds,vfmovfv,vfmovfv")
(set_attr "mode" "")])
(define_insn "*pred_broadcast_extended_scalar"
[(set (match_operand:VI_D 0 "register_operand" "=vr, vr, vr, vr")
(if_then_else:VI_D
(unspec:
[(match_operand: 1 "vector_broadcast_mask_operand" "Wc1,Wc1,Wb1,Wb1")
(match_operand 4 "vector_length_operand" " rK, rK, rK, rK")
(match_operand 5 "const_int_operand" " i, i, i, i")
(match_operand 6 "const_int_operand" " i, i, i, i")
(match_operand 7 "const_int_operand" " i, i, i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(vec_duplicate:VI_D
(sign_extend:
(match_operand: 3 "register_operand" " r, r, r, r")))
(match_operand:VI_D 2 "vector_merge_operand" "vu, 0, vu, 0")))]
"TARGET_VECTOR"
"@
vmv.v.x\t%0,%3
vmv.v.x\t%0,%3
vmv.s.x\t%0,%3
vmv.s.x\t%0,%3"
[(set_attr "type" "vimov,vimov,vimovxv,vimovxv")
(set_attr "mode" "")])
;; -------------------------------------------------------------------------------
;; ---- Predicated Strided loads/stores
;; -------------------------------------------------------------------------------
;; Includes:
;; - 7.5. Vector Strided Instructions
;; -------------------------------------------------------------------------------
(define_insn "@pred_strided_load"
[(set (match_operand:V 0 "register_operand" "=vr, vr, vd")
(if_then_else:V
(unspec:
[(match_operand: 1 "vector_mask_operand" "vmWc1, Wc1, vm")
(match_operand 5 "vector_length_operand" " rK, rK, rK")
(match_operand 6 "const_int_operand" " i, i, i")
(match_operand 7 "const_int_operand" " i, i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(unspec:V
[(match_operand:V 3 "memory_operand" " m, m, m")
(match_operand 4 "pmode_reg_or_0_operand" " rJ, rJ, rJ")] UNSPEC_STRIDED)
(match_operand:V 2 "vector_merge_operand" " 0, vu, vu")))]
"TARGET_VECTOR"
"vlse.v\t%0,%3,%z4%p1"
[(set_attr "type" "vlds")
(set_attr "mode" "")])
(define_insn "@pred_strided_store"
[(set (match_operand:V 0 "memory_operand" "+m")
(if_then_else:V
(unspec:
[(match_operand: 1 "vector_mask_operand" "vmWc1")
(match_operand 4 "vector_length_operand" " rK")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(unspec:V
[(match_operand 2 "pmode_reg_or_0_operand" " rJ")
(match_operand:V 3 "register_operand" " vr")] UNSPEC_STRIDED)
(match_dup 0)))]
"TARGET_VECTOR"
"vsse.v\t%3,%0,%z2%p1"
[(set_attr "type" "vsts")
(set_attr "mode" "")])
;; -------------------------------------------------------------------------------
;; ---- Predicated indexed loads/stores
;; -------------------------------------------------------------------------------
;; Includes:
;; - 7.6. Vector Indexed Instructions
;; -------------------------------------------------------------------------------
;; DEST eew is same as SOURCE eew, DEST register can overlap SOURCE.
(define_insn "@pred_indexed_load_same_eew"
[(set (match_operand:V 0 "register_operand" "=vd, vr,vd, vr")
(if_then_else:V
(unspec:
[(match_operand: 1 "vector_mask_operand" " vm,Wc1,vm,Wc1")
(match_operand 5 "vector_length_operand" " rK, rK,rK, rK")
(match_operand 6 "const_int_operand" " i, i, i, i")
(match_operand 7 "const_int_operand" " i, i, i, i")
(match_operand 8 "const_int_operand" " i, i, i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(unspec:V
[(match_operand 3 "pmode_reg_or_0_operand" " rJ, rJ,rJ, rJ")
(mem:BLK (scratch))
(match_operand: 4 "register_operand" " vr, vr,vr, vr")] ORDER)
(match_operand:V 2 "vector_merge_operand" " vu, vu, 0, 0")))]
"TARGET_VECTOR"
"vlxei.v\t%0,(%z3),%4%p1"
[(set_attr "type" "vldx")
(set_attr "mode" "")])
;; DEST eew is greater than SOURCE eew.
(define_insn "@pred_indexed_load_x2_greater_eew"
[(set (match_operand:VEEWEXT2 0 "register_operand" "=&vr, &vr")
(if_then_else:VEEWEXT2
(unspec:
[(match_operand: 1 "vector_mask_operand" "vmWc1,vmWc1")
(match_operand 5 "vector_length_operand" " rK, rK")
(match_operand 6 "const_int_operand" " i, i")
(match_operand 7 "const_int_operand" " i, i")
(match_operand 8 "const_int_operand" " i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(unspec:VEEWEXT2
[(match_operand 3 "pmode_reg_or_0_operand" " rJ, rJ")
(mem:BLK (scratch))
(match_operand: 4 "register_operand" " vr, vr")] ORDER)
(match_operand:VEEWEXT2 2 "vector_merge_operand" " vu, 0")))]
"TARGET_VECTOR"
"vlxei.v\t%0,(%z3),%4%p1"
[(set_attr "type" "vldx")
(set_attr "mode" "")])
(define_insn "@pred_indexed_load_x4_greater_eew"
[(set (match_operand:VEEWEXT4 0 "register_operand" "=&vr, &vr")
(if_then_else:VEEWEXT4
(unspec:
[(match_operand: 1 "vector_mask_operand" "vmWc1,vmWc1")
(match_operand 5 "vector_length_operand" " rK, rK")
(match_operand 6 "const_int_operand" " i, i")
(match_operand 7 "const_int_operand" " i, i")
(match_operand 8 "const_int_operand" " i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(unspec:VEEWEXT4
[(match_operand 3 "pmode_reg_or_0_operand" " rJ, rJ")
(mem:BLK (scratch))
(match_operand: 4 "register_operand" " vr, vr")] ORDER)
(match_operand:VEEWEXT4 2 "vector_merge_operand" " vu, 0")))]
"TARGET_VECTOR"
"vlxei.v\t%0,(%z3),%4%p1"
[(set_attr "type" "vldx")
(set_attr "mode" "")])
(define_insn "@pred_indexed_load_x8_greater_eew"
[(set (match_operand:VEEWEXT8 0 "register_operand" "=&vr, &vr")
(if_then_else:VEEWEXT8
(unspec:
[(match_operand: 1 "vector_mask_operand" "vmWc1,vmWc1")
(match_operand 5 "vector_length_operand" " rK, rK")
(match_operand 6 "const_int_operand" " i, i")
(match_operand 7 "const_int_operand" " i, i")
(match_operand 8 "const_int_operand" " i, i")
(reg:SI VL_REGNUM)
(reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
(unspec:VEEWEXT8
[(match_operand 3 "pmode_reg_or_0_operand" " rJ, rJ")
(mem:BLK (scratch))
(match_operand: 4 "register_operand" " vr, vr")] ORDER)
(match_operand:VEEWEXT8 2 "vector_merge_operand" " vu, 0")))]
"TARGET_VECTOR"
"vlxei.v\t%0,(%z3),%4%p1"
[(set_attr "type" "vldx")
(set_attr "mode" "")])
;; DEST eew is smaller than SOURCE eew.
(define_insn "@pred_indexed_load_x2_smaller_eew"
[(set (match_operand:VEEWTRUNC2 0 "register_operand" "=vd, vd, vr, vr, &vr, &vr")
(if_then_else:VEEWTRUNC2
(unspec: