target/arm: Add mte helpers for sve scatter/gather memory ops

+285

target/arm/helper-sve.h

··· 1617 1617 DEF_HELPER_FLAGS_6(sve_ldsds_be_zd, TCG_CALL_NO_WG, 1618 1618 void, env, ptr, ptr, ptr, tl, i32) 1619 1619 1620 + DEF_HELPER_FLAGS_6(sve_ldbsu_zsu_mte, TCG_CALL_NO_WG, 1621 + void, env, ptr, ptr, ptr, tl, i32) 1622 + DEF_HELPER_FLAGS_6(sve_ldhsu_le_zsu_mte, TCG_CALL_NO_WG, 1623 + void, env, ptr, ptr, ptr, tl, i32) 1624 + DEF_HELPER_FLAGS_6(sve_ldhsu_be_zsu_mte, TCG_CALL_NO_WG, 1625 + void, env, ptr, ptr, ptr, tl, i32) 1626 + DEF_HELPER_FLAGS_6(sve_ldss_le_zsu_mte, TCG_CALL_NO_WG, 1627 + void, env, ptr, ptr, ptr, tl, i32) 1628 + DEF_HELPER_FLAGS_6(sve_ldss_be_zsu_mte, TCG_CALL_NO_WG, 1629 + void, env, ptr, ptr, ptr, tl, i32) 1630 + DEF_HELPER_FLAGS_6(sve_ldbss_zsu_mte, TCG_CALL_NO_WG, 1631 + void, env, ptr, ptr, ptr, tl, i32) 1632 + DEF_HELPER_FLAGS_6(sve_ldhss_le_zsu_mte, TCG_CALL_NO_WG, 1633 + void, env, ptr, ptr, ptr, tl, i32) 1634 + DEF_HELPER_FLAGS_6(sve_ldhss_be_zsu_mte, TCG_CALL_NO_WG, 1635 + void, env, ptr, ptr, ptr, tl, i32) 1636 + 1637 + DEF_HELPER_FLAGS_6(sve_ldbsu_zss_mte, TCG_CALL_NO_WG, 1638 + void, env, ptr, ptr, ptr, tl, i32) 1639 + DEF_HELPER_FLAGS_6(sve_ldhsu_le_zss_mte, TCG_CALL_NO_WG, 1640 + void, env, ptr, ptr, ptr, tl, i32) 1641 + DEF_HELPER_FLAGS_6(sve_ldhsu_be_zss_mte, TCG_CALL_NO_WG, 1642 + void, env, ptr, ptr, ptr, tl, i32) 1643 + DEF_HELPER_FLAGS_6(sve_ldss_le_zss_mte, TCG_CALL_NO_WG, 1644 + void, env, ptr, ptr, ptr, tl, i32) 1645 + DEF_HELPER_FLAGS_6(sve_ldss_be_zss_mte, TCG_CALL_NO_WG, 1646 + void, env, ptr, ptr, ptr, tl, i32) 1647 + DEF_HELPER_FLAGS_6(sve_ldbss_zss_mte, TCG_CALL_NO_WG, 1648 + void, env, ptr, ptr, ptr, tl, i32) 1649 + DEF_HELPER_FLAGS_6(sve_ldhss_le_zss_mte, TCG_CALL_NO_WG, 1650 + void, env, ptr, ptr, ptr, tl, i32) 1651 + DEF_HELPER_FLAGS_6(sve_ldhss_be_zss_mte, TCG_CALL_NO_WG, 1652 + void, env, ptr, ptr, ptr, tl, i32) 1653 + 1654 + DEF_HELPER_FLAGS_6(sve_ldbdu_zsu_mte, TCG_CALL_NO_WG, 1655 + void, env, ptr, ptr, ptr, tl, i32) 1656 + DEF_HELPER_FLAGS_6(sve_ldhdu_le_zsu_mte, TCG_CALL_NO_WG, 1657 + void, env, ptr, ptr, ptr, tl, i32) 1658 + DEF_HELPER_FLAGS_6(sve_ldhdu_be_zsu_mte, TCG_CALL_NO_WG, 1659 + void, env, ptr, ptr, ptr, tl, i32) 1660 + DEF_HELPER_FLAGS_6(sve_ldsdu_le_zsu_mte, TCG_CALL_NO_WG, 1661 + void, env, ptr, ptr, ptr, tl, i32) 1662 + DEF_HELPER_FLAGS_6(sve_ldsdu_be_zsu_mte, TCG_CALL_NO_WG, 1663 + void, env, ptr, ptr, ptr, tl, i32) 1664 + DEF_HELPER_FLAGS_6(sve_lddd_le_zsu_mte, TCG_CALL_NO_WG, 1665 + void, env, ptr, ptr, ptr, tl, i32) 1666 + DEF_HELPER_FLAGS_6(sve_lddd_be_zsu_mte, TCG_CALL_NO_WG, 1667 + void, env, ptr, ptr, ptr, tl, i32) 1668 + DEF_HELPER_FLAGS_6(sve_ldbds_zsu_mte, TCG_CALL_NO_WG, 1669 + void, env, ptr, ptr, ptr, tl, i32) 1670 + DEF_HELPER_FLAGS_6(sve_ldhds_le_zsu_mte, TCG_CALL_NO_WG, 1671 + void, env, ptr, ptr, ptr, tl, i32) 1672 + DEF_HELPER_FLAGS_6(sve_ldhds_be_zsu_mte, TCG_CALL_NO_WG, 1673 + void, env, ptr, ptr, ptr, tl, i32) 1674 + DEF_HELPER_FLAGS_6(sve_ldsds_le_zsu_mte, TCG_CALL_NO_WG, 1675 + void, env, ptr, ptr, ptr, tl, i32) 1676 + DEF_HELPER_FLAGS_6(sve_ldsds_be_zsu_mte, TCG_CALL_NO_WG, 1677 + void, env, ptr, ptr, ptr, tl, i32) 1678 + 1679 + DEF_HELPER_FLAGS_6(sve_ldbdu_zss_mte, TCG_CALL_NO_WG, 1680 + void, env, ptr, ptr, ptr, tl, i32) 1681 + DEF_HELPER_FLAGS_6(sve_ldhdu_le_zss_mte, TCG_CALL_NO_WG, 1682 + void, env, ptr, ptr, ptr, tl, i32) 1683 + DEF_HELPER_FLAGS_6(sve_ldhdu_be_zss_mte, TCG_CALL_NO_WG, 1684 + void, env, ptr, ptr, ptr, tl, i32) 1685 + DEF_HELPER_FLAGS_6(sve_ldsdu_le_zss_mte, TCG_CALL_NO_WG, 1686 + void, env, ptr, ptr, ptr, tl, i32) 1687 + DEF_HELPER_FLAGS_6(sve_ldsdu_be_zss_mte, TCG_CALL_NO_WG, 1688 + void, env, ptr, ptr, ptr, tl, i32) 1689 + DEF_HELPER_FLAGS_6(sve_lddd_le_zss_mte, TCG_CALL_NO_WG, 1690 + void, env, ptr, ptr, ptr, tl, i32) 1691 + DEF_HELPER_FLAGS_6(sve_lddd_be_zss_mte, TCG_CALL_NO_WG, 1692 + void, env, ptr, ptr, ptr, tl, i32) 1693 + DEF_HELPER_FLAGS_6(sve_ldbds_zss_mte, TCG_CALL_NO_WG, 1694 + void, env, ptr, ptr, ptr, tl, i32) 1695 + DEF_HELPER_FLAGS_6(sve_ldhds_le_zss_mte, TCG_CALL_NO_WG, 1696 + void, env, ptr, ptr, ptr, tl, i32) 1697 + DEF_HELPER_FLAGS_6(sve_ldhds_be_zss_mte, TCG_CALL_NO_WG, 1698 + void, env, ptr, ptr, ptr, tl, i32) 1699 + DEF_HELPER_FLAGS_6(sve_ldsds_le_zss_mte, TCG_CALL_NO_WG, 1700 + void, env, ptr, ptr, ptr, tl, i32) 1701 + DEF_HELPER_FLAGS_6(sve_ldsds_be_zss_mte, TCG_CALL_NO_WG, 1702 + void, env, ptr, ptr, ptr, tl, i32) 1703 + 1704 + DEF_HELPER_FLAGS_6(sve_ldbdu_zd_mte, TCG_CALL_NO_WG, 1705 + void, env, ptr, ptr, ptr, tl, i32) 1706 + DEF_HELPER_FLAGS_6(sve_ldhdu_le_zd_mte, TCG_CALL_NO_WG, 1707 + void, env, ptr, ptr, ptr, tl, i32) 1708 + DEF_HELPER_FLAGS_6(sve_ldhdu_be_zd_mte, TCG_CALL_NO_WG, 1709 + void, env, ptr, ptr, ptr, tl, i32) 1710 + DEF_HELPER_FLAGS_6(sve_ldsdu_le_zd_mte, TCG_CALL_NO_WG, 1711 + void, env, ptr, ptr, ptr, tl, i32) 1712 + DEF_HELPER_FLAGS_6(sve_ldsdu_be_zd_mte, TCG_CALL_NO_WG, 1713 + void, env, ptr, ptr, ptr, tl, i32) 1714 + DEF_HELPER_FLAGS_6(sve_lddd_le_zd_mte, TCG_CALL_NO_WG, 1715 + void, env, ptr, ptr, ptr, tl, i32) 1716 + DEF_HELPER_FLAGS_6(sve_lddd_be_zd_mte, TCG_CALL_NO_WG, 1717 + void, env, ptr, ptr, ptr, tl, i32) 1718 + DEF_HELPER_FLAGS_6(sve_ldbds_zd_mte, TCG_CALL_NO_WG, 1719 + void, env, ptr, ptr, ptr, tl, i32) 1720 + DEF_HELPER_FLAGS_6(sve_ldhds_le_zd_mte, TCG_CALL_NO_WG, 1721 + void, env, ptr, ptr, ptr, tl, i32) 1722 + DEF_HELPER_FLAGS_6(sve_ldhds_be_zd_mte, TCG_CALL_NO_WG, 1723 + void, env, ptr, ptr, ptr, tl, i32) 1724 + DEF_HELPER_FLAGS_6(sve_ldsds_le_zd_mte, TCG_CALL_NO_WG, 1725 + void, env, ptr, ptr, ptr, tl, i32) 1726 + DEF_HELPER_FLAGS_6(sve_ldsds_be_zd_mte, TCG_CALL_NO_WG, 1727 + void, env, ptr, ptr, ptr, tl, i32) 1728 + 1620 1729 DEF_HELPER_FLAGS_6(sve_ldffbsu_zsu, TCG_CALL_NO_WG, 1621 1730 void, env, ptr, ptr, ptr, tl, i32) 1622 1731 DEF_HELPER_FLAGS_6(sve_ldffhsu_le_zsu, TCG_CALL_NO_WG, ··· 1726 1835 DEF_HELPER_FLAGS_6(sve_ldffsds_be_zd, TCG_CALL_NO_WG, 1727 1836 void, env, ptr, ptr, ptr, tl, i32) 1728 1837 1838 + DEF_HELPER_FLAGS_6(sve_ldffbsu_zsu_mte, TCG_CALL_NO_WG, 1839 + void, env, ptr, ptr, ptr, tl, i32) 1840 + DEF_HELPER_FLAGS_6(sve_ldffhsu_le_zsu_mte, TCG_CALL_NO_WG, 1841 + void, env, ptr, ptr, ptr, tl, i32) 1842 + DEF_HELPER_FLAGS_6(sve_ldffhsu_be_zsu_mte, TCG_CALL_NO_WG, 1843 + void, env, ptr, ptr, ptr, tl, i32) 1844 + DEF_HELPER_FLAGS_6(sve_ldffss_le_zsu_mte, TCG_CALL_NO_WG, 1845 + void, env, ptr, ptr, ptr, tl, i32) 1846 + DEF_HELPER_FLAGS_6(sve_ldffss_be_zsu_mte, TCG_CALL_NO_WG, 1847 + void, env, ptr, ptr, ptr, tl, i32) 1848 + DEF_HELPER_FLAGS_6(sve_ldffbss_zsu_mte, TCG_CALL_NO_WG, 1849 + void, env, ptr, ptr, ptr, tl, i32) 1850 + DEF_HELPER_FLAGS_6(sve_ldffhss_le_zsu_mte, TCG_CALL_NO_WG, 1851 + void, env, ptr, ptr, ptr, tl, i32) 1852 + DEF_HELPER_FLAGS_6(sve_ldffhss_be_zsu_mte, TCG_CALL_NO_WG, 1853 + void, env, ptr, ptr, ptr, tl, i32) 1854 + 1855 + DEF_HELPER_FLAGS_6(sve_ldffbsu_zss_mte, TCG_CALL_NO_WG, 1856 + void, env, ptr, ptr, ptr, tl, i32) 1857 + DEF_HELPER_FLAGS_6(sve_ldffhsu_le_zss_mte, TCG_CALL_NO_WG, 1858 + void, env, ptr, ptr, ptr, tl, i32) 1859 + DEF_HELPER_FLAGS_6(sve_ldffhsu_be_zss_mte, TCG_CALL_NO_WG, 1860 + void, env, ptr, ptr, ptr, tl, i32) 1861 + DEF_HELPER_FLAGS_6(sve_ldffss_le_zss_mte, TCG_CALL_NO_WG, 1862 + void, env, ptr, ptr, ptr, tl, i32) 1863 + DEF_HELPER_FLAGS_6(sve_ldffss_be_zss_mte, TCG_CALL_NO_WG, 1864 + void, env, ptr, ptr, ptr, tl, i32) 1865 + DEF_HELPER_FLAGS_6(sve_ldffbss_zss_mte, TCG_CALL_NO_WG, 1866 + void, env, ptr, ptr, ptr, tl, i32) 1867 + DEF_HELPER_FLAGS_6(sve_ldffhss_le_zss_mte, TCG_CALL_NO_WG, 1868 + void, env, ptr, ptr, ptr, tl, i32) 1869 + DEF_HELPER_FLAGS_6(sve_ldffhss_be_zss_mte, TCG_CALL_NO_WG, 1870 + void, env, ptr, ptr, ptr, tl, i32) 1871 + 1872 + DEF_HELPER_FLAGS_6(sve_ldffbdu_zsu_mte, TCG_CALL_NO_WG, 1873 + void, env, ptr, ptr, ptr, tl, i32) 1874 + DEF_HELPER_FLAGS_6(sve_ldffhdu_le_zsu_mte, TCG_CALL_NO_WG, 1875 + void, env, ptr, ptr, ptr, tl, i32) 1876 + DEF_HELPER_FLAGS_6(sve_ldffhdu_be_zsu_mte, TCG_CALL_NO_WG, 1877 + void, env, ptr, ptr, ptr, tl, i32) 1878 + DEF_HELPER_FLAGS_6(sve_ldffsdu_le_zsu_mte, TCG_CALL_NO_WG, 1879 + void, env, ptr, ptr, ptr, tl, i32) 1880 + DEF_HELPER_FLAGS_6(sve_ldffsdu_be_zsu_mte, TCG_CALL_NO_WG, 1881 + void, env, ptr, ptr, ptr, tl, i32) 1882 + DEF_HELPER_FLAGS_6(sve_ldffdd_le_zsu_mte, TCG_CALL_NO_WG, 1883 + void, env, ptr, ptr, ptr, tl, i32) 1884 + DEF_HELPER_FLAGS_6(sve_ldffdd_be_zsu_mte, TCG_CALL_NO_WG, 1885 + void, env, ptr, ptr, ptr, tl, i32) 1886 + DEF_HELPER_FLAGS_6(sve_ldffbds_zsu_mte, TCG_CALL_NO_WG, 1887 + void, env, ptr, ptr, ptr, tl, i32) 1888 + DEF_HELPER_FLAGS_6(sve_ldffhds_le_zsu_mte, TCG_CALL_NO_WG, 1889 + void, env, ptr, ptr, ptr, tl, i32) 1890 + DEF_HELPER_FLAGS_6(sve_ldffhds_be_zsu_mte, TCG_CALL_NO_WG, 1891 + void, env, ptr, ptr, ptr, tl, i32) 1892 + DEF_HELPER_FLAGS_6(sve_ldffsds_le_zsu_mte, TCG_CALL_NO_WG, 1893 + void, env, ptr, ptr, ptr, tl, i32) 1894 + DEF_HELPER_FLAGS_6(sve_ldffsds_be_zsu_mte, TCG_CALL_NO_WG, 1895 + void, env, ptr, ptr, ptr, tl, i32) 1896 + 1897 + DEF_HELPER_FLAGS_6(sve_ldffbdu_zss_mte, TCG_CALL_NO_WG, 1898 + void, env, ptr, ptr, ptr, tl, i32) 1899 + DEF_HELPER_FLAGS_6(sve_ldffhdu_le_zss_mte, TCG_CALL_NO_WG, 1900 + void, env, ptr, ptr, ptr, tl, i32) 1901 + DEF_HELPER_FLAGS_6(sve_ldffhdu_be_zss_mte, TCG_CALL_NO_WG, 1902 + void, env, ptr, ptr, ptr, tl, i32) 1903 + DEF_HELPER_FLAGS_6(sve_ldffsdu_le_zss_mte, TCG_CALL_NO_WG, 1904 + void, env, ptr, ptr, ptr, tl, i32) 1905 + DEF_HELPER_FLAGS_6(sve_ldffsdu_be_zss_mte, TCG_CALL_NO_WG, 1906 + void, env, ptr, ptr, ptr, tl, i32) 1907 + DEF_HELPER_FLAGS_6(sve_ldffdd_le_zss_mte, TCG_CALL_NO_WG, 1908 + void, env, ptr, ptr, ptr, tl, i32) 1909 + DEF_HELPER_FLAGS_6(sve_ldffdd_be_zss_mte, TCG_CALL_NO_WG, 1910 + void, env, ptr, ptr, ptr, tl, i32) 1911 + DEF_HELPER_FLAGS_6(sve_ldffbds_zss_mte, TCG_CALL_NO_WG, 1912 + void, env, ptr, ptr, ptr, tl, i32) 1913 + DEF_HELPER_FLAGS_6(sve_ldffhds_le_zss_mte, TCG_CALL_NO_WG, 1914 + void, env, ptr, ptr, ptr, tl, i32) 1915 + DEF_HELPER_FLAGS_6(sve_ldffhds_be_zss_mte, TCG_CALL_NO_WG, 1916 + void, env, ptr, ptr, ptr, tl, i32) 1917 + DEF_HELPER_FLAGS_6(sve_ldffsds_le_zss_mte, TCG_CALL_NO_WG, 1918 + void, env, ptr, ptr, ptr, tl, i32) 1919 + DEF_HELPER_FLAGS_6(sve_ldffsds_be_zss_mte, TCG_CALL_NO_WG, 1920 + void, env, ptr, ptr, ptr, tl, i32) 1921 + 1922 + DEF_HELPER_FLAGS_6(sve_ldffbdu_zd_mte, TCG_CALL_NO_WG, 1923 + void, env, ptr, ptr, ptr, tl, i32) 1924 + DEF_HELPER_FLAGS_6(sve_ldffhdu_le_zd_mte, TCG_CALL_NO_WG, 1925 + void, env, ptr, ptr, ptr, tl, i32) 1926 + DEF_HELPER_FLAGS_6(sve_ldffhdu_be_zd_mte, TCG_CALL_NO_WG, 1927 + void, env, ptr, ptr, ptr, tl, i32) 1928 + DEF_HELPER_FLAGS_6(sve_ldffsdu_le_zd_mte, TCG_CALL_NO_WG, 1929 + void, env, ptr, ptr, ptr, tl, i32) 1930 + DEF_HELPER_FLAGS_6(sve_ldffsdu_be_zd_mte, TCG_CALL_NO_WG, 1931 + void, env, ptr, ptr, ptr, tl, i32) 1932 + DEF_HELPER_FLAGS_6(sve_ldffdd_le_zd_mte, TCG_CALL_NO_WG, 1933 + void, env, ptr, ptr, ptr, tl, i32) 1934 + DEF_HELPER_FLAGS_6(sve_ldffdd_be_zd_mte, TCG_CALL_NO_WG, 1935 + void, env, ptr, ptr, ptr, tl, i32) 1936 + DEF_HELPER_FLAGS_6(sve_ldffbds_zd_mte, TCG_CALL_NO_WG, 1937 + void, env, ptr, ptr, ptr, tl, i32) 1938 + DEF_HELPER_FLAGS_6(sve_ldffhds_le_zd_mte, TCG_CALL_NO_WG, 1939 + void, env, ptr, ptr, ptr, tl, i32) 1940 + DEF_HELPER_FLAGS_6(sve_ldffhds_be_zd_mte, TCG_CALL_NO_WG, 1941 + void, env, ptr, ptr, ptr, tl, i32) 1942 + DEF_HELPER_FLAGS_6(sve_ldffsds_le_zd_mte, TCG_CALL_NO_WG, 1943 + void, env, ptr, ptr, ptr, tl, i32) 1944 + DEF_HELPER_FLAGS_6(sve_ldffsds_be_zd_mte, TCG_CALL_NO_WG, 1945 + void, env, ptr, ptr, ptr, tl, i32) 1946 + 1729 1947 DEF_HELPER_FLAGS_6(sve_stbs_zsu, TCG_CALL_NO_WG, 1730 1948 void, env, ptr, ptr, ptr, tl, i32) 1731 1949 DEF_HELPER_FLAGS_6(sve_sths_le_zsu, TCG_CALL_NO_WG, ··· 1791 2009 DEF_HELPER_FLAGS_6(sve_stdd_le_zd, TCG_CALL_NO_WG, 1792 2010 void, env, ptr, ptr, ptr, tl, i32) 1793 2011 DEF_HELPER_FLAGS_6(sve_stdd_be_zd, TCG_CALL_NO_WG, 2012 + void, env, ptr, ptr, ptr, tl, i32) 2013 + 2014 + DEF_HELPER_FLAGS_6(sve_stbs_zsu_mte, TCG_CALL_NO_WG, 2015 + void, env, ptr, ptr, ptr, tl, i32) 2016 + DEF_HELPER_FLAGS_6(sve_sths_le_zsu_mte, TCG_CALL_NO_WG, 2017 + void, env, ptr, ptr, ptr, tl, i32) 2018 + DEF_HELPER_FLAGS_6(sve_sths_be_zsu_mte, TCG_CALL_NO_WG, 2019 + void, env, ptr, ptr, ptr, tl, i32) 2020 + DEF_HELPER_FLAGS_6(sve_stss_le_zsu_mte, TCG_CALL_NO_WG, 2021 + void, env, ptr, ptr, ptr, tl, i32) 2022 + DEF_HELPER_FLAGS_6(sve_stss_be_zsu_mte, TCG_CALL_NO_WG, 2023 + void, env, ptr, ptr, ptr, tl, i32) 2024 + 2025 + DEF_HELPER_FLAGS_6(sve_stbs_zss_mte, TCG_CALL_NO_WG, 2026 + void, env, ptr, ptr, ptr, tl, i32) 2027 + DEF_HELPER_FLAGS_6(sve_sths_le_zss_mte, TCG_CALL_NO_WG, 2028 + void, env, ptr, ptr, ptr, tl, i32) 2029 + DEF_HELPER_FLAGS_6(sve_sths_be_zss_mte, TCG_CALL_NO_WG, 2030 + void, env, ptr, ptr, ptr, tl, i32) 2031 + DEF_HELPER_FLAGS_6(sve_stss_le_zss_mte, TCG_CALL_NO_WG, 2032 + void, env, ptr, ptr, ptr, tl, i32) 2033 + DEF_HELPER_FLAGS_6(sve_stss_be_zss_mte, TCG_CALL_NO_WG, 2034 + void, env, ptr, ptr, ptr, tl, i32) 2035 + 2036 + DEF_HELPER_FLAGS_6(sve_stbd_zsu_mte, TCG_CALL_NO_WG, 2037 + void, env, ptr, ptr, ptr, tl, i32) 2038 + DEF_HELPER_FLAGS_6(sve_sthd_le_zsu_mte, TCG_CALL_NO_WG, 2039 + void, env, ptr, ptr, ptr, tl, i32) 2040 + DEF_HELPER_FLAGS_6(sve_sthd_be_zsu_mte, TCG_CALL_NO_WG, 2041 + void, env, ptr, ptr, ptr, tl, i32) 2042 + DEF_HELPER_FLAGS_6(sve_stsd_le_zsu_mte, TCG_CALL_NO_WG, 2043 + void, env, ptr, ptr, ptr, tl, i32) 2044 + DEF_HELPER_FLAGS_6(sve_stsd_be_zsu_mte, TCG_CALL_NO_WG, 2045 + void, env, ptr, ptr, ptr, tl, i32) 2046 + DEF_HELPER_FLAGS_6(sve_stdd_le_zsu_mte, TCG_CALL_NO_WG, 2047 + void, env, ptr, ptr, ptr, tl, i32) 2048 + DEF_HELPER_FLAGS_6(sve_stdd_be_zsu_mte, TCG_CALL_NO_WG, 2049 + void, env, ptr, ptr, ptr, tl, i32) 2050 + 2051 + DEF_HELPER_FLAGS_6(sve_stbd_zss_mte, TCG_CALL_NO_WG, 2052 + void, env, ptr, ptr, ptr, tl, i32) 2053 + DEF_HELPER_FLAGS_6(sve_sthd_le_zss_mte, TCG_CALL_NO_WG, 2054 + void, env, ptr, ptr, ptr, tl, i32) 2055 + DEF_HELPER_FLAGS_6(sve_sthd_be_zss_mte, TCG_CALL_NO_WG, 2056 + void, env, ptr, ptr, ptr, tl, i32) 2057 + DEF_HELPER_FLAGS_6(sve_stsd_le_zss_mte, TCG_CALL_NO_WG, 2058 + void, env, ptr, ptr, ptr, tl, i32) 2059 + DEF_HELPER_FLAGS_6(sve_stsd_be_zss_mte, TCG_CALL_NO_WG, 2060 + void, env, ptr, ptr, ptr, tl, i32) 2061 + DEF_HELPER_FLAGS_6(sve_stdd_le_zss_mte, TCG_CALL_NO_WG, 2062 + void, env, ptr, ptr, ptr, tl, i32) 2063 + DEF_HELPER_FLAGS_6(sve_stdd_be_zss_mte, TCG_CALL_NO_WG, 2064 + void, env, ptr, ptr, ptr, tl, i32) 2065 + 2066 + DEF_HELPER_FLAGS_6(sve_stbd_zd_mte, TCG_CALL_NO_WG, 2067 + void, env, ptr, ptr, ptr, tl, i32) 2068 + DEF_HELPER_FLAGS_6(sve_sthd_le_zd_mte, TCG_CALL_NO_WG, 2069 + void, env, ptr, ptr, ptr, tl, i32) 2070 + DEF_HELPER_FLAGS_6(sve_sthd_be_zd_mte, TCG_CALL_NO_WG, 2071 + void, env, ptr, ptr, ptr, tl, i32) 2072 + DEF_HELPER_FLAGS_6(sve_stsd_le_zd_mte, TCG_CALL_NO_WG, 2073 + void, env, ptr, ptr, ptr, tl, i32) 2074 + DEF_HELPER_FLAGS_6(sve_stsd_be_zd_mte, TCG_CALL_NO_WG, 2075 + void, env, ptr, ptr, ptr, tl, i32) 2076 + DEF_HELPER_FLAGS_6(sve_stdd_le_zd_mte, TCG_CALL_NO_WG, 2077 + void, env, ptr, ptr, ptr, tl, i32) 2078 + DEF_HELPER_FLAGS_6(sve_stdd_be_zd_mte, TCG_CALL_NO_WG, 1794 2079 void, env, ptr, ptr, ptr, tl, i32) 1795 2080 1796 2081 DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)

+152 -31

target/arm/sve_helper.c

··· 5354 5354 static inline QEMU_ALWAYS_INLINE 5355 5355 void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, 5356 5356 target_ulong base, uint32_t desc, uintptr_t retaddr, 5357 - int esize, int msize, zreg_off_fn *off_fn, 5357 + uint32_t mtedesc, int esize, int msize, 5358 + zreg_off_fn *off_fn, 5358 5359 sve_ldst1_host_fn *host_fn, 5359 5360 sve_ldst1_tlb_fn *tlb_fn) 5360 5361 { ··· 5382 5383 cpu_check_watchpoint(env_cpu(env), addr, msize, 5383 5384 info.attrs, BP_MEM_READ, retaddr); 5384 5385 } 5385 - /* TODO: MTE check */ 5386 + if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) { 5387 + mte_check1(env, mtedesc, addr, retaddr); 5388 + } 5386 5389 host_fn(&scratch, reg_off, info.host); 5387 5390 } else { 5388 5391 /* Element crosses the page boundary. */ ··· 5393 5396 msize, info.attrs, 5394 5397 BP_MEM_READ, retaddr); 5395 5398 } 5396 - /* TODO: MTE check */ 5399 + if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) { 5400 + mte_check1(env, mtedesc, addr, retaddr); 5401 + } 5397 5402 tlb_fn(env, &scratch, reg_off, addr, retaddr); 5398 5403 } 5399 5404 } ··· 5406 5411 memcpy(vd, &scratch, reg_max); 5407 5412 } 5408 5413 5414 + static inline QEMU_ALWAYS_INLINE 5415 + void sve_ld1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm, 5416 + target_ulong base, uint32_t desc, uintptr_t retaddr, 5417 + int esize, int msize, zreg_off_fn *off_fn, 5418 + sve_ldst1_host_fn *host_fn, 5419 + sve_ldst1_tlb_fn *tlb_fn) 5420 + { 5421 + uint32_t mtedesc = desc >> (SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT); 5422 + /* Remove mtedesc from the normal sve descriptor. */ 5423 + desc = extract32(desc, 0, SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT); 5424 + 5425 + /* 5426 + * ??? TODO: For the 32-bit offset extractions, base + ofs cannot 5427 + * offset base entirely over the address space hole to change the 5428 + * pointer tag, or change the bit55 selector. So we could here 5429 + * examine TBI + TCMA like we do for sve_ldN_r_mte(). 5430 + */ 5431 + sve_ld1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc, 5432 + esize, msize, off_fn, host_fn, tlb_fn); 5433 + } 5434 + 5409 5435 #define DO_LD1_ZPZ_S(MEM, OFS, MSZ) \ 5410 5436 void HELPER(sve_ld##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \ 5411 5437 void *vm, target_ulong base, uint32_t desc) \ 5412 5438 { \ 5413 - sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \ 5439 + sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 4, 1 << MSZ, \ 5414 5440 off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5441 + } \ 5442 + void HELPER(sve_ld##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \ 5443 + void *vm, target_ulong base, uint32_t desc) \ 5444 + { \ 5445 + sve_ld1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \ 5446 + off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5415 5447 } 5416 5448 5417 5449 #define DO_LD1_ZPZ_D(MEM, OFS, MSZ) \ 5418 5450 void HELPER(sve_ld##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \ 5419 5451 void *vm, target_ulong base, uint32_t desc) \ 5420 5452 { \ 5421 - sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \ 5453 + sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 8, 1 << MSZ, \ 5422 5454 off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5455 + } \ 5456 + void HELPER(sve_ld##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \ 5457 + void *vm, target_ulong base, uint32_t desc) \ 5458 + { \ 5459 + sve_ld1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \ 5460 + off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5423 5461 } 5424 5462 5425 5463 DO_LD1_ZPZ_S(bsu, zsu, MO_8) ··· 5498 5536 static inline QEMU_ALWAYS_INLINE 5499 5537 void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, 5500 5538 target_ulong base, uint32_t desc, uintptr_t retaddr, 5501 - const int esz, const int msz, zreg_off_fn *off_fn, 5539 + uint32_t mtedesc, const int esz, const int msz, 5540 + zreg_off_fn *off_fn, 5502 5541 sve_ldst1_host_fn *host_fn, 5503 5542 sve_ldst1_tlb_fn *tlb_fn) 5504 5543 { ··· 5523 5562 * Probe the first element, allowing faults. 5524 5563 */ 5525 5564 addr = base + (off_fn(vm, reg_off) << scale); 5565 + if (mtedesc) { 5566 + mte_check1(env, mtedesc, addr, retaddr); 5567 + } 5526 5568 tlb_fn(env, vd, reg_off, addr, retaddr); 5527 5569 5528 5570 /* After any fault, zero the other elements. */ ··· 5555 5597 (env_cpu(env), addr, msize) & BP_MEM_READ)) { 5556 5598 goto fault; 5557 5599 } 5558 - /* TODO: MTE check. */ 5600 + if (mtedesc && 5601 + arm_tlb_mte_tagged(&info.attrs) && 5602 + !mte_probe1(env, mtedesc, addr)) { 5603 + goto fault; 5604 + } 5559 5605 5560 5606 host_fn(vd, reg_off, info.host); 5561 5607 } ··· 5568 5614 record_fault(env, reg_off, reg_max); 5569 5615 } 5570 5616 5571 - #define DO_LDFF1_ZPZ_S(MEM, OFS, MSZ) \ 5572 - void HELPER(sve_ldff##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \ 5573 - void *vm, target_ulong base, uint32_t desc) \ 5574 - { \ 5575 - sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), MO_32, MSZ, \ 5576 - off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5617 + static inline QEMU_ALWAYS_INLINE 5618 + void sve_ldff1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm, 5619 + target_ulong base, uint32_t desc, uintptr_t retaddr, 5620 + const int esz, const int msz, 5621 + zreg_off_fn *off_fn, 5622 + sve_ldst1_host_fn *host_fn, 5623 + sve_ldst1_tlb_fn *tlb_fn) 5624 + { 5625 + uint32_t mtedesc = desc >> (SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT); 5626 + /* Remove mtedesc from the normal sve descriptor. */ 5627 + desc = extract32(desc, 0, SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT); 5628 + 5629 + /* 5630 + * ??? TODO: For the 32-bit offset extractions, base + ofs cannot 5631 + * offset base entirely over the address space hole to change the 5632 + * pointer tag, or change the bit55 selector. So we could here 5633 + * examine TBI + TCMA like we do for sve_ldN_r_mte(). 5634 + */ 5635 + sve_ldff1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc, 5636 + esz, msz, off_fn, host_fn, tlb_fn); 5637 + } 5638 + 5639 + #define DO_LDFF1_ZPZ_S(MEM, OFS, MSZ) \ 5640 + void HELPER(sve_ldff##MEM##_##OFS) \ 5641 + (CPUARMState *env, void *vd, void *vg, \ 5642 + void *vm, target_ulong base, uint32_t desc) \ 5643 + { \ 5644 + sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), 0, MO_32, MSZ, \ 5645 + off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5646 + } \ 5647 + void HELPER(sve_ldff##MEM##_##OFS##_mte) \ 5648 + (CPUARMState *env, void *vd, void *vg, \ 5649 + void *vm, target_ulong base, uint32_t desc) \ 5650 + { \ 5651 + sve_ldff1_z_mte(env, vd, vg, vm, base, desc, GETPC(), MO_32, MSZ, \ 5652 + off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5577 5653 } 5578 5654 5579 - #define DO_LDFF1_ZPZ_D(MEM, OFS, MSZ) \ 5580 - void HELPER(sve_ldff##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \ 5581 - void *vm, target_ulong base, uint32_t desc) \ 5582 - { \ 5583 - sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), MO_64, MSZ, \ 5584 - off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5655 + #define DO_LDFF1_ZPZ_D(MEM, OFS, MSZ) \ 5656 + void HELPER(sve_ldff##MEM##_##OFS) \ 5657 + (CPUARMState *env, void *vd, void *vg, \ 5658 + void *vm, target_ulong base, uint32_t desc) \ 5659 + { \ 5660 + sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), 0, MO_64, MSZ, \ 5661 + off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5662 + } \ 5663 + void HELPER(sve_ldff##MEM##_##OFS##_mte) \ 5664 + (CPUARMState *env, void *vd, void *vg, \ 5665 + void *vm, target_ulong base, uint32_t desc) \ 5666 + { \ 5667 + sve_ldff1_z_mte(env, vd, vg, vm, base, desc, GETPC(), MO_64, MSZ, \ 5668 + off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \ 5585 5669 } 5586 5670 5587 5671 DO_LDFF1_ZPZ_S(bsu, zsu, MO_8) ··· 5653 5737 static inline QEMU_ALWAYS_INLINE 5654 5738 void sve_st1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, 5655 5739 target_ulong base, uint32_t desc, uintptr_t retaddr, 5656 - int esize, int msize, zreg_off_fn *off_fn, 5740 + uint32_t mtedesc, int esize, int msize, 5741 + zreg_off_fn *off_fn, 5657 5742 sve_ldst1_host_fn *host_fn, 5658 5743 sve_ldst1_tlb_fn *tlb_fn) 5659 5744 { ··· 5697 5782 cpu_check_watchpoint(env_cpu(env), addr, msize, 5698 5783 info.attrs, BP_MEM_WRITE, retaddr); 5699 5784 } 5700 - /* TODO: MTE check. */ 5785 + 5786 + if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) { 5787 + mte_check1(env, mtedesc, addr, retaddr); 5788 + } 5701 5789 } 5702 5790 i += 1; 5703 5791 reg_off += esize; ··· 5727 5815 } while (reg_off < reg_max); 5728 5816 } 5729 5817 5730 - #define DO_ST1_ZPZ_S(MEM, OFS, MSZ) \ 5731 - void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \ 5818 + static inline QEMU_ALWAYS_INLINE 5819 + void sve_st1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm, 5820 + target_ulong base, uint32_t desc, uintptr_t retaddr, 5821 + int esize, int msize, zreg_off_fn *off_fn, 5822 + sve_ldst1_host_fn *host_fn, 5823 + sve_ldst1_tlb_fn *tlb_fn) 5824 + { 5825 + uint32_t mtedesc = desc >> (SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT); 5826 + /* Remove mtedesc from the normal sve descriptor. */ 5827 + desc = extract32(desc, 0, SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT); 5828 + 5829 + /* 5830 + * ??? TODO: For the 32-bit offset extractions, base + ofs cannot 5831 + * offset base entirely over the address space hole to change the 5832 + * pointer tag, or change the bit55 selector. So we could here 5833 + * examine TBI + TCMA like we do for sve_ldN_r_mte(). 5834 + */ 5835 + sve_st1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc, 5836 + esize, msize, off_fn, host_fn, tlb_fn); 5837 + } 5838 + 5839 + #define DO_ST1_ZPZ_S(MEM, OFS, MSZ) \ 5840 + void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \ 5732 5841 void *vm, target_ulong base, uint32_t desc) \ 5733 - { \ 5734 - sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \ 5735 - off_##OFS##_s, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \ 5842 + { \ 5843 + sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 4, 1 << MSZ, \ 5844 + off_##OFS##_s, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \ 5845 + } \ 5846 + void HELPER(sve_st##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \ 5847 + void *vm, target_ulong base, uint32_t desc) \ 5848 + { \ 5849 + sve_st1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \ 5850 + off_##OFS##_s, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \ 5736 5851 } 5737 5852 5738 - #define DO_ST1_ZPZ_D(MEM, OFS, MSZ) \ 5739 - void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \ 5853 + #define DO_ST1_ZPZ_D(MEM, OFS, MSZ) \ 5854 + void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \ 5740 5855 void *vm, target_ulong base, uint32_t desc) \ 5741 - { \ 5742 - sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \ 5743 - off_##OFS##_d, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \ 5856 + { \ 5857 + sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 8, 1 << MSZ, \ 5858 + off_##OFS##_d, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \ 5859 + } \ 5860 + void HELPER(sve_st##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \ 5861 + void *vm, target_ulong base, uint32_t desc) \ 5862 + { \ 5863 + sve_st1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \ 5864 + off_##OFS##_d, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \ 5744 5865 } 5745 5866 5746 5867 DO_ST1_ZPZ_S(bs, zsu, MO_8)

+434 -216

target/arm/translate-sve.c

··· 5261 5261 */ 5262 5262 5263 5263 static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm, 5264 - int scale, TCGv_i64 scalar, int msz, 5264 + int scale, TCGv_i64 scalar, int msz, bool is_write, 5265 5265 gen_helper_gvec_mem_scatter *fn) 5266 5266 { 5267 5267 unsigned vsz = vec_full_reg_size(s); ··· 5269 5269 TCGv_ptr t_pg = tcg_temp_new_ptr(); 5270 5270 TCGv_ptr t_zt = tcg_temp_new_ptr(); 5271 5271 TCGv_i32 t_desc; 5272 - int desc; 5272 + int desc = 0; 5273 5273 5274 + if (s->mte_active[0]) { 5275 + desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s)); 5276 + desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); 5277 + desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); 5278 + desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); 5279 + desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << msz); 5280 + desc <<= SVE_MTEDESC_SHIFT; 5281 + } 5274 5282 desc = simd_desc(vsz, vsz, scale); 5275 5283 t_desc = tcg_const_i32(desc); 5276 5284 ··· 5285 5293 tcg_temp_free_i32(t_desc); 5286 5294 } 5287 5295 5288 - /* Indexed by [be][ff][xs][u][msz]. */ 5289 - static gen_helper_gvec_mem_scatter * const gather_load_fn32[2][2][2][2][3] = { 5290 - /* Little-endian */ 5291 - { { { { gen_helper_sve_ldbss_zsu, 5292 - gen_helper_sve_ldhss_le_zsu, 5293 - NULL, }, 5294 - { gen_helper_sve_ldbsu_zsu, 5295 - gen_helper_sve_ldhsu_le_zsu, 5296 - gen_helper_sve_ldss_le_zsu, } }, 5297 - { { gen_helper_sve_ldbss_zss, 5298 - gen_helper_sve_ldhss_le_zss, 5299 - NULL, }, 5300 - { gen_helper_sve_ldbsu_zss, 5301 - gen_helper_sve_ldhsu_le_zss, 5302 - gen_helper_sve_ldss_le_zss, } } }, 5296 + /* Indexed by [mte][be][ff][xs][u][msz]. */ 5297 + static gen_helper_gvec_mem_scatter * const 5298 + gather_load_fn32[2][2][2][2][2][3] = { 5299 + { /* MTE Inactive */ 5300 + { /* Little-endian */ 5301 + { { { gen_helper_sve_ldbss_zsu, 5302 + gen_helper_sve_ldhss_le_zsu, 5303 + NULL, }, 5304 + { gen_helper_sve_ldbsu_zsu, 5305 + gen_helper_sve_ldhsu_le_zsu, 5306 + gen_helper_sve_ldss_le_zsu, } }, 5307 + { { gen_helper_sve_ldbss_zss, 5308 + gen_helper_sve_ldhss_le_zss, 5309 + NULL, }, 5310 + { gen_helper_sve_ldbsu_zss, 5311 + gen_helper_sve_ldhsu_le_zss, 5312 + gen_helper_sve_ldss_le_zss, } } }, 5313 + 5314 + /* First-fault */ 5315 + { { { gen_helper_sve_ldffbss_zsu, 5316 + gen_helper_sve_ldffhss_le_zsu, 5317 + NULL, }, 5318 + { gen_helper_sve_ldffbsu_zsu, 5319 + gen_helper_sve_ldffhsu_le_zsu, 5320 + gen_helper_sve_ldffss_le_zsu, } }, 5321 + { { gen_helper_sve_ldffbss_zss, 5322 + gen_helper_sve_ldffhss_le_zss, 5323 + NULL, }, 5324 + { gen_helper_sve_ldffbsu_zss, 5325 + gen_helper_sve_ldffhsu_le_zss, 5326 + gen_helper_sve_ldffss_le_zss, } } } }, 5327 + 5328 + { /* Big-endian */ 5329 + { { { gen_helper_sve_ldbss_zsu, 5330 + gen_helper_sve_ldhss_be_zsu, 5331 + NULL, }, 5332 + { gen_helper_sve_ldbsu_zsu, 5333 + gen_helper_sve_ldhsu_be_zsu, 5334 + gen_helper_sve_ldss_be_zsu, } }, 5335 + { { gen_helper_sve_ldbss_zss, 5336 + gen_helper_sve_ldhss_be_zss, 5337 + NULL, }, 5338 + { gen_helper_sve_ldbsu_zss, 5339 + gen_helper_sve_ldhsu_be_zss, 5340 + gen_helper_sve_ldss_be_zss, } } }, 5341 + 5342 + /* First-fault */ 5343 + { { { gen_helper_sve_ldffbss_zsu, 5344 + gen_helper_sve_ldffhss_be_zsu, 5345 + NULL, }, 5346 + { gen_helper_sve_ldffbsu_zsu, 5347 + gen_helper_sve_ldffhsu_be_zsu, 5348 + gen_helper_sve_ldffss_be_zsu, } }, 5349 + { { gen_helper_sve_ldffbss_zss, 5350 + gen_helper_sve_ldffhss_be_zss, 5351 + NULL, }, 5352 + { gen_helper_sve_ldffbsu_zss, 5353 + gen_helper_sve_ldffhsu_be_zss, 5354 + gen_helper_sve_ldffss_be_zss, } } } } }, 5355 + { /* MTE Active */ 5356 + { /* Little-endian */ 5357 + { { { gen_helper_sve_ldbss_zsu_mte, 5358 + gen_helper_sve_ldhss_le_zsu_mte, 5359 + NULL, }, 5360 + { gen_helper_sve_ldbsu_zsu_mte, 5361 + gen_helper_sve_ldhsu_le_zsu_mte, 5362 + gen_helper_sve_ldss_le_zsu_mte, } }, 5363 + { { gen_helper_sve_ldbss_zss_mte, 5364 + gen_helper_sve_ldhss_le_zss_mte, 5365 + NULL, }, 5366 + { gen_helper_sve_ldbsu_zss_mte, 5367 + gen_helper_sve_ldhsu_le_zss_mte, 5368 + gen_helper_sve_ldss_le_zss_mte, } } }, 5303 5369 5304 - /* First-fault */ 5305 - { { { gen_helper_sve_ldffbss_zsu, 5306 - gen_helper_sve_ldffhss_le_zsu, 5307 - NULL, }, 5308 - { gen_helper_sve_ldffbsu_zsu, 5309 - gen_helper_sve_ldffhsu_le_zsu, 5310 - gen_helper_sve_ldffss_le_zsu, } }, 5311 - { { gen_helper_sve_ldffbss_zss, 5312 - gen_helper_sve_ldffhss_le_zss, 5313 - NULL, }, 5314 - { gen_helper_sve_ldffbsu_zss, 5315 - gen_helper_sve_ldffhsu_le_zss, 5316 - gen_helper_sve_ldffss_le_zss, } } } }, 5370 + /* First-fault */ 5371 + { { { gen_helper_sve_ldffbss_zsu_mte, 5372 + gen_helper_sve_ldffhss_le_zsu_mte, 5373 + NULL, }, 5374 + { gen_helper_sve_ldffbsu_zsu_mte, 5375 + gen_helper_sve_ldffhsu_le_zsu_mte, 5376 + gen_helper_sve_ldffss_le_zsu_mte, } }, 5377 + { { gen_helper_sve_ldffbss_zss_mte, 5378 + gen_helper_sve_ldffhss_le_zss_mte, 5379 + NULL, }, 5380 + { gen_helper_sve_ldffbsu_zss_mte, 5381 + gen_helper_sve_ldffhsu_le_zss_mte, 5382 + gen_helper_sve_ldffss_le_zss_mte, } } } }, 5317 5383 5318 - /* Big-endian */ 5319 - { { { { gen_helper_sve_ldbss_zsu, 5320 - gen_helper_sve_ldhss_be_zsu, 5321 - NULL, }, 5322 - { gen_helper_sve_ldbsu_zsu, 5323 - gen_helper_sve_ldhsu_be_zsu, 5324 - gen_helper_sve_ldss_be_zsu, } }, 5325 - { { gen_helper_sve_ldbss_zss, 5326 - gen_helper_sve_ldhss_be_zss, 5327 - NULL, }, 5328 - { gen_helper_sve_ldbsu_zss, 5329 - gen_helper_sve_ldhsu_be_zss, 5330 - gen_helper_sve_ldss_be_zss, } } }, 5384 + { /* Big-endian */ 5385 + { { { gen_helper_sve_ldbss_zsu_mte, 5386 + gen_helper_sve_ldhss_be_zsu_mte, 5387 + NULL, }, 5388 + { gen_helper_sve_ldbsu_zsu_mte, 5389 + gen_helper_sve_ldhsu_be_zsu_mte, 5390 + gen_helper_sve_ldss_be_zsu_mte, } }, 5391 + { { gen_helper_sve_ldbss_zss_mte, 5392 + gen_helper_sve_ldhss_be_zss_mte, 5393 + NULL, }, 5394 + { gen_helper_sve_ldbsu_zss_mte, 5395 + gen_helper_sve_ldhsu_be_zss_mte, 5396 + gen_helper_sve_ldss_be_zss_mte, } } }, 5331 5397 5332 - /* First-fault */ 5333 - { { { gen_helper_sve_ldffbss_zsu, 5334 - gen_helper_sve_ldffhss_be_zsu, 5335 - NULL, }, 5336 - { gen_helper_sve_ldffbsu_zsu, 5337 - gen_helper_sve_ldffhsu_be_zsu, 5338 - gen_helper_sve_ldffss_be_zsu, } }, 5339 - { { gen_helper_sve_ldffbss_zss, 5340 - gen_helper_sve_ldffhss_be_zss, 5341 - NULL, }, 5342 - { gen_helper_sve_ldffbsu_zss, 5343 - gen_helper_sve_ldffhsu_be_zss, 5344 - gen_helper_sve_ldffss_be_zss, } } } }, 5398 + /* First-fault */ 5399 + { { { gen_helper_sve_ldffbss_zsu_mte, 5400 + gen_helper_sve_ldffhss_be_zsu_mte, 5401 + NULL, }, 5402 + { gen_helper_sve_ldffbsu_zsu_mte, 5403 + gen_helper_sve_ldffhsu_be_zsu_mte, 5404 + gen_helper_sve_ldffss_be_zsu_mte, } }, 5405 + { { gen_helper_sve_ldffbss_zss_mte, 5406 + gen_helper_sve_ldffhss_be_zss_mte, 5407 + NULL, }, 5408 + { gen_helper_sve_ldffbsu_zss_mte, 5409 + gen_helper_sve_ldffhsu_be_zss_mte, 5410 + gen_helper_sve_ldffss_be_zss_mte, } } } } }, 5345 5411 }; 5346 5412 5347 5413 /* Note that we overload xs=2 to indicate 64-bit offset. */ 5348 - static gen_helper_gvec_mem_scatter * const gather_load_fn64[2][2][3][2][4] = { 5349 - /* Little-endian */ 5350 - { { { { gen_helper_sve_ldbds_zsu, 5351 - gen_helper_sve_ldhds_le_zsu, 5352 - gen_helper_sve_ldsds_le_zsu, 5353 - NULL, }, 5354 - { gen_helper_sve_ldbdu_zsu, 5355 - gen_helper_sve_ldhdu_le_zsu, 5356 - gen_helper_sve_ldsdu_le_zsu, 5357 - gen_helper_sve_lddd_le_zsu, } }, 5358 - { { gen_helper_sve_ldbds_zss, 5359 - gen_helper_sve_ldhds_le_zss, 5360 - gen_helper_sve_ldsds_le_zss, 5361 - NULL, }, 5362 - { gen_helper_sve_ldbdu_zss, 5363 - gen_helper_sve_ldhdu_le_zss, 5364 - gen_helper_sve_ldsdu_le_zss, 5365 - gen_helper_sve_lddd_le_zss, } }, 5366 - { { gen_helper_sve_ldbds_zd, 5367 - gen_helper_sve_ldhds_le_zd, 5368 - gen_helper_sve_ldsds_le_zd, 5369 - NULL, }, 5370 - { gen_helper_sve_ldbdu_zd, 5371 - gen_helper_sve_ldhdu_le_zd, 5372 - gen_helper_sve_ldsdu_le_zd, 5373 - gen_helper_sve_lddd_le_zd, } } }, 5414 + static gen_helper_gvec_mem_scatter * const 5415 + gather_load_fn64[2][2][2][3][2][4] = { 5416 + { /* MTE Inactive */ 5417 + { /* Little-endian */ 5418 + { { { gen_helper_sve_ldbds_zsu, 5419 + gen_helper_sve_ldhds_le_zsu, 5420 + gen_helper_sve_ldsds_le_zsu, 5421 + NULL, }, 5422 + { gen_helper_sve_ldbdu_zsu, 5423 + gen_helper_sve_ldhdu_le_zsu, 5424 + gen_helper_sve_ldsdu_le_zsu, 5425 + gen_helper_sve_lddd_le_zsu, } }, 5426 + { { gen_helper_sve_ldbds_zss, 5427 + gen_helper_sve_ldhds_le_zss, 5428 + gen_helper_sve_ldsds_le_zss, 5429 + NULL, }, 5430 + { gen_helper_sve_ldbdu_zss, 5431 + gen_helper_sve_ldhdu_le_zss, 5432 + gen_helper_sve_ldsdu_le_zss, 5433 + gen_helper_sve_lddd_le_zss, } }, 5434 + { { gen_helper_sve_ldbds_zd, 5435 + gen_helper_sve_ldhds_le_zd, 5436 + gen_helper_sve_ldsds_le_zd, 5437 + NULL, }, 5438 + { gen_helper_sve_ldbdu_zd, 5439 + gen_helper_sve_ldhdu_le_zd, 5440 + gen_helper_sve_ldsdu_le_zd, 5441 + gen_helper_sve_lddd_le_zd, } } }, 5374 5442 5375 - /* First-fault */ 5376 - { { { gen_helper_sve_ldffbds_zsu, 5377 - gen_helper_sve_ldffhds_le_zsu, 5378 - gen_helper_sve_ldffsds_le_zsu, 5379 - NULL, }, 5380 - { gen_helper_sve_ldffbdu_zsu, 5381 - gen_helper_sve_ldffhdu_le_zsu, 5382 - gen_helper_sve_ldffsdu_le_zsu, 5383 - gen_helper_sve_ldffdd_le_zsu, } }, 5384 - { { gen_helper_sve_ldffbds_zss, 5385 - gen_helper_sve_ldffhds_le_zss, 5386 - gen_helper_sve_ldffsds_le_zss, 5387 - NULL, }, 5388 - { gen_helper_sve_ldffbdu_zss, 5389 - gen_helper_sve_ldffhdu_le_zss, 5390 - gen_helper_sve_ldffsdu_le_zss, 5391 - gen_helper_sve_ldffdd_le_zss, } }, 5392 - { { gen_helper_sve_ldffbds_zd, 5393 - gen_helper_sve_ldffhds_le_zd, 5394 - gen_helper_sve_ldffsds_le_zd, 5395 - NULL, }, 5396 - { gen_helper_sve_ldffbdu_zd, 5397 - gen_helper_sve_ldffhdu_le_zd, 5398 - gen_helper_sve_ldffsdu_le_zd, 5399 - gen_helper_sve_ldffdd_le_zd, } } } }, 5443 + /* First-fault */ 5444 + { { { gen_helper_sve_ldffbds_zsu, 5445 + gen_helper_sve_ldffhds_le_zsu, 5446 + gen_helper_sve_ldffsds_le_zsu, 5447 + NULL, }, 5448 + { gen_helper_sve_ldffbdu_zsu, 5449 + gen_helper_sve_ldffhdu_le_zsu, 5450 + gen_helper_sve_ldffsdu_le_zsu, 5451 + gen_helper_sve_ldffdd_le_zsu, } }, 5452 + { { gen_helper_sve_ldffbds_zss, 5453 + gen_helper_sve_ldffhds_le_zss, 5454 + gen_helper_sve_ldffsds_le_zss, 5455 + NULL, }, 5456 + { gen_helper_sve_ldffbdu_zss, 5457 + gen_helper_sve_ldffhdu_le_zss, 5458 + gen_helper_sve_ldffsdu_le_zss, 5459 + gen_helper_sve_ldffdd_le_zss, } }, 5460 + { { gen_helper_sve_ldffbds_zd, 5461 + gen_helper_sve_ldffhds_le_zd, 5462 + gen_helper_sve_ldffsds_le_zd, 5463 + NULL, }, 5464 + { gen_helper_sve_ldffbdu_zd, 5465 + gen_helper_sve_ldffhdu_le_zd, 5466 + gen_helper_sve_ldffsdu_le_zd, 5467 + gen_helper_sve_ldffdd_le_zd, } } } }, 5468 + { /* Big-endian */ 5469 + { { { gen_helper_sve_ldbds_zsu, 5470 + gen_helper_sve_ldhds_be_zsu, 5471 + gen_helper_sve_ldsds_be_zsu, 5472 + NULL, }, 5473 + { gen_helper_sve_ldbdu_zsu, 5474 + gen_helper_sve_ldhdu_be_zsu, 5475 + gen_helper_sve_ldsdu_be_zsu, 5476 + gen_helper_sve_lddd_be_zsu, } }, 5477 + { { gen_helper_sve_ldbds_zss, 5478 + gen_helper_sve_ldhds_be_zss, 5479 + gen_helper_sve_ldsds_be_zss, 5480 + NULL, }, 5481 + { gen_helper_sve_ldbdu_zss, 5482 + gen_helper_sve_ldhdu_be_zss, 5483 + gen_helper_sve_ldsdu_be_zss, 5484 + gen_helper_sve_lddd_be_zss, } }, 5485 + { { gen_helper_sve_ldbds_zd, 5486 + gen_helper_sve_ldhds_be_zd, 5487 + gen_helper_sve_ldsds_be_zd, 5488 + NULL, }, 5489 + { gen_helper_sve_ldbdu_zd, 5490 + gen_helper_sve_ldhdu_be_zd, 5491 + gen_helper_sve_ldsdu_be_zd, 5492 + gen_helper_sve_lddd_be_zd, } } }, 5400 5493 5401 - /* Big-endian */ 5402 - { { { { gen_helper_sve_ldbds_zsu, 5403 - gen_helper_sve_ldhds_be_zsu, 5404 - gen_helper_sve_ldsds_be_zsu, 5405 - NULL, }, 5406 - { gen_helper_sve_ldbdu_zsu, 5407 - gen_helper_sve_ldhdu_be_zsu, 5408 - gen_helper_sve_ldsdu_be_zsu, 5409 - gen_helper_sve_lddd_be_zsu, } }, 5410 - { { gen_helper_sve_ldbds_zss, 5411 - gen_helper_sve_ldhds_be_zss, 5412 - gen_helper_sve_ldsds_be_zss, 5413 - NULL, }, 5414 - { gen_helper_sve_ldbdu_zss, 5415 - gen_helper_sve_ldhdu_be_zss, 5416 - gen_helper_sve_ldsdu_be_zss, 5417 - gen_helper_sve_lddd_be_zss, } }, 5418 - { { gen_helper_sve_ldbds_zd, 5419 - gen_helper_sve_ldhds_be_zd, 5420 - gen_helper_sve_ldsds_be_zd, 5421 - NULL, }, 5422 - { gen_helper_sve_ldbdu_zd, 5423 - gen_helper_sve_ldhdu_be_zd, 5424 - gen_helper_sve_ldsdu_be_zd, 5425 - gen_helper_sve_lddd_be_zd, } } }, 5494 + /* First-fault */ 5495 + { { { gen_helper_sve_ldffbds_zsu, 5496 + gen_helper_sve_ldffhds_be_zsu, 5497 + gen_helper_sve_ldffsds_be_zsu, 5498 + NULL, }, 5499 + { gen_helper_sve_ldffbdu_zsu, 5500 + gen_helper_sve_ldffhdu_be_zsu, 5501 + gen_helper_sve_ldffsdu_be_zsu, 5502 + gen_helper_sve_ldffdd_be_zsu, } }, 5503 + { { gen_helper_sve_ldffbds_zss, 5504 + gen_helper_sve_ldffhds_be_zss, 5505 + gen_helper_sve_ldffsds_be_zss, 5506 + NULL, }, 5507 + { gen_helper_sve_ldffbdu_zss, 5508 + gen_helper_sve_ldffhdu_be_zss, 5509 + gen_helper_sve_ldffsdu_be_zss, 5510 + gen_helper_sve_ldffdd_be_zss, } }, 5511 + { { gen_helper_sve_ldffbds_zd, 5512 + gen_helper_sve_ldffhds_be_zd, 5513 + gen_helper_sve_ldffsds_be_zd, 5514 + NULL, }, 5515 + { gen_helper_sve_ldffbdu_zd, 5516 + gen_helper_sve_ldffhdu_be_zd, 5517 + gen_helper_sve_ldffsdu_be_zd, 5518 + gen_helper_sve_ldffdd_be_zd, } } } } }, 5519 + { /* MTE Active */ 5520 + { /* Little-endian */ 5521 + { { { gen_helper_sve_ldbds_zsu_mte, 5522 + gen_helper_sve_ldhds_le_zsu_mte, 5523 + gen_helper_sve_ldsds_le_zsu_mte, 5524 + NULL, }, 5525 + { gen_helper_sve_ldbdu_zsu_mte, 5526 + gen_helper_sve_ldhdu_le_zsu_mte, 5527 + gen_helper_sve_ldsdu_le_zsu_mte, 5528 + gen_helper_sve_lddd_le_zsu_mte, } }, 5529 + { { gen_helper_sve_ldbds_zss_mte, 5530 + gen_helper_sve_ldhds_le_zss_mte, 5531 + gen_helper_sve_ldsds_le_zss_mte, 5532 + NULL, }, 5533 + { gen_helper_sve_ldbdu_zss_mte, 5534 + gen_helper_sve_ldhdu_le_zss_mte, 5535 + gen_helper_sve_ldsdu_le_zss_mte, 5536 + gen_helper_sve_lddd_le_zss_mte, } }, 5537 + { { gen_helper_sve_ldbds_zd_mte, 5538 + gen_helper_sve_ldhds_le_zd_mte, 5539 + gen_helper_sve_ldsds_le_zd_mte, 5540 + NULL, }, 5541 + { gen_helper_sve_ldbdu_zd_mte, 5542 + gen_helper_sve_ldhdu_le_zd_mte, 5543 + gen_helper_sve_ldsdu_le_zd_mte, 5544 + gen_helper_sve_lddd_le_zd_mte, } } }, 5545 + 5546 + /* First-fault */ 5547 + { { { gen_helper_sve_ldffbds_zsu_mte, 5548 + gen_helper_sve_ldffhds_le_zsu_mte, 5549 + gen_helper_sve_ldffsds_le_zsu_mte, 5550 + NULL, }, 5551 + { gen_helper_sve_ldffbdu_zsu_mte, 5552 + gen_helper_sve_ldffhdu_le_zsu_mte, 5553 + gen_helper_sve_ldffsdu_le_zsu_mte, 5554 + gen_helper_sve_ldffdd_le_zsu_mte, } }, 5555 + { { gen_helper_sve_ldffbds_zss_mte, 5556 + gen_helper_sve_ldffhds_le_zss_mte, 5557 + gen_helper_sve_ldffsds_le_zss_mte, 5558 + NULL, }, 5559 + { gen_helper_sve_ldffbdu_zss_mte, 5560 + gen_helper_sve_ldffhdu_le_zss_mte, 5561 + gen_helper_sve_ldffsdu_le_zss_mte, 5562 + gen_helper_sve_ldffdd_le_zss_mte, } }, 5563 + { { gen_helper_sve_ldffbds_zd_mte, 5564 + gen_helper_sve_ldffhds_le_zd_mte, 5565 + gen_helper_sve_ldffsds_le_zd_mte, 5566 + NULL, }, 5567 + { gen_helper_sve_ldffbdu_zd_mte, 5568 + gen_helper_sve_ldffhdu_le_zd_mte, 5569 + gen_helper_sve_ldffsdu_le_zd_mte, 5570 + gen_helper_sve_ldffdd_le_zd_mte, } } } }, 5571 + { /* Big-endian */ 5572 + { { { gen_helper_sve_ldbds_zsu_mte, 5573 + gen_helper_sve_ldhds_be_zsu_mte, 5574 + gen_helper_sve_ldsds_be_zsu_mte, 5575 + NULL, }, 5576 + { gen_helper_sve_ldbdu_zsu_mte, 5577 + gen_helper_sve_ldhdu_be_zsu_mte, 5578 + gen_helper_sve_ldsdu_be_zsu_mte, 5579 + gen_helper_sve_lddd_be_zsu_mte, } }, 5580 + { { gen_helper_sve_ldbds_zss_mte, 5581 + gen_helper_sve_ldhds_be_zss_mte, 5582 + gen_helper_sve_ldsds_be_zss_mte, 5583 + NULL, }, 5584 + { gen_helper_sve_ldbdu_zss_mte, 5585 + gen_helper_sve_ldhdu_be_zss_mte, 5586 + gen_helper_sve_ldsdu_be_zss_mte, 5587 + gen_helper_sve_lddd_be_zss_mte, } }, 5588 + { { gen_helper_sve_ldbds_zd_mte, 5589 + gen_helper_sve_ldhds_be_zd_mte, 5590 + gen_helper_sve_ldsds_be_zd_mte, 5591 + NULL, }, 5592 + { gen_helper_sve_ldbdu_zd_mte, 5593 + gen_helper_sve_ldhdu_be_zd_mte, 5594 + gen_helper_sve_ldsdu_be_zd_mte, 5595 + gen_helper_sve_lddd_be_zd_mte, } } }, 5426 5596 5427 - /* First-fault */ 5428 - { { { gen_helper_sve_ldffbds_zsu, 5429 - gen_helper_sve_ldffhds_be_zsu, 5430 - gen_helper_sve_ldffsds_be_zsu, 5431 - NULL, }, 5432 - { gen_helper_sve_ldffbdu_zsu, 5433 - gen_helper_sve_ldffhdu_be_zsu, 5434 - gen_helper_sve_ldffsdu_be_zsu, 5435 - gen_helper_sve_ldffdd_be_zsu, } }, 5436 - { { gen_helper_sve_ldffbds_zss, 5437 - gen_helper_sve_ldffhds_be_zss, 5438 - gen_helper_sve_ldffsds_be_zss, 5439 - NULL, }, 5440 - { gen_helper_sve_ldffbdu_zss, 5441 - gen_helper_sve_ldffhdu_be_zss, 5442 - gen_helper_sve_ldffsdu_be_zss, 5443 - gen_helper_sve_ldffdd_be_zss, } }, 5444 - { { gen_helper_sve_ldffbds_zd, 5445 - gen_helper_sve_ldffhds_be_zd, 5446 - gen_helper_sve_ldffsds_be_zd, 5447 - NULL, }, 5448 - { gen_helper_sve_ldffbdu_zd, 5449 - gen_helper_sve_ldffhdu_be_zd, 5450 - gen_helper_sve_ldffsdu_be_zd, 5451 - gen_helper_sve_ldffdd_be_zd, } } } }, 5597 + /* First-fault */ 5598 + { { { gen_helper_sve_ldffbds_zsu_mte, 5599 + gen_helper_sve_ldffhds_be_zsu_mte, 5600 + gen_helper_sve_ldffsds_be_zsu_mte, 5601 + NULL, }, 5602 + { gen_helper_sve_ldffbdu_zsu_mte, 5603 + gen_helper_sve_ldffhdu_be_zsu_mte, 5604 + gen_helper_sve_ldffsdu_be_zsu_mte, 5605 + gen_helper_sve_ldffdd_be_zsu_mte, } }, 5606 + { { gen_helper_sve_ldffbds_zss_mte, 5607 + gen_helper_sve_ldffhds_be_zss_mte, 5608 + gen_helper_sve_ldffsds_be_zss_mte, 5609 + NULL, }, 5610 + { gen_helper_sve_ldffbdu_zss_mte, 5611 + gen_helper_sve_ldffhdu_be_zss_mte, 5612 + gen_helper_sve_ldffsdu_be_zss_mte, 5613 + gen_helper_sve_ldffdd_be_zss_mte, } }, 5614 + { { gen_helper_sve_ldffbds_zd_mte, 5615 + gen_helper_sve_ldffhds_be_zd_mte, 5616 + gen_helper_sve_ldffsds_be_zd_mte, 5617 + NULL, }, 5618 + { gen_helper_sve_ldffbdu_zd_mte, 5619 + gen_helper_sve_ldffhdu_be_zd_mte, 5620 + gen_helper_sve_ldffsdu_be_zd_mte, 5621 + gen_helper_sve_ldffdd_be_zd_mte, } } } } }, 5452 5622 }; 5453 5623 5454 5624 static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a) 5455 5625 { 5456 5626 gen_helper_gvec_mem_scatter *fn = NULL; 5457 - int be = s->be_data == MO_BE; 5627 + bool be = s->be_data == MO_BE; 5628 + bool mte = s->mte_active[0]; 5458 5629 5459 5630 if (!sve_access_check(s)) { 5460 5631 return true; ··· 5462 5633 5463 5634 switch (a->esz) { 5464 5635 case MO_32: 5465 - fn = gather_load_fn32[be][a->ff][a->xs][a->u][a->msz]; 5636 + fn = gather_load_fn32[mte][be][a->ff][a->xs][a->u][a->msz]; 5466 5637 break; 5467 5638 case MO_64: 5468 - fn = gather_load_fn64[be][a->ff][a->xs][a->u][a->msz]; 5639 + fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz]; 5469 5640 break; 5470 5641 } 5471 5642 assert(fn != NULL); 5472 5643 5473 5644 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, 5474 - cpu_reg_sp(s, a->rn), a->msz, fn); 5645 + cpu_reg_sp(s, a->rn), a->msz, false, fn); 5475 5646 return true; 5476 5647 } 5477 5648 5478 5649 static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a) 5479 5650 { 5480 5651 gen_helper_gvec_mem_scatter *fn = NULL; 5481 - int be = s->be_data == MO_BE; 5652 + bool be = s->be_data == MO_BE; 5653 + bool mte = s->mte_active[0]; 5482 5654 TCGv_i64 imm; 5483 5655 5484 5656 if (a->esz < a->msz || (a->esz == a->msz && !a->u)) { ··· 5490 5662 5491 5663 switch (a->esz) { 5492 5664 case MO_32: 5493 - fn = gather_load_fn32[be][a->ff][0][a->u][a->msz]; 5665 + fn = gather_load_fn32[mte][be][a->ff][0][a->u][a->msz]; 5494 5666 break; 5495 5667 case MO_64: 5496 - fn = gather_load_fn64[be][a->ff][2][a->u][a->msz]; 5668 + fn = gather_load_fn64[mte][be][a->ff][2][a->u][a->msz]; 5497 5669 break; 5498 5670 } 5499 5671 assert(fn != NULL); ··· 5502 5674 * by loading the immediate into the scalar parameter. 5503 5675 */ 5504 5676 imm = tcg_const_i64(a->imm << a->msz); 5505 - do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, fn); 5677 + do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, false, fn); 5506 5678 tcg_temp_free_i64(imm); 5507 5679 return true; 5508 5680 } 5509 5681 5510 - /* Indexed by [be][xs][msz]. */ 5511 - static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][3] = { 5512 - /* Little-endian */ 5513 - { { gen_helper_sve_stbs_zsu, 5514 - gen_helper_sve_sths_le_zsu, 5515 - gen_helper_sve_stss_le_zsu, }, 5516 - { gen_helper_sve_stbs_zss, 5517 - gen_helper_sve_sths_le_zss, 5518 - gen_helper_sve_stss_le_zss, } }, 5519 - /* Big-endian */ 5520 - { { gen_helper_sve_stbs_zsu, 5521 - gen_helper_sve_sths_be_zsu, 5522 - gen_helper_sve_stss_be_zsu, }, 5523 - { gen_helper_sve_stbs_zss, 5524 - gen_helper_sve_sths_be_zss, 5525 - gen_helper_sve_stss_be_zss, } }, 5682 + /* Indexed by [mte][be][xs][msz]. */ 5683 + static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][2][3] = { 5684 + { /* MTE Inactive */ 5685 + { /* Little-endian */ 5686 + { gen_helper_sve_stbs_zsu, 5687 + gen_helper_sve_sths_le_zsu, 5688 + gen_helper_sve_stss_le_zsu, }, 5689 + { gen_helper_sve_stbs_zss, 5690 + gen_helper_sve_sths_le_zss, 5691 + gen_helper_sve_stss_le_zss, } }, 5692 + { /* Big-endian */ 5693 + { gen_helper_sve_stbs_zsu, 5694 + gen_helper_sve_sths_be_zsu, 5695 + gen_helper_sve_stss_be_zsu, }, 5696 + { gen_helper_sve_stbs_zss, 5697 + gen_helper_sve_sths_be_zss, 5698 + gen_helper_sve_stss_be_zss, } } }, 5699 + { /* MTE Active */ 5700 + { /* Little-endian */ 5701 + { gen_helper_sve_stbs_zsu_mte, 5702 + gen_helper_sve_sths_le_zsu_mte, 5703 + gen_helper_sve_stss_le_zsu_mte, }, 5704 + { gen_helper_sve_stbs_zss_mte, 5705 + gen_helper_sve_sths_le_zss_mte, 5706 + gen_helper_sve_stss_le_zss_mte, } }, 5707 + { /* Big-endian */ 5708 + { gen_helper_sve_stbs_zsu_mte, 5709 + gen_helper_sve_sths_be_zsu_mte, 5710 + gen_helper_sve_stss_be_zsu_mte, }, 5711 + { gen_helper_sve_stbs_zss_mte, 5712 + gen_helper_sve_sths_be_zss_mte, 5713 + gen_helper_sve_stss_be_zss_mte, } } }, 5526 5714 }; 5527 5715 5528 5716 /* Note that we overload xs=2 to indicate 64-bit offset. */ 5529 - static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][3][4] = { 5530 - /* Little-endian */ 5531 - { { gen_helper_sve_stbd_zsu, 5532 - gen_helper_sve_sthd_le_zsu, 5533 - gen_helper_sve_stsd_le_zsu, 5534 - gen_helper_sve_stdd_le_zsu, }, 5535 - { gen_helper_sve_stbd_zss, 5536 - gen_helper_sve_sthd_le_zss, 5537 - gen_helper_sve_stsd_le_zss, 5538 - gen_helper_sve_stdd_le_zss, }, 5539 - { gen_helper_sve_stbd_zd, 5540 - gen_helper_sve_sthd_le_zd, 5541 - gen_helper_sve_stsd_le_zd, 5542 - gen_helper_sve_stdd_le_zd, } }, 5543 - /* Big-endian */ 5544 - { { gen_helper_sve_stbd_zsu, 5545 - gen_helper_sve_sthd_be_zsu, 5546 - gen_helper_sve_stsd_be_zsu, 5547 - gen_helper_sve_stdd_be_zsu, }, 5548 - { gen_helper_sve_stbd_zss, 5549 - gen_helper_sve_sthd_be_zss, 5550 - gen_helper_sve_stsd_be_zss, 5551 - gen_helper_sve_stdd_be_zss, }, 5552 - { gen_helper_sve_stbd_zd, 5553 - gen_helper_sve_sthd_be_zd, 5554 - gen_helper_sve_stsd_be_zd, 5555 - gen_helper_sve_stdd_be_zd, } }, 5717 + static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][2][3][4] = { 5718 + { /* MTE Inactive */ 5719 + { /* Little-endian */ 5720 + { gen_helper_sve_stbd_zsu, 5721 + gen_helper_sve_sthd_le_zsu, 5722 + gen_helper_sve_stsd_le_zsu, 5723 + gen_helper_sve_stdd_le_zsu, }, 5724 + { gen_helper_sve_stbd_zss, 5725 + gen_helper_sve_sthd_le_zss, 5726 + gen_helper_sve_stsd_le_zss, 5727 + gen_helper_sve_stdd_le_zss, }, 5728 + { gen_helper_sve_stbd_zd, 5729 + gen_helper_sve_sthd_le_zd, 5730 + gen_helper_sve_stsd_le_zd, 5731 + gen_helper_sve_stdd_le_zd, } }, 5732 + { /* Big-endian */ 5733 + { gen_helper_sve_stbd_zsu, 5734 + gen_helper_sve_sthd_be_zsu, 5735 + gen_helper_sve_stsd_be_zsu, 5736 + gen_helper_sve_stdd_be_zsu, }, 5737 + { gen_helper_sve_stbd_zss, 5738 + gen_helper_sve_sthd_be_zss, 5739 + gen_helper_sve_stsd_be_zss, 5740 + gen_helper_sve_stdd_be_zss, }, 5741 + { gen_helper_sve_stbd_zd, 5742 + gen_helper_sve_sthd_be_zd, 5743 + gen_helper_sve_stsd_be_zd, 5744 + gen_helper_sve_stdd_be_zd, } } }, 5745 + { /* MTE Inactive */ 5746 + { /* Little-endian */ 5747 + { gen_helper_sve_stbd_zsu_mte, 5748 + gen_helper_sve_sthd_le_zsu_mte, 5749 + gen_helper_sve_stsd_le_zsu_mte, 5750 + gen_helper_sve_stdd_le_zsu_mte, }, 5751 + { gen_helper_sve_stbd_zss_mte, 5752 + gen_helper_sve_sthd_le_zss_mte, 5753 + gen_helper_sve_stsd_le_zss_mte, 5754 + gen_helper_sve_stdd_le_zss_mte, }, 5755 + { gen_helper_sve_stbd_zd_mte, 5756 + gen_helper_sve_sthd_le_zd_mte, 5757 + gen_helper_sve_stsd_le_zd_mte, 5758 + gen_helper_sve_stdd_le_zd_mte, } }, 5759 + { /* Big-endian */ 5760 + { gen_helper_sve_stbd_zsu_mte, 5761 + gen_helper_sve_sthd_be_zsu_mte, 5762 + gen_helper_sve_stsd_be_zsu_mte, 5763 + gen_helper_sve_stdd_be_zsu_mte, }, 5764 + { gen_helper_sve_stbd_zss_mte, 5765 + gen_helper_sve_sthd_be_zss_mte, 5766 + gen_helper_sve_stsd_be_zss_mte, 5767 + gen_helper_sve_stdd_be_zss_mte, }, 5768 + { gen_helper_sve_stbd_zd_mte, 5769 + gen_helper_sve_sthd_be_zd_mte, 5770 + gen_helper_sve_stsd_be_zd_mte, 5771 + gen_helper_sve_stdd_be_zd_mte, } } }, 5556 5772 }; 5557 5773 5558 5774 static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a) 5559 5775 { 5560 5776 gen_helper_gvec_mem_scatter *fn; 5561 - int be = s->be_data == MO_BE; 5777 + bool be = s->be_data == MO_BE; 5778 + bool mte = s->mte_active[0]; 5562 5779 5563 5780 if (a->esz < a->msz || (a->msz == 0 && a->scale)) { 5564 5781 return false; ··· 5568 5785 } 5569 5786 switch (a->esz) { 5570 5787 case MO_32: 5571 - fn = scatter_store_fn32[be][a->xs][a->msz]; 5788 + fn = scatter_store_fn32[mte][be][a->xs][a->msz]; 5572 5789 break; 5573 5790 case MO_64: 5574 - fn = scatter_store_fn64[be][a->xs][a->msz]; 5791 + fn = scatter_store_fn64[mte][be][a->xs][a->msz]; 5575 5792 break; 5576 5793 default: 5577 5794 g_assert_not_reached(); 5578 5795 } 5579 5796 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, 5580 - cpu_reg_sp(s, a->rn), a->msz, fn); 5797 + cpu_reg_sp(s, a->rn), a->msz, true, fn); 5581 5798 return true; 5582 5799 } 5583 5800 5584 5801 static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a) 5585 5802 { 5586 5803 gen_helper_gvec_mem_scatter *fn = NULL; 5587 - int be = s->be_data == MO_BE; 5804 + bool be = s->be_data == MO_BE; 5805 + bool mte = s->mte_active[0]; 5588 5806 TCGv_i64 imm; 5589 5807 5590 5808 if (a->esz < a->msz) { ··· 5596 5814 5597 5815 switch (a->esz) { 5598 5816 case MO_32: 5599 - fn = scatter_store_fn32[be][0][a->msz]; 5817 + fn = scatter_store_fn32[mte][be][0][a->msz]; 5600 5818 break; 5601 5819 case MO_64: 5602 - fn = scatter_store_fn64[be][2][a->msz]; 5820 + fn = scatter_store_fn64[mte][be][2][a->msz]; 5603 5821 break; 5604 5822 } 5605 5823 assert(fn != NULL); ··· 5608 5826 * by loading the immediate into the scalar parameter. 5609 5827 */ 5610 5828 imm = tcg_const_i64(a->imm << a->msz); 5611 - do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, fn); 5829 + do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, true, fn); 5612 5830 tcg_temp_free_i64(imm); 5613 5831 return true; 5614 5832 }