rustc_target: Add various aarch64 features

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -521,13 +521,20 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
 
     let function_features = function_features
         .iter()
-        .flat_map(|feat| {
-            llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{f}"))
-        })
+        // Convert to LLVMFeatures and filter out unavailable ones
+        .flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat))
+        // Convert LLVMFeatures & dependencies to +<feats>s
+        .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}")))
         .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x {
             InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(),
             InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(),
         }))
+        // HACK: LLVM versions 19+ do not have the FPMR feature and treat it as always enabled
+        // It only exists as a feature in LLVM 18, cannot be passed down for any other version
+        .chain(match &*cx.tcx.sess.target.arch {
+            "aarch64" if llvm_util::get_version().0 == 18 => vec!["+fpmr".to_string()],
+            _ => vec![],
+        })
         .collect::<Vec<String>>();
 
     if cx.tcx.sess.target.is_like_wasm {

compiler/rustc_codegen_llvm/src/llvm_util.rs

@@ -209,7 +209,7 @@ impl<'a> IntoIterator for LLVMFeature<'a> {
 // Though note that Rust can also be build with an external precompiled version of LLVM
 // which might lead to failures if the oldest tested / supported LLVM version
 // doesn't yet support the relevant intrinsics
-pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> {
+pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> {
     let arch = if sess.target.arch == "x86_64" {
         "x86"
     } else if sess.target.arch == "arm64ec" {
@@ -218,40 +218,59 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a
         &*sess.target.arch
     };
     match (arch, s) {
-        ("x86", "sse4.2") => {
-            LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32"))
-        }
-        ("x86", "pclmulqdq") => LLVMFeature::new("pclmul"),
-        ("x86", "rdrand") => LLVMFeature::new("rdrnd"),
-        ("x86", "bmi1") => LLVMFeature::new("bmi"),
-        ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"),
-        ("x86", "lahfsahf") => LLVMFeature::new("sahf"),
-        ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"),
-        ("aarch64", "dpb") => LLVMFeature::new("ccpp"),
-        ("aarch64", "dpb2") => LLVMFeature::new("ccdp"),
-        ("aarch64", "frintts") => LLVMFeature::new("fptoint"),
-        ("aarch64", "fcma") => LLVMFeature::new("complxnum"),
-        ("aarch64", "pmuv3") => LLVMFeature::new("perfmon"),
-        ("aarch64", "paca") => LLVMFeature::new("pauth"),
-        ("aarch64", "pacg") => LLVMFeature::new("pauth"),
+        ("x86", "sse4.2") => Some(LLVMFeature::with_dependency(
+            "sse4.2",
+            TargetFeatureFoldStrength::EnableOnly("crc32"),
+        )),
+        ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")),
+        ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")),
+        ("x86", "bmi1") => Some(LLVMFeature::new("bmi")),
+        ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")),
+        ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")),
+        ("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")),
+        ("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")),
+        ("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")),
+        ("aarch64", "frintts") => Some(LLVMFeature::new("fptoint")),
+        ("aarch64", "fcma") => Some(LLVMFeature::new("complxnum")),
+        ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")),
+        ("aarch64", "paca") => Some(LLVMFeature::new("pauth")),
+        ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")),
+        ("aarch64", "sve-b16b16") => Some(LLVMFeature::new("b16b16")),
+        ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")),
         // Rust ties fp and neon together.
         ("aarch64", "neon") => {
-            LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8"))
+            Some(LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8")))
         }
         // In LLVM neon implicitly enables fp, but we manually enable
         // neon when a feature only implicitly enables fp
-        ("aarch64", "fhm") => LLVMFeature::new("fp16fml"),
-        ("aarch64", "fp16") => LLVMFeature::new("fullfp16"),
+        ("aarch64", "fhm") => Some(LLVMFeature::new("fp16fml")),
+        ("aarch64", "fp16") => Some(LLVMFeature::new("fullfp16")),
+        // Filter out features that are not supported by the current LLVM version
+        ("aarch64", "faminmax") if get_version().0 < 18 => None,
+        ("aarch64", "fp8") if get_version().0 < 18 => None,
+        ("aarch64", "fp8dot2") if get_version().0 < 18 => None,
+        ("aarch64", "fp8dot4") if get_version().0 < 18 => None,
+        ("aarch64", "fp8fma") if get_version().0 < 18 => None,
+        ("aarch64", "fpmr") if get_version().0 != 18 => None,
+        ("aarch64", "lut") if get_version().0 < 18 => None,
+        ("aarch64", "sme-f8f16") if get_version().0 < 18 => None,
+        ("aarch64", "sme-f8f32") if get_version().0 < 18 => None,
+        ("aarch64", "sme-fa64") if get_version().0 < 18 => None,
+        ("aarch64", "sme-lutv2") if get_version().0 < 18 => None,
+        ("aarch64", "ssve-fp8dot2") if get_version().0 < 18 => None,
+        ("aarch64", "ssve-fp8dot4") if get_version().0 < 18 => None,
+        ("aarch64", "ssve-fp8fma") if get_version().0 < 18 => None,
+        ("aarch64", "v9.5a") if get_version().0 < 18 => None,
         // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single feature called
         // `fast-unaligned-access`. In LLVM 19, it was split back out.
         ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => {
-            LLVMFeature::new("fast-unaligned-access")
+            Some(LLVMFeature::new("fast-unaligned-access"))
         }
         // For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled.
         ("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => {
-            LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512"))
+            Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")))
         }
-        (_, s) => LLVMFeature::new(s),
+        (_, s) => Some(LLVMFeature::new(s)),
     }
 }
 
@@ -291,13 +310,17 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
                     return true;
                 }
                 // check that all features in a given smallvec are enabled
-                for llvm_feature in to_llvm_features(sess, feature) {
-                    let cstr = SmallCStr::new(llvm_feature);
-                    if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } {
-                        return false;
+                if let Some(feat) = to_llvm_features(sess, feature) {
+                    for llvm_feature in feat {
+                        let cstr = SmallCStr::new(llvm_feature);
+                        if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } {
+                            return false;
+                        }
                     }
+                    true
+                } else {
+                    false
                 }
-                true
             })
             .map(|(feature, _, _)| Symbol::intern(feature)),
     );
@@ -386,9 +409,9 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach
         .target
         .supported_target_features()
         .iter()
-        .map(|(feature, _gate, _implied)| {
+        .filter_map(|(feature, _gate, _implied)| {
             // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings.
-            let llvm_feature = to_llvm_features(sess, *feature).llvm_feature_name;
+            let llvm_feature = to_llvm_features(sess, *feature)?.llvm_feature_name;
             let desc =
                 match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() {
                     Some(index) => {
@@ -398,7 +421,7 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach
                     None => "",
                 };
 
-            (*feature, desc)
+            Some((*feature, desc))
         })
         .collect::<Vec<_>>();
 
@@ -595,7 +618,7 @@ pub(crate) fn global_llvm_features(
                     if feature_state.is_none() {
                         let rust_feature =
                             supported_features.iter().find_map(|&(rust_feature, _, _)| {
-                                let llvm_features = to_llvm_features(sess, rust_feature);
+                                let llvm_features = to_llvm_features(sess, rust_feature)?;
                                 if llvm_features.contains(feature)
                                     && !llvm_features.contains(rust_feature)
                                 {
@@ -641,7 +664,7 @@ pub(crate) fn global_llvm_features(
                 // passing requests down to LLVM. This means that all in-language
                 // features also work on the command line instead of having two
                 // different names when the LLVM name and the Rust name differ.
-                let llvm_feature = to_llvm_features(sess, feature);
+                let llvm_feature = to_llvm_features(sess, feature)?;
 
                 Some(
                     std::iter::once(format!(
@@ -691,6 +714,9 @@ fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> {
     let feature = s
         .strip_prefix(&['+', '-'][..])
         .unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s }));
+    if s.is_empty() {
+        return None;
+    }
     // Rustc-specific feature requests like `+crt-static` or `-crt-static`
     // are not passed down to LLVM.
     if RUSTC_SPECIFIC_FEATURES.contains(&feature) {

compiler/rustc_feature/src/unstable.rs

@@ -302,6 +302,7 @@ declare_features! (
     // FIXME: Document these and merge with the list below.
 
     // Unstable `#[target_feature]` directives.
+    (unstable, aarch64_unstable_target_feature, "CURRENT_RUSTC_VERSION", Some(44839)),
     (unstable, aarch64_ver_target_feature, "1.27.0", Some(44839)),
     (unstable, arm_target_feature, "1.27.0", Some(44839)),
     (unstable, avx512_target_feature, "1.27.0", Some(44839)),

compiler/rustc_span/src/symbol.rs

@@ -356,6 +356,7 @@ symbols! {
         _task_context,
         a32,
         aarch64_target_feature,
+        aarch64_unstable_target_feature,
         aarch64_ver_target_feature,
         abi,
         abi_amdgpu_kernel,

compiler/rustc_target/src/target_features.rs

@@ -99,6 +99,8 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("bti", Stable, &[]),
     // FEAT_CRC
     ("crc", Stable, &[]),
+    // FEAT_CSSC
+    ("cssc", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_DIT
     ("dit", Stable, &[]),
     // FEAT_DotProd
@@ -107,21 +109,37 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("dpb", Stable, &[]),
     // FEAT_DPB2
     ("dpb2", Stable, &["dpb"]),
+    // FEAT_ECV
+    ("ecv", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_F32MM
     ("f32mm", Stable, &["sve"]),
     // FEAT_F64MM
     ("f64mm", Stable, &["sve"]),
+    // FEAT_FAMINMAX
+    ("faminmax", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_FCMA
     ("fcma", Stable, &["neon"]),
     // FEAT_FHM
     ("fhm", Stable, &["fp16"]),
     // FEAT_FLAGM
     ("flagm", Stable, &[]),
+    // FEAT_FLAGM2
+    ("flagm2", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_FP16
     // Rust ties FP and Neon: https://github.com/rust-lang/rust/pull/91608
     ("fp16", Stable, &["neon"]),
+    // FEAT_FP8
+    ("fp8", Unstable(sym::aarch64_unstable_target_feature), &["faminmax", "lut", "bf16"]),
+    // FEAT_FP8DOT2
+    ("fp8dot2", Unstable(sym::aarch64_unstable_target_feature), &["fp8dot4"]),
+    // FEAT_FP8DOT4
+    ("fp8dot4", Unstable(sym::aarch64_unstable_target_feature), &["fp8fma"]),
+    // FEAT_FP8FMA
+    ("fp8fma", Unstable(sym::aarch64_unstable_target_feature), &["fp8"]),
     // FEAT_FRINTTS
     ("frintts", Stable, &[]),
+    // FEAT_HBC
+    ("hbc", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_I8MM
     ("i8mm", Stable, &[]),
     // FEAT_JSCVT
@@ -131,6 +149,14 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("lor", Stable, &[]),
     // FEAT_LSE
     ("lse", Stable, &[]),
+    // FEAT_LSE128
+    ("lse128", Unstable(sym::aarch64_unstable_target_feature), &["lse"]),
+    // FEAT_LSE2
+    ("lse2", Unstable(sym::aarch64_unstable_target_feature), &[]),
+    // FEAT_LUT
+    ("lut", Unstable(sym::aarch64_unstable_target_feature), &[]),
+    // FEAT_MOPS
+    ("mops", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // FEAT_MTE & FEAT_MTE2
     ("mte", Stable, &[]),
     // FEAT_AdvSimd & FEAT_FP
@@ -143,14 +169,16 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("pan", Stable, &[]),
     // FEAT_PMUv3
     ("pmuv3", Stable, &[]),
-    // FEAT_RAND
+    // FEAT_RNG
     ("rand", Stable, &[]),
     // FEAT_RAS & FEAT_RASv1p1
     ("ras", Stable, &[]),
-    // FEAT_RCPC
+    // FEAT_LRCPC
     ("rcpc", Stable, &[]),
-    // FEAT_RCPC2
+    // FEAT_LRCPC2
     ("rcpc2", Stable, &["rcpc"]),
+    // FEAT_LRCPC3
+    ("rcpc3", Unstable(sym::aarch64_unstable_target_feature), &["rcpc2"]),
     // FEAT_RDM
     ("rdm", Stable, &["neon"]),
     // FEAT_SB
@@ -161,10 +189,36 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("sha3", Stable, &["sha2"]),
     // FEAT_SM3 & FEAT_SM4
     ("sm4", Stable, &["neon"]),
+    // FEAT_SME
+    ("sme", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]),
+    // FEAT_SME_F16F16
+    ("sme-f16f16", Unstable(sym::aarch64_unstable_target_feature), &["sme2"]),
+    // FEAT_SME_F64F64
+    ("sme-f64f64", Unstable(sym::aarch64_unstable_target_feature), &["sme"]),
+    // FEAT_SME_F8F16
+    ("sme-f8f16", Unstable(sym::aarch64_unstable_target_feature), &["sme-f8f32"]),
+    // FEAT_SME_F8F32
+    ("sme-f8f32", Unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]),
+    // FEAT_SME_FA64
+    ("sme-fa64", Unstable(sym::aarch64_unstable_target_feature), &["sme", "sve2"]),
+    // FEAT_SME_I16I64
+    ("sme-i16i64", Unstable(sym::aarch64_unstable_target_feature), &["sme"]),
+    // FEAT_SME_LUTv2
+    ("sme-lutv2", Unstable(sym::aarch64_unstable_target_feature), &[]),
+    // FEAT_SME2
+    ("sme2", Unstable(sym::aarch64_unstable_target_feature), &["sme"]),
+    // FEAT_SME2p1
+    ("sme2p1", Unstable(sym::aarch64_unstable_target_feature), &["sme2"]),
     // FEAT_SPE
     ("spe", Stable, &[]),
     // FEAT_SSBS & FEAT_SSBS2
     ("ssbs", Stable, &[]),
+    // FEAT_SSVE_FP8FDOT2
+    ("ssve-fp8dot2", Unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8dot4"]),
+    // FEAT_SSVE_FP8FDOT4
+    ("ssve-fp8dot4", Unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8fma"]),
+    // FEAT_SSVE_FP8FMA
+    ("ssve-fp8fma", Unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]),
     // FEAT_SVE
     // It was decided that SVE requires Neon: https://github.com/rust-lang/rust/pull/91608
     //
@@ -173,16 +227,20 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     //
     // "For backwards compatibility, Neon and VFP are required in the latest architectures."
     ("sve", Stable, &["neon"]),
+    // FEAT_SVE_B16B16 (SVE or SME Instructions)
+    ("sve-b16b16", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]),
     // FEAT_SVE2
     ("sve2", Stable, &["sve"]),
-    // FEAT_SVE2_AES
+    // FEAT_SVE_AES & FEAT_SVE_PMULL128
     ("sve2-aes", Stable, &["sve2", "aes"]),
     // FEAT_SVE2_BitPerm
     ("sve2-bitperm", Stable, &["sve2"]),
     // FEAT_SVE2_SHA3
     ("sve2-sha3", Stable, &["sve2", "sha3"]),
     // FEAT_SVE2_SM4
     ("sve2-sm4", Stable, &["sve2", "sm4"]),
+    // FEAT_SVE2p1
+    ("sve2p1", Unstable(sym::aarch64_unstable_target_feature), &["sve2"]),
     // FEAT_TME
     ("tme", Stable, &[]),
     (
@@ -199,9 +257,19 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     ("v8.4a", Unstable(sym::aarch64_ver_target_feature), &["v8.3a", "dotprod", "dit", "flagm"]),
     ("v8.5a", Unstable(sym::aarch64_ver_target_feature), &["v8.4a", "ssbs", "sb", "dpb2", "bti"]),
     ("v8.6a", Unstable(sym::aarch64_ver_target_feature), &["v8.5a", "bf16", "i8mm"]),
-    ("v8.7a", Unstable(sym::aarch64_ver_target_feature), &[]),
+    ("v8.7a", Unstable(sym::aarch64_ver_target_feature), &["v8.6a", "wfxt"]),
+    ("v8.8a", Unstable(sym::aarch64_ver_target_feature), &["v8.7a", "hbc", "mops"]),
+    ("v8.9a", Unstable(sym::aarch64_ver_target_feature), &["v8.8a", "cssc"]),
+    ("v9.1a", Unstable(sym::aarch64_ver_target_feature), &["v9a", "v8.6a"]),
+    ("v9.2a", Unstable(sym::aarch64_ver_target_feature), &["v9.1a", "v8.7a"]),
+    ("v9.3a", Unstable(sym::aarch64_ver_target_feature), &["v9.2a", "v8.8a"]),
+    ("v9.4a", Unstable(sym::aarch64_ver_target_feature), &["v9.3a", "v8.9a"]),
+    ("v9.5a", Unstable(sym::aarch64_ver_target_feature), &["v9.4a"]),
+    ("v9a", Unstable(sym::aarch64_ver_target_feature), &["v8.5a", "sve2"]),
     // FEAT_VHE
     ("vh", Stable, &[]),
+    // FEAT_WFxT
+    ("wfxt", Unstable(sym::aarch64_unstable_target_feature), &[]),
     // tidy-alphabetical-end
 ];