From 38ce4572eea2a325cb648b465d9d65640ab51138 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 14 Aug 2024 10:28:45 +0400
Subject: [PATCH] AArch64: Add tests for atomicrmw fp operations

There were only codegen tests for the fadd vector case,
so round out the test coverage for the scalar cases
and all the other operations.
---
 .../AArch64/atomicrmw-fadd-fp-vector.ll       | 115 ---
 llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll   | 706 ++++++++++++++++
 llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll   | 766 ++++++++++++++++++
 llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll   | 766 ++++++++++++++++++
 llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll   | 706 ++++++++++++++++
 5 files changed, 2944 insertions(+), 115 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fadd-fp-vector.ll
 create mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
 create mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
 create mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
 create mode 100644 llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll

diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fadd-fp-vector.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fadd-fp-vector.ll
deleted file mode 100644
index a7539ac3cce802..00000000000000
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fadd-fp-vector.ll
+++ /dev/null
@@ -1,115 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=aarch64-- -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefixes=CHECK,NOLSE %s
-; RUN: llc -mtriple=aarch64-- -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefixes=CHECK,LSE %s
-
-define <2 x half> @test_atomicrmw_fadd_v2f16_align4(ptr addrspace(1) %ptr, <2 x half> %value) #0 {
-; NOLSE-LABEL: test_atomicrmw_fadd_v2f16_align4:
-; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    fcvtl v1.4s, v0.4h
-; NOLSE-NEXT:    ldr s0, [x0]
-; NOLSE-NEXT:    b .LBB0_2
-; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB0_5
-; NOLSE-NEXT:  .LBB0_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB0_3 Depth 2
-; NOLSE-NEXT:    fcvtl v2.4s, v0.4h
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fadd v2.4s, v2.4s, v1.4s
-; NOLSE-NEXT:    fcvtn v2.4h, v2.4s
-; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB0_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB0_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB0_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB0_3
-; NOLSE-NEXT:    b .LBB0_1
-; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
-; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; NOLSE-NEXT:    ret
-;
-; LSE-LABEL: test_atomicrmw_fadd_v2f16_align4:
-; LSE:       // %bb.0:
-; LSE-NEXT:    fcvtl v1.4s, v0.4h
-; LSE-NEXT:    ldr s0, [x0]
-; LSE-NEXT:  .LBB0_1: // %atomicrmw.start
-; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
-; LSE-NEXT:    fcvtl v2.4s, v0.4h
-; LSE-NEXT:    fmov w8, s0
-; LSE-NEXT:    mov w10, w8
-; LSE-NEXT:    fadd v2.4s, v2.4s, v1.4s
-; LSE-NEXT:    fcvtn v2.4h, v2.4s
-; LSE-NEXT:    fmov w9, s2
-; LSE-NEXT:    casal w10, w9, [x0]
-; LSE-NEXT:    fmov s0, w10
-; LSE-NEXT:    cmp w10, w8
-; LSE-NEXT:    b.ne .LBB0_1
-; LSE-NEXT:  // %bb.2: // %atomicrmw.end
-; LSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; LSE-NEXT:    ret
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value seq_cst, align 4
-  ret <2 x half> %res
-}
-
-define <2 x float> @test_atomicrmw_fadd_v2f32_align8(ptr addrspace(1) %ptr, <2 x float> %value) #0 {
-; NOLSE-LABEL: test_atomicrmw_fadd_v2f32_align8:
-; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr d1, [x0]
-; NOLSE-NEXT:    b .LBB1_2
-; NOLSE-NEXT:  .LBB1_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x10
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.eq .LBB1_5
-; NOLSE-NEXT:  .LBB1_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB1_3 Depth 2
-; NOLSE-NEXT:    fadd v2.2s, v1.2s, v0.2s
-; NOLSE-NEXT:    fmov x9, d1
-; NOLSE-NEXT:    fmov x8, d2
-; NOLSE-NEXT:  .LBB1_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB1_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr x10, [x0]
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.ne .LBB1_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, x8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB1_3
-; NOLSE-NEXT:    b .LBB1_1
-; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
-; NOLSE-NEXT:    fmov d0, d1
-; NOLSE-NEXT:    ret
-;
-; LSE-LABEL: test_atomicrmw_fadd_v2f32_align8:
-; LSE:       // %bb.0:
-; LSE-NEXT:    ldr d1, [x0]
-; LSE-NEXT:  .LBB1_1: // %atomicrmw.start
-; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
-; LSE-NEXT:    fadd v2.2s, v1.2s, v0.2s
-; LSE-NEXT:    fmov x8, d1
-; LSE-NEXT:    mov x10, x8
-; LSE-NEXT:    fmov x9, d2
-; LSE-NEXT:    casal x10, x9, [x0]
-; LSE-NEXT:    fmov d1, x10
-; LSE-NEXT:    cmp x10, x8
-; LSE-NEXT:    b.ne .LBB1_1
-; LSE-NEXT:  // %bb.2: // %atomicrmw.end
-; LSE-NEXT:    fmov d0, d1
-; LSE-NEXT:    ret
-  %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x float> %value seq_cst, align 8
-  ret <2 x float> %res
-}
-
-attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
new file mode 100644
index 00000000000000..605c7d5e0a55ea
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
@@ -0,0 +1,706 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-- -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=NOLSE %s
+; RUN: llc -mtriple=aarch64-- -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s
+
+define half @test_atomicrmw_fadd_f16_seq_cst_align2(ptr %ptr, half %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align2:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvt s1, h0
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    b .LBB0_2
+; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB0_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB0_5
+; NOLSE-NEXT:  .LBB0_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB0_3 Depth 2
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fadd s2, s2, s1
+; NOLSE-NEXT:    fcvt h2, s2
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB0_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB0_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB0_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB0_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB0_3
+; NOLSE-NEXT:    b .LBB0_1
+; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align2:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvt s1, h0
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:  .LBB0_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fadd s2, s2, s1
+; LSE-NEXT:    fcvt h2, s2
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casalh w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8, uxth
+; LSE-NEXT:    b.ne .LBB0_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+  ret half %res
+}
+
+define half @test_atomicrmw_fadd_f16_seq_cst_align4(ptr %ptr, half %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvt s1, h0
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    b .LBB1_2
+; NOLSE-NEXT:  .LBB1_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB1_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB1_5
+; NOLSE-NEXT:  .LBB1_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB1_3 Depth 2
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fadd s2, s2, s1
+; NOLSE-NEXT:    fcvt h2, s2
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB1_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB1_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB1_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB1_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB1_3
+; NOLSE-NEXT:    b .LBB1_1
+; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvt s1, h0
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:  .LBB1_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fadd s2, s2, s1
+; LSE-NEXT:    fcvt h2, s2
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casalh w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8, uxth
+; LSE-NEXT:    b.ne .LBB1_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, half %value seq_cst, align 4
+  ret half %res
+}
+
+define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align2(ptr %ptr, bfloat %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align2:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB2_2
+; NOLSE-NEXT:  .LBB2_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB2_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB2_5
+; NOLSE-NEXT:  .LBB2_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB2_3 Depth 2
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fadd s2, s2, s1
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    ubfx w10, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    add w9, w10, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:  .LBB2_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB2_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB2_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB2_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB2_3
+; NOLSE-NEXT:    b .LBB2_1
+; NOLSE-NEXT:  .LBB2_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align2:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB2_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s2, w9
+; LSE-NEXT:    fadd s2, s2, s1
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    ubfx w10, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    add w9, w10, w9
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    mov w11, w10
+; LSE-NEXT:    casalh w11, w9, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w10, uxth
+; LSE-NEXT:    b.ne .LBB2_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
+  ret bfloat %res
+}
+
+define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align4(ptr %ptr, bfloat %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB3_2
+; NOLSE-NEXT:  .LBB3_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB3_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB3_5
+; NOLSE-NEXT:  .LBB3_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB3_3 Depth 2
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fadd s2, s2, s1
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    ubfx w10, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    add w9, w10, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:  .LBB3_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB3_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB3_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB3_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB3_3
+; NOLSE-NEXT:    b .LBB3_1
+; NOLSE-NEXT:  .LBB3_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_bf16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB3_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s2, w9
+; LSE-NEXT:    fadd s2, s2, s1
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    ubfx w10, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    add w9, w10, w9
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    mov w11, w10
+; LSE-NEXT:    casalh w11, w9, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w10, uxth
+; LSE-NEXT:    b.ne .LBB3_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 4
+  ret bfloat %res
+}
+
+define float @test_atomicrmw_fadd_f32_seq_cst_align4(ptr %ptr, float %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr s1, [x0]
+; NOLSE-NEXT:    b .LBB4_2
+; NOLSE-NEXT:  .LBB4_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=1
+; NOLSE-NEXT:    fmov s1, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB4_5
+; NOLSE-NEXT:  .LBB4_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB4_3 Depth 2
+; NOLSE-NEXT:    fadd s2, s1, s0
+; NOLSE-NEXT:    fmov w9, s1
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB4_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB4_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB4_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB4_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB4_3
+; NOLSE-NEXT:    b .LBB4_1
+; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov s0, s1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr s1, [x0]
+; LSE-NEXT:  .LBB4_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fadd s2, s1, s0
+; LSE-NEXT:    fmov w8, s1
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s1, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB4_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov s0, s1
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+  ret float %res
+}
+
+define double @test_atomicrmw_fadd_f32_seq_cst_align8(ptr %ptr, double %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr d1, [x0]
+; NOLSE-NEXT:    b .LBB5_2
+; NOLSE-NEXT:  .LBB5_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x10
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.eq .LBB5_5
+; NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
+; NOLSE-NEXT:    fadd d2, d1, d0
+; NOLSE-NEXT:    fmov x9, d1
+; NOLSE-NEXT:    fmov x8, d2
+; NOLSE-NEXT:  .LBB5_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr x10, [x0]
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.ne .LBB5_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, x8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB5_3
+; NOLSE-NEXT:    b .LBB5_1
+; NOLSE-NEXT:  .LBB5_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, d1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr d1, [x0]
+; LSE-NEXT:  .LBB5_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fadd d2, d1, d0
+; LSE-NEXT:    fmov x8, d1
+; LSE-NEXT:    mov x10, x8
+; LSE-NEXT:    fmov x9, d2
+; LSE-NEXT:    casal x10, x9, [x0]
+; LSE-NEXT:    fmov d1, x10
+; LSE-NEXT:    cmp x10, x8
+; LSE-NEXT:    b.ne .LBB5_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov d0, d1
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+  ret double %res
+}
+
+define fp128 @test_atomicrmw_fadd_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_fp128_seq_cst_align16:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    sub sp, sp, #96
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; NOLSE-NEXT:    mov x19, x0
+; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
+; NOLSE-NEXT:    stp x12, x13, [sp, #32]
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    ldr q1, [sp, #32]
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    b.eq .LBB6_6
+; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    bl __addtf3
+; NOLSE-NEXT:    str q0, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp, #64]
+; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
+; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x19]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB6_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    add sp, sp, #96
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_fp128_seq_cst_align16:
+; LSE:       // %bb.0:
+; LSE-NEXT:    sub sp, sp, #96
+; LSE-NEXT:    ldr q1, [x0]
+; LSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; LSE-NEXT:    mov x19, x0
+; LSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; LSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; LSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; LSE-NEXT:    bl __addtf3
+; LSE-NEXT:    str q0, [sp, #48]
+; LSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; LSE-NEXT:    ldp x0, x1, [sp, #48]
+; LSE-NEXT:    str q0, [sp, #64]
+; LSE-NEXT:    ldp x2, x3, [sp, #64]
+; LSE-NEXT:    mov x4, x2
+; LSE-NEXT:    mov x5, x3
+; LSE-NEXT:    caspal x4, x5, x0, x1, [x19]
+; LSE-NEXT:    stp x4, x5, [sp, #32]
+; LSE-NEXT:    cmp x5, x3
+; LSE-NEXT:    ldr q1, [sp, #32]
+; LSE-NEXT:    ccmp x4, x2, #0, eq
+; LSE-NEXT:    b.ne .LBB6_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    add sp, sp, #96
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, fp128 %value seq_cst, align 16
+  ret fp128 %res
+}
+
+define <2 x half> @test_atomicrmw_fadd_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_v2f16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvtl v1.4s, v0.4h
+; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    b .LBB7_2
+; NOLSE-NEXT:  .LBB7_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB7_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB7_5
+; NOLSE-NEXT:  .LBB7_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB7_3 Depth 2
+; NOLSE-NEXT:    fcvtl v2.4s, v0.4h
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fadd v2.4s, v2.4s, v1.4s
+; NOLSE-NEXT:    fcvtn v2.4h, v2.4s
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB7_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB7_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB7_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB7_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB7_3
+; NOLSE-NEXT:    b .LBB7_1
+; NOLSE-NEXT:  .LBB7_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_v2f16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvtl v1.4s, v0.4h
+; LSE-NEXT:    ldr s0, [x0]
+; LSE-NEXT:  .LBB7_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvtl v2.4s, v0.4h
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fadd v2.4s, v2.4s, v1.4s
+; LSE-NEXT:    fcvtn v2.4h, v2.4s
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB7_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, <2 x half> %value seq_cst, align 4
+  ret <2 x half> %res
+}
+
+define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_seq_cst_align4(ptr %ptr, <2 x bfloat> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_v2bf16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    movi v1.4s, #1
+; NOLSE-NEXT:    movi v2.4s, #127, msl #8
+; NOLSE-NEXT:    shll v3.4s, v0.4h, #16
+; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    b .LBB8_2
+; NOLSE-NEXT:  .LBB8_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB8_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB8_5
+; NOLSE-NEXT:  .LBB8_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB8_3 Depth 2
+; NOLSE-NEXT:    shll v4.4s, v0.4h, #16
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fadd v4.4s, v4.4s, v3.4s
+; NOLSE-NEXT:    ushr v5.4s, v4.4s, #16
+; NOLSE-NEXT:    and v5.16b, v5.16b, v1.16b
+; NOLSE-NEXT:    add v4.4s, v5.4s, v4.4s
+; NOLSE-NEXT:    addhn v4.4h, v4.4s, v2.4s
+; NOLSE-NEXT:    fmov w8, s4
+; NOLSE-NEXT:  .LBB8_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB8_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB8_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB8_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB8_3
+; NOLSE-NEXT:    b .LBB8_1
+; NOLSE-NEXT:  .LBB8_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_v2bf16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    movi v1.4s, #1
+; LSE-NEXT:    movi v2.4s, #127, msl #8
+; LSE-NEXT:    shll v3.4s, v0.4h, #16
+; LSE-NEXT:    ldr s0, [x0]
+; LSE-NEXT:  .LBB8_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    shll v4.4s, v0.4h, #16
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    fadd v4.4s, v4.4s, v3.4s
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    ushr v5.4s, v4.4s, #16
+; LSE-NEXT:    and v5.16b, v5.16b, v1.16b
+; LSE-NEXT:    add v4.4s, v5.4s, v4.4s
+; LSE-NEXT:    addhn v4.4h, v4.4s, v2.4s
+; LSE-NEXT:    fmov w9, s4
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB8_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value seq_cst, align 4
+  ret <2 x bfloat> %res
+}
+
+define <2 x float> @test_atomicrmw_fadd_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_v2f32_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr d1, [x0]
+; NOLSE-NEXT:    b .LBB9_2
+; NOLSE-NEXT:  .LBB9_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x10
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.eq .LBB9_5
+; NOLSE-NEXT:  .LBB9_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB9_3 Depth 2
+; NOLSE-NEXT:    fadd v2.2s, v1.2s, v0.2s
+; NOLSE-NEXT:    fmov x9, d1
+; NOLSE-NEXT:    fmov x8, d2
+; NOLSE-NEXT:  .LBB9_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB9_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr x10, [x0]
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.ne .LBB9_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB9_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, x8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB9_3
+; NOLSE-NEXT:    b .LBB9_1
+; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, d1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_v2f32_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr d1, [x0]
+; LSE-NEXT:  .LBB9_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fadd v2.2s, v1.2s, v0.2s
+; LSE-NEXT:    fmov x8, d1
+; LSE-NEXT:    mov x10, x8
+; LSE-NEXT:    fmov x9, d2
+; LSE-NEXT:    casal x10, x9, [x0]
+; LSE-NEXT:    fmov d1, x10
+; LSE-NEXT:    cmp x10, x8
+; LSE-NEXT:    b.ne .LBB9_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov d0, d1
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, <2 x float> %value seq_cst, align 8
+  ret <2 x float> %res
+}
+
+define <2 x double> @test_atomicrmw_fadd_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fadd_v2f64_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    b .LBB10_2
+; NOLSE-NEXT:  .LBB10_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x12
+; NOLSE-NEXT:    cmp x13, x9
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    mov v1.d[1], x13
+; NOLSE-NEXT:    b.eq .LBB10_6
+; NOLSE-NEXT:  .LBB10_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB10_3 Depth 2
+; NOLSE-NEXT:    fadd v2.2d, v1.2d, v0.2d
+; NOLSE-NEXT:    mov x9, v1.d[1]
+; NOLSE-NEXT:    fmov x11, d1
+; NOLSE-NEXT:    mov x8, v2.d[1]
+; NOLSE-NEXT:    fmov x10, d2
+; NOLSE-NEXT:  .LBB10_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB10_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x0]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x9
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB10_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x0]
+; NOLSE-NEXT:    cbnz w14, .LBB10_3
+; NOLSE-NEXT:    b .LBB10_1
+; NOLSE-NEXT:  .LBB10_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x10, x8, [x0]
+; NOLSE-NEXT:    cbnz w14, .LBB10_3
+; NOLSE-NEXT:    b .LBB10_1
+; NOLSE-NEXT:  .LBB10_6: // %atomicrmw.end
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fadd_v2f64_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr q1, [x0]
+; LSE-NEXT:  .LBB10_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fadd v2.2d, v1.2d, v0.2d
+; LSE-NEXT:    mov x3, v1.d[1]
+; LSE-NEXT:    fmov x2, d1
+; LSE-NEXT:    mov x7, x3
+; LSE-NEXT:    mov x5, v2.d[1]
+; LSE-NEXT:    mov x6, x2
+; LSE-NEXT:    fmov x4, d2
+; LSE-NEXT:    caspal x6, x7, x4, x5, [x0]
+; LSE-NEXT:    fmov d1, x6
+; LSE-NEXT:    cmp x7, x3
+; LSE-NEXT:    ccmp x6, x2, #0, eq
+; LSE-NEXT:    mov v1.d[1], x7
+; LSE-NEXT:    b.ne .LBB10_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    ret
+  %res = atomicrmw fadd ptr %ptr, <2 x double> %value seq_cst, align 16
+  ret <2 x double> %res
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
new file mode 100644
index 00000000000000..43d556d7193522
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
@@ -0,0 +1,766 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-- -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=NOLSE %s
+; RUN: llc -mtriple=aarch64-- -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s
+
+define half @test_atomicrmw_fmax_f16_seq_cst_align2(ptr %ptr, half %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align2:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvt s1, h0
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    b .LBB0_2
+; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB0_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB0_5
+; NOLSE-NEXT:  .LBB0_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB0_3 Depth 2
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmaxnm s2, s2, s1
+; NOLSE-NEXT:    fcvt h2, s2
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB0_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB0_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB0_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB0_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB0_3
+; NOLSE-NEXT:    b .LBB0_1
+; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align2:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvt s1, h0
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:  .LBB0_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fmaxnm s2, s2, s1
+; LSE-NEXT:    fcvt h2, s2
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casalh w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8, uxth
+; LSE-NEXT:    b.ne .LBB0_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+  ret half %res
+}
+
+define half @test_atomicrmw_fmax_f16_seq_cst_align4(ptr %ptr, half %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvt s1, h0
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    b .LBB1_2
+; NOLSE-NEXT:  .LBB1_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB1_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB1_5
+; NOLSE-NEXT:  .LBB1_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB1_3 Depth 2
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmaxnm s2, s2, s1
+; NOLSE-NEXT:    fcvt h2, s2
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB1_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB1_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB1_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB1_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB1_3
+; NOLSE-NEXT:    b .LBB1_1
+; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvt s1, h0
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:  .LBB1_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fmaxnm s2, s2, s1
+; LSE-NEXT:    fcvt h2, s2
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casalh w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8, uxth
+; LSE-NEXT:    b.ne .LBB1_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, half %value seq_cst, align 4
+  ret half %res
+}
+
+define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align2(ptr %ptr, bfloat %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align2:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB2_2
+; NOLSE-NEXT:  .LBB2_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB2_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB2_5
+; NOLSE-NEXT:  .LBB2_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB2_3 Depth 2
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmaxnm s2, s2, s1
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    ubfx w10, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    add w9, w10, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:  .LBB2_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB2_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB2_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB2_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB2_3
+; NOLSE-NEXT:    b .LBB2_1
+; NOLSE-NEXT:  .LBB2_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align2:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB2_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s2, w9
+; LSE-NEXT:    fmaxnm s2, s2, s1
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    ubfx w10, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    add w9, w10, w9
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    mov w11, w10
+; LSE-NEXT:    casalh w11, w9, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w10, uxth
+; LSE-NEXT:    b.ne .LBB2_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
+  ret bfloat %res
+}
+
+define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align4(ptr %ptr, bfloat %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB3_2
+; NOLSE-NEXT:  .LBB3_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB3_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB3_5
+; NOLSE-NEXT:  .LBB3_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB3_3 Depth 2
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmaxnm s2, s2, s1
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    ubfx w10, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    add w9, w10, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:  .LBB3_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB3_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB3_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB3_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB3_3
+; NOLSE-NEXT:    b .LBB3_1
+; NOLSE-NEXT:  .LBB3_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_bf16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB3_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s2, w9
+; LSE-NEXT:    fmaxnm s2, s2, s1
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    ubfx w10, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    add w9, w10, w9
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    mov w11, w10
+; LSE-NEXT:    casalh w11, w9, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w10, uxth
+; LSE-NEXT:    b.ne .LBB3_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 4
+  ret bfloat %res
+}
+
+define float @test_atomicrmw_fmax_f32_seq_cst_align4(ptr %ptr, float %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr s1, [x0]
+; NOLSE-NEXT:    b .LBB4_2
+; NOLSE-NEXT:  .LBB4_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=1
+; NOLSE-NEXT:    fmov s1, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB4_5
+; NOLSE-NEXT:  .LBB4_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB4_3 Depth 2
+; NOLSE-NEXT:    fmaxnm s2, s1, s0
+; NOLSE-NEXT:    fmov w9, s1
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB4_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB4_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB4_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB4_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB4_3
+; NOLSE-NEXT:    b .LBB4_1
+; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov s0, s1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr s1, [x0]
+; LSE-NEXT:  .LBB4_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmaxnm s2, s1, s0
+; LSE-NEXT:    fmov w8, s1
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s1, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB4_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov s0, s1
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+  ret float %res
+}
+
+define double @test_atomicrmw_fmax_f32_seq_cst_align8(ptr %ptr, double %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr d1, [x0]
+; NOLSE-NEXT:    b .LBB5_2
+; NOLSE-NEXT:  .LBB5_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x10
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.eq .LBB5_5
+; NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
+; NOLSE-NEXT:    fmaxnm d2, d1, d0
+; NOLSE-NEXT:    fmov x9, d1
+; NOLSE-NEXT:    fmov x8, d2
+; NOLSE-NEXT:  .LBB5_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr x10, [x0]
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.ne .LBB5_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, x8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB5_3
+; NOLSE-NEXT:    b .LBB5_1
+; NOLSE-NEXT:  .LBB5_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, d1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr d1, [x0]
+; LSE-NEXT:  .LBB5_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmaxnm d2, d1, d0
+; LSE-NEXT:    fmov x8, d1
+; LSE-NEXT:    mov x10, x8
+; LSE-NEXT:    fmov x9, d2
+; LSE-NEXT:    casal x10, x9, [x0]
+; LSE-NEXT:    fmov d1, x10
+; LSE-NEXT:    cmp x10, x8
+; LSE-NEXT:    b.ne .LBB5_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov d0, d1
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+  ret double %res
+}
+
+define fp128 @test_atomicrmw_fmax_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    sub sp, sp, #96
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; NOLSE-NEXT:    mov x19, x0
+; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
+; NOLSE-NEXT:    stp x12, x13, [sp, #32]
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    ldr q1, [sp, #32]
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    b.eq .LBB6_6
+; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    bl fmaxl
+; NOLSE-NEXT:    str q0, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp, #64]
+; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
+; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x19]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB6_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    add sp, sp, #96
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16:
+; LSE:       // %bb.0:
+; LSE-NEXT:    sub sp, sp, #96
+; LSE-NEXT:    ldr q1, [x0]
+; LSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; LSE-NEXT:    mov x19, x0
+; LSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; LSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; LSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; LSE-NEXT:    bl fmaxl
+; LSE-NEXT:    str q0, [sp, #48]
+; LSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; LSE-NEXT:    ldp x0, x1, [sp, #48]
+; LSE-NEXT:    str q0, [sp, #64]
+; LSE-NEXT:    ldp x2, x3, [sp, #64]
+; LSE-NEXT:    mov x4, x2
+; LSE-NEXT:    mov x5, x3
+; LSE-NEXT:    caspal x4, x5, x0, x1, [x19]
+; LSE-NEXT:    stp x4, x5, [sp, #32]
+; LSE-NEXT:    cmp x5, x3
+; LSE-NEXT:    ldr q1, [sp, #32]
+; LSE-NEXT:    ccmp x4, x2, #0, eq
+; LSE-NEXT:    b.ne .LBB6_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    add sp, sp, #96
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, fp128 %value seq_cst, align 16
+  ret fp128 %res
+}
+
+define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_v2f16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOLSE-NEXT:    mov h1, v0.h[1]
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    fcvt s1, h1
+; NOLSE-NEXT:    b .LBB7_2
+; NOLSE-NEXT:  .LBB7_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB7_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB7_5
+; NOLSE-NEXT:  .LBB7_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB7_3 Depth 2
+; NOLSE-NEXT:    mov h3, v0.h[1]
+; NOLSE-NEXT:    fcvt s4, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fcvt s3, h3
+; NOLSE-NEXT:    fmaxnm s4, s4, s2
+; NOLSE-NEXT:    fmaxnm s3, s3, s1
+; NOLSE-NEXT:    fcvt h4, s4
+; NOLSE-NEXT:    fcvt h3, s3
+; NOLSE-NEXT:    mov v4.h[1], v3.h[0]
+; NOLSE-NEXT:    fmov w8, s4
+; NOLSE-NEXT:  .LBB7_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB7_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB7_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB7_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB7_3
+; NOLSE-NEXT:    b .LBB7_1
+; NOLSE-NEXT:  .LBB7_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_v2f16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $d0 killed $d0 def $q0
+; LSE-NEXT:    mov h1, v0.h[1]
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    ldr s0, [x0]
+; LSE-NEXT:    fcvt s1, h1
+; LSE-NEXT:  .LBB7_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    mov h3, v0.h[1]
+; LSE-NEXT:    fcvt s4, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fcvt s3, h3
+; LSE-NEXT:    fmaxnm s4, s4, s2
+; LSE-NEXT:    fmaxnm s3, s3, s1
+; LSE-NEXT:    fcvt h4, s4
+; LSE-NEXT:    fcvt h3, s3
+; LSE-NEXT:    mov v4.h[1], v3.h[0]
+; LSE-NEXT:    fmov w9, s4
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB7_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, <2 x half> %value seq_cst, align 4
+  ret <2 x half> %res
+}
+
+define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bfloat> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_v2bf16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOLSE-NEXT:    mov h1, v0.h[1]
+; NOLSE-NEXT:    fmov w10, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    lsl w10, w10, #16
+; NOLSE-NEXT:    fmov w9, s1
+; NOLSE-NEXT:    fmov s2, w10
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB8_2
+; NOLSE-NEXT:  .LBB8_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB8_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9
+; NOLSE-NEXT:    b.eq .LBB8_5
+; NOLSE-NEXT:  .LBB8_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB8_3 Depth 2
+; NOLSE-NEXT:    mov h3, v0.h[1]
+; NOLSE-NEXT:    fmov w10, s0
+; NOLSE-NEXT:    lsl w10, w10, #16
+; NOLSE-NEXT:    fmov w9, s3
+; NOLSE-NEXT:    fmov s4, w10
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmaxnm s4, s4, s2
+; NOLSE-NEXT:    fmov s3, w9
+; NOLSE-NEXT:    fmaxnm s3, s3, s1
+; NOLSE-NEXT:    fmov w10, s4
+; NOLSE-NEXT:    ubfx w12, w10, #16, #1
+; NOLSE-NEXT:    add w10, w10, w8
+; NOLSE-NEXT:    fmov w9, s3
+; NOLSE-NEXT:    add w10, w12, w10
+; NOLSE-NEXT:    lsr w10, w10, #16
+; NOLSE-NEXT:    ubfx w11, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    fmov s4, w10
+; NOLSE-NEXT:    add w9, w11, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s3, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov v4.h[1], v3.h[0]
+; NOLSE-NEXT:    fmov w10, s4
+; NOLSE-NEXT:  .LBB8_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB8_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9
+; NOLSE-NEXT:    b.ne .LBB8_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB8_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB8_3
+; NOLSE-NEXT:    b .LBB8_1
+; NOLSE-NEXT:  .LBB8_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_v2bf16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $d0 killed $d0 def $q0
+; LSE-NEXT:    mov h1, v0.h[1]
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr s0, [x0]
+; LSE-NEXT:    lsl w10, w10, #16
+; LSE-NEXT:    fmov w9, s1
+; LSE-NEXT:    fmov s2, w10
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB8_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    mov h3, v0.h[1]
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsl w10, w10, #16
+; LSE-NEXT:    fmov w9, s3
+; LSE-NEXT:    fmov s4, w10
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmaxnm s4, s4, s2
+; LSE-NEXT:    fmov s3, w9
+; LSE-NEXT:    fmaxnm s3, s3, s1
+; LSE-NEXT:    fmov w10, s4
+; LSE-NEXT:    ubfx w12, w10, #16, #1
+; LSE-NEXT:    add w10, w10, w8
+; LSE-NEXT:    fmov w9, s3
+; LSE-NEXT:    add w10, w12, w10
+; LSE-NEXT:    lsr w10, w10, #16
+; LSE-NEXT:    ubfx w11, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    fmov s4, w10
+; LSE-NEXT:    add w9, w11, w9
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    fmov s3, w9
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov v4.h[1], v3.h[0]
+; LSE-NEXT:    mov w11, w9
+; LSE-NEXT:    fmov w10, s4
+; LSE-NEXT:    casal w11, w10, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w9
+; LSE-NEXT:    b.ne .LBB8_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, <2 x bfloat> %value seq_cst, align 4
+  ret <2 x bfloat> %res
+}
+
+define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_v2f32_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr d1, [x0]
+; NOLSE-NEXT:    b .LBB9_2
+; NOLSE-NEXT:  .LBB9_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x10
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.eq .LBB9_5
+; NOLSE-NEXT:  .LBB9_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB9_3 Depth 2
+; NOLSE-NEXT:    fmaxnm v2.2s, v1.2s, v0.2s
+; NOLSE-NEXT:    fmov x9, d1
+; NOLSE-NEXT:    fmov x8, d2
+; NOLSE-NEXT:  .LBB9_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB9_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr x10, [x0]
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.ne .LBB9_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB9_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, x8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB9_3
+; NOLSE-NEXT:    b .LBB9_1
+; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, d1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_v2f32_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr d1, [x0]
+; LSE-NEXT:  .LBB9_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmaxnm v2.2s, v1.2s, v0.2s
+; LSE-NEXT:    fmov x8, d1
+; LSE-NEXT:    mov x10, x8
+; LSE-NEXT:    fmov x9, d2
+; LSE-NEXT:    casal x10, x9, [x0]
+; LSE-NEXT:    fmov d1, x10
+; LSE-NEXT:    cmp x10, x8
+; LSE-NEXT:    b.ne .LBB9_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov d0, d1
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, <2 x float> %value seq_cst, align 8
+  ret <2 x float> %res
+}
+
+define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmax_v2f64_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    b .LBB10_2
+; NOLSE-NEXT:  .LBB10_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x12
+; NOLSE-NEXT:    cmp x13, x9
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    mov v1.d[1], x13
+; NOLSE-NEXT:    b.eq .LBB10_6
+; NOLSE-NEXT:  .LBB10_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB10_3 Depth 2
+; NOLSE-NEXT:    fmaxnm v2.2d, v1.2d, v0.2d
+; NOLSE-NEXT:    mov x9, v1.d[1]
+; NOLSE-NEXT:    fmov x11, d1
+; NOLSE-NEXT:    mov x8, v2.d[1]
+; NOLSE-NEXT:    fmov x10, d2
+; NOLSE-NEXT:  .LBB10_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB10_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x0]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x9
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB10_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x0]
+; NOLSE-NEXT:    cbnz w14, .LBB10_3
+; NOLSE-NEXT:    b .LBB10_1
+; NOLSE-NEXT:  .LBB10_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x10, x8, [x0]
+; NOLSE-NEXT:    cbnz w14, .LBB10_3
+; NOLSE-NEXT:    b .LBB10_1
+; NOLSE-NEXT:  .LBB10_6: // %atomicrmw.end
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmax_v2f64_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr q1, [x0]
+; LSE-NEXT:  .LBB10_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmaxnm v2.2d, v1.2d, v0.2d
+; LSE-NEXT:    mov x3, v1.d[1]
+; LSE-NEXT:    fmov x2, d1
+; LSE-NEXT:    mov x7, x3
+; LSE-NEXT:    mov x5, v2.d[1]
+; LSE-NEXT:    mov x6, x2
+; LSE-NEXT:    fmov x4, d2
+; LSE-NEXT:    caspal x6, x7, x4, x5, [x0]
+; LSE-NEXT:    fmov d1, x6
+; LSE-NEXT:    cmp x7, x3
+; LSE-NEXT:    ccmp x6, x2, #0, eq
+; LSE-NEXT:    mov v1.d[1], x7
+; LSE-NEXT:    b.ne .LBB10_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    ret
+  %res = atomicrmw fmax ptr %ptr, <2 x double> %value seq_cst, align 16
+  ret <2 x double> %res
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
new file mode 100644
index 00000000000000..533d0d8cd6a4fc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
@@ -0,0 +1,766 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-- -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=NOLSE %s
+; RUN: llc -mtriple=aarch64-- -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s
+
+define half @test_atomicrmw_fmin_f16_seq_cst_align2(ptr %ptr, half %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align2:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvt s1, h0
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    b .LBB0_2
+; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB0_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB0_5
+; NOLSE-NEXT:  .LBB0_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB0_3 Depth 2
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fminnm s2, s2, s1
+; NOLSE-NEXT:    fcvt h2, s2
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB0_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB0_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB0_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB0_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB0_3
+; NOLSE-NEXT:    b .LBB0_1
+; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align2:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvt s1, h0
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:  .LBB0_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fminnm s2, s2, s1
+; LSE-NEXT:    fcvt h2, s2
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casalh w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8, uxth
+; LSE-NEXT:    b.ne .LBB0_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+  ret half %res
+}
+
+define half @test_atomicrmw_fmin_f16_seq_cst_align4(ptr %ptr, half %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvt s1, h0
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    b .LBB1_2
+; NOLSE-NEXT:  .LBB1_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB1_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB1_5
+; NOLSE-NEXT:  .LBB1_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB1_3 Depth 2
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fminnm s2, s2, s1
+; NOLSE-NEXT:    fcvt h2, s2
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB1_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB1_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB1_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB1_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB1_3
+; NOLSE-NEXT:    b .LBB1_1
+; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvt s1, h0
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:  .LBB1_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fminnm s2, s2, s1
+; LSE-NEXT:    fcvt h2, s2
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casalh w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8, uxth
+; LSE-NEXT:    b.ne .LBB1_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, half %value seq_cst, align 4
+  ret half %res
+}
+
+define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align2(ptr %ptr, bfloat %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align2:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB2_2
+; NOLSE-NEXT:  .LBB2_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB2_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB2_5
+; NOLSE-NEXT:  .LBB2_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB2_3 Depth 2
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fminnm s2, s2, s1
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    ubfx w10, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    add w9, w10, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:  .LBB2_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB2_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB2_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB2_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB2_3
+; NOLSE-NEXT:    b .LBB2_1
+; NOLSE-NEXT:  .LBB2_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align2:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB2_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s2, w9
+; LSE-NEXT:    fminnm s2, s2, s1
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    ubfx w10, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    add w9, w10, w9
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    mov w11, w10
+; LSE-NEXT:    casalh w11, w9, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w10, uxth
+; LSE-NEXT:    b.ne .LBB2_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
+  ret bfloat %res
+}
+
+define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align4(ptr %ptr, bfloat %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB3_2
+; NOLSE-NEXT:  .LBB3_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB3_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB3_5
+; NOLSE-NEXT:  .LBB3_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB3_3 Depth 2
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fminnm s2, s2, s1
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    ubfx w10, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    add w9, w10, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:  .LBB3_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB3_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB3_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB3_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB3_3
+; NOLSE-NEXT:    b .LBB3_1
+; NOLSE-NEXT:  .LBB3_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_bf16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB3_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s2, w9
+; LSE-NEXT:    fminnm s2, s2, s1
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    ubfx w10, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    add w9, w10, w9
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    mov w11, w10
+; LSE-NEXT:    casalh w11, w9, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w10, uxth
+; LSE-NEXT:    b.ne .LBB3_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 4
+  ret bfloat %res
+}
+
+define float @test_atomicrmw_fmin_f32_seq_cst_align4(ptr %ptr, float %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr s1, [x0]
+; NOLSE-NEXT:    b .LBB4_2
+; NOLSE-NEXT:  .LBB4_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=1
+; NOLSE-NEXT:    fmov s1, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB4_5
+; NOLSE-NEXT:  .LBB4_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB4_3 Depth 2
+; NOLSE-NEXT:    fminnm s2, s1, s0
+; NOLSE-NEXT:    fmov w9, s1
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB4_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB4_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB4_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB4_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB4_3
+; NOLSE-NEXT:    b .LBB4_1
+; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov s0, s1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr s1, [x0]
+; LSE-NEXT:  .LBB4_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fminnm s2, s1, s0
+; LSE-NEXT:    fmov w8, s1
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s1, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB4_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov s0, s1
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+  ret float %res
+}
+
+define double @test_atomicrmw_fmin_f32_seq_cst_align8(ptr %ptr, double %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr d1, [x0]
+; NOLSE-NEXT:    b .LBB5_2
+; NOLSE-NEXT:  .LBB5_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x10
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.eq .LBB5_5
+; NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
+; NOLSE-NEXT:    fminnm d2, d1, d0
+; NOLSE-NEXT:    fmov x9, d1
+; NOLSE-NEXT:    fmov x8, d2
+; NOLSE-NEXT:  .LBB5_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr x10, [x0]
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.ne .LBB5_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, x8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB5_3
+; NOLSE-NEXT:    b .LBB5_1
+; NOLSE-NEXT:  .LBB5_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, d1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr d1, [x0]
+; LSE-NEXT:  .LBB5_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fminnm d2, d1, d0
+; LSE-NEXT:    fmov x8, d1
+; LSE-NEXT:    mov x10, x8
+; LSE-NEXT:    fmov x9, d2
+; LSE-NEXT:    casal x10, x9, [x0]
+; LSE-NEXT:    fmov d1, x10
+; LSE-NEXT:    cmp x10, x8
+; LSE-NEXT:    b.ne .LBB5_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov d0, d1
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+  ret double %res
+}
+
+define fp128 @test_atomicrmw_fmin_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    sub sp, sp, #96
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; NOLSE-NEXT:    mov x19, x0
+; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
+; NOLSE-NEXT:    stp x12, x13, [sp, #32]
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    ldr q1, [sp, #32]
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    b.eq .LBB6_6
+; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    bl fminl
+; NOLSE-NEXT:    str q0, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp, #64]
+; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
+; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x19]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB6_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    add sp, sp, #96
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16:
+; LSE:       // %bb.0:
+; LSE-NEXT:    sub sp, sp, #96
+; LSE-NEXT:    ldr q1, [x0]
+; LSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; LSE-NEXT:    mov x19, x0
+; LSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; LSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; LSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; LSE-NEXT:    bl fminl
+; LSE-NEXT:    str q0, [sp, #48]
+; LSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; LSE-NEXT:    ldp x0, x1, [sp, #48]
+; LSE-NEXT:    str q0, [sp, #64]
+; LSE-NEXT:    ldp x2, x3, [sp, #64]
+; LSE-NEXT:    mov x4, x2
+; LSE-NEXT:    mov x5, x3
+; LSE-NEXT:    caspal x4, x5, x0, x1, [x19]
+; LSE-NEXT:    stp x4, x5, [sp, #32]
+; LSE-NEXT:    cmp x5, x3
+; LSE-NEXT:    ldr q1, [sp, #32]
+; LSE-NEXT:    ccmp x4, x2, #0, eq
+; LSE-NEXT:    b.ne .LBB6_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    add sp, sp, #96
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, fp128 %value seq_cst, align 16
+  ret fp128 %res
+}
+
+define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_v2f16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOLSE-NEXT:    mov h1, v0.h[1]
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    fcvt s1, h1
+; NOLSE-NEXT:    b .LBB7_2
+; NOLSE-NEXT:  .LBB7_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB7_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB7_5
+; NOLSE-NEXT:  .LBB7_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB7_3 Depth 2
+; NOLSE-NEXT:    mov h3, v0.h[1]
+; NOLSE-NEXT:    fcvt s4, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fcvt s3, h3
+; NOLSE-NEXT:    fminnm s4, s4, s2
+; NOLSE-NEXT:    fminnm s3, s3, s1
+; NOLSE-NEXT:    fcvt h4, s4
+; NOLSE-NEXT:    fcvt h3, s3
+; NOLSE-NEXT:    mov v4.h[1], v3.h[0]
+; NOLSE-NEXT:    fmov w8, s4
+; NOLSE-NEXT:  .LBB7_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB7_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB7_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB7_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB7_3
+; NOLSE-NEXT:    b .LBB7_1
+; NOLSE-NEXT:  .LBB7_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_v2f16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $d0 killed $d0 def $q0
+; LSE-NEXT:    mov h1, v0.h[1]
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    ldr s0, [x0]
+; LSE-NEXT:    fcvt s1, h1
+; LSE-NEXT:  .LBB7_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    mov h3, v0.h[1]
+; LSE-NEXT:    fcvt s4, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fcvt s3, h3
+; LSE-NEXT:    fminnm s4, s4, s2
+; LSE-NEXT:    fminnm s3, s3, s1
+; LSE-NEXT:    fcvt h4, s4
+; LSE-NEXT:    fcvt h3, s3
+; LSE-NEXT:    mov v4.h[1], v3.h[0]
+; LSE-NEXT:    fmov w9, s4
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB7_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, <2 x half> %value seq_cst, align 4
+  ret <2 x half> %res
+}
+
+define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bfloat> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_v2bf16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 def $q0
+; NOLSE-NEXT:    mov h1, v0.h[1]
+; NOLSE-NEXT:    fmov w10, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    lsl w10, w10, #16
+; NOLSE-NEXT:    fmov w9, s1
+; NOLSE-NEXT:    fmov s2, w10
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB8_2
+; NOLSE-NEXT:  .LBB8_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB8_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9
+; NOLSE-NEXT:    b.eq .LBB8_5
+; NOLSE-NEXT:  .LBB8_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB8_3 Depth 2
+; NOLSE-NEXT:    mov h3, v0.h[1]
+; NOLSE-NEXT:    fmov w10, s0
+; NOLSE-NEXT:    lsl w10, w10, #16
+; NOLSE-NEXT:    fmov w9, s3
+; NOLSE-NEXT:    fmov s4, w10
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fminnm s4, s4, s2
+; NOLSE-NEXT:    fmov s3, w9
+; NOLSE-NEXT:    fminnm s3, s3, s1
+; NOLSE-NEXT:    fmov w10, s4
+; NOLSE-NEXT:    ubfx w12, w10, #16, #1
+; NOLSE-NEXT:    add w10, w10, w8
+; NOLSE-NEXT:    fmov w9, s3
+; NOLSE-NEXT:    add w10, w12, w10
+; NOLSE-NEXT:    lsr w10, w10, #16
+; NOLSE-NEXT:    ubfx w11, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    fmov s4, w10
+; NOLSE-NEXT:    add w9, w11, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s3, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov v4.h[1], v3.h[0]
+; NOLSE-NEXT:    fmov w10, s4
+; NOLSE-NEXT:  .LBB8_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB8_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9
+; NOLSE-NEXT:    b.ne .LBB8_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB8_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB8_3
+; NOLSE-NEXT:    b .LBB8_1
+; NOLSE-NEXT:  .LBB8_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_v2bf16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $d0 killed $d0 def $q0
+; LSE-NEXT:    mov h1, v0.h[1]
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr s0, [x0]
+; LSE-NEXT:    lsl w10, w10, #16
+; LSE-NEXT:    fmov w9, s1
+; LSE-NEXT:    fmov s2, w10
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB8_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    mov h3, v0.h[1]
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsl w10, w10, #16
+; LSE-NEXT:    fmov w9, s3
+; LSE-NEXT:    fmov s4, w10
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fminnm s4, s4, s2
+; LSE-NEXT:    fmov s3, w9
+; LSE-NEXT:    fminnm s3, s3, s1
+; LSE-NEXT:    fmov w10, s4
+; LSE-NEXT:    ubfx w12, w10, #16, #1
+; LSE-NEXT:    add w10, w10, w8
+; LSE-NEXT:    fmov w9, s3
+; LSE-NEXT:    add w10, w12, w10
+; LSE-NEXT:    lsr w10, w10, #16
+; LSE-NEXT:    ubfx w11, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    fmov s4, w10
+; LSE-NEXT:    add w9, w11, w9
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    fmov s3, w9
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov v4.h[1], v3.h[0]
+; LSE-NEXT:    mov w11, w9
+; LSE-NEXT:    fmov w10, s4
+; LSE-NEXT:    casal w11, w10, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w9
+; LSE-NEXT:    b.ne .LBB8_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, <2 x bfloat> %value seq_cst, align 4
+  ret <2 x bfloat> %res
+}
+
+define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_v2f32_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr d1, [x0]
+; NOLSE-NEXT:    b .LBB9_2
+; NOLSE-NEXT:  .LBB9_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x10
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.eq .LBB9_5
+; NOLSE-NEXT:  .LBB9_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB9_3 Depth 2
+; NOLSE-NEXT:    fminnm v2.2s, v1.2s, v0.2s
+; NOLSE-NEXT:    fmov x9, d1
+; NOLSE-NEXT:    fmov x8, d2
+; NOLSE-NEXT:  .LBB9_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB9_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr x10, [x0]
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.ne .LBB9_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB9_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, x8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB9_3
+; NOLSE-NEXT:    b .LBB9_1
+; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, d1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_v2f32_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr d1, [x0]
+; LSE-NEXT:  .LBB9_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fminnm v2.2s, v1.2s, v0.2s
+; LSE-NEXT:    fmov x8, d1
+; LSE-NEXT:    mov x10, x8
+; LSE-NEXT:    fmov x9, d2
+; LSE-NEXT:    casal x10, x9, [x0]
+; LSE-NEXT:    fmov d1, x10
+; LSE-NEXT:    cmp x10, x8
+; LSE-NEXT:    b.ne .LBB9_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov d0, d1
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, <2 x float> %value seq_cst, align 8
+  ret <2 x float> %res
+}
+
+define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fmin_v2f64_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    b .LBB10_2
+; NOLSE-NEXT:  .LBB10_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x12
+; NOLSE-NEXT:    cmp x13, x9
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    mov v1.d[1], x13
+; NOLSE-NEXT:    b.eq .LBB10_6
+; NOLSE-NEXT:  .LBB10_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB10_3 Depth 2
+; NOLSE-NEXT:    fminnm v2.2d, v1.2d, v0.2d
+; NOLSE-NEXT:    mov x9, v1.d[1]
+; NOLSE-NEXT:    fmov x11, d1
+; NOLSE-NEXT:    mov x8, v2.d[1]
+; NOLSE-NEXT:    fmov x10, d2
+; NOLSE-NEXT:  .LBB10_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB10_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x0]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x9
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB10_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x0]
+; NOLSE-NEXT:    cbnz w14, .LBB10_3
+; NOLSE-NEXT:    b .LBB10_1
+; NOLSE-NEXT:  .LBB10_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x10, x8, [x0]
+; NOLSE-NEXT:    cbnz w14, .LBB10_3
+; NOLSE-NEXT:    b .LBB10_1
+; NOLSE-NEXT:  .LBB10_6: // %atomicrmw.end
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fmin_v2f64_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr q1, [x0]
+; LSE-NEXT:  .LBB10_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fminnm v2.2d, v1.2d, v0.2d
+; LSE-NEXT:    mov x3, v1.d[1]
+; LSE-NEXT:    fmov x2, d1
+; LSE-NEXT:    mov x7, x3
+; LSE-NEXT:    mov x5, v2.d[1]
+; LSE-NEXT:    mov x6, x2
+; LSE-NEXT:    fmov x4, d2
+; LSE-NEXT:    caspal x6, x7, x4, x5, [x0]
+; LSE-NEXT:    fmov d1, x6
+; LSE-NEXT:    cmp x7, x3
+; LSE-NEXT:    ccmp x6, x2, #0, eq
+; LSE-NEXT:    mov v1.d[1], x7
+; LSE-NEXT:    b.ne .LBB10_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    ret
+  %res = atomicrmw fmin ptr %ptr, <2 x double> %value seq_cst, align 16
+  ret <2 x double> %res
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
new file mode 100644
index 00000000000000..3380cbef6f4a2d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
@@ -0,0 +1,706 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-- -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=NOLSE %s
+; RUN: llc -mtriple=aarch64-- -mattr=+lse -O1 -fast-isel=0 -global-isel=false %s -o - | FileCheck -check-prefix=LSE %s
+
+define half @test_atomicrmw_fsub_f16_seq_cst_align2(ptr %ptr, half %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align2:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvt s1, h0
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    b .LBB0_2
+; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB0_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB0_5
+; NOLSE-NEXT:  .LBB0_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB0_3 Depth 2
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fsub s2, s2, s1
+; NOLSE-NEXT:    fcvt h2, s2
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB0_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB0_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB0_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB0_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB0_3
+; NOLSE-NEXT:    b .LBB0_1
+; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align2:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvt s1, h0
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:  .LBB0_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fsub s2, s2, s1
+; LSE-NEXT:    fcvt h2, s2
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casalh w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8, uxth
+; LSE-NEXT:    b.ne .LBB0_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+  ret half %res
+}
+
+define half @test_atomicrmw_fsub_f16_seq_cst_align4(ptr %ptr, half %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvt s1, h0
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    b .LBB1_2
+; NOLSE-NEXT:  .LBB1_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB1_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB1_5
+; NOLSE-NEXT:  .LBB1_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB1_3 Depth 2
+; NOLSE-NEXT:    fcvt s2, h0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fsub s2, s2, s1
+; NOLSE-NEXT:    fcvt h2, s2
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB1_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB1_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB1_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB1_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB1_3
+; NOLSE-NEXT:    b .LBB1_1
+; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvt s1, h0
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:  .LBB1_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvt s2, h0
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fsub s2, s2, s1
+; LSE-NEXT:    fcvt h2, s2
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casalh w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8, uxth
+; LSE-NEXT:    b.ne .LBB1_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, half %value seq_cst, align 4
+  ret half %res
+}
+
+define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align2(ptr %ptr, bfloat %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align2:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB2_2
+; NOLSE-NEXT:  .LBB2_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB2_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB2_5
+; NOLSE-NEXT:  .LBB2_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB2_3 Depth 2
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fsub s2, s2, s1
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    ubfx w10, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    add w9, w10, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:  .LBB2_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB2_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB2_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB2_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB2_3
+; NOLSE-NEXT:    b .LBB2_1
+; NOLSE-NEXT:  .LBB2_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align2:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB2_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s2, w9
+; LSE-NEXT:    fsub s2, s2, s1
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    ubfx w10, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    add w9, w10, w9
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    mov w11, w10
+; LSE-NEXT:    casalh w11, w9, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w10, uxth
+; LSE-NEXT:    b.ne .LBB2_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2
+  ret bfloat %res
+}
+
+define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align4(ptr %ptr, bfloat %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
+; NOLSE-NEXT:    ldr h0, [x0]
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s1, w9
+; NOLSE-NEXT:    b .LBB3_2
+; NOLSE-NEXT:  .LBB3_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB3_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w11
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.eq .LBB3_5
+; NOLSE-NEXT:  .LBB3_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB3_3 Depth 2
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    lsl w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fsub s2, s2, s1
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    ubfx w10, w9, #16, #1
+; NOLSE-NEXT:    add w9, w9, w8
+; NOLSE-NEXT:    add w9, w10, w9
+; NOLSE-NEXT:    lsr w9, w9, #16
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:  .LBB3_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB3_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxrh w11, [x0]
+; NOLSE-NEXT:    cmp w11, w9, uxth
+; NOLSE-NEXT:    b.ne .LBB3_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB3_3 Depth=2
+; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB3_3
+; NOLSE-NEXT:    b .LBB3_1
+; NOLSE-NEXT:  .LBB3_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_bf16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    // kill: def $h0 killed $h0 def $s0
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    mov w8, #32767 // =0x7fff
+; LSE-NEXT:    ldr h0, [x0]
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s1, w9
+; LSE-NEXT:  .LBB3_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fmov w9, s0
+; LSE-NEXT:    lsl w9, w9, #16
+; LSE-NEXT:    fmov s2, w9
+; LSE-NEXT:    fsub s2, s2, s1
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    ubfx w10, w9, #16, #1
+; LSE-NEXT:    add w9, w9, w8
+; LSE-NEXT:    add w9, w10, w9
+; LSE-NEXT:    fmov w10, s0
+; LSE-NEXT:    lsr w9, w9, #16
+; LSE-NEXT:    mov w11, w10
+; LSE-NEXT:    casalh w11, w9, [x0]
+; LSE-NEXT:    fmov s0, w11
+; LSE-NEXT:    cmp w11, w10, uxth
+; LSE-NEXT:    b.ne .LBB3_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 4
+  ret bfloat %res
+}
+
+define float @test_atomicrmw_fsub_f32_seq_cst_align4(ptr %ptr, float %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr s1, [x0]
+; NOLSE-NEXT:    b .LBB4_2
+; NOLSE-NEXT:  .LBB4_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=1
+; NOLSE-NEXT:    fmov s1, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB4_5
+; NOLSE-NEXT:  .LBB4_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB4_3 Depth 2
+; NOLSE-NEXT:    fsub s2, s1, s0
+; NOLSE-NEXT:    fmov w9, s1
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB4_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB4_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB4_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB4_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB4_3
+; NOLSE-NEXT:    b .LBB4_1
+; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov s0, s1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr s1, [x0]
+; LSE-NEXT:  .LBB4_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fsub s2, s1, s0
+; LSE-NEXT:    fmov w8, s1
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s1, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB4_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov s0, s1
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+  ret float %res
+}
+
+define double @test_atomicrmw_fsub_f32_seq_cst_align8(ptr %ptr, double %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr d1, [x0]
+; NOLSE-NEXT:    b .LBB5_2
+; NOLSE-NEXT:  .LBB5_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x10
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.eq .LBB5_5
+; NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
+; NOLSE-NEXT:    fsub d2, d1, d0
+; NOLSE-NEXT:    fmov x9, d1
+; NOLSE-NEXT:    fmov x8, d2
+; NOLSE-NEXT:  .LBB5_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr x10, [x0]
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.ne .LBB5_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, x8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB5_3
+; NOLSE-NEXT:    b .LBB5_1
+; NOLSE-NEXT:  .LBB5_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, d1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr d1, [x0]
+; LSE-NEXT:  .LBB5_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fsub d2, d1, d0
+; LSE-NEXT:    fmov x8, d1
+; LSE-NEXT:    mov x10, x8
+; LSE-NEXT:    fmov x9, d2
+; LSE-NEXT:    casal x10, x9, [x0]
+; LSE-NEXT:    fmov d1, x10
+; LSE-NEXT:    cmp x10, x8
+; LSE-NEXT:    b.ne .LBB5_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov d0, d1
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+  ret double %res
+}
+
+define fp128 @test_atomicrmw_fsub_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_fp128_seq_cst_align16:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    sub sp, sp, #96
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; NOLSE-NEXT:    mov x19, x0
+; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
+; NOLSE-NEXT:    stp x12, x13, [sp, #32]
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    ldr q1, [sp, #32]
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    b.eq .LBB6_6
+; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    bl __subtf3
+; NOLSE-NEXT:    str q0, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp, #64]
+; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
+; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x19]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB6_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    add sp, sp, #96
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_fp128_seq_cst_align16:
+; LSE:       // %bb.0:
+; LSE-NEXT:    sub sp, sp, #96
+; LSE-NEXT:    ldr q1, [x0]
+; LSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; LSE-NEXT:    mov x19, x0
+; LSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; LSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; LSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; LSE-NEXT:    bl __subtf3
+; LSE-NEXT:    str q0, [sp, #48]
+; LSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; LSE-NEXT:    ldp x0, x1, [sp, #48]
+; LSE-NEXT:    str q0, [sp, #64]
+; LSE-NEXT:    ldp x2, x3, [sp, #64]
+; LSE-NEXT:    mov x4, x2
+; LSE-NEXT:    mov x5, x3
+; LSE-NEXT:    caspal x4, x5, x0, x1, [x19]
+; LSE-NEXT:    stp x4, x5, [sp, #32]
+; LSE-NEXT:    cmp x5, x3
+; LSE-NEXT:    ldr q1, [sp, #32]
+; LSE-NEXT:    ccmp x4, x2, #0, eq
+; LSE-NEXT:    b.ne .LBB6_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    add sp, sp, #96
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, fp128 %value seq_cst, align 16
+  ret fp128 %res
+}
+
+define <2 x half> @test_atomicrmw_fsub_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_v2f16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    fcvtl v1.4s, v0.4h
+; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    b .LBB7_2
+; NOLSE-NEXT:  .LBB7_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB7_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB7_5
+; NOLSE-NEXT:  .LBB7_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB7_3 Depth 2
+; NOLSE-NEXT:    fcvtl v2.4s, v0.4h
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fsub v2.4s, v2.4s, v1.4s
+; NOLSE-NEXT:    fcvtn v2.4h, v2.4s
+; NOLSE-NEXT:    fmov w8, s2
+; NOLSE-NEXT:  .LBB7_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB7_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB7_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB7_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB7_3
+; NOLSE-NEXT:    b .LBB7_1
+; NOLSE-NEXT:  .LBB7_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_v2f16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    fcvtl v1.4s, v0.4h
+; LSE-NEXT:    ldr s0, [x0]
+; LSE-NEXT:  .LBB7_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fcvtl v2.4s, v0.4h
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    fsub v2.4s, v2.4s, v1.4s
+; LSE-NEXT:    fcvtn v2.4h, v2.4s
+; LSE-NEXT:    fmov w9, s2
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB7_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, <2 x half> %value seq_cst, align 4
+  ret <2 x half> %res
+}
+
+define <2 x bfloat> @test_atomicrmw_fsub_v2bf16_seq_cst_align4(ptr %ptr, <2 x bfloat> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_v2bf16_seq_cst_align4:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    movi v1.4s, #1
+; NOLSE-NEXT:    movi v2.4s, #127, msl #8
+; NOLSE-NEXT:    shll v3.4s, v0.4h, #16
+; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    b .LBB8_2
+; NOLSE-NEXT:  .LBB8_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB8_2 Depth=1
+; NOLSE-NEXT:    fmov s0, w10
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.eq .LBB8_5
+; NOLSE-NEXT:  .LBB8_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB8_3 Depth 2
+; NOLSE-NEXT:    shll v4.4s, v0.4h, #16
+; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    fsub v4.4s, v4.4s, v3.4s
+; NOLSE-NEXT:    ushr v5.4s, v4.4s, #16
+; NOLSE-NEXT:    and v5.16b, v5.16b, v1.16b
+; NOLSE-NEXT:    add v4.4s, v5.4s, v4.4s
+; NOLSE-NEXT:    addhn v4.4h, v4.4s, v2.4s
+; NOLSE-NEXT:    fmov w8, s4
+; NOLSE-NEXT:  .LBB8_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB8_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr w10, [x0]
+; NOLSE-NEXT:    cmp w10, w9
+; NOLSE-NEXT:    b.ne .LBB8_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB8_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, w8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB8_3
+; NOLSE-NEXT:    b .LBB8_1
+; NOLSE-NEXT:  .LBB8_5: // %atomicrmw.end
+; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_v2bf16_seq_cst_align4:
+; LSE:       // %bb.0:
+; LSE-NEXT:    movi v1.4s, #1
+; LSE-NEXT:    movi v2.4s, #127, msl #8
+; LSE-NEXT:    shll v3.4s, v0.4h, #16
+; LSE-NEXT:    ldr s0, [x0]
+; LSE-NEXT:  .LBB8_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    shll v4.4s, v0.4h, #16
+; LSE-NEXT:    fmov w8, s0
+; LSE-NEXT:    fsub v4.4s, v4.4s, v3.4s
+; LSE-NEXT:    mov w10, w8
+; LSE-NEXT:    ushr v5.4s, v4.4s, #16
+; LSE-NEXT:    and v5.16b, v5.16b, v1.16b
+; LSE-NEXT:    add v4.4s, v5.4s, v4.4s
+; LSE-NEXT:    addhn v4.4h, v4.4s, v2.4s
+; LSE-NEXT:    fmov w9, s4
+; LSE-NEXT:    casal w10, w9, [x0]
+; LSE-NEXT:    fmov s0, w10
+; LSE-NEXT:    cmp w10, w8
+; LSE-NEXT:    b.ne .LBB8_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, <2 x bfloat> %value seq_cst, align 4
+  ret <2 x bfloat> %res
+}
+
+define <2 x float> @test_atomicrmw_fsub_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_v2f32_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr d1, [x0]
+; NOLSE-NEXT:    b .LBB9_2
+; NOLSE-NEXT:  .LBB9_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x10
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.eq .LBB9_5
+; NOLSE-NEXT:  .LBB9_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB9_3 Depth 2
+; NOLSE-NEXT:    fsub v2.2s, v1.2s, v0.2s
+; NOLSE-NEXT:    fmov x9, d1
+; NOLSE-NEXT:    fmov x8, d2
+; NOLSE-NEXT:  .LBB9_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB9_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxr x10, [x0]
+; NOLSE-NEXT:    cmp x10, x9
+; NOLSE-NEXT:    b.ne .LBB9_1
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB9_3 Depth=2
+; NOLSE-NEXT:    stlxr wzr, x8, [x0]
+; NOLSE-NEXT:    cbnz wzr, .LBB9_3
+; NOLSE-NEXT:    b .LBB9_1
+; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, d1
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_v2f32_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr d1, [x0]
+; LSE-NEXT:  .LBB9_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fsub v2.2s, v1.2s, v0.2s
+; LSE-NEXT:    fmov x8, d1
+; LSE-NEXT:    mov x10, x8
+; LSE-NEXT:    fmov x9, d2
+; LSE-NEXT:    casal x10, x9, [x0]
+; LSE-NEXT:    fmov d1, x10
+; LSE-NEXT:    cmp x10, x8
+; LSE-NEXT:    b.ne .LBB9_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    fmov d0, d1
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, <2 x float> %value seq_cst, align 8
+  ret <2 x float> %res
+}
+
+define <2 x double> @test_atomicrmw_fsub_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 {
+; NOLSE-LABEL: test_atomicrmw_fsub_v2f64_seq_cst_align8:
+; NOLSE:       // %bb.0:
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    b .LBB10_2
+; NOLSE-NEXT:  .LBB10_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_2 Depth=1
+; NOLSE-NEXT:    fmov d1, x12
+; NOLSE-NEXT:    cmp x13, x9
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    mov v1.d[1], x13
+; NOLSE-NEXT:    b.eq .LBB10_6
+; NOLSE-NEXT:  .LBB10_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB10_3 Depth 2
+; NOLSE-NEXT:    fsub v2.2d, v1.2d, v0.2d
+; NOLSE-NEXT:    mov x9, v1.d[1]
+; NOLSE-NEXT:    fmov x11, d1
+; NOLSE-NEXT:    mov x8, v2.d[1]
+; NOLSE-NEXT:    fmov x10, d2
+; NOLSE-NEXT:  .LBB10_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB10_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x0]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x9
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB10_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x0]
+; NOLSE-NEXT:    cbnz w14, .LBB10_3
+; NOLSE-NEXT:    b .LBB10_1
+; NOLSE-NEXT:  .LBB10_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x10, x8, [x0]
+; NOLSE-NEXT:    cbnz w14, .LBB10_3
+; NOLSE-NEXT:    b .LBB10_1
+; NOLSE-NEXT:  .LBB10_6: // %atomicrmw.end
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    ret
+;
+; LSE-LABEL: test_atomicrmw_fsub_v2f64_seq_cst_align8:
+; LSE:       // %bb.0:
+; LSE-NEXT:    ldr q1, [x0]
+; LSE-NEXT:  .LBB10_1: // %atomicrmw.start
+; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; LSE-NEXT:    fsub v2.2d, v1.2d, v0.2d
+; LSE-NEXT:    mov x3, v1.d[1]
+; LSE-NEXT:    fmov x2, d1
+; LSE-NEXT:    mov x7, x3
+; LSE-NEXT:    mov x5, v2.d[1]
+; LSE-NEXT:    mov x6, x2
+; LSE-NEXT:    fmov x4, d2
+; LSE-NEXT:    caspal x6, x7, x4, x5, [x0]
+; LSE-NEXT:    fmov d1, x6
+; LSE-NEXT:    cmp x7, x3
+; LSE-NEXT:    ccmp x6, x2, #0, eq
+; LSE-NEXT:    mov v1.d[1], x7
+; LSE-NEXT:    b.ne .LBB10_1
+; LSE-NEXT:  // %bb.2: // %atomicrmw.end
+; LSE-NEXT:    mov v0.16b, v1.16b
+; LSE-NEXT:    ret
+  %res = atomicrmw fsub ptr %ptr, <2 x double> %value seq_cst, align 16
+  ret <2 x double> %res
+}
+
+attributes #0 = { nounwind }