diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index 9dae026535ccfb..80def6dfc24b1a 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -708,7 +708,7 @@ static void AttemptToFoldSymbolOffsetDifference(
       if (DF) {
         Displacement += DF->getContents().size();
       } else if (auto *AF = dyn_cast<MCAlignFragment>(FI);
-                 AF && Layout &&
+                 AF && Layout && AF->hasEmitNops() &&
                  !Asm->getBackend().shouldInsertExtraNopBytesForCodeAlign(
                      *AF, Count)) {
         Displacement += Asm->computeFragmentSize(*Layout, *AF);
diff --git a/llvm/test/MC/RISCV/align-non-executable.s b/llvm/test/MC/RISCV/align-non-executable.s
new file mode 100644
index 00000000000000..95f91d93369f2e
--- /dev/null
+++ b/llvm/test/MC/RISCV/align-non-executable.s
@@ -0,0 +1,25 @@
+## A label difference separated by an alignment directive, when the
+## referenced symbols are in a non-executable section with instructions,
+## should generate ADD/SUB relocations.
+## https://github.com/llvm/llvm-project/pull/76552
+
+# RUN: llvm-mc --filetype=obj --triple=riscv64 --mattr=+relax %s \
+# RUN:     | llvm-readobj -r - | FileCheck --check-prefixes=CHECK,RELAX %s
+# RUN: llvm-mc --filetype=obj --triple=riscv64 --mattr=-relax %s \
+# RUN:     | llvm-readobj -r - | FileCheck %s
+
+.section ".dummy", "a"
+.L1:
+  call func
+.p2align 3
+.L2:
+.dword .L2 - .L1
+
+# CHECK:       Relocations [
+# CHECK-NEXT:    Section ({{.*}}) .rela.dummy {
+# CHECK-NEXT:      0x0 R_RISCV_CALL_PLT func 0x0
+# RELAX-NEXT:      0x0 R_RISCV_RELAX - 0x0
+# CHECK-NEXT:      0x8 R_RISCV_ADD64 .L2 0x0
+# CHECK-NEXT:      0x8 R_RISCV_SUB64 .L1 0x0
+# CHECK-NEXT:    }
+# CHECK-NEXT:  ]