diff --git a/src/mkdocstrings/references.py b/src/mkdocstrings/references.py
index ae5affe2..fdd404db 100644
--- a/src/mkdocstrings/references.py
+++ b/src/mkdocstrings/references.py
@@ -67,10 +67,6 @@ def relative_url(url_a: str, url_b: str) -> str:
     Returns:
         The relative URL to go from A to B.
     """
-    directory_url = False
-    if url_a[-1] == "/":
-        url_a = url_a.rstrip("/")
-        directory_url = True
     parts_a = url_a.split("/")
     url_b, anchor = url_b.split("#", 1)
     parts_b = url_b.split("/")
@@ -81,9 +77,7 @@ def relative_url(url_a: str, url_b: str) -> str:
         parts_b.pop(0)
 
     # go up as many times as remaining a parts' depth
-    levels = len(parts_a)
-    if not directory_url:
-        levels -= 1
+    levels = len(parts_a) - 1
     parts_relative = [".."] * levels + parts_b  # noqa: WPS435 (list multiply ok)
     relative = "/".join(parts_relative)
     return f"{relative}#{anchor}"
diff --git a/tests/test_references.py b/tests/test_references.py
index 2300c035..7e18037a 100644
--- a/tests/test_references.py
+++ b/tests/test_references.py
@@ -24,6 +24,11 @@
         ("a/b/c.html", "d.html#e", "../../d.html#e"),
         ("a/b.html", "c/d.html#e", "../c/d.html#e"),
         ("a/b/index.html", "a/b/c/d.html#e", "c/d.html#e"),
+        ("", "#x", "#x"),
+        ('a/', "#x", "../#x"),
+        ('a/b.html', "#x", "../#x"),
+        ("", "a/#x", "a/#x"),
+        ("", "a/b.html#x", "a/b.html#x"),
     ],
 )
 def test_relative_url(current_url, to_url, href_url):