diff --git a/oras/auth/__init__.py b/oras/auth/__init__.py
index ff6b65f..cfbd0a1 100644
--- a/oras/auth/__init__.py
+++ b/oras/auth/__init__.py
@@ -14,10 +14,11 @@ class AuthenticationException(Exception):
     pass
 
 
-def get_auth_backend(name="token", session=None, **kwargs):
+def get_auth_backend(name="token", session=None, prefix="https", **kwargs):
     backend = auth_backends.get(name)
     if not backend:
         raise ValueError(f"Authentication backend {backend} is not known.")
     backend = backend(**kwargs)
     backend.session = session or requests.Session()
+    backend.prefix = prefix
     return backend
diff --git a/oras/auth/base.py b/oras/auth/base.py
index 37098e5..8dd7ad8 100644
--- a/oras/auth/base.py
+++ b/oras/auth/base.py
@@ -19,6 +19,7 @@ class AuthBackend:
 
     def __init__(self, *args, **kwargs):
         self._auths: dict = {}
+        self.prefix: str = "https"
 
     def get_auth_header(self):
         raise NotImplementedError
diff --git a/oras/provider.py b/oras/provider.py
index ff2e950..c9bfd4c 100644
--- a/oras/provider.py
+++ b/oras/provider.py
@@ -78,7 +78,7 @@ def __init__(
         self.session.cookies.set_policy(DefaultCookiePolicy(allowed_domains=[]))
 
         # Get custom backend, pass on session to share
-        self.auth = oras.auth.get_auth_backend(auth_backend, self.session)
+        self.auth = oras.auth.get_auth_backend(auth_backend, self.session, self.prefix)
 
     def __repr__(self) -> str:
         return str(self)