diff --git a/conans/client/rest/uploader_downloader.py b/conans/client/rest/uploader_downloader.py
index cff66cbfa1c..7f3a7a5a93a 100644
--- a/conans/client/rest/uploader_downloader.py
+++ b/conans/client/rest/uploader_downloader.py
@@ -239,6 +239,8 @@ def call_with_retry(out, retry, retry_wait, method, *args, **kwargs):
     for counter in range(retry):
         try:
             return method(*args, **kwargs)
+        except NotFoundException:
+            raise
         except ConanException as exc:
             if counter == (retry - 1):
                 raise
diff --git a/conans/test/functional/download_retries_test.py b/conans/test/functional/download_retries_test.py
index 58c2f91fc99..5f553426ac8 100644
--- a/conans/test/functional/download_retries_test.py
+++ b/conans/test/functional/download_retries_test.py
@@ -1,11 +1,26 @@
 import unittest
 
+from conans import API_V2
 from conans.paths import CONANFILE
 from conans.test.utils.tools import TestClient, TestServer
 
 
 class DownloadRetriesTest(unittest.TestCase):
 
+    def test_do_not_retry_when_missing_file(self):
+
+        test_server = TestServer(server_capabilities=[API_V2])
+        client = TestClient(servers={"default": test_server},
+                            users={"default": [("lasote", "mypass")]})
+        conanfile = '''from conans import ConanFile
+class MyConanfile(ConanFile):
+    pass
+'''
+        client.save({CONANFILE: conanfile})
+        client.run("create . Pkg/0.1@lasote/stable")
+        client.run("upload '*' -c --all")
+        self.assertEquals(str(client.out).count("seconds to retry..."), 0)
+
     def test_recipe_download_retry(self):
         test_server = TestServer()
         client = TestClient(servers={"default": test_server},