diff --git a/jenkins/jobs/bmo_e2e_tests.pipeline b/jenkins/jobs/bmo_e2e_tests.pipeline
index 380d9330..eede5435 100644
--- a/jenkins/jobs/bmo_e2e_tests.pipeline
+++ b/jenkins/jobs/bmo_e2e_tests.pipeline
@@ -4,8 +4,10 @@ import java.text.SimpleDateFormat
 def TIMEOUT = 3600
 
 // Set defaults for non-PR jobs
-def branch = env.ghprbActualCommit ?: "main"
-def repoUrl = env.ghprbAuthorRepoGitUrl ?: "https://github.com/metal3-io/baremetal-operator.git"
+def pullSha = (env.PULL_PULL_SHA) ?: (env.ghprbActualCommit) ?: "main"
+def repoUrl = "https://github.com/metal3-io/baremetal-operator.git"
+// Fetch the main branch and the pullSha, nothing else
+def refspec = '+refs/heads/main:refs/remotes/origin/main ' + pullSha
 
 pipeline {
   agent none
@@ -26,11 +28,12 @@ pipeline {
           stage("Checkout source code") {
             steps {
               checkout scmGit(
-                  branches: [[name: branch]],
-                  userRemoteConfigs: [[url: repoUrl, credentialsId: "metal3-jenkins-github-token"]],
+                  branches: [[name: pullSha]],
+                  userRemoteConfigs: [[url: repoUrl, credentialsId: "metal3-jenkins-github-token", refspec: refspec]],
                   extensions: [[$class: "WipeWorkspace"],
                   [$class: "CleanCheckout"],
-                  [$class: "CleanBeforeCheckout"]],
+                  [$class: "CleanBeforeCheckout"],
+                  cloneOption(honorRefspec: true)],
                   submoduleCfg: [],)
               script {
                 CURRENT_START_TIME = System.currentTimeMillis()
diff --git a/prow/manifests/overlays/metal3/config.yaml b/prow/manifests/overlays/metal3/config.yaml
index 18dfcb86..ef170739 100644
--- a/prow/manifests/overlays/metal3/config.yaml
+++ b/prow/manifests/overlays/metal3/config.yaml
@@ -540,6 +540,16 @@ presubmits:
                 value: "/"
             image: ghcr.io/yannh/kubeconform:v0.6.2-alpine@sha256:49b5f6b320d30c1b8b72a7abdf02740ac9dc36a3ba23b934d1c02f7b37e6e740
             imagePullPolicy: Always
+    - name: metal3-bmo-e2e-test-pull
+      # E2e tests do not exist before release-0.5
+      branches:
+        - main
+        - release-0.5
+      agent: jenkins
+      # Don't run unless triggered to avoid wasting resources
+      always_run: false
+      # Until we have checked that it works, keep it optional
+      optional: true
 
   metal3-io/cluster-api-provider-metal3:
     - name: gomod