diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index efcf3ed7c64..0f6c95b0f90 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -16,6 +16,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: github/issue-labeler@v2.6
+        # We continue-on-error because the action is buggy and tries to remove labels when they don't exist
+        continue-on-error: true
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           configuration-path: .github/labeler.yml