diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index c3bb83f9a5..e8e58015a2 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -17,6 +17,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/labeler@v5
+      if: ${{ github.event.action != 'labeled' && github.event.action != 'unlabeled' }}
+    - run: echo "Done adding labels"
   # Run this after labeler applied labels
   check-labels:
     needs: