diff --git a/redash/tasks/queries.py b/redash/tasks/queries.py
index 292a664c58..f5ecdd0ba9 100644
--- a/redash/tasks/queries.py
+++ b/redash/tasks/queries.py
@@ -123,7 +123,7 @@ def prune(cls, list_name, keep_count):
 
         remove_count = count - keep_count
         keys = redis_connection.zrange(list_name, 0, remove_count - 1)
-        redis_connection.delete(keys)
+        redis_connection.delete(*keys)
         redis_connection.zremrangebyrank(list_name, 0, remove_count - 1)
 
         return remove_count
diff --git a/tests/tasks/test_queries.py b/tests/tasks/test_queries.py
index f53624aa99..a8819a0f77 100644
--- a/tests/tasks/test_queries.py
+++ b/tests/tasks/test_queries.py
@@ -7,8 +7,12 @@ class TestPrune(TestCase):
     def setUp(self):
         self.list = "test_list"
         redis_connection.delete(self.list)
+        self.keys = []
         for score in range(0, 100):
-            redis_connection.zadd(self.list, score, 'k:{}'.format(score))
+            key = 'k:{}'.format(score)
+            self.keys.append(key)
+            redis_connection.zadd(self.list, score, key)
+            redis_connection.set(key, 1)
 
     def test_does_nothing_when_below_threshold(self):
         remove_count = QueryTaskTracker.prune(self.list, 100)
@@ -22,3 +26,6 @@ def test_removes_oldest_items_first(self):
 
         self.assertEqual(redis_connection.zscore(self.list, 'k:99'), 99.0)
         self.assertIsNone(redis_connection.zscore(self.list, 'k:1'))
+
+        for k in self.keys[0:50]:
+            self.assertFalse(redis_connection.exists(k))