princeton-nlp / SWE-bench

[ICLR 2024] SWE-Bench: Can Language Models Resolve Real-world Github Issues?
https://www.swebench.com
MIT License
1.45k stars 240 forks source link

Passed test case count as failure? #165

Closed xingyaoww closed 2 days ago

xingyaoww commented 3 days ago

Describe the bug

For a test output like this:

test_has_key_race_handling (cache.tests.FileBasedCachePathLibTests) ... File /tmp/django_ufmcv9q2/tmpnp0fac7m/ca5ad7d0e17c2bb52c770052ee1ef5ad.djcache does not exist.
File /tmp/django_ufmcv9q2/tmpnp0fac7m/ca5ad7d0e17c2bb52c770052ee1ef5ad.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmpo7gh9d8q/41cb952daaf0006edddd487f3bd3ba14.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp0p5bke4_/432577f9fdcd8847d23e4a1d814e2311.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp1l94wblq/7093c6a4d04df5136608ac8cc30d03f4.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp1l94wblq/7093c6a4d04df5136608ac8cc30d03f4.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp1l94wblq/e78e6e76824c789d3c63fd50b0833ff9.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp1l94wblq/7bff30732ed894b4d439e957d4f3b4bc.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp1l94wblq/7bff30732ed894b4d439e957d4f3b4bc.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp1l94wblq/3449aff77f8fbc7e1253f5e0dc744e9d.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp1l94wblq/ff9befa22c734a739cd8635e948c7843.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp1l94wblq/ff9befa22c734a739cd8635e948c7843.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp1l94wblq/c51afb26eaa02a2b928f0cedc9dc57e5.djcache does not exist.
File /tmp/django_ufmcv9q2/tmpxkn3j3rx/e78e6e76824c789d3c63fd50b0833ff9.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmpxkn3j3rx/e78e6e76824c789d3c63fd50b0833ff9.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmpxkn3j3rx/7093c6a4d04df5136608ac8cc30d03f4.djcache does not exist.
File /tmp/django_ufmcv9q2/tmpxkn3j3rx/7093c6a4d04df5136608ac8cc30d03f4.djcache does not exist.
File /tmp/django_ufmcv9q2/tmpxkn3j3rx/e78e6e76824c789d3c63fd50b0833ff9.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmpxkn3j3rx/7093c6a4d04df5136608ac8cc30d03f4.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/f4941b866cf285ca9b1ec88904ee7346.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/8ba8f8da2249e3f73844d7e0c0f8092f.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/4375e95cdb119dbc6687b87516723235.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/45d4341786505628cdda44c23a449b01.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/48803088ec7c368cd989b38be9c9d609.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/e8cfeaf2dddcd5d086f464d812302237.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/8678b43495f0c608293fe0a2ef69dced.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/0849249ca32fb8838b9597d9d0bc1c7c.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/4be8dbd92f93a3e5fa22c2581604fefa.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/9047ad55492d54cf1abd2e242722ee72.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/240966f22aa94c42a6f59a81f457cac3.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/eaa98e292bc47bf9187546ee92a3c312.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/111c013efb2c207351657adb67429a17.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/3aaf6185810470873c0aad0cab0b1495.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/01600ecde6ed2973a0815db3333b40a6.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/733c5dc2bba66f9c1027d2f31a071330.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/e3d47f0e792204ad9616b8089e717f8f.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/015cf8a29a8b55d96ab10242b3fdebf7.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/01b903e2f2b12e8af30a73f3efb91f07.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/755c823d802d9fd66c0778671b9ad66a.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/b958160e966d99446e9a70478ee339bf.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/aa53ed53f4d56b9bbcbd9af5f1be67d8.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/15837dab371cb021499afba99d1f1c5e.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/aa3eeff1509358b73bc8432801eb94c9.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/15450370898cf05f8cfe71b70527057c.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/0c33b187196bd1404edced15104c0db8.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/f945d12a6188a33dfcacce22a3c0eb91.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/265e3b590efd498b1dcea955354dd17c.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/fb50f8a9f8e5716443dacb5d0f992434.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/b8c2742ed2bdc5d223cb2d885c6f8361.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/011ef412a6e3c2bf6000337bbd3441e6.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/c24ee8770cc57f44377a37ca8ecb9324.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/031b12f7900b88ad10fbc64caa29749d.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/f713c49bf0756b538872b0a5a31767f1.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/07165b5c2d6dbc880cdff5844fb3ae27.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/547e17c23209f75494ca511d3da69a19.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/ced60cad06ab6aab25a9192d6e5bd025.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/52724e5c0b29ff79ae96b461b1de9ba4.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/523366a6bb65773ebbf6c874329f7ff1.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/07781502dda2cc23636b72875c9cc3cc.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp838ypmt_/681fa3262d1d4e15df89f715c08fe3d5.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/67189d2e69aa96e99fbf0be2a5c55b1a.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/b72b1fcfa3b170a649a314e453c39796.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/69299f5583f390cd9c59752db6d0fce6.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/82e459138636ceb4543a0e59a598a7a3.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/9dca55e91db08f30e8b570f6218fcbde.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/395ee1dd54b3551fa2dda43453f7b399.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/5678642f8e21d5199e2851586f58313e.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp838ypmt_/b45d43de0cc13ae8cf22bddaad83e7c0.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp9goyq_tc/911be56704babedfae9cb52936c2e073.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmphdd38p7x/1c3606b8a3d5c508ab817bf376d262c9.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmphdd38p7x/d57763f46729bfd02f5d873e34ef8047.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp7h5aj18f/f6382bebc1c31754c06cd51510b07eb6.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp7h5aj18f/f9c7b7675d9da5a80bc92eb60e78dbf0.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmp8c4n5471/c000ea28ae50a63101f5f6d131080760.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp8c4n5471/ec48461ff84f546cebabcc74df7047dd.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp7j7rplg4/1ba415fce8ec8158c0e46bc81771044c.djcache does not exist.
File /tmp/django_ufmcv9q2/tmp7j7rplg4/ec48461ff84f546cebabcc74df7047dd.djcache does not exist.
File /tmp/django_ufmcv9q2/tmpf3s27w_h/493a2a92b5516592c80896546d31dbbb.djcache does not exist.
File /tmp/django_ufmcv9q2/tmpf3s27w_h/cdbffd16642f8bb2fec5bd2a7a147fb0.djcache does not exist.
File /tmp/django_ufmcv9q2/tmpe73g2wdu/5f5c7905b38d16747bdb10938dcf4d37.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmpe73g2wdu/de18da09eccb6481bde9a8a99ecae44f.djcache does not exist.
File /tmp/django_ufmcv9q2/tmpe73g2wdu/e4b54c66fde8f269cd060000b3399aed.djcache opened successfully.
File /tmp/django_ufmcv9q2/tmpe73g2wdu/ec48461ff84f546cebabcc74df7047dd.djcache opened successfully.
ok

The test case will actually be deemed as failure. It seems to be impacted by the output printed by running the testcase. Any idea we could fix this?

Steps/Code to Reproduce

Evaluate on django__django-16379 for the following patch:

diff --git a/django/core/cache/backends/.backup.filebased.py b/django/core/cache/backends/.backup.filebased.py
new file mode 100644
index 0000000000..f4f5564efc
--- /dev/null
+++ b/django/core/cache/backends/.backup.filebased.py
@@ -0,0 +1,173 @@
+"File-based cache backend"
+import glob
+import os
+import pickle
+import random
+import tempfile
+import time
+import zlib
+
+from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache
+from django.core.files import locks
+from django.core.files.move import file_move_safe
+from django.utils.crypto import md5
+
+
+class FileBasedCache(BaseCache):
+    cache_suffix = ".djcache"
+    pickle_protocol = pickle.HIGHEST_PROTOCOL
+
+    def __init__(self, dir, params):
+        super().__init__(params)
+        self._dir = os.path.abspath(dir)
+        self._createdir()
+
+    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
+        if self.has_key(key, version):
+            return False
+        self.set(key, value, timeout, version)
+        return True
+
+    def get(self, key, default=None, version=None):
+        fname = self._key_to_file(key, version)
+        try:
+            with open(fname, "rb") as f:
+                if not self._is_expired(f):
+                    return pickle.loads(zlib.decompress(f.read()))
+        except FileNotFoundError:
+            pass
+        return default
+
+    def _write_content(self, file, timeout, value):
+        expiry = self.get_backend_timeout(timeout)
+        file.write(pickle.dumps(expiry, self.pickle_protocol))
+        file.write(zlib.compress(pickle.dumps(value, self.pickle_protocol)))
+
+    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
+        self._createdir()  # Cache dir can be deleted at any time.
+        fname = self._key_to_file(key, version)
+        self._cull()  # make some room if necessary
+        fd, tmp_path = tempfile.mkstemp(dir=self._dir)
+        renamed = False
+        try:
+            with open(fd, "wb") as f:
+                self._write_content(f, timeout, value)
+            file_move_safe(tmp_path, fname, allow_overwrite=True)
+            renamed = True
+        finally:
+            if not renamed:
+                os.remove(tmp_path)
+
+    def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
+        try:
+            with open(self._key_to_file(key, version), "r+b") as f:
+                try:
+                    locks.lock(f, locks.LOCK_EX)
+                    if self._is_expired(f):
+                        return False
+                    else:
+                        previous_value = pickle.loads(zlib.decompress(f.read()))
+                        f.seek(0)
+                        self._write_content(f, timeout, previous_value)
+                        return True
+                finally:
+                    locks.unlock(f)
+        except FileNotFoundError:
+            return False
+
+    def delete(self, key, version=None):
+        return self._delete(self._key_to_file(key, version))
+
+    def _delete(self, fname):
+        if not fname.startswith(self._dir) or not os.path.exists(fname):
+            return False
+        try:
+            os.remove(fname)
+        except FileNotFoundError:
+            # The file may have been removed by another process.
+            return False
+        return True
+
+    def has_key(self, key, version=None):
+        fname = self._key_to_file(key, version)
+        if os.path.exists(fname):
+            try:
+                with open(fname, "rb") as f:
+                    return not self._is_expired(f)
+            except FileNotFoundError:
+                return False
+        return False
+        return False
+
+    def _cull(self):
+        """
+        Remove random cache entries if max_entries is reached at a ratio
+        of num_entries / cull_frequency. A value of 0 for CULL_FREQUENCY means
+        that the entire cache will be purged.
+        """
+        filelist = self._list_cache_files()
+        num_entries = len(filelist)
+        if num_entries < self._max_entries:
+            return  # return early if no culling is required
+        if self._cull_frequency == 0:
+            return self.clear()  # Clear the cache when CULL_FREQUENCY = 0
+        # Delete a random selection of entries
+        filelist = random.sample(filelist, int(num_entries / self._cull_frequency))
+        for fname in filelist:
+            self._delete(fname)
+
+    def _createdir(self):
+        # Set the umask because os.makedirs() doesn't apply the "mode" argument
+        # to intermediate-level directories.
+        old_umask = os.umask(0o077)
+        try:
+            os.makedirs(self._dir, 0o700, exist_ok=True)
+        finally:
+            os.umask(old_umask)
+
+    def _key_to_file(self, key, version=None):
+        """
+        Convert a key into a cache file path. Basically this is the
+        root cache path joined with the md5sum of the key and a suffix.
+        """
+        key = self.make_and_validate_key(key, version=version)
+        return os.path.join(
+            self._dir,
+            "".join(
+                [
+                    md5(key.encode(), usedforsecurity=False).hexdigest(),
+                    self.cache_suffix,
+                ]
+            ),
+        )
+
+    def clear(self):
+        """
+        Remove all the cache files.
+        """
+        for fname in self._list_cache_files():
+            self._delete(fname)
+
+    def _is_expired(self, f):
+        """
+        Take an open cache file `f` and delete it if it's expired.
+        """
+        try:
+            exp = pickle.load(f)
+        except EOFError:
+            exp = 0  # An empty file is considered expired.
+        if exp is not None and exp < time.time():
+            f.close()  # On Windows a file has to be closed before deleting
+            self._delete(f.name)
+            return True
+        return False
+
+    def _list_cache_files(self):
+        """
+        Get a list of paths to all the cache files. These are all the files
+        in the root cache dir that end on the cache_suffix.
+        """
+        return [
+            os.path.join(self._dir, fname)
+            for fname in glob.glob1(self._dir, "*%s" % self.cache_suffix)
+        ]
diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py
index 631da49444..fab927d6cc 100644
--- a/django/core/cache/backends/filebased.py
+++ b/django/core/cache/backends/filebased.py
@@ -91,8 +91,15 @@ class FileBasedCache(BaseCache):
     def has_key(self, key, version=None):
         fname = self._key_to_file(key, version)
         if os.path.exists(fname):
-            with open(fname, "rb") as f:
-                return not self._is_expired(f)
+            try:
+                with open(fname, "rb") as f:
+                    print(f"File {fname} opened successfully.")
+                    return not self._is_expired(f)
+            except FileNotFoundError:
+                print(f"File {fname} not found.")
+                return False
+        print(f"File {fname} does not exist.")
+        return False
         return False

     def _cull(self):
diff --git a/tests/.backup.test_filebasedcache_race_condition.py b/tests/.backup.test_filebasedcache_race_condition.py
new file mode 100644
index 0000000000..2404886e91
--- /dev/null
+++ b/tests/.backup.test_filebasedcache_race_condition.py
@@ -0,0 +1,43 @@
+import os
+import django
+from django.conf import settings
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_settings'
+django.setup()
+
+
+
+import threading
+import time
+from django.core.cache.backends.filebased import FileBasedCache
+from django.test import TestCase
+
+class FileBasedCacheRaceConditionTest(TestCase):
+    def setUp(self):
+        self.cache_dir = '/tmp/test_cache'
+        self.cache = FileBasedCache(self.cache_dir, {})
+
+    def tearDown(self):
+        for filename in os.listdir(self.cache_dir):
+            file_path = os.path.join(self.cache_dir, filename)
+            if os.path.isfile(file_path):
+                os.unlink(file_path)
+            os.remove(self.cache._key_to_file('test_key'))
+    def test_has_key_race_condition(self):
+        key = 'test_key'
+        value = 'test_value'
+        self.cache.set(key, value, timeout=1)  # Set a short timeout to expire the cache quickly
+
+        def delete_cache():
+            time.sleep(0.1)
+            os.remove(self.cache._key_to_file(key))
+
+        delete_thread = threading.Thread(target=delete_cache)
+        delete_thread.start()
+
+        # Try to access the cache key while it is being deleted
+        result = self.cache.has_key(key)
+        delete_thread.join()
+
+        # The result should be False because the key should be deleted
+        self.assertFalse(result)
diff --git a/tests/.backup.test_settings.py b/tests/.backup.test_settings.py
new file mode 100644
index 0000000000..864d87dbbc
--- /dev/null
+++ b/tests/.backup.test_settings.py
@@ -0,0 +1,6 @@
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': '/tmp/test_cache',
+    }
+}
diff --git a/tests/test_filebasedcache_race_condition.py b/tests/test_filebasedcache_race_condition.py
new file mode 100644
index 0000000000..1f7e265a95
--- /dev/null
+++ b/tests/test_filebasedcache_race_condition.py
@@ -0,0 +1,42 @@
+import os
+import django
+from django.conf import settings
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_settings'
+django.setup()
+
+
+
+import threading
+import time
+from django.core.cache.backends.filebased import FileBasedCache
+from django.test import TestCase
+
+class FileBasedCacheRaceConditionTest(TestCase):
+    def setUp(self):
+        self.cache_dir = '/tmp/test_cache'
+        self.cache = FileBasedCache(self.cache_dir, {})
+
+    def tearDown(self):
+        for filename in os.listdir(self.cache_dir):
+            file_path = os.path.join(self.cache_dir, filename)
+            if os.path.isfile(file_path):
+                os.unlink(file_path)
+            os.remove(self.cache._key_to_file('test_key'))
+    def test_has_key_race_condition(self):
+        key = 'test_key'
+        value = 'test_value'
+        self.cache.set(key, value, timeout=1)  # Set a short timeout to expire the cache quickly
+
+        def delete_cache():
+            os.remove(self.cache._key_to_file(key))
+
+        delete_thread = threading.Thread(target=delete_cache)
+        delete_thread.start()
+
+        # Try to access the cache key while it is being deleted
+        result = self.cache.has_key(key)
+        delete_thread.join()
+
+        # The result should be False because the key should be deleted
+        self.assertFalse(result)
diff --git a/tests/test_settings.py b/tests/test_settings.py
new file mode 100644
index 0000000000..9a52e22f34
--- /dev/null
+++ b/tests/test_settings.py
@@ -0,0 +1,13 @@
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': '/tmp/test_cache',
+    }
+
+}
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': ':memory:',
+    }
+}

Expected Results

The patch should be considered as pass.

Actual Results

{
    "django__django-16379": {
        "patch_is_None": false,
        "patch_exists": true,
        "patch_successfully_applied": true,
        "resolved": false,
        "tests_status": {
            "FAIL_TO_PASS": {
                "success": [
                    "test_has_key_race_handling (cache.tests.FileBasedCacheTests)"
                ],
                "failure": [
                    "test_has_key_race_handling (cache.tests.FileBasedCachePathLibTests)"
                ]
            },
            "PASS_TO_PASS": {
                "success": [
                    "If None is cached, get() returns it instead of the default.",
                    "Passing in None into timeout results in a value that is cached forever",
                    "Follow memcached's convention where a timeout greater than 30 days is",
                    "Nonexistent cache keys return as None/default.",
                    "set_many() returns an empty list when all keys are inserted.",
                    "Passing in zero into timeout results in a value that is not cached",
                    "test_createcachetable_observes_database_router (cache.tests.CreateCacheTableForDBCacheTests)",
                    "Memory caches that have the TIMEOUT parameter set to `None` in the",
                    "Memory caches that have the TIMEOUT parameter set to `None` will set",
                    "Caches that have the TIMEOUT parameter undefined in the default",
                    "Memory caches that have the TIMEOUT parameter unset will set cache",
                    "The default expiration time of a cache key is 5 minutes.",
                    "test_head_caches_correctly (cache.tests.CacheHEADTest)",
                    "test_head_with_cached_get (cache.tests.CacheHEADTest)",
                    "test_long_vary_on (cache.tests.TestMakeTemplateFragmentKey)",
                    "test_proper_escaping (cache.tests.TestMakeTemplateFragmentKey)",
                    "test_with_ints_vary_on (cache.tests.TestMakeTemplateFragmentKey)",
                    "test_with_many_vary_on (cache.tests.TestMakeTemplateFragmentKey)",
                    "test_with_one_vary_on (cache.tests.TestMakeTemplateFragmentKey)",
                    "test_with_unicode_vary_on (cache.tests.TestMakeTemplateFragmentKey)",
                    "test_without_vary_on (cache.tests.TestMakeTemplateFragmentKey)",
                    "get_cache_key keys differ by fully-qualified URL instead of path",
                    "test_get_cache_key (cache.tests.PrefixedCacheUtils)",
                    "test_get_cache_key_with_query (cache.tests.PrefixedCacheUtils)",
                    "test_learn_cache_key (cache.tests.PrefixedCacheUtils)",
                    "test_patch_cache_control (cache.tests.PrefixedCacheUtils)",
                    "test_patch_vary_headers (cache.tests.PrefixedCacheUtils)",
                    "test_get_cache_key (cache.tests.TestWithTemplateResponse)",
                    "test_get_cache_key_with_query (cache.tests.TestWithTemplateResponse)",
                    "test_patch_vary_headers (cache.tests.TestWithTemplateResponse)",
                    "test_get_cache_key (cache.tests.CacheUtils)",
                    "test_get_cache_key_with_query (cache.tests.CacheUtils)",
                    "test_learn_cache_key (cache.tests.CacheUtils)",
                    "test_patch_cache_control (cache.tests.CacheUtils)",
                    "test_patch_vary_headers (cache.tests.CacheUtils)",
                    "test_close (cache.tests.CacheClosingTests)",
                    "test_close_only_initialized (cache.tests.CacheClosingTests)",
                    "test_custom_key_validation (cache.tests.CustomCacheKeyValidationTests)",
                    "test_cache_key_i18n_timezone (cache.tests.PrefixedCacheI18nTest)",
                    "test_cache_key_i18n_translation (cache.tests.PrefixedCacheI18nTest)",
                    "test_cache_key_i18n_translation_accept_language (cache.tests.PrefixedCacheI18nTest)",
                    "test_cache_key_no_i18n (cache.tests.PrefixedCacheI18nTest)",
                    "test_middleware (cache.tests.PrefixedCacheI18nTest)",
                    "test_middleware_doesnt_cache_streaming_response (cache.tests.PrefixedCacheI18nTest)",
                    "test_cache_key_i18n_timezone (cache.tests.CacheI18nTest)",
                    "test_cache_key_i18n_translation (cache.tests.CacheI18nTest)",
                    "test_cache_key_i18n_translation_accept_language (cache.tests.CacheI18nTest)",
                    "test_cache_key_no_i18n (cache.tests.CacheI18nTest)",
                    "test_middleware (cache.tests.CacheI18nTest)",
                    "test_middleware_doesnt_cache_streaming_response (cache.tests.CacheI18nTest)",
                    "test_all (cache.tests.CacheHandlerTest)",
                    "test_nonexistent_alias (cache.tests.CacheHandlerTest)",
                    "test_nonexistent_backend (cache.tests.CacheHandlerTest)",
                    "Requesting the same alias from separate threads should yield separate",
                    "Attempting to retrieve the same alias should yield the same instance.",
                    "Add doesn't do anything in dummy cache backend",
                    "clear does nothing for the dummy cache backend",
                    "All data types are ignored equally by the dummy cache",
                    "Dummy cache values can't be decremented",
                    "Dummy cache versions can't be decremented",
                    "Cache deletion is transparently ignored on the dummy cache backend",
                    "delete_many does nothing for the dummy cache backend",
                    "test_delete_many_invalid_key (cache.tests.DummyCacheTests)",
                    "Expiration has no effect on the dummy cache",
                    "get_many returns nothing for the dummy cache backend",
                    "test_get_many_invalid_key (cache.tests.DummyCacheTests)",
                    "test_get_or_set (cache.tests.DummyCacheTests)",
                    "test_get_or_set_callable (cache.tests.DummyCacheTests)",
                    "The has_key method doesn't ever return True for the dummy cache backend",
                    "The in operator doesn't ever return True for the dummy cache backend",
                    "Dummy cache values can't be incremented",
                    "Dummy cache versions can't be incremented",
                    "Nonexistent keys aren't found in the dummy cache backend",
                    "set_many does nothing for the dummy cache backend",
                    "test_set_many_invalid_key (cache.tests.DummyCacheTests)",
                    "Dummy cache backend ignores cache set calls",
                    "Dummy cache can't do touch().",
                    "Unicode values are ignored by the dummy cache",
                    "test_304_response_has_http_caching_headers_but_not_cached (cache.tests.CacheMiddlewareTest)",
                    "test_cache_page_timeout (cache.tests.CacheMiddlewareTest)",
                    "Responses with 'Cache-Control: private' are not cached.",
                    "The constructor is correctly distinguishing between usage of",
                    "test_fetch_cache_middleware_constructor (cache.tests.CacheMiddlewareTest)",
                    "test_middleware (cache.tests.CacheMiddlewareTest)",
                    "The cache instance is different for each thread.",
                    "Django must prevent caching of responses that set a user-specific (and",
                    "test_update_cache_middleware_constructor (cache.tests.CacheMiddlewareTest)",
                    "test_view_decorator (cache.tests.CacheMiddlewareTest)",
                    "test_add (cache.tests.LocMemCacheTests)",
                    "test_add_fail_on_pickleerror (cache.tests.LocMemCacheTests)",
                    "test_binary_string (cache.tests.LocMemCacheTests)",
                    "test_cache_read_for_model_instance (cache.tests.LocMemCacheTests)",
                    "test_cache_read_for_model_instance_with_deferred (cache.tests.LocMemCacheTests)",
                    "test_cache_versioning_add (cache.tests.LocMemCacheTests)",
                    "test_cache_versioning_delete (cache.tests.LocMemCacheTests)",
                    "test_cache_versioning_get_set (cache.tests.LocMemCacheTests)",
                    "test_cache_versioning_get_set_many (cache.tests.LocMemCacheTests)",
                    "test_cache_versioning_has_key (cache.tests.LocMemCacheTests)",
                    "test_cache_versioning_incr_decr (cache.tests.LocMemCacheTests)",
                    "test_cache_write_for_model_instance_with_deferred (cache.tests.LocMemCacheTests)",
                    "test_cache_write_unpicklable_object (cache.tests.LocMemCacheTests)",
                    "test_clear (cache.tests.LocMemCacheTests)",
                    "test_close (cache.tests.LocMemCacheTests)",
                    "test_cull (cache.tests.LocMemCacheTests)",
                    "test_cull_delete_when_store_empty (cache.tests.LocMemCacheTests)",
                    "test_custom_key_func (cache.tests.LocMemCacheTests)",
                    "test_data_types (cache.tests.LocMemCacheTests)",
                    "test_decr (cache.tests.LocMemCacheTests)",
                    "test_decr_version (cache.tests.LocMemCacheTests)",
                    "test_delete (cache.tests.LocMemCacheTests)",
                    "test_delete_many (cache.tests.LocMemCacheTests)",
                    "test_delete_many_no_keys (cache.tests.LocMemCacheTests)",
                    "test_delete_nonexistent (cache.tests.LocMemCacheTests)",
                    "test_expiration (cache.tests.LocMemCacheTests)",
                    "test_float_timeout (cache.tests.LocMemCacheTests)",
                    "test_get_many (cache.tests.LocMemCacheTests)",
                    "test_get_or_set (cache.tests.LocMemCacheTests)",
                    "test_get_or_set_callable (cache.tests.LocMemCacheTests)",
                    "test_get_or_set_racing (cache.tests.LocMemCacheTests)",
                    "test_get_or_set_version (cache.tests.LocMemCacheTests)",
                    "test_has_key (cache.tests.LocMemCacheTests)",
                    "test_in (cache.tests.LocMemCacheTests)",
                    "test_incr (cache.tests.LocMemCacheTests)",
                    "incr/decr does not modify expiry time (matches memcached behavior)",
                    "test_incr_version (cache.tests.LocMemCacheTests)",
                    "test_invalid_key_characters (cache.tests.LocMemCacheTests)",
                    "test_invalid_key_length (cache.tests.LocMemCacheTests)",
                    "test_invalid_with_version_key_length (cache.tests.LocMemCacheTests)",
                    "#20613/#18541 -- Ensures pickling is done outside of the lock.",
                    "get() moves cache keys.",
                    "incr() moves cache keys.",
                    "set() moves cache keys.",
                    "Multiple locmem caches are isolated",
                    "test_prefix (cache.tests.LocMemCacheTests)",
                    "test_set_fail_on_pickleerror (cache.tests.LocMemCacheTests)",
                    "test_set_many (cache.tests.LocMemCacheTests)",
                    "test_set_many_empty_data (cache.tests.LocMemCacheTests)",
                    "test_set_many_expiration (cache.tests.LocMemCacheTests)",
                    "test_simple (cache.tests.LocMemCacheTests)",
                    "test_touch (cache.tests.LocMemCacheTests)",
                    "test_unicode (cache.tests.LocMemCacheTests)",
                    "test_zero_cull (cache.tests.LocMemCacheTests)",
                    "test_add (cache.tests.FileBasedCachePathLibTests)",
                    "test_add_fail_on_pickleerror (cache.tests.FileBasedCachePathLibTests)",
                    "test_binary_string (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_dir_permissions (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_read_for_model_instance (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_read_for_model_instance_with_deferred (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_versioning_add (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_versioning_delete (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_versioning_get_set (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_versioning_get_set_many (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_versioning_has_key (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_versioning_incr_decr (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_write_for_model_instance_with_deferred (cache.tests.FileBasedCachePathLibTests)",
                    "test_cache_write_unpicklable_object (cache.tests.FileBasedCachePathLibTests)",
                    "test_clear (cache.tests.FileBasedCachePathLibTests)",
                    "test_clear_does_not_remove_cache_dir (cache.tests.FileBasedCachePathLibTests)",
                    "test_close (cache.tests.FileBasedCachePathLibTests)",
                    "test_creates_cache_dir_if_nonexistent (cache.tests.FileBasedCachePathLibTests)",
                    "test_cull (cache.tests.FileBasedCachePathLibTests)",
                    "test_cull_delete_when_store_empty (cache.tests.FileBasedCachePathLibTests)",
                    "test_custom_key_func (cache.tests.FileBasedCachePathLibTests)",
                    "test_data_types (cache.tests.FileBasedCachePathLibTests)",
                    "test_decr (cache.tests.FileBasedCachePathLibTests)",
                    "test_decr_version (cache.tests.FileBasedCachePathLibTests)",
                    "test_delete (cache.tests.FileBasedCachePathLibTests)",
                    "test_delete_many (cache.tests.FileBasedCachePathLibTests)",
                    "test_delete_many_no_keys (cache.tests.FileBasedCachePathLibTests)",
                    "test_delete_nonexistent (cache.tests.FileBasedCachePathLibTests)",
                    "test_empty_cache_file_considered_expired (cache.tests.FileBasedCachePathLibTests)",
                    "test_expiration (cache.tests.FileBasedCachePathLibTests)",
                    "test_float_timeout (cache.tests.FileBasedCachePathLibTests)",
                    "test_get_does_not_ignore_non_filenotfound_exceptions (cache.tests.FileBasedCachePathLibTests)",
                    "test_get_ignores_enoent (cache.tests.FileBasedCachePathLibTests)",
                    "test_get_many (cache.tests.FileBasedCachePathLibTests)",
                    "test_get_or_set (cache.tests.FileBasedCachePathLibTests)",
                    "test_get_or_set_callable (cache.tests.FileBasedCachePathLibTests)",
                    "test_get_or_set_racing (cache.tests.FileBasedCachePathLibTests)",
                    "test_get_or_set_version (cache.tests.FileBasedCachePathLibTests)",
                    "test_has_key (cache.tests.FileBasedCachePathLibTests)",
                    "test_ignores_non_cache_files (cache.tests.FileBasedCachePathLibTests)",
                    "test_in (cache.tests.FileBasedCachePathLibTests)",
                    "test_incr (cache.tests.FileBasedCachePathLibTests)",
                    "test_incr_version (cache.tests.FileBasedCachePathLibTests)",
                    "test_invalid_key_characters (cache.tests.FileBasedCachePathLibTests)",
                    "test_invalid_key_length (cache.tests.FileBasedCachePathLibTests)",
                    "test_invalid_with_version_key_length (cache.tests.FileBasedCachePathLibTests)",
                    "test_prefix (cache.tests.FileBasedCachePathLibTests)",
                    "test_set_fail_on_pickleerror (cache.tests.FileBasedCachePathLibTests)",
                    "test_set_many (cache.tests.FileBasedCachePathLibTests)",
                    "test_set_many_empty_data (cache.tests.FileBasedCachePathLibTests)",
                    "test_set_many_expiration (cache.tests.FileBasedCachePathLibTests)",
                    "test_simple (cache.tests.FileBasedCachePathLibTests)",
                    "test_touch (cache.tests.FileBasedCachePathLibTests)",
                    "test_unicode (cache.tests.FileBasedCachePathLibTests)",
                    "test_zero_cull (cache.tests.FileBasedCachePathLibTests)",
                    "test_add (cache.tests.FileBasedCacheTests)",
                    "test_add_fail_on_pickleerror (cache.tests.FileBasedCacheTests)",
                    "test_binary_string (cache.tests.FileBasedCacheTests)",
                    "test_cache_dir_permissions (cache.tests.FileBasedCacheTests)",
                    "test_cache_read_for_model_instance (cache.tests.FileBasedCacheTests)",
                    "test_cache_read_for_model_instance_with_deferred (cache.tests.FileBasedCacheTests)",
                    "test_cache_versioning_add (cache.tests.FileBasedCacheTests)",
                    "test_cache_versioning_delete (cache.tests.FileBasedCacheTests)",
                    "test_cache_versioning_get_set (cache.tests.FileBasedCacheTests)",
                    "test_cache_versioning_get_set_many (cache.tests.FileBasedCacheTests)",
                    "test_cache_versioning_incr_decr (cache.tests.FileBasedCacheTests)",
                    "test_cache_write_for_model_instance_with_deferred (cache.tests.FileBasedCacheTests)",
                    "test_cache_write_unpicklable_object (cache.tests.FileBasedCacheTests)",
                    "test_clear (cache.tests.FileBasedCacheTests)",
                    "test_clear_does_not_remove_cache_dir (cache.tests.FileBasedCacheTests)",
                    "test_close (cache.tests.FileBasedCacheTests)",
                    "test_creates_cache_dir_if_nonexistent (cache.tests.FileBasedCacheTests)",
                    "test_cull (cache.tests.FileBasedCacheTests)",
                    "test_cull_delete_when_store_empty (cache.tests.FileBasedCacheTests)",
                    "test_custom_key_func (cache.tests.FileBasedCacheTests)",
                    "test_data_types (cache.tests.FileBasedCacheTests)",
                    "test_decr (cache.tests.FileBasedCacheTests)",
                    "test_decr_version (cache.tests.FileBasedCacheTests)",
                    "test_delete (cache.tests.FileBasedCacheTests)",
                    "test_delete_many (cache.tests.FileBasedCacheTests)",
                    "test_delete_many_no_keys (cache.tests.FileBasedCacheTests)",
                    "test_delete_nonexistent (cache.tests.FileBasedCacheTests)",
                    "test_empty_cache_file_considered_expired (cache.tests.FileBasedCacheTests)",
                    "test_expiration (cache.tests.FileBasedCacheTests)",
                    "test_float_timeout (cache.tests.FileBasedCacheTests)",
                    "test_get_does_not_ignore_non_filenotfound_exceptions (cache.tests.FileBasedCacheTests)",
                    "test_get_ignores_enoent (cache.tests.FileBasedCacheTests)",
                    "test_get_many (cache.tests.FileBasedCacheTests)",
                    "test_get_or_set (cache.tests.FileBasedCacheTests)",
                    "test_get_or_set_callable (cache.tests.FileBasedCacheTests)",
                    "test_get_or_set_racing (cache.tests.FileBasedCacheTests)",
                    "test_get_or_set_version (cache.tests.FileBasedCacheTests)",
                    "test_has_key (cache.tests.FileBasedCacheTests)",
                    "test_ignores_non_cache_files (cache.tests.FileBasedCacheTests)",
                    "test_in (cache.tests.FileBasedCacheTests)",
                    "test_incr (cache.tests.FileBasedCacheTests)",
                    "test_incr_version (cache.tests.FileBasedCacheTests)",
                    "test_invalid_key_characters (cache.tests.FileBasedCacheTests)",
                    "test_invalid_key_length (cache.tests.FileBasedCacheTests)",
                    "test_invalid_with_version_key_length (cache.tests.FileBasedCacheTests)",
                    "test_prefix (cache.tests.FileBasedCacheTests)",
                    "test_set_fail_on_pickleerror (cache.tests.FileBasedCacheTests)",
                    "test_set_many (cache.tests.FileBasedCacheTests)",
                    "test_set_many_empty_data (cache.tests.FileBasedCacheTests)",
                    "test_set_many_expiration (cache.tests.FileBasedCacheTests)",
                    "test_simple (cache.tests.FileBasedCacheTests)",
                    "test_touch (cache.tests.FileBasedCacheTests)",
                    "test_unicode (cache.tests.FileBasedCacheTests)",
                    "test_add (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_add_fail_on_pickleerror (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_binary_string (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_read_for_model_instance (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_read_for_model_instance_with_deferred (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_versioning_add (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_versioning_delete (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_versioning_get_set (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_versioning_get_set_many (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_versioning_has_key (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_versioning_incr_decr (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_write_for_model_instance_with_deferred (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cache_write_unpicklable_object (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_clear (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_close (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_createcachetable_dry_run_mode (cache.tests.DBCacheWithTimeZoneTests)",
                    "Delete and recreate cache table with legacy behavior (explicitly",
                    "test_cull (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cull_delete_when_store_empty (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_cull_queries (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_custom_key_func (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_data_types (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_decr (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_decr_version (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_delete (cache.tests.DBCacheWithTimeZoneTests)",
                    "The rowcount attribute should not be checked on a closed cursor.",
                    "test_delete_many (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_delete_many_no_keys (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_delete_many_num_queries (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_delete_nonexistent (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_expiration (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_float_timeout (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_get_many (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_get_many_num_queries (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_get_or_set (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_get_or_set_callable (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_get_or_set_racing (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_get_or_set_version (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_has_key (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_has_key_query_columns_quoted (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_in (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_incr (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_incr_version (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_invalid_key_characters (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_invalid_key_length (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_invalid_with_version_key_length (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_prefix (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_second_call_doesnt_crash (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_set_fail_on_pickleerror (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_set_many (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_set_many_empty_data (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_set_many_expiration (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_simple (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_touch (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_unicode (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_zero_cull (cache.tests.DBCacheWithTimeZoneTests)",
                    "test_add_fail_on_pickleerror (cache.tests.DBCacheTests)",
                    "test_binary_string (cache.tests.DBCacheTests)",
                    "test_cache_read_for_model_instance (cache.tests.DBCacheTests)",
                    "test_cache_read_for_model_instance_with_deferred (cache.tests.DBCacheTests)",
                    "test_cache_versioning_add (cache.tests.DBCacheTests)",
                    "test_cache_versioning_delete (cache.tests.DBCacheTests)",
                    "test_cache_versioning_get_set (cache.tests.DBCacheTests)",
                    "test_cache_versioning_get_set_many (cache.tests.DBCacheTests)",
                    "test_cache_versioning_has_key (cache.tests.DBCacheTests)",
                    "test_cache_versioning_incr_decr (cache.tests.DBCacheTests)",
                    "test_cache_write_for_model_instance_with_deferred (cache.tests.DBCacheTests)",
                    "test_cache_write_unpicklable_object (cache.tests.DBCacheTests)",
                    "test_clear (cache.tests.DBCacheTests)",
                    "test_close (cache.tests.DBCacheTests)",
                    "test_createcachetable_dry_run_mode (cache.tests.DBCacheTests)",
                    "test_cull (cache.tests.DBCacheTests)",
                    "test_cull_delete_when_store_empty (cache.tests.DBCacheTests)",
                    "test_cull_queries (cache.tests.DBCacheTests)",
                    "test_custom_key_func (cache.tests.DBCacheTests)",
                    "test_data_types (cache.tests.DBCacheTests)",
                    "test_decr (cache.tests.DBCacheTests)",
                    "test_decr_version (cache.tests.DBCacheTests)",
                    "test_delete (cache.tests.DBCacheTests)",
                    "test_delete_many (cache.tests.DBCacheTests)",
                    "test_delete_many_no_keys (cache.tests.DBCacheTests)",
                    "test_delete_many_num_queries (cache.tests.DBCacheTests)",
                    "test_delete_nonexistent (cache.tests.DBCacheTests)",
                    "test_expiration (cache.tests.DBCacheTests)",
                    "test_float_timeout (cache.tests.DBCacheTests)",
                    "test_get_many (cache.tests.DBCacheTests)",
                    "test_get_many_num_queries (cache.tests.DBCacheTests)",
                    "test_get_or_set (cache.tests.DBCacheTests)",
                    "test_get_or_set_callable (cache.tests.DBCacheTests)",
                    "test_get_or_set_racing (cache.tests.DBCacheTests)",
                    "test_get_or_set_version (cache.tests.DBCacheTests)",
                    "test_has_key (cache.tests.DBCacheTests)",
                    "test_has_key_query_columns_quoted (cache.tests.DBCacheTests)",
                    "test_in (cache.tests.DBCacheTests)",
                    "test_incr (cache.tests.DBCacheTests)",
                    "test_incr_version (cache.tests.DBCacheTests)",
                    "test_invalid_key_characters (cache.tests.DBCacheTests)",
                    "test_invalid_key_length (cache.tests.DBCacheTests)",
                    "test_invalid_with_version_key_length (cache.tests.DBCacheTests)",
                    "test_prefix (cache.tests.DBCacheTests)",
                    "test_second_call_doesnt_crash (cache.tests.DBCacheTests)",
                    "test_set_fail_on_pickleerror (cache.tests.DBCacheTests)",
                    "test_set_many (cache.tests.DBCacheTests)",
                    "test_set_many_empty_data (cache.tests.DBCacheTests)",
                    "test_set_many_expiration (cache.tests.DBCacheTests)",
                    "test_simple (cache.tests.DBCacheTests)",
                    "test_touch (cache.tests.DBCacheTests)",
                    "test_unicode (cache.tests.DBCacheTests)",
                    "test_zero_cull (cache.tests.DBCacheTests)"
                ],
                "failure": [
                    "test_cache_versioning_has_key (cache.tests.FileBasedCacheTests)",
                    "test_zero_cull (cache.tests.FileBasedCacheTests)",
                    "test_add (cache.tests.DBCacheTests)"
                ]
            },
            "FAIL_TO_FAIL": {
                "success": [],
                "failure": []
            },
            "PASS_TO_FAIL": {
                "success": [],
                "failure": []
            }
        }
    }
}

System Information

No response