[Draft] [Fail on pytest warnings 2/n] Failing on pytest warnings

dashboard/modules/job/tests/test_job_agent.py

@@ -40,6 +40,10 @@
 
 logger = logging.getLogger(__name__)
 
+# Disable fail-on-warnings for this file as some of the tests
+# fail due to warnings.
+pytestmark = pytest.mark.filterwarnings("default")
+
 DRIVER_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "subprocess_driver_scripts")
 EVENT_LOOP = get_or_create_event_loop()
 

pytest.ini

@@ -1,10 +1,188 @@
 [pytest]
-
+# Configure pytest to fail any test that raises an unexpected warning.
+# This will enable us to catch usages of deprecated features in downstream libraries,
+# before they are hard-deprecated.
+# Furthermore, they will catch any test issues which may hide bugs.
+# The format is `action:message_regex:category:module:line`.
 filterwarnings =
-    # This means: treat every warning as error, except for warnings that match .* (aka all warnings), ignore those.
-    # This triggers the SyntaxError described in https://github.com/ray-project/ray/pull/31523 but keeps the status quo
-    # warning behavior until https://github.com/ray-project/ray/pull/31219 .
+    # Fail builds on any unexpected warnings.
+    # See https://docs.google.com/document/d/1TVMfmhO0vD1MdkVrqRS9t4om2shMTY9nqdqqOopNv0M for more details.
     error
 
-    # The format is `action:message_regex:category:module:line`.
-    ignore:.*:
+    # Fixed by https://github.com/benjaminp/six/pull/343, doesn't appear to impact anything as-is.
+    # This must go before RayDeprecationWarnings...
+    # Can I change the RayDeprecationWarning references to simple DeprecationWarnings?
+    ignore:_SixMetaPathImporter.*\(\) not found; falling back to .*\(\):ImportWarning
+
+    # Need to only ignore this when it comes from botocore. Intended behavior, should be covered by botocore upgrade. https://github.com/boto/botocore/issues/2744
+    # TODO(cade) can I ignore this specifically from botocore?
+    ignore:'urllib3.contrib.pyopenssl' module is deprecated and will be removed in a future release of urllib3 2.x:DeprecationWarning:botocore
+
+    # TODO(cade) create issue for this, p0
+    ignore:async def functions are not natively supported and have been skipped:pytest.PytestUnhandledCoroutineWarning
+
+    # Pytest installs import hooks to add visibility to assertions.
+    # If modules are imported before pytest is started, it will raise this warning.
+    # https://docs.pytest.org/en/7.1.x/how-to/assert.html#assertion-introspection-details
+    ignore::pytest.PytestAssertRewriteWarning
+
+    # torchvision. 
+    # Deprecation https://pytorch.org/blog/introducing-torchvision-new-multi-weight-support-api/#deprecations
+    # https://github.com/pytorch/vision/releases/tag/v0.12.0
+    # TODO(cade) create issue
+    ignore:Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and .* be removed .*:UserWarning:torchvision
+    ignore:The parameter 'pretrained' is deprecated since 0.13 and .* be removed .*, please use 'weights' instead:UserWarning:torchvision
+
+    # numpy https://numpy.org/doc/stable/release/1.19.0-notes.html#deprecate-automatic-dtype-object-for-ragged-input
+    # TODO(cade) create issue
+    # Used in _create_possibly_ragged_ndarray.
+    ignore:Creating an ndarray from ragged nested sequences \(which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes\) is deprecated.:numpy.VisibleDeprecationWarning
+    ignore:`np.*` is a deprecated alias for the builtin `.*`.:DeprecationWarning:python.ray.data.tests.test_dataset_pipeline
+    ignore:`np.*` is a deprecated alias for the builtin `.*`.:DeprecationWarning:python.ray.data.tests.test_dataset_tfrecords
+    ignore:using `dtype=` in comparisons is only useful for `dtype=object` \(and will do nothing for bool\). This operation will fail in the future.:DeprecationWarning
+
+    # Ray deprecation/user warnings.
+    # TODO should we blanket ignore all RayDeprecationWarnings?
+    ignore:placement_group parameter is deprecated:DeprecationWarning
+    ignore:placement_group_bundle_index parameter is deprecated. Use scheduling_strategy:DeprecationWarning
+    ignore:The 'native' batch format has been renamed 'default'.:DeprecationWarning
+    ignore:Setting 'object_store_memory' for .* is deprecated since:DeprecationWarning
+    ignore:DeprecationWarning. local mode is an experimental feature that is no longer maintained and will be removed in the future:DeprecationWarning
+    ignore:Accessing values through ctx\[".*"\] is deprecated. Use ctx.address_info\[".*"\] instead.:DeprecationWarning
+    ignore:More than 10MB of messages have been created to schedule tasks on the server:UserWarning
+    ignore:Please use timeout=None if you expect ray.get\(\) to block. Setting timeout=0 in future ray releases will raise GetTimeoutError if the objects references are not available. You could suppress this warning by setting RAY_WARN_RAY_GET_TIMEOUT_ZERO=0.:UserWarning
+
+    # ray.util.annotations.RayDeprecationWarning
+    ignore:This API is deprecated and may be removed in future Ray releases.*\nUse get_.*_id\(\) instead:DeprecationWarning
+    # ray.util.annotations.RayDeprecationWarning
+    ignore:This API is deprecated and may be removed in future Ray releases.*\n.*removed by Ray 2.4. Please use Runtime Environments:DeprecationWarning
+    # ray.util.annotations.RayDeprecationWarning
+    ignore:This API is deprecated and may be removed in future Ray releases.*\nFor stateless/task processing, use ray.util.multiprocessing:DeprecationWarning:python.ray.tests.test_actor_group
+    #ignore:This API is deprecated and may be removed in future Ray releases.*\nFor stateless/task processing, use ray.util.multiprocessing:DeprecationWarning
+    ignore:Executing `.fit\(\)` may leave less than 20% of CPUs in this cluster for Dataset execution:UserWarning
+    ignore:The wandb_mixin/WandbTrainableMixin is deprecated. Use `ray.air.integrations.wandb.setup_wandb` instead.:DeprecationWarning
+    # python.ray.air.tests.test_integration_mlflow
+    ignore:The mlflow_mixin/MLflowTrainableMixin is deprecated. Use `ray.air.integrations.mlflow.setup_mlflow` instead:DeprecationWarning
+
+    # Keras uses deprecated Pillow APIs https://github.com/keras-team/keras/issues/16639
+    # TODO(cade) add keras label
+    ignore:.* is deprecated and will be removed in Pillow 10 \(2023-07-01\). Use .* instead.:DeprecationWarning:keras
+    ignore:.* is deprecated and will be removed in Pillow 10 \(2023-07-01\). Use .* instead.:DeprecationWarning:torchvision
+
+    # Used by test_parquet_deserialize_pieces_with_retry
+    # TODO(cade) create issue
+    # Both categories necessary, pytest doesn't support regex on category.
+    # Raised as a FutureWarning and a DeprecationWarning. I omitted the category to catch both.
+    ignore:'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Use the '.fragments' attribute instead:FutureWarning
+    ignore:'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Use the '.fragments' attribute instead:DeprecationWarning
+
+    # Used by test_callsite_warning.
+    # Will be removed in Pytest 8.
+    # TODO(cade) create issue
+    ignore:Passing None has been deprecated.:pytest.PytestRemovedIn8Warning
+
+    ignore:Please install grpcio-status to obtain helpful grpc error messages:ImportWarning
+
+    # TODO(cade) investigate ray client issues
+    ignore:Ray Client connection timed out.:UserWarning
+    ignore:Starting a connection through `ray.client` will be deprecated:DeprecationWarning
+
+    # Used in test_tensors_in_tables_parquet_bytes_manual_serde_col_schema
+    # Pyarrow deprecation notice.
+    # TODO(cade) create issue
+    ignore:The 'field_by_name' method is deprecated, use 'field' instead:FutureWarning
+
+    # Python logging deprecation warning.
+    # Deprecated since Python 3.3, unclear when it will break.
+    # https://docs.python.org/3/library/logging.html#logging.Logger.warning
+    # https://bugs.python.org/issue13235
+    ignore:The 'warn' method is deprecated, use 'warning' instead:DeprecationWarning
+
+    # Used in test_groupby_map_groups_for_pandas
+    # Unclear when Pandas will hard deprecate.
+    # https://pandas.pydata.org/docs/whatsnew/v1.0.0.html#default-dtype-of-empty-pandas-series
+    ignore:The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.:DeprecationWarning
+
+    # Low-impact test hygiene warnings.
+    ignore:Value of environment variable KUBERNETES_SERVICE_HOST type should be str, but got 1 \(type. int\); converted to str implicitly:pytest.PytestWarning
+    ignore:Value of environment variable RAY_WORKER_TIMEOUT_S type should be str, but got .* \(type. int\); converted to str implicitly:pytest.PytestWarning
+
+    # Modin uses some deprecated APIs.
+    ignore:`np.*` is a deprecated alias for the builtin `.*`.:DeprecationWarning:modin
+    ignore:The pandas.datetime class is deprecated and will be removed from pandas in a future version. Import from datetime module instead.:FutureWarning:modin
+
+    # Modin warns when distributing an object could take a while.
+    # TODO(cade) replace type with regex
+    #ignore:Distributing <class 'list'> object. This may take some time.:UserWarning:modin
+    #ignore:Distributing <class 'NoneType'> object. This may take some time.:UserWarning:modin
+    ignore:Distributing <class '.*'> object. This may take some time.:UserWarning:modin
+    ignore:.*defaulting to pandas implementation.:UserWarning:modin
+    ignore:.*implementation has mismatches with pandas.:UserWarning:modin
+
+    # pymongoarrow uses deprecated APIs.
+    ignore:`np.*` is a deprecated alias for the builtin `.*`.:DeprecationWarning:pymongoarrow
+
+    # Used by test_sklearn_benchmarks.
+    ignore:n_components > n_samples. This is not possible:UserWarning
+
+    # Caused by `ResourceWarning: unclosed file <filename>` by the following file paths:
+    # * /tmp/ray/<session>/logs/log_monitor.err from File "/ray/python/ray/_private/node.py", line 1183, in start_ray_processes
+    # * /tmp/ray/<session>/logs/raylet.1.err from File "/ray/python/ray/_private/node.py", line 1181, in start_ray_processes
+    # * /tmp/ray/<session>/logs/dashboard.err from File "/ray/python/ray/_private/node.py", line 1139, in start_head_processes
+    # See also https://docs.python.org/3/library/sys.html#sys.unraisablehook
+    ignore:Exception ignored in. <_io.FileIO \[closed\]>:pytest.PytestUnraisableExceptionWarning
+
+    # This is caused by "ResourceWarning: subprocess 17289 is still running".
+    ignore:Exception ignored in. <function Popen.__del__ at 0x.*:pytest.PytestUnraisableExceptionWarning
+    ignore:cannot collect test class .* because it has a __init__ constructor:pytest.PytestCollectionWarning
+    ignore:Unknown pytest.mark.asyncio - is this a typo?:pytest.PytestUnknownMarkWarning
+
+    # From python.ray.tests.horovod.test_horovod
+    ignore:CUDA initialization. Found no NVIDIA driver on your system:UserWarning
+
+    # From torch, in python.ray.tests.horovod.test_horovod
+    # If I keep seeing new errors, need to prototype and get the module ignore working for tests.
+    ignore:Named tensors and all their associated APIs are an experimental feature and subject to change:UserWarning
+
+    # TODO(cade) create issue for this? See how bad it is
+    # Fails test_tensors_in_tables_to_torch
+    # Looks like the code catches an Exception, which also catches warnings.
+    # python/ray/train/tests/test_horovod_trainer.py::test_horovod
+    ignore:The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior.:UserWarning
+
+    # python.ray.train.tests.test_horovod_trainer
+    #ignore:The given NumPy array is not writeable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior.:UserWarning
+    ignore:The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors.:UserWarning
+
+    # python/ray/data/tests/test_dataset_tf.py:168
+    ignore:crc32c.crc32 will be eventually removed, use crc32c.crc32c instead:DeprecationWarning:tensorboardX
+
+    ignore:Please use assertEqual instead:DeprecationWarning:release.ray_release.tests.test_buildkite
+
+    ignore:The use of label encoder in XGBClassifier is deprecated and will be removed in a future release:UserWarning
+
+    # TestWandbClassMixin.test_wandb_mixin_api_key_file
+    ignore:Passing a `.*` key in the config dict is deprecated and will raise an error in the future. Please pass the actual arguments to `.*\(\)` instead:DeprecationWarning
+
+    # python.ray.tests.test_runtime_env
+    ignore:ssl.PROTOCOL_TLS is deprecated:DeprecationWarning
+
+    # python.ray.air.tests.test_integration_mlflow
+    ignore:`.*` is meant to only be called inside a function that is executed by a Tuner or Trainer. Returning `.*`.:UserWarning
+
+    ignore:load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead:FutureWarning:python.ray.train.tests.test_huggingface_trainer
+
+    ignore:This test must be run on large machines.:UserWarning:python.ray.tests.test_object_manager
+
+    # python.ray.air.tests.test_integration_wandb
+    # f"\x1b[{attrs}m{string}\x1b[0m" from https://github.com/Farama-Foundation/Gymnasium/blob/6ba886abce531aa7e6804a1bd18f0afadb625789/gymnasium/utils/colorize.py#L19
+    ignore:.*WARN. The environment CartPole-v0 is out of date. You should consider upgrading to version `v1`:UserWarning
+
+    # python/ray/tests/test_runtime_env
+    ignore:ssl.match_hostname\(\) is deprecated:DeprecationWarning
+
+    ignore:Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 50:UserWarning:transformers
+
+    ignore:DataFrame.sql_ctx is an internal property, and will be removed in future releases. Use DataFrame.sparkSession instead.:UserWarning
+
+    ignore:Callsite is not being recorded. To record callsite information for each ObjectRef created, set env variable RAY_record_ref_creation_sites=1 during `ray start` and `ray.init`.:UserWarning

python/ray/data/grouped_dataset.py

@@ -459,7 +459,7 @@ def sum(
     def min(
         self, on: Union[KeyFn, List[KeyFn]] = None, ignore_nulls: bool = True
     ) -> Dataset[U]:
-        """Compute grouped min aggregation.
+        r"""Compute grouped min aggregation.
 
         Examples:
             >>> import ray
@@ -518,7 +518,7 @@ def min(
     def max(
         self, on: Union[KeyFn, List[KeyFn]] = None, ignore_nulls: bool = True
     ) -> Dataset[U]:
-        """Compute grouped max aggregation.
+        r"""Compute grouped max aggregation.
 
         Examples:
             >>> import ray
@@ -577,7 +577,7 @@ def max(
     def mean(
         self, on: Union[KeyFn, List[KeyFn]] = None, ignore_nulls: bool = True
     ) -> Dataset[U]:
-        """Compute grouped mean aggregation.
+        r"""Compute grouped mean aggregation.
 
         Examples:
             >>> import ray
@@ -640,7 +640,7 @@ def std(
         ddof: int = 1,
         ignore_nulls: bool = True,
     ) -> Dataset[U]:
-        """Compute grouped standard deviation aggregation.
+        r"""Compute grouped standard deviation aggregation.
 
         Examples:
             >>> import ray

python/ray/data/preprocessors/hasher.py

@@ -11,7 +11,7 @@
 
 @PublicAPI(stability="alpha")
 class FeatureHasher(Preprocessor):
-    """Apply the `hashing trick <https://en.wikipedia.org/wiki/Feature_hashing>`_ to a
+    r"""Apply the `hashing trick <https://en.wikipedia.org/wiki/Feature_hashing>`_ to a
     table that describes token frequencies.
 
     :class:`FeatureHasher` creates ``num_features`` columns named ``hash_{index}``,

python/ray/data/tests/test_stats.py

@@ -16,11 +16,11 @@
 
 def canonicalize(stats: str) -> str:
     # Time expressions.
-    s1 = re.sub("[0-9\.]+(ms|us|s)", "T", stats)
+    s1 = re.sub(r"[0-9\.]+(ms|us|s)", "T", stats)
     # Handle zero values specially so we can check for missing values.
-    s2 = re.sub(" [0]+(\.[0]+)?", " Z", s1)
+    s2 = re.sub(r" [0]+(\.[0]+)?", " Z", s1)
     # Other numerics.
-    s3 = re.sub("[0-9]+(\.[0-9]+)?", "N", s2)
+    s3 = re.sub(r"[0-9]+(\.[0-9]+)?", "N", s2)
     # Replace tabs with spaces.
     s4 = re.sub("\t", "    ", s3)
     return s4