diff --git a/ivy/data_classes/array/experimental/losses.py b/ivy/data_classes/array/experimental/losses.py index f676338c11c89..e06132caf1638 100644 --- a/ivy/data_classes/array/experimental/losses.py +++ b/ivy/data_classes/array/experimental/losses.py @@ -365,3 +365,84 @@ def poisson_nll_loss( eps=eps, reduction=reduction, ) + + def hinge_embedding_loss( + self: Union[ivy.Array, ivy.NativeArray], + target: Union[ivy.Array, ivy.NativeArray], + *, + margin: float = 1.0, + reduction: str = "mean", + ) -> ivy.Array: + r"""Measures loss from input `x` and label `y` with values 1 or -1. It + evaluates if two inputs are similar or not, often used for embedding or + semi-supervised learning. + + Loss for the `n`-th sample: + .. math:: + l_n = \begin{cases} + x_n, & \text{if}\; y_n = 1,\\ + \max \{0, margin - x_n\}, & \text{if}\; y_n = -1, + \end{cases} + + Total loss: + .. math:: + \ell(x, y) = \begin{cases} + \operatorname{mean}(L), & \text{if reduction} = \text{`mean';}\\ + \operatorname{sum}(L), & \text{if reduction} = \text{`sum'.} + \end{cases} + + where :math:`L = \{l_1,\dots,l_N\}^\top` + + Parameters + ---------- + input + Input tensor with dtype float. + The shape is [N, \*], where N is batch size and `\*` represents + any number of additional dimensions. + label + Label tensor containing 1 or -1 with dtype float32 or float64. + Its shape matches that of the input. + margin + Sets the hyperparameter margin. Determines the necessary input size + for hinge_embedding_loss calculations when label is -1. Inputs smaller + than the margin are minimized with hinge_embedding_loss. + Default is 1.0. + reduction + Specifies how to aggregate the loss across the batch. Options are: + - ``'none'``: Returns the unreduced loss. + - ``'mean'``: Returns the mean loss. + - ``'sum'``: Returns the summed loss. + Default is ``'mean'``. + + Shape + ----- + - Input: :math:`(*)` where :math:`*` means, any number of dimensions. \ + The sum operation operates over all the elements. + - Target: :math:`(*)`, same shape as the input + - Output: scalar. If :attr:`reduction` is ``'none'``, + then same shape as the input + + Returns + ------- + ret + Hinge embedding loss calculated from the input and label, + shaped based on the reduction method. + + Examples + -------- + >>> input_tensor = ivy.array([1, 2, 3, 4], dtype=ivy.float64) + >>> target_tensor = ivy.array([1, 1, 1, 1], dtype=ivy.float64) + >>> input_tensor.hinge_embedding_loss(target_tensor,reduction="sum") + ivy.array(10.) + + >>> input_tensor = ivy.array([1, 2, 3], dtype=ivy.float64) + >>> target_tensor = ivy.array([1, -1, -1], dtype=ivy.float64) + >>> input_tensor.hinge_embedding_loss(target_tensor, margin=2.0) + ivy.array(0.33333333) + """ + return ivy.hinge_embedding_loss( + self._data, + target, + margin=margin, + reduction=reduction, + ) diff --git a/ivy/data_classes/container/experimental/losses.py b/ivy/data_classes/container/experimental/losses.py index 799c44adfcb25..26fc89086aadc 100644 --- a/ivy/data_classes/container/experimental/losses.py +++ b/ivy/data_classes/container/experimental/losses.py @@ -1089,3 +1089,186 @@ def poisson_nll_loss( prune_unapplied=prune_unapplied, map_sequences=map_sequences, ) + + @staticmethod + def _static_hinge_embedding_loss( + input: Union[ivy.Container, ivy.Array, ivy.NativeArray], + target: Union[ivy.Container, ivy.Array, ivy.NativeArray], + *, + margin: [Union[float, ivy.Container]] = 1.0, + reduction: [Union[str, ivy.Container]] = "mean", + key_chains: Optional[Union[List[str], Dict[str, str], ivy.Container]] = None, + to_apply: Union[bool, ivy.Container] = True, + prune_unapplied: Union[bool, ivy.Container] = False, + map_sequences: Union[bool, ivy.Container] = False, + ) -> ivy.Container: + r"""ivy.Container static method variant of ivy.hinge_embedding_loss. + This method simplywraps the function, and so the docstring for + ivy.hinge_embedding_loss also applies to this method with minimal + changes. + + Parameters + ---------- + input + input array or container containing input labels. + target + input array or container containing the target labels. + margin + Sets the hyperparameter margin. Determines the necessary input size + for hinge_embedding_loss calculations when label is -1. Inputs smaller + than the margin are minimized with hinge_embedding_loss. + Default is 1.0. + reduction + Specifies how to aggregate the loss across the batch. Options are: + - ``'none'``: Returns the unreduced loss. + - ``'mean'``: Returns the mean loss. + - ``'sum'``: Returns the summed loss. + Default is ``'mean'``. + key_chains + The key-chains to apply or not apply the method to. Default is ``None``. + to_apply + If input, the method will be applied to key_chains, otherwise key_chains + will be skipped. Default is ``input``. + prune_unapplied + Whether to prune key_chains for which the function was not applied. + Default is ``False``. + map_sequences + Whether to also map method to sequences (lists, tuples). + Default is ``False``. + + Shape + ----- + - Input: :math:`(*)` where :math:`*` means, any number of dimensions. \ + The sum operation operates over all the elements. + - Target: :math:`(*)`, same shape as the input + - Output: scalar. If :attr:`reduction` is ``'none'``, + then same shape as the input + + Returns + ------- + ret + Hinge embedding loss calculated from the input and label, + shaped based on the reduction method. + + Examples + -------- + With :class:`ivy.Container` inputs: + + >>> x = ivy.Container(a=ivy.array([[1, 0, 2]], dtype=ivy.float32), + ... b=ivy.array([[-1, 1, 1]], dtype=ivy.float32)) + >>> y = ivy.Container(a=ivy.array([[0.6, 0.2, 0.3]], dtype=ivy.float32), + ... b=ivy.array([[1, 1, 1]], dtype=ivy.float32)) + >>> z = ivy.Container._static_hinge_embedding_loss(x, y, reduction="none") + >>> z + { + a: ivy.array([[0., 0., 0.]]), + b: ivy.array([[-1., 1., 1.]]) + } + + With a mix of :class:`ivy.Array` and :class:`ivy.Container` inputs: + + >>> x = ivy.array([[10, 20, 32]], dtype=ivy.float32) + >>> y = ivy.Container(a=ivy.array([[-1, -1, -1]], dtype=ivy.float32), + ... b=ivy.array([[1, 1, 1]], dtype=ivy.float32)) + >>> z = ivy.Container._static_hinge_embedding_loss(x, y, + ... reduction="sum", margin=2.0) + >>> z + { + a: ivy.array(0.), + b: ivy.array(62.) + } + """ + return ContainerBase.cont_multi_map_in_function( + "hinge_embedding_loss", + input, + target, + margin=margin, + reduction=reduction, + key_chains=key_chains, + to_apply=to_apply, + prune_unapplied=prune_unapplied, + map_sequences=map_sequences, + ) + + def hinge_embedding_loss( + self: Union[ivy.Container, ivy.Array, ivy.NativeArray], + target: Union[ivy.Container, ivy.Array, ivy.NativeArray], + *, + margin: [Union[float, ivy.Container]] = 1.0, + reduction: [Union[str, ivy.Container]] = "mean", + key_chains: Optional[Union[List[str], Dict[str, str], ivy.Container]] = None, + to_apply: Union[bool, ivy.Container] = True, + prune_unapplied: Union[bool, ivy.Container] = False, + map_sequences: Union[bool, ivy.Container] = False, + ) -> ivy.Container: + r"""ivy.Container instance method variant of ivy.hinge_embedding_loss. + This method simply wraps the function, and so the docstring for + ivy.hinge_embedding_loss also applies to this method with minimal + changes. + + Parameters + ---------- + input + input array or container containing input labels. + target + input array or container containing the target labels. + margin + Sets the hyperparameter margin. Determines the necessary input size + for hinge_embedding_loss calculations when label is -1. Inputs smaller + than the margin are minimized with hinge_embedding_loss. + Default is 1.0. + reduction + Specifies how to aggregate the loss across the batch. Options are: + - ``'none'``: Returns the unreduced loss. + - ``'mean'``: Returns the mean loss. + - ``'sum'``: Returns the summed loss. + Default is ``'mean'``. + key_chains + The key-chains to apply or not apply the method to. Default is ``None``. + to_apply + If input, the method will be applied to key_chains, otherwise key_chains + will be skipped. Default is ``input``. + prune_unapplied + Whether to prune key_chains for which the function was not applied. + Default is ``False``. + map_sequences + Whether to also map method to sequences (lists, tuples). + Default is ``False``. + + Shape + ----- + - Input: :math:`(*)` where :math:`*` means, any number of dimensions. \ + The sum operation operates over all the elements. + - Target: :math:`(*)`, same shape as the input + - Output: scalar. If :attr:`reduction` is ``'none'``, + then same shape as the input + + Returns + ------- + ret + Hinge embedding loss calculated from the input and label, + shaped based on the reduction method. + + + Examples + -------- + >>> x = ivy.Container(a=ivy.array([[1, 0, 2]], dtype=ivy.float32), + ... b=ivy.array([[3, 2, 1]], dtype=ivy.float32)) + >>> y = ivy.Container(a=ivy.array([[-1, -1, -1]], dtype=ivy.float32), + ... b=ivy.array([[1, 1, 1]], dtype=ivy.float32)) + >>> x.hinge_embedding_loss(y, reduction="none", margin=0.5) + { + a: ivy.array([[0., 0.5, 0.]]), + b: ivy.array([[3., 2., 1.]]) + } + """ + return self._static_hinge_embedding_loss( + self, + target, + margin=margin, + reduction=reduction, + key_chains=key_chains, + to_apply=to_apply, + prune_unapplied=prune_unapplied, + map_sequences=map_sequences, + ) diff --git a/ivy/functional/backends/jax/experimental/losses.py b/ivy/functional/backends/jax/experimental/losses.py index 24359a26366be..91aa08c222779 100644 --- a/ivy/functional/backends/jax/experimental/losses.py +++ b/ivy/functional/backends/jax/experimental/losses.py @@ -64,11 +64,11 @@ def soft_margin_loss( return loss -def _apply_loss_reduction(loss: JaxArray, reduction: str, axis=None) -> JaxArray: +def _apply_loss_reduction(loss: JaxArray, reduction: str) -> JaxArray: if reduction == "sum": - return jnp.sum(loss, axis=axis) + return jnp.sum(loss) elif reduction == "mean": - return jnp.mean(loss, axis=axis) + return jnp.mean(loss) else: # reduction == "none" return loss @@ -114,7 +114,7 @@ def _validate_poisson_nll_params( @with_supported_device_and_dtypes( { - "0.4.14 and below": { + "0.4.18 and below": { "cpu": ("float16", "float32", "float64"), } }, @@ -153,3 +153,28 @@ def poisson_nll_loss( cond = jnp.logical_and(target_arr >= zeroes, target_arr <= ones) loss = loss + jnp.where(cond, zeroes, striling_approx_term) return _apply_loss_reduction(loss, reduction) + + +@with_supported_device_and_dtypes( + { + "0.4.18 and below": { + "cpu": ("float32", "float64"), + } + }, + backend_version, +) +def hinge_embedding_loss( + input: JaxArray, + target: JaxArray, + *, + margin: float = 1.0, + reduction: str = "mean", +) -> JaxArray: + zero_ = jnp.zeros([1], dtype=input.dtype) + + relu_part = jnp.maximum(margin - input, 0) + + loss = jnp.where(target == 1.0, input, zero_) + jnp.where( + target == -1.0, relu_part, zero_ + ) + return _apply_loss_reduction(loss, reduction) diff --git a/ivy/functional/backends/numpy/experimental/losses.py b/ivy/functional/backends/numpy/experimental/losses.py index 35cc06af9c8c4..873bc604d8a6e 100644 --- a/ivy/functional/backends/numpy/experimental/losses.py +++ b/ivy/functional/backends/numpy/experimental/losses.py @@ -75,15 +75,12 @@ def soft_margin_loss( return loss -def _apply_loss_reduction(loss: np.ndarray, reduction: str, axis, out) -> np.ndarray: +def _apply_loss_reduction(loss: np.ndarray, reduction: str) -> np.ndarray: if reduction == "sum": - return np.sum(loss, axis=axis, out=out) + return np.sum(loss) elif reduction == "mean": - return np.mean(loss, axis=axis, out=out) + return np.mean(loss) else: # reduction == "none" - if out is not None: - out[...] = loss - return out return loss @@ -128,7 +125,7 @@ def _validate_poisson_nll_params( @with_supported_device_and_dtypes( { - "1.25.2 and below": { + "1.26.0 and below": { "cpu": ("float16", "float32", "float64"), } }, @@ -167,3 +164,29 @@ def poisson_nll_loss( cond = np.logical_and(target_arr >= zeroes, target_arr <= ones) loss = loss + np.where(cond, zeroes, striling_approx_term) return _apply_loss_reduction(loss, reduction) + + +@with_supported_device_and_dtypes( + { + "1.26.0 and below": { + "cpu": ("float32", "float64"), + } + }, + backend_version, +) +def hinge_embedding_loss( + input: np.ndarray, + target: np.ndarray, + *, + margin: float = 1.0, + reduction: str = "mean", +) -> np.ndarray: + zero_ = np.zeros([1], dtype=input.dtype) + + relu_part = np.maximum(margin - input, 0) + + loss = np.where(target == 1.0, input, zero_) + np.where( + target == -1.0, relu_part, zero_ + ) + + return _apply_loss_reduction(loss, reduction) diff --git a/ivy/functional/backends/paddle/experimental/losses.py b/ivy/functional/backends/paddle/experimental/losses.py index 7d23b31362e40..ec01230538bae 100644 --- a/ivy/functional/backends/paddle/experimental/losses.py +++ b/ivy/functional/backends/paddle/experimental/losses.py @@ -239,3 +239,27 @@ def poisson_nll_loss( cond = paddle.logical_and(target_arr >= zeroes, target_arr <= ones) loss = loss + paddle.where(cond, zeroes, striling_approx_term) return _apply_loss_reduction(loss, reduction) + + +@with_supported_device_and_dtypes( + { + "2.5.1 and below": { + "cpu": ("float32", "float64"), + "gpu": ("float16", "float32", "float64"), + } + }, + backend_version, +) +def hinge_embedding_loss( + input: paddle.Tensor, + target: paddle.Tensor, + *, + margin: float = 1.0, + reduction: str = "mean", +) -> paddle.Tensor: + return paddle.nn.functional.hinge_embedding_loss( + input, + target, + margin=margin, + reduction=reduction, + ) diff --git a/ivy/functional/backends/tensorflow/experimental/losses.py b/ivy/functional/backends/tensorflow/experimental/losses.py index ecc812b53c085..404e2764b8d61 100644 --- a/ivy/functional/backends/tensorflow/experimental/losses.py +++ b/ivy/functional/backends/tensorflow/experimental/losses.py @@ -68,11 +68,11 @@ def soft_margin_loss( return loss -def _apply_loss_reduction(loss: tf.Tensor, reduction: str, axis) -> tf.Tensor: +def _apply_loss_reduction(loss: tf.Tensor, reduction: str) -> tf.Tensor: if reduction == "sum": - return tf.math.reduce_sum(loss, axis=axis) + return tf.math.reduce_sum(loss) elif reduction == "mean": - return tf.reduce_mean(loss, axis=axis) + return tf.reduce_mean(loss) else: # reduction == "none" return loss @@ -156,3 +156,30 @@ def poisson_nll_loss( cond = tf.math.logical_and(target_tensor >= zeros, target_tensor <= ones) loss = loss + tf.where(cond, zeros, stirling_approx) return _apply_loss_reduction(loss, reduction) + + +@with_supported_device_and_dtypes( + { + "2.14.0 and below": { + "cpu": ("float32", "float64"), + "gpu": ("float32", "float64"), + } + }, + backend_version, +) +def hinge_embedding_loss( + input: tf.Tensor, + target: tf.Tensor, + *, + margin: float = 1.0, + reduction: str = "mean", +) -> tf.Tensor: + zero_ = tf.zeros([1], dtype=input.dtype) + + relu_part = tf.math.maximum(margin - input, 0) + + loss = tf.where(tf.equal(target, 1.0), input, zero_) + tf.where( + tf.equal(target, -1.0), relu_part, zero_ + ) + + return _apply_loss_reduction(loss, reduction) diff --git a/ivy/functional/backends/torch/experimental/losses.py b/ivy/functional/backends/torch/experimental/losses.py index fbd5a899d964e..42a65a78451fe 100644 --- a/ivy/functional/backends/torch/experimental/losses.py +++ b/ivy/functional/backends/torch/experimental/losses.py @@ -152,3 +152,24 @@ def poisson_nll_loss( return torch.nn.functional.poisson_nll_loss( input, target, log_input=log_input, full=full, eps=eps, reduction=reduction ) + + +@with_supported_device_and_dtypes( + { + "2.1.2 and below": { + "cpu": ("float16", "float32", "float64"), + "gpu": ("float16", "float32", "float64"), + } + }, + backend_version, +) +def hinge_embedding_loss( + input: torch.Tensor, + target: torch.Tensor, + *, + margin: float = 1.0, + reduction: str = "mean", +) -> torch.Tensor: + return torch.nn.functional.hinge_embedding_loss( + input, target, margin=margin, reduction=reduction + ) diff --git a/ivy/functional/ivy/experimental/losses.py b/ivy/functional/ivy/experimental/losses.py index 0824093ca069d..394863b5ddc26 100644 --- a/ivy/functional/ivy/experimental/losses.py +++ b/ivy/functional/ivy/experimental/losses.py @@ -571,3 +571,92 @@ def poisson_nll_loss( eps=eps, reduction=reduction, ) + + +@handle_exceptions +@handle_nestable +@handle_array_like_without_promotion +@to_native_arrays_and_back +def hinge_embedding_loss( + input: Union[ivy.Array, ivy.NativeArray], + target: Union[ivy.Array, ivy.NativeArray], + *, + margin: float = 1.0, + reduction: str = "mean", +) -> ivy.Array: + r"""Measures loss from input `x` and label `y` with values 1 or -1. It + evaluates if two inputs are similar or not, often used for embedding or + semi-supervised learning. + + Loss for the `n`-th sample: + .. math:: + l_n = \begin{cases} + x_n, & \text{if}\; y_n = 1,\\ + \max \{0, margin - x_n\}, & \text{if}\; y_n = -1, + \end{cases} + + Total loss: + .. math:: + \ell(x, y) = \begin{cases} + \operatorname{mean}(L), & \text{if reduction} = \text{`mean';}\\ + \operatorname{sum}(L), & \text{if reduction} = \text{`sum'.} + \end{cases} + + where :math:`L = \{l_1,\dots,l_N\}^\top` . + + Parameters + ---------- + input + Input tensor with dtype float. + The shape is [N, \*], where N is batch size and `\*` represents + any number of additional dimensions. + label + Label tensor containing 1 or -1 with dtype float32 or float64. + Its shape matches that of the input. + margin + Sets the hyperparameter margin. Determines the necessary input size + for hinge_embedding_loss calculations when label is -1. Inputs smaller + than the margin are minimized with hinge_embedding_loss. + Default is 1.0. + reduction + Specifies how to aggregate the loss across the batch. Options are: + - ``'none'``: Returns the unreduced loss. + - ``'mean'``: Returns the mean loss. + - ``'sum'``: Returns the summed loss. + Default is ``'mean'``. + + Shape + ----- + - Input: :math:`(*)` where :math:`*` means, any number of dimensions. \ + The sum operation operates over all the elements. + - Target: :math:`(*)`, same shape as the input + - Output: scalar. If :attr:`reduction` is ``'none'``, + then same shape as the input + + Returns + ------- + ret + Hinge embedding loss calculated from the input and label, + shaped based on the reduction method. + + Examples + -------- + >>> input_tensor = ivy.array([1, 2, 3, 4], dtype=ivy.float64) + >>> target_tensor = ivy.array([1, 1, 1, 1], dtype=ivy.float64) + >>> loss = ivy.hinge_embedding_loss(input_tensor, target_tensor, reduction="none") + >>> loss + ivy.array([1., 2., 3., 4.]) + + >>> input_tensor = ivy.array([21, 22], dtype=ivy.float32) + >>> target_tensor = ivy.array([-1, 1], dtype=ivy.float32) + >>> loss = ivy.hinge_embedding_loss(input_tensor,target_tensor, + ... margin=2.0, reduction="sum") + >>> loss + ivy.array(22.) + """ + return ivy.current_backend().hinge_embedding_loss( + input, + target, + margin=margin, + reduction=reduction, + ) diff --git a/ivy_tests/test_ivy/test_functional/test_experimental/test_nn/test_losses.py b/ivy_tests/test_ivy/test_functional/test_experimental/test_nn/test_losses.py index 2161ee814304b..8c0028d457e24 100644 --- a/ivy_tests/test_ivy/test_functional/test_experimental/test_nn/test_losses.py +++ b/ivy_tests/test_ivy/test_functional/test_experimental/test_nn/test_losses.py @@ -7,6 +7,100 @@ from ivy_tests.test_ivy.helpers import handle_test +# --- Helpers --- # +# --------------- # + + +@st.composite +def _hinge_embedding_loss_input( + draw, min_num_dims=1, max_num_dims=5, min_dim_size=1, max_dim_size=10 +): + # determine the shape for both arrays (input and target) + shape = draw( + st.shared( + helpers.get_shape( + min_num_dims=min_num_dims, + max_num_dims=max_num_dims, + min_dim_size=min_dim_size, + max_dim_size=max_dim_size, + ), + key="shared_shape", + ) + ) + + # Generate an array of -1 and 1 with the given shape (target_array) + def _arrays_of_neg1_and_1(shape): + value_strategy = st.sampled_from([-1, 1]) + prod_shape = int(np.prod(shape)) # Convert np.int64 to int + array_data = draw( + st.lists(value_strategy, min_size=prod_shape, max_size=prod_shape) + ) + return np.asarray(array_data).reshape(shape) + + # input_array + dtype, xx = draw( + helpers.dtype_and_values( + shape=shape, + available_dtypes=helpers.get_dtypes("valid"), + safety_factor_scale="linear", + large_abs_safety_factor=2, + small_abs_safety_factor=2, + min_value=1, + max_value=10, + min_dim_size=1, + min_num_dims=1, + max_num_dims=5, + max_dim_size=5, + ) + ) + + # generate the target array 'yy' containing either 1 or -1 + yy = _arrays_of_neg1_and_1(shape=shape) + + return dtype, xx, yy + + +# --- Main --- # +# ------------ # + + +# hinge_embedding_loss +@handle_test( + fn_tree="functional.ivy.experimental.hinge_embedding_loss", + dtype_and_inputs=_hinge_embedding_loss_input(), + margin=st.floats(min_value=1, max_value=5), + reduction=st.sampled_from(["none", "sum", "mean"]), + test_gradients=st.just( + False + ), # Gradients are failing for "jax" and "paddle" backend. + test_with_out=st.just(False), + ground_truth_backend="torch", +) +def test_hinge_embedding_loss( + dtype_and_inputs, + margin, + reduction, + test_flags, + backend_fw, + fn_name, + on_device, +): + dtype, xx, yy = dtype_and_inputs + helpers.test_function( + input_dtypes=dtype, + test_flags=test_flags, + backend_to_test=backend_fw, + fn_name=fn_name, + on_device=on_device, + input=xx[0], + target=yy, + margin=margin, + reduction=reduction, + rtol_=1e-05, + atol_=1e-05, + ) + + # huber_loss @handle_test( fn_tree="functional.ivy.experimental.huber_loss",