Add sparse_sigmoid activation (#21175)

shashaka · web-flow · commit c7b6b424feef · 2025-04-17T08:52:21.000-07:00
* Add sparse_sigmoid activation layer

* Correct typo
diff --git a/keras/api/_tf_keras/keras/activations/__init__.py b/keras/api/_tf_keras/keras/activations/__init__.py
@@ -33,6 +33,7 @@
 from keras.src.activations.activations import softplus
 from keras.src.activations.activations import softsign
 from keras.src.activations.activations import sparse_plus
+from keras.src.activations.activations import sparse_sigmoid
 from keras.src.activations.activations import sparsemax
 from keras.src.activations.activations import squareplus
 from keras.src.activations.activations import tanh
diff --git a/keras/api/_tf_keras/keras/ops/__init__.py b/keras/api/_tf_keras/keras/ops/__init__.py
@@ -103,6 +103,7 @@
 from keras.src.ops.nn import softsign
 from keras.src.ops.nn import sparse_categorical_crossentropy
 from keras.src.ops.nn import sparse_plus
+from keras.src.ops.nn import sparse_sigmoid
 from keras.src.ops.nn import sparsemax
 from keras.src.ops.nn import squareplus
 from keras.src.ops.nn import tanh_shrink
diff --git a/keras/api/_tf_keras/keras/ops/nn/__init__.py b/keras/api/_tf_keras/keras/ops/nn/__init__.py
@@ -47,6 +47,7 @@
 from keras.src.ops.nn import softsign
 from keras.src.ops.nn import sparse_categorical_crossentropy
 from keras.src.ops.nn import sparse_plus
+from keras.src.ops.nn import sparse_sigmoid
 from keras.src.ops.nn import sparsemax
 from keras.src.ops.nn import squareplus
 from keras.src.ops.nn import tanh_shrink
diff --git a/keras/api/activations/__init__.py b/keras/api/activations/__init__.py
@@ -33,6 +33,7 @@
 from keras.src.activations.activations import softplus
 from keras.src.activations.activations import softsign
 from keras.src.activations.activations import sparse_plus
+from keras.src.activations.activations import sparse_sigmoid
 from keras.src.activations.activations import sparsemax
 from keras.src.activations.activations import squareplus
 from keras.src.activations.activations import tanh
diff --git a/keras/api/ops/__init__.py b/keras/api/ops/__init__.py
@@ -103,6 +103,7 @@
 from keras.src.ops.nn import softsign
 from keras.src.ops.nn import sparse_categorical_crossentropy
 from keras.src.ops.nn import sparse_plus
+from keras.src.ops.nn import sparse_sigmoid
 from keras.src.ops.nn import sparsemax
 from keras.src.ops.nn import squareplus
 from keras.src.ops.nn import tanh_shrink
diff --git a/keras/api/ops/nn/__init__.py b/keras/api/ops/nn/__init__.py
@@ -47,6 +47,7 @@
 from keras.src.ops.nn import softsign
 from keras.src.ops.nn import sparse_categorical_crossentropy
 from keras.src.ops.nn import sparse_plus
+from keras.src.ops.nn import sparse_sigmoid
 from keras.src.ops.nn import sparsemax
 from keras.src.ops.nn import squareplus
 from keras.src.ops.nn import tanh_shrink
diff --git a/keras/src/activations/__init__.py b/keras/src/activations/__init__.py
@@ -24,6 +24,7 @@
 from keras.src.activations.activations import softplus
 from keras.src.activations.activations import softsign
 from keras.src.activations.activations import sparse_plus
+from keras.src.activations.activations import sparse_sigmoid
 from keras.src.activations.activations import sparsemax
 from keras.src.activations.activations import squareplus
 from keras.src.activations.activations import tanh
@@ -53,6 +54,7 @@
     tanh_shrink,
     threshold,
     sigmoid,
+    sparse_sigmoid,
     exponential,
     hard_sigmoid,
     hard_silu,
diff --git a/keras/src/activations/activations.py b/keras/src/activations/activations.py
@@ -552,6 +552,27 @@ def log_sigmoid(x):
     return ops.log_sigmoid(x)
 
 
+@keras_export("keras.activations.sparse_sigmoid")
+def sparse_sigmoid(x):
+    """Sparse sigmoid activation function.
+
+    It is defined as
+
+    `f(x) = 0` for `x <= -1`,
+    `f(x) = 0.5 * (x + 1)` for `-1 < x < 1`,
+    `f(x) = 1` for `x >= 1`.
+
+    Args:
+        x: Input tensor.
+
+    Reference:
+
+    - [M. Blondel, A. F. T. Martins, V. Niculae, 2019](https://arxiv.org/pdf/1901.02324)
+
+    """
+    return ops.sparse_sigmoid(x)
+
+
 @keras_export(["keras.activations.hard_silu", "keras.activations.hard_swish"])
 def hard_silu(x):
     """Hard SiLU activation function, also known as Hard Swish.
diff --git a/keras/src/activations/activations_test.py b/keras/src/activations/activations_test.py
@@ -40,6 +40,10 @@ def _ref_hard_sigmoid(x):
     return z
 
 
+def _ref_sparse_sigmoid(x):
+    return np.where(x <= -1, 0, np.where(x >= 1, 1, 0.5 * (x + 1)))
+
+
 def _ref_log_sigmoid(x):
     return -1 * _ref_softplus(-x)
 
@@ -341,6 +345,45 @@ def test_hard_sigmoid(self):
             result_positive_above_1, expected_positive_above_1, rtol=1e-05
         )
 
+    def test_sparse_sigmoid(self):
+        # Basic test for random values between 0 and 1
+        x = np.random.uniform(0, 1, (2, 5))
+        result = activations.sparse_sigmoid(x[np.newaxis, :])[0]
+        expected = np.vectorize(_ref_sparse_sigmoid)(x)
+        self.assertAllClose(result, expected, rtol=1e-05)
+
+        # Test with 1D array
+        x_1d = np.random.uniform(-10, 10, 5)
+        result_1d = activations.sparse_sigmoid(x_1d)
+        expected_1d = np.vectorize(_ref_sparse_sigmoid)(x_1d)
+        self.assertAllClose(result_1d, expected_1d, rtol=1e-05)
+
+        # Test with 3D array
+        x_3d = np.random.uniform(-10, 10, (3, 3, 3))
+        result_3d = activations.sparse_sigmoid(x_3d)
+        expected_3d = np.vectorize(_ref_sparse_sigmoid)(x_3d)
+        self.assertAllClose(result_3d, expected_3d, rtol=1e-05)
+
+        # Test large positive values
+        x_large_positive = np.random.uniform(10, 100, (2, 5))
+        result_large_positive = activations.sparse_sigmoid(x_large_positive)
+        expected_large_positive = np.vectorize(_ref_sparse_sigmoid)(
+            x_large_positive
+        )
+        self.assertAllClose(
+            result_large_positive, expected_large_positive, rtol=1e-05
+        )
+
+        # Test large negative values
+        x_large_negative = np.random.uniform(-100, -10, (2, 5))
+        result_large_negative = activations.sparse_sigmoid(x_large_negative)
+        expected_large_negative = np.vectorize(_ref_sparse_sigmoid)(
+            x_large_negative
+        )
+        self.assertAllClose(
+            result_large_negative, expected_large_negative, rtol=1e-05
+        )
+
     def test_log_sigmoid(self):
         # Basic test for random values between 0 and 1
         x = np.random.uniform(0, 1, (2, 5))
diff --git a/keras/src/backend/jax/nn.py b/keras/src/backend/jax/nn.py
@@ -36,6 +36,11 @@ def sigmoid(x):
     return jnn.sigmoid(x)
 
 
+def sparse_sigmoid(x):
+    x = convert_to_tensor(x)
+    return jnn.sparse_sigmoid(x)
+
+
 def tanh(x):
     x = convert_to_tensor(x)
     return jnn.tanh(x)
diff --git a/keras/src/backend/numpy/nn.py b/keras/src/backend/numpy/nn.py
@@ -31,6 +31,17 @@ def sigmoid(x):
     return np.array(1.0, x.dtype) / (np.array(1.0, x.dtype) + np.exp(-x))
 
 
+def sparse_sigmoid(x):
+    x = convert_to_tensor(x)
+    return np.where(
+        x <= -1,
+        np.array(0.0, x.dtype),
+        np.where(
+            x >= 1, np.array(1.0, x.dtype), np.array(0.5 * (x + 1), x.dtype)
+        ),
+    )
+
+
 def tanh(x):
     return np.tanh(x)
 
diff --git a/keras/src/backend/tensorflow/nn.py b/keras/src/backend/tensorflow/nn.py
@@ -26,6 +26,15 @@ def sigmoid(x):
     return output
 
 
+def sparse_sigmoid(x):
+    x = convert_to_tensor(x)
+    return tf.where(
+        x <= -1,
+        tf.constant(0.0, dtype=x.dtype),
+        tf.where(x >= 1, tf.constant(1.0, dtype=x.dtype), 0.5 * (x + 1)),
+    )
+
+
 def tanh(x):
     return tf.nn.tanh(x)
 
diff --git a/keras/src/backend/torch/nn.py b/keras/src/backend/torch/nn.py
@@ -29,6 +29,19 @@ def sigmoid(x):
     return tnn.sigmoid(x)
 
 
+def sparse_sigmoid(x):
+    x = convert_to_tensor(x)
+    return torch.where(
+        x <= -1,
+        torch.tensor(0.0, device=x.device, dtype=x.dtype),
+        torch.where(
+            x >= 1,
+            torch.tensor(1.0, device=x.device, dtype=x.dtype),
+            0.5 * (x + 1),
+        ),
+    )
+
+
 def tanh(x):
     x = convert_to_tensor(x)
     return tnn.tanh(x)
diff --git a/keras/src/ops/nn.py b/keras/src/ops/nn.py
@@ -110,6 +110,42 @@ def sigmoid(x):
     return backend.nn.sigmoid(x)
 
 
+class SparseSigmoid(Operation):
+    def call(self, x):
+        return backend.nn.sparse_sigmoid(x)
+
+    def compute_output_spec(self, x):
+        return KerasTensor(x.shape, dtype=x.dtype)
+
+
+@keras_export(["keras.ops.sparse_sigmoid", "keras.ops.nn.sparse_sigmoid"])
+def sparse_sigmoid(x):
+    """Sparse sigmoid activation function.
+
+    It is defined as
+
+    `f(x) = 0` for `x <= -1`,
+    `f(x) = 0.5 * (x + 1)` for `-1 < x < 1`,
+    `f(x) = 1` for `x >= 1`.
+
+    Args:
+        x: Input tensor.
+
+    Returns:
+        A tensor with the same shape as `x`.
+
+    Example:
+
+    >>> x = keras.ops.convert_to_tensor([-6.0, 1.0, 0.0, 1.0, 6.0])
+    >>> keras.ops.sparse_sigmoid(x)
+    array([0. , 1. , 0.5, 1. , 1. ], dtype=float32)
+
+    """
+    if any_symbolic_tensors((x,)):
+        return SparseSigmoid().symbolic_call(x)
+    return backend.nn.sparse_sigmoid(x)
+
+
 class Softplus(Operation):
     def call(self, x):
         return backend.nn.softplus(x)
diff --git a/keras/src/ops/nn_test.py b/keras/src/ops/nn_test.py
@@ -100,6 +100,10 @@ def test_sigmoid(self):
         x = KerasTensor([None, 2, 3])
         self.assertEqual(knn.sigmoid(x).shape, (None, 2, 3))
 
+    def test_sparse_sigmoid(self):
+        x = KerasTensor([None, 2, 3])
+        self.assertEqual(knn.sparse_sigmoid(x).shape, (None, 2, 3))
+
     def test_softplus(self):
         x = KerasTensor([None, 2, 3])
         self.assertEqual(knn.softplus(x).shape, (None, 2, 3))
@@ -785,6 +789,10 @@ def test_sigmoid(self):
         x = KerasTensor([1, 2, 3])
         self.assertEqual(knn.sigmoid(x).shape, (1, 2, 3))
 
+    def test_sparse_sigmoid(self):
+        x = KerasTensor([1, 2, 3])
+        self.assertEqual(knn.sparse_sigmoid(x).shape, (1, 2, 3))
+
     def test_softplus(self):
         x = KerasTensor([1, 2, 3])
         self.assertEqual(knn.softplus(x).shape, (1, 2, 3))
@@ -1306,6 +1314,10 @@ def test_sigmoid(self):
             knn.sigmoid(x), [0.26894143, 0.5, 0.7310586, 0.880797, 0.95257413]
         )
 
+    def test_sparse_sigmoid(self):
+        x = np.array([-1, 0, 1, 2, 3], dtype=np.float32)
+        self.assertAllClose(knn.sparse_sigmoid(x), [0.0, 0.5, 1.0, 1.0, 1.0])
+
     def test_softplus(self):
         x = np.array([-1, 0, 1, 2, 3], dtype=np.float32)
         self.assertAllClose(
@@ -2883,6 +2895,24 @@ def test_sigmoid(self, dtype):
             expected_dtype,
         )
 
+    @parameterized.named_parameters(named_product(dtype=FLOAT_DTYPES))
+    def test_sparse_sigmoid(self, dtype):
+        import jax.nn as jnn
+        import jax.numpy as jnp
+
+        x = knp.ones((), dtype=dtype)
+        x_jax = jnp.ones((), dtype=dtype)
+        expected_dtype = standardize_dtype(jnn.sparse_sigmoid(x_jax).dtype)
+
+        self.assertEqual(
+            standardize_dtype(knn.sparse_sigmoid(x).dtype),
+            expected_dtype,
+        )
+        self.assertEqual(
+            standardize_dtype(knn.SparseSigmoid().symbolic_call(x).dtype),
+            expected_dtype,
+        )
+
     @parameterized.named_parameters(named_product(dtype=FLOAT_DTYPES))
     def test_silu(self, dtype):
         import jax.nn as jnn
