Add 6 new methods

Author: antoinedemathelin

adapt/base.py

@@ -141,6 +141,9 @@ def insert_base_doc(func):
             new_doc = doc
 
         func.__doc__ = new_doc
+        
+        if str(inspect.signature(func)) == "(*args, **kwargs)":
+            func.__signature__ = inspect.signature(func.__init__)
 
         return func
     return insert_base_doc

adapt/feature_based/__init__.py

@@ -11,5 +11,9 @@
 from ._mdd import MDD
 from ._wdgrl import WDGRL
 from ._cdan import CDAN
+from ._sa import SA
+from ._fmmd import fMMD
+from ._ccsa import CCSA
 
-__all__ = ["FE", "CORAL", "DeepCORAL", "ADDA", "DANN", "MCD", "MDD", "WDGRL", "CDAN"]
+__all__ = ["FE", "CORAL", "DeepCORAL", "ADDA", "DANN",
+           "MCD", "MDD", "WDGRL", "CDAN", "SA", "fMMD", "CCSA"]

adapt/feature_based/_ccsa.py

@@ -0,0 +1,128 @@
+import numpy as np
+import tensorflow as tf
+
+from adapt.base import BaseAdaptDeep, make_insert_doc
+from adapt.utils import set_random_seed
+
+
+EPS = np.finfo(np.float32).eps
+
+def pairwise_y(X, Y):
+    batch_size = tf.shape(X)[0]
+    dim = tf.reduce_prod(tf.shape(X)[1:])
+    X = tf.reshape(X, (batch_size, dim))
+    Y = tf.reshape(Y, (batch_size, dim))
+    X = tf.tile(tf.expand_dims(X, -1), [1, 1, batch_size])
+    Y = tf.tile(tf.expand_dims(Y, -1), [1, 1, batch_size])
+    return tf.reduce_sum(tf.abs(X-tf.transpose(Y)), 1)/2
+
+def pairwise_X(X, Y):
+    batch_size = tf.shape(X)[0]
+    dim = tf.reduce_prod(tf.shape(X)[1:])
+    X = tf.reshape(X, (batch_size, dim))
+    Y = tf.reshape(Y, (batch_size, dim))
+    X = tf.tile(tf.expand_dims(X, -1), [1, 1, batch_size])
+    Y = tf.tile(tf.expand_dims(Y, -1), [1, 1, batch_size])
+    return tf.reduce_sum(tf.square(X-tf.transpose(Y)), 1)
+
+
+@make_insert_doc(["encoder", "task"])
+class CCSA(BaseAdaptDeep):
+    """
+    CCSA : 
+    
+    Parameters
+    ----------
+    margin : float (default=1.)
+        Margin for the inter-class separation.
+        The higher the margin, the more the classes
+        will be separated in the encoded space.
+        
+    gamma : float  (default=0.5)
+        Trade-off parameter. ``0<gamma<1``
+        If ``gamma`` is close to 1 more
+        importance are given to the task. If
+        gamma is close to 0, more importance
+        are given to the contrastive loss.
+    
+    Attributes
+    ----------    
+    encoder_ : tensorflow Model
+        encoder network.
+        
+    task_ : tensorflow Model
+        task network.
+        
+    history_ : dict
+        history of the losses and metrics across the epochs.
+        If ``yt`` is given in ``fit`` method, target metrics
+        and losses are recorded too.
+    """
+    
+    def __init__(self,
+                 encoder=None,
+                 task=None,
+                 Xt=None,
+                 yt=None,
+                 margin=1.,
+                 gamma=0.5,
+                 copy=True,
+                 verbose=1,
+                 random_state=None,
+                 **params):
+
+        names = self._get_param_names()
+        kwargs = {k: v for k, v in locals().items() if k in names}
+        kwargs.update(params)
+        super().__init__(**kwargs)
+        
+        
+    def train_step(self, data):
+        # Unpack the data.
+        Xs, Xt, ys, yt = self._unpack_data(data)
+       
+        # loss
+        with tf.GradientTape() as task_tape, tf.GradientTape() as enc_tape:
+            # Forward pass
+            Xs_enc = self.encoder_(Xs, training=True)
+            ys_pred = self.task_(Xs_enc, training=True)
+            
+            Xt_enc = self.encoder_(Xt, training=True)
+            
+            dist_y = pairwise_y(ys, yt)
+            dist_X = pairwise_X(Xs_enc, Xt_enc)
+            
+            contrastive_loss = tf.reduce_sum(dist_y * tf.maximum(0., self.margin - dist_X), 1) / (tf.reduce_sum(dist_y, 1) + EPS)
+            contrastive_loss += tf.reduce_sum((1-dist_y) * dist_X, 1) / (tf.reduce_sum(1-dist_y, 1) + EPS)
+            contrastive_loss = tf.reduce_mean(contrastive_loss)
+            contrastive_loss *= 0.5
+            
+            # Reshape
+            ys_pred = tf.reshape(ys_pred, tf.shape(ys))
+            
+            # Compute the loss value
+            task_loss = tf.reduce_mean(self.task_loss_(ys, ys_pred))
+                        
+            enc_loss = self.gamma * task_loss + (1-self.gamma) * contrastive_loss
+            
+            task_loss += sum(self.task_.losses)
+            enc_loss += sum(self.encoder_.losses)
+            
+        # Compute gradients
+        trainable_vars_task = self.task_.trainable_variables
+        trainable_vars_enc = self.encoder_.trainable_variables
+        
+        gradients_task = task_tape.gradient(task_loss, trainable_vars_task)
+        gradients_enc = enc_tape.gradient(enc_loss, trainable_vars_enc)
+        
+        # Update weights
+        self.optimizer.apply_gradients(zip(gradients_task, trainable_vars_task))
+        self.optimizer_enc.apply_gradients(zip(gradients_enc, trainable_vars_enc))
+        
+        # Update metrics
+        self.compiled_metrics.update_state(ys, ys_pred)
+        self.compiled_loss(ys, ys_pred)
+        # Return a dict mapping metric names to current value
+        logs = {m.name: m.result() for m in self.metrics}
+        logs.update({"contrast": contrastive_loss})
+        return logs

adapt/feature_based/_fe.py

@@ -59,6 +59,11 @@ class FE(BaseAdaptEstimator):
 
     Attributes
     ----------
+    estimator_ : object
+        Fitted estimator.
+        
+    n_domains_ : int
+        Number of domains given in fit.
         
     See also
     --------

adapt/feature_based/_fmmd.py

@@ -0,0 +1,221 @@
+import numpy as np
+import tensorflow as tf
+from sklearn.base import check_array
+from cvxopt import solvers, matrix
+
+from adapt.base import BaseAdaptEstimator, make_insert_doc
+from adapt.utils import set_random_seed
+
+
+def pairwise_X(X, Y):
+    batch_size = tf.shape(X)[0]
+    dim = tf.reduce_prod(tf.shape(X)[1:])
+    X = tf.reshape(X, (batch_size, dim))
+    Y = tf.reshape(Y, (batch_size, dim))
+    X = tf.tile(tf.expand_dims(X, -1), [1, 1, batch_size])
+    Y = tf.tile(tf.expand_dims(Y, -1), [1, 1, batch_size])
+    return tf.reduce_sum(tf.square(X-tf.transpose(Y)), 1)
+
+
+def _get_optim_function(Xs, Xt, kernel="linear", gamma=1., degree=2, coef=1.):
+    
+    n = len(Xs)
+    m = len(Xt)
+    p = Xs.shape[1]
+    
+    Lxx = tf.ones((n,n), dtype=tf.float64) * (1./(n**2))
+    Lxy = tf.ones((n,m), dtype=tf.float64) * (-1./(n*m))
+    Lyy = tf.ones((m,m), dtype=tf.float64) * (1./(m**2))
+    Lyx = tf.ones((m,n), dtype=tf.float64) * (-1./(n*m))
+
+    L = tf.concat((Lxx, Lxy), axis=1)
+    L = tf.concat((L, tf.concat((Lyx, Lyy), axis=1)), axis=0)
+    
+    if kernel == "linear":
+        
+        @tf.function
+        def func(W):
+            Kxx = tf.matmul(tf.matmul(Xs, tf.linalg.diag(W**1)), tf.transpose(Xs))
+            Kyy = tf.matmul(tf.matmul(Xt, tf.linalg.diag(W**1)), tf.transpose(Xt))
+            Kxy = tf.matmul(tf.matmul(Xs, tf.linalg.diag(W**1)), tf.transpose(Xt))
+
+            K = tf.concat((Kxx, Kxy), axis=1)
+            K = tf.concat((K, tf.concat((Kyy, tf.transpose(Kxy)), axis=1)), axis=0)
+
+            f = -tf.linalg.trace(tf.matmul(K, L))
+            Df = tf.gradients(f, W)
+            H = tf.hessians(f, W)
+            return f, Df, H
+        
+    elif kernel == "rbf":
+        
+        @tf.function
+        def func(W):
+            Kxx = pairwise_X(tf.matmul(Xs, tf.linalg.diag(W**1)), Xs)
+            Kyy = pairwise_X(tf.matmul(Xt, tf.linalg.diag(W**1)), Xt)
+            Kxy = pairwise_X(tf.matmul(Xs, tf.linalg.diag(W**1)), Xt)
+
+            K = tf.concat((Kxx, Kxy), axis=1)
+            K = tf.concat((K, tf.concat((Kyy, tf.transpose(Kxy)), axis=1)), axis=0)
+            K = tf.exp(-gamma * K)
+
+            f = -tf.linalg.trace(tf.matmul(K, L))
+            Df = tf.gradients(f, W)
+            H = tf.hessians(f, W)
+            return f, Df, H
+        
+    elif kernel == "poly":
+        
+        @tf.function
+        def func(W):
+            Kxx = tf.matmul(tf.matmul(Xs, tf.linalg.diag(W**1)), tf.transpose(Xs))
+            Kyy = tf.matmul(tf.matmul(Xt, tf.linalg.diag(W**1)), tf.transpose(Xt))
+            Kxy = tf.matmul(tf.matmul(Xs, tf.linalg.diag(W**1)), tf.transpose(Xt))
+
+            K = tf.concat((Kxx, Kxy), axis=1)
+            K = tf.concat((K, tf.concat((Kyy, tf.transpose(Kxy)), axis=1)), axis=0)
+            K = (gamma * K + coef)**degree
+
+            f = -tf.linalg.trace(tf.matmul(K, L))
+            Df = tf.gradients(f, W)
+            H = tf.hessians(f, W)
+            return f, Df, H
+        
+    else:
+        raise ValueError("kernel param should be in ['linear', 'rbf', 'poly']")
+        
+    return func
+    
+
+@make_insert_doc()
+class fMMD(BaseAdaptEstimator):
+    """
+    fMMD : feature Selection with MMD
+    
+    LDM selects input features inorder to minimize the
+    maximum mean discrepancy (MMD) between the source and
+    the target data.
+    
+    Parameters
+    ----------
+    threshold : float or 'auto' (default='auto')
+        Threshold on ``features_scores_`` all
+        feature with score above threshold will be
+        removed.
+        If 'auto' the threshold is chosen to maximize
+        the difference between scores of selected features
+        and removed ones.
+        
+    kernel : str (default='linear')
+        Choose the kernel between
+        ['linear', 'rbf', 'poly'].
+        The kernels are computed as follows:
+        ``rbf(X, Y) = exp(gamma * ||X-Y||^2)``
+        ``poly(X, Y) = (gamma * <X, Y> + coef)^degree``
+        
+    gamma : float (default=1.)
+        Gamma multiplier for the 'rbf' and 
+        'poly' kernels.
+        
+    degree : int (default=2)
+        Degree of the 'poly' kernel
+        
+    coef : float (default=1.)
+        Coef of the 'poly' kernel
+    
+    Attributes
+    ----------
+    estimator_ : object
+        Estimator.
+    
+    selected_features_ : numpy array
+        The selected features
+        
+    features_scores_ : numpy array
+        The score attributed to each feature
+    
+    See also
+    --------
+    CORAL
+    FE
+    """
+
+    def __init__(self,
+                 estimator=None,
+                 Xt=None,
+                 yt=None,
+                 threshold="auto",
+                 kernel="linear",
+                 gamma=1.,
+                 degree=2,
+                 coef=1.,
+                 copy=True,
+                 verbose=1,
+                 random_state=None,
+                 **params):
+        
+        names = self._get_param_names()
+        kwargs = {k: v for k, v in locals().items() if k in names}
+        kwargs.update(params)
+        super().__init__(**kwargs)
+        
+    
+    def fit_transform(self, Xs, Xt, **fit_params):
+        Xs = check_array(Xs)
+        Xt = check_array(Xt)
+        set_random_seed(self.random_state)
+        
+        n = len(Xs)
+        m = len(Xt)
+        p = Xs.shape[1]
+        
+        optim_func = _get_optim_function(tf.identity(Xs),
+                                         tf.identity(Xt),
+                                         self.kernel,
+                                         self.gamma,
+                                         self.degree,
+                                         self.coef)
+        
+        def F(x=None, z=None):
+            if x is None: return 0, matrix(1.0, (p,1))
+            x = tf.identity(np.array(x).ravel())
+            f, Df, H = optim_func(x)
+            f = f.numpy()
+            Df = Df[0].numpy().reshape(1, -1)
+            H = H[0].numpy()
+            if z is None: return matrix(f), matrix(Df)
+            return matrix(f), matrix(Df), matrix(H)
+        
+        linear_const_G = -np.eye(p)
+        squared_constraint_G = np.concatenate((np.zeros((1, p)), -np.eye(p)), axis=0)
+
+        linear_const_h = np.zeros(p)
+        squared_constraint_h = np.concatenate((np.ones(1), np.zeros(p)))
+
+        G = matrix(np.concatenate((linear_const_G, squared_constraint_G)))
+        h = matrix(np.concatenate((linear_const_h, squared_constraint_h)))
+        dims = {'l': p, 'q': [p+1], 's':  []}
+        sol = solvers.cp(F, G, h, dims)
+        
+        W = np.array(sol["x"]).ravel()
+        
+        self.selected_features_ = np.zeros(p, dtype=bool)
+        
+        if self.threshold == "auto":
+            args = np.argsort(W).ravel()
+            max_diff_arg = np.argmax(np.diff(W[args]))
+            threshold = W[args[max_diff_arg]]
+            self.selected_features_[W<=threshold] = 1
+        else:
+            self.selected_features_[W<=self.threshold] = 1
+            
+        if np.sum(self.selected_features_) == 0:
+            raise Exception("No features selected")
+            
+        self.features_scores_ = W
+        return Xs[:, self.selected_features_]
+            
+    
+    def transform(self, X):
+        X = check_array(X)
+        return X[:, self.selected_features_]