"""
Scikit-learn-compatible Kernel Discriminant Analysis.
Used in
David Diaz-Vico, Jose R. Dorronsoro
"Deep vs Kernel Fisher Discriminant Analysis"
Based on algorithm 5 in
Zhihua Zhang, Guang Dai, Congfu Xu, Michael I. Jordan
"Regularized Discriminant Analysis, Ridge Regression and Beyond"
http://www.jmlr.org/papers/v11/zhang10b.html
@author: David Diaz Vico
@license: MIT
"""
import numpy as np
from sklearn.base import BaseEstimator, ClassifierMixin, TransformerMixin
from sklearn.metrics.pairwise import (chi2_kernel, laplacian_kernel,
linear_kernel, polynomial_kernel,
rbf_kernel, sigmoid_kernel)
from sklearn.preprocessing import OneHotEncoder
from sklearn.utils.validation import check_array, check_is_fitted, check_X_y
[docs]
class KernelDiscriminantAnalysis(BaseEstimator, ClassifierMixin,
TransformerMixin):
"""Kernel Discriminant Analysis.
Parameters
----------
lmb: float (>= 0.0), default=0.001
Regularization parameter
kernel: {"chi2", "laplacian", "linear", "polynomial", "rbf", "sigmoid"},
default='rbf'
Kernel.
degree: integer, default=3
gamma: float, default=None
coef0: integer, default=1
"""
def __init__(self, lmb=0.001, kernel='rbf', degree=3, gamma=None, coef0=1):
self.lmb = lmb
self.kernel = kernel
self.degree = degree
self.gamma = gamma
self.coef0 = coef0
def _kernel(self, X, Y=None):
"""Kernel"""
kernel = None
if self.kernel == 'chi2':
kernel = chi2_kernel(X, Y, gamma=self.gamma)
elif self.kernel == 'laplacian':
kernel = laplacian_kernel(X, Y, gamma=self.gamma)
elif self.kernel == 'linear':
kernel = linear_kernel(X, Y)
elif self.kernel == 'polynomial':
kernel = polynomial_kernel(X, Y, degree=self.degree,
gamma=self.gamma, coef0=self.coef0)
elif self.kernel == 'rbf':
kernel = rbf_kernel(X, Y, gamma=self.gamma)
elif self.kernel == 'sigmoid':
kernel = sigmoid_kernel(X, Y, gamma=self.gamma, coef0=self.coef0)
return kernel
[docs]
def fit(self, X, y):
"""Fit KFDA model.
Parameters
----------
X: numpy array of shape [n_samples, n_features]
Training set.
y: numpy array of shape [n_samples]
Target values. Only works for 2 classes.
Returns
-------
self
"""
n = len(X)
self._X = X
self._H = np.identity(n) - 1 / n * np.ones(n) @ np.ones(n).T
self._E = OneHotEncoder().fit_transform(y.reshape(n, 1))
_, counts = np.unique(y, return_counts=True)
K = self._kernel(X)
C = self._H @ K @ self._H
self._Delta = np.linalg.inv(C + self.lmb * np.identity(n))
A = self._E.T @ C
B = self._Delta @ self._E
self._Pi_12 = np.diag(np.sqrt(1.0 / counts))
P = self._Pi_12 @ A
Q = B @ self._Pi_12
R = P @ Q
V, self._Gamma, self._U = np.linalg.svd(R, full_matrices=False)
return self