%FEATEVAL Evaluation of feature set for classification
%
% J = FEATEVAL(A,CRIT,T)
% J = A*FEATEVAL([],CRIT,T)
% J = A*FEATEVAL(CRIT,T)
% J = FEATEVAL(A,CRIT,N)
% J = A*FEATEVAL([],CRIT,N)
% J = A*FEATEVAL(CRIT,N)
%
% INPUT
% A input dataset
% CRIT string name of a method or untrained mapping, default 'NN'
% T validation dataset (optional)
% N number of cross-validations (optional)
%
% OUTPUT
% J scalar criterion value
%
% DESCRIPTION
% Evaluation of features by the criterion CRIT, using objects in the
% dataset A. The larger J, the better. Resulting J-values are
% incomparable over the various methods.
% The following methods are supported:
%
% crit='in-in' : inter-intra distance.
% crit='maha-s': sum of estimated Mahalanobis distances.
% crit='maha-m': minimum of estimated Mahalanobis distances.
% crit='eucl-s': sum of squared Euclidean distances.
% crit='eucl-m': minimum of squared Euclidean distances.
% crit='NN' : 1-Nearest Neighbour leave-one-out
% classification performance (default).
% (performance = 1 - error).
% crit='mad' : mean absolute deviation (only for regression!)
% crit='mse' : mean squared error (only for regression!)
%
% For classification problems, CRIT can also be any untrained
% classifier, e.g. LDC([],1e-6,1e-6). Then the classification error is
% used for a performance estimate. If supplied, the dataset T is used
% for obtaining an unbiased estimate of the performance of classifiers
%trained with the dataset A. If a number of cross-validations N is
% supplied, the routine is run for N times with different training and
% test sets generated from A by cross-validation. Results are averaged.
% If T nor N are given, the apparent performance on A is used.
%
% SEE ALSO (<a href="http://37steps.com/prtools">PRTools Guide</a>)
% DATASETS, FEATSELO, FEATSELB, FEATSELF, FEATSELP, FEATSELM, FEATRANK
% Copyright: R.P.W. Duin, r.p.w.duin@37steps.com
% Faculty EWI, Delft University of Technology
% P.O. Box 5031, 2600 GA Delft, The Netherlands
% REVISIONS
% DXD1: David Tax, 08-05-2003
% I added the inter/intra distance criterion.
% DXD2: David Tax, 10-06-2011
% I added the MAD and MSE criteria for regression.
% RD : Conversion to mapping, 08-08-2012
function J = feateval(varargin)
argin = shiftargin(varargin,'char');
argin = setdefaults(argin,[],'NN',[]);
if mapping_task(argin,'definition')
J = define_mapping(argin,'fixed');
else
[a,crit,t] = deal(argin{:});
[ma,k,c] = getsize(a);
if is_scalar(t) & ~isdataset(t) % cross-validation desired, t rotations
if ~ismapping(crit) | ~isuntrained(crit)
error('Cross-validation only possible with untrained classifiers')
end
J = 1-prcrossval(a,crit,t);
return
end
% islabtype(a,'crisp');
isvaldfile(a,1,2); % at least 1 object per class, 2 classes
a = testdatasize(a);
iscomdset(a,t);
if isstr(crit)
%DXD1
if strcmp(crit,'in-in') % inter/intra distances
islabtype(a,'crisp','soft');
if isempty(t)
[U,G] = meancov(a);
else
[U,G] = meancov(t);
end
S_b = prcov(+U); % between scatter
prior = getprior(a);
S_w = reshape(sum(reshape(G,k*k,c)*prior',2),k,k); % within scatter
J = trace(prinv(S_w)*S_b);
elseif strcmp(crit,'maha-s') | strcmp(crit,'maha-m') % Mahalanobis distances
islabtype(a,'crisp','soft');
if isempty(t)
D = distmaha(a);
else
[U,G] = meancov(a);
D = distmaha(t,U,G);
D = meancov(D);
end
if strcmp(crit,'maha-m')
D = D + realmax*eye(c);
J = min(min(D));
else
J = sum(sum(D))/2;
end
elseif strcmp(crit,'eucl-s') | strcmp(crit,'eucl-m') % Euclidean distances
islabtype(a,'crisp','soft');
U = meancov(a);
if isempty(t)
D = distm(U);
else
D = distm(t,U);
D = meancov(D);
end
if strcmp(crit,'eucl-m')
D = D + realmax*eye(c);
J = min(min(D));
else
J = sum(sum(D))/2;
end
elseif strcmp(crit,'NN') % 1-NN performance
islabtype(a,'crisp','soft');
if isempty(t)
J = 1 - testk(a,1);
else
J = 1 - testk(a,1,t);
end
elseif strcmp(crit,'kcentres') || strcmp(crit,'kcenters') % data radius, unsupervised
% assumes disrep, so experimental
J = max(min(+a,[],2));
if J == 0, J = inf; else J = 1/J; end
elseif strcmp(crit,'representation_error') || strcmp(crit,'reperror')% also unsupervised
J = mean(min(+a,[],2));
if J == 0, J = inf; else J = 1/J; end
elseif strcmp(crit,'mad') % Mean Absolute Deviation for regression
J = 1-testr(a*linearr(a,0.001,1),'mad');
elseif strcmp(crit,'mse') % Mean Squared Error for regression
J = 1-testr(a*linearr(a,0.001,1),'mse');
else
error('Criterion undefined');
end
else
ismapping(crit);
if isuntrained(crit)
if isempty(t)
J = 1 - (a * (a * crit) * testc);
else
J = 1 - (t * (a * crit) * testc);
end
elseif isfixed(crit)
J = a * crit;
else
error('Criterion should be defined by an untrained or fixed mapping')
end
end
end
return