Hola tienes que hacer un archivo .m con las restricciones, te paso un ejemplo de una funcion que hice para maximizar Epsilon y Beta.
function [c, ceq] = confun(x)
% Nonlinear inequality constraints
c = [-x(1)+0.0001; -x(2)+0.0001];
% Nonlinear equality constraints
ceq = [];
end
------------------------------
function f= myfun(x,y,u)
h = log( (1/x(1)) * ((1+ (x(2)*(y(1)-u) ) / x(1) ) ^ (-1-1/x(2))))+log( (1/x(1)) * ((1+ (x(2)*(y(2)-u) ) / x(1) ) ^ (-1-1/x(2))));
h = h + log( (1/x(1)) * ((1+ (x(2)*(y(3)-u) ) / x(1) ) ^ (-1-1/x(2))))+log( (1/x(1)) * ((1+ (x(2)*(y(4)-u) ) / x(1) ) ^ (-1-1/x(2))));
h = h + log( (1/x(1)) * ((1+ (x(2)*(y(5)-u) ) / x(1) ) ^ (-1-1/x(2))))+ log( (1/x(1)) * ((1+ (x(2)*(y(6)-u) ) / x(1) ) ^ (-1-1/x(2))));
h = h + log( (1/x(1)) * ((1+ (x(2)*(y(7)-u) ) / x(1) ) ^ (-1-1/x(2))))+ log( (1/x(1)) * ((1+ (x(2)*(y(8)-u) ) / x(1) ) ^ (-1-1/x(2))));
h = h + log( (1/x(1)) * ((1+ (x(2)*(y(9)-u) ) / x(1) ) ^ (-1-1/x(2))))+ log( (1/x(1)) * ((1+ (x(2)*(y(10)-u) ) / x(1) ) ^ (-1-1/x(2))));
f = -h;
end
-------------------------------------
options = optimoptions('fmincon','Algorithm','interior-point','MaxFunEvals',100,'MaxIter',200);
f = @(x)myfun(x,y,u);
[x,fval]=fmincon(f,x0,[],[],[],[],[],[],@confun,options);
beta = x(1);
epsilon = x(2);
q=0.95;
nu=10;
n=500;