cjw/geMoldInsight
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
cjw
2026-02-12 23:22:11 +08:00
parent 7b09eb3d89
commit 89660bba4e
5988 changed files with 2517516 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
venv/lib/python3.11/site-packages/scipy/_lib/pyprima/cobyla/__init__.py
View File
venv/lib/python3.11/site-packages/scipy/_lib/pyprima/cobyla/cobyla.py
+559
View File
@@ -0,0 +1,559 @@
'''
This module provides Powell's COBYLA algorithm.
Translated from Zaikun Zhang's modern-Fortran reference implementation in PRIMA.
Dedicated to late Professor M. J. D. Powell FRS (1936--2015).
Python translation by Nickolai Belakovski.
N.B.:
1. The modern-Fortran reference implementation in PRIMA contains bug fixes and improvements over the
original Fortran 77 implementation by Powell. Consequently, the PRIMA implementation behaves differently
from the original Fortran 77 implementation by Powell. Therefore, it is important to point out that
you are using PRIMA rather than the original solvers if you want your results to be reproducible.
2. Compared to Powell's Fortran 77 implementation, the modern-Fortran implementation and hence any
faithful translation like this one generally produce better solutions with fewer function evaluations,
making them preferable for applications with expensive function evaluations. However, if function
evaluations are not the dominant cost in your application, the Fortran 77 solvers are likely to be
faster, as they are more efficient in terms of memory usage and flops thanks to the careful and
ingenious (but unmaintained and unmaintainable) implementation by Powell.
See the PRIMA documentation (www.libprima.net) for more information.
'''
from ..common.evaluate import evaluate, moderatex, moderatef, moderatec
from ..common.consts import (EPS, RHOBEG_DEFAULT, RHOEND_DEFAULT, CTOL_DEFAULT,
CWEIGHT_DEFAULT, FTARGET_DEFAULT, IPRINT_DEFAULT,
MAXFUN_DIM_DEFAULT, DEBUGGING, BOUNDMAX,
ETA1_DEFAULT, ETA2_DEFAULT, GAMMA1_DEFAULT,
GAMMA2_DEFAULT)
from ..common.preproc import preproc
from ..common.present import present
from ..common.linalg import matprod
from .cobylb import cobylb
import numpy as np
from dataclasses import dataclass
from copy import copy
@dataclass
class COBYLAResult:
x: np.ndarray
f: float
constr: np.ndarray
cstrv: float
nf: int
xhist: np.ndarray | None
fhist: np.ndarray | None
chist: np.ndarray | None
conhist: np.ndarray | None
info: int
def cobyla(calcfc, m_nlcon, x, Aineq=None, bineq=None, Aeq=None, beq=None,
xl=None, xu=None, f0=None, nlconstr0=None, rhobeg=None, rhoend=None,
ftarget=FTARGET_DEFAULT, ctol=CTOL_DEFAULT, cweight=CWEIGHT_DEFAULT,
maxfun=None, iprint=IPRINT_DEFAULT, eta1=None, eta2=None,
gamma1=GAMMA1_DEFAULT, gamma2=GAMMA2_DEFAULT, maxhist=None, maxfilt=2000,
callback=None):
"""
Among all the arguments, only CALCFC, M_NLCON, and X are obligatory. The others are
OPTIONAL and you can neglect them unless you are familiar with the algorithm. Any
unspecified optional input will take the default value detailed below. For
instance, we may invoke the solver as follows.
# First define CALCFC, M_NLCON, and X, and then do the following.
result = cobyla(calcfc, m_nlcon, x)
or
# First define CALCFC, M_NLCON, X, Aineq, and Bineq, and then do the following.
result = cobyla(calcfc, m_nlcon, x, Aineq=Aineq, bineq=bineq, rhobeg=1.0e0,
rhoend=1.0e-6)
####################################################################################
# IMPORTANT NOTICE: The user must set M_NLCON correctly to the number of nonlinear
# constraints, namely the size of NLCONSTR introduced below. Set it to 0 if there
# is no nonlinear constraint.
####################################################################################
See examples/cobyla/cobyla_example.py for a concrete example.
A detailed introduction to the arguments is as follows.
####################################################################################
# INPUTS
####################################################################################
CALCFC
Input, function.
f, nlconstr = CALCFC(X) should evaluate the objective function and nonlinear
constraints at the given vector X; it should return a tuple consisting of the
objective function value and the nonlinear constraint value. It must be provided
by the user, and its definition must conform to the following interface:
#-------------------------------------------------------------------------#
def calcfc(x):
f = 0.0
nlconstr = np.zeros(m_nlcon)
return f, nlconstr
#-------------------------------------------------------------------------#
M_NLCON
Input, scalar.
M_NLCON must be set to the number of nonlinear constraints, namely the size of
NLCONSTR(X).
N.B.:
1. Why don't we define M_NLCON as optional and default it to 0 when it is absent?
This is because we need to allocate memory for CONSTR_LOC using M_NLCON. To
ensure that the size of CONSTR_LOC is correct, we require the user to specify
M_NLCON explicitly.
X
Input, vector.
As an input, X should be an N-dimensional vector that contains the starting
point, N being the dimension of the problem.
Aineq, Bineq
Input, matrix of size [Mineq, N] and vector of size Mineq unless they are both
empty, default: None and None.
Aineq and Bineq represent the linear inequality constraints: Aineq*X <= Bineq.
Aeq, Beq
Input, matrix of size [Meq, N] and vector of size Meq unless they are both
empty, default: None and None.
Aeq and Beq represent the linear equality constraints: Aeq*X = Beq.
XL, XU
Input, vectors of size N unless they are both None, default: None and None.
XL is the lower bound for X. If XL is None, X has no
lower bound. Any entry of XL that is NaN or below -BOUNDMAX will be taken as
-BOUNDMAX, which effectively means there is no lower bound for the corresponding
entry of X. The value of BOUNDMAX is 0.25*HUGE(X), which is about 8.6E37 for
single precision and 4.5E307 for double precision. XU is similar.
F0
Input, scalar.
F0, if present, should be set to the objective function value of the starting X.
NLCONSTR0
Input, vector.
NLCONSTR0, if present, should be set to the nonlinear constraint value at the
starting X; in addition, SIZE(NLCONSTR0) must be M_NLCON, or the solver will
abort.
RHOBEG, RHOEND
Inputs, scalars, default: RHOBEG = 1, RHOEND = 10^-6. RHOBEG and RHOEND must be
set to the initial and final values of a trust-region radius, both being positive
and RHOEND <= RHOBEG. Typically RHOBEG should be about one tenth of the greatest
expected change to a variable, and RHOEND should indicate the accuracy that is
required in the final values of the variables.
FTARGET
Input, scalar, default: -Inf.
FTARGET is the target function value. The algorithm will terminate when a
feasible point with a function value <= FTARGET is found.
CTOL
Input, scalar, default: sqrt(machine epsilon).
CTOL is the tolerance of constraint violation. X is considered feasible if
CSTRV(X) <= CTOL.
N.B.:
1. CTOL is absolute, not relative.
2. CTOL is used only when selecting the returned X. It does not affect the
iterations of the algorithm.
CWEIGHT
Input, scalar, default: CWEIGHT_DFT defined in common/consts.py.
CWEIGHT is the weight that the constraint violation takes in the selection of the
returned X.
MAXFUN
Input, integer scalar, default: MAXFUN_DIM_DFT*N with MAXFUN_DIM_DFT defined in
common/consts.py. MAXFUN is the maximal number of calls of CALCFC.
IPRINT
Input, integer scalar, default: 0.
The value of IPRINT should be set to 0, 1, -1, 2, -2, 3, or -3, which controls
how much information will be printed during the computation:
0: there will be no printing;
1: a message will be printed to the screen at the return, showing the best vector
of variables found and its objective function value;
2: in addition to 1, each new value of RHO is printed to the screen, with the
best vector of variables so far and its objective function value; each new
value of CPEN is also printed;
3: in addition to 2, each function evaluation with its variables will be printed
to the screen; -1, -2, -3: the same information as 1, 2, 3 will be printed,
not to the screen but to a file named COBYLA_output.txt; the file will be
created if it does not exist; the new output will be appended to the end of
this file if it already exists.
Note that IPRINT = +/-3 can be costly in terms of time and/or space.
ETA1, ETA2, GAMMA1, GAMMA2
Input, scalars, default: ETA1 = 0.1, ETA2 = 0.7, GAMMA1 = 0.5, and GAMMA2 = 2.
ETA1, ETA2, GAMMA1, and GAMMA2 are parameters in the updating scheme of the
trust-region radius detailed in the subroutine TRRAD in trustregion.py. Roughly
speaking, the trust-region radius is contracted by a factor of GAMMA1 when the
reduction ratio is below ETA1, and enlarged by a factor of GAMMA2 when the
reduction ratio is above ETA2. It is required that 0 < ETA1 <= ETA2 < 1 and
0 < GAMMA1 < 1 < GAMMA2. Normally, ETA1 <= 0.25. It is NOT advised to set
ETA1 >= 0.5.
MAXFILT
Input, scalar.
MAXFILT is a nonnegative integer indicating the maximal length of the filter used
for selecting the returned solution; default: MAXFILT_DFT (a value lower than
MIN_MAXFILT is not recommended);
see common/consts.py for the definitions of MAXFILT_DFT and MIN_MAXFILT.
CALLBACK
Input, function to report progress and optionally request termination.
####################################################################################
# OUTPUTS
####################################################################################
The output is a single data structure, COBYLAResult, with the following fields:
X
Output, vector.
As an output, X will be set to an approximate minimizer.
F
Output, scalar.
F will be set to the objective function value of X at exit.
CONSTR
Output, vector.
CONSTR will be set to the constraint value of X at exit.
CSTRV
Output, scalar.
CSTRV will be set to the constraint violation of X at exit, i.e.,
max([0, XL - X, X - XU, Aineq*X - Bineq, ABS(Aeq*X -Beq), NLCONSTR(X)]).
NF
Output, scalar.
NF will be set to the number of calls of CALCFC at exit.
XHIST, FHIST, CHIST, CONHIST, MAXHIST
XHIST: Output, rank 2 array;
FHIST: Output, rank 1 array;
CHIST: Output, rank 1 array;
CONHIST: Output, rank 2 array;
MAXHIST: Input, scalar, default: MAXFUN
XHIST, if present, will output the history of iterates; FHIST, if present, will
output the history function values; CHIST, if present, will output the history of
constraint violations; CONHIST, if present, will output the history of constraint
values; MAXHIST should be a nonnegative integer, and XHIST/FHIST/CHIST/CONHIST
will output only the history of the last MAXHIST iterations.
Therefore, MAXHIST= 0 means XHIST/FHIST/CONHIST/CHIST will output
nothing, while setting MAXHIST = MAXFUN requests XHIST/FHIST/CHIST/CONHIST to
output all the history. If XHIST is present, its size at exit will be
(N, min(NF, MAXHIST)); if FHIST is present, its size at exit will be
min(NF, MAXHIST); if CHIST is present, its size at exit will be min(NF, MAXHIST);
if CONHIST is present, its size at exit will be (M, min(NF, MAXHIST)).
IMPORTANT NOTICE:
Setting MAXHIST to a large value can be costly in terms of memory for large
problems.
MAXHIST will be reset to a smaller value if the memory needed exceeds MAXHISTMEM
defined in common/consts.py
Use *HIST with caution!!! (N.B.: the algorithm is NOT designed for large
problems).
INFO
Output, scalar.
INFO is the exit flag. It will be set to one of the following values defined in
common/infos.py:
SMALL_TR_RADIUS: the lower bound for the trust region radius is reached;
FTARGET_ACHIEVED: the target function value is reached;
MAXFUN_REACHED: the objective function has been evaluated MAXFUN times;
MAXTR_REACHED: the trust region iteration has been performed MAXTR times (MAXTR = 2*MAXFUN);
NAN_INF_X: NaN or Inf occurs in X;
DAMAGING_ROUNDING: rounding errors are becoming damaging.
#--------------------------------------------------------------------------#
The following case(s) should NEVER occur unless there is a bug.
NAN_INF_F: the objective function returns NaN or +Inf;
NAN_INF_MODEL: NaN or Inf occurs in the model;
TRSUBP_FAILED: a trust region step failed to reduce the model
#--------------------------------------------------------------------------#
"""
# Local variables
solver = "COBYLA"
srname = "COBYLA"
# Sizes
mineq = len(bineq) if present(bineq) else 0
meq = len(beq) if present(beq) else 0
mxl = sum(xl > -BOUNDMAX) if present(xl) else 0
mxu = sum(xu < BOUNDMAX) if present(xu) else 0
mmm = mxu + mxl + 2*meq + mineq + m_nlcon
num_vars = len(x)
# Preconditions
if DEBUGGING:
assert m_nlcon >= 0, f'{srname} M_NLCON >= 0'
assert num_vars >= 1, f'{srname} N >= 1'
assert present(Aineq) == present(bineq), \
f'{srname} Aineq and Bineq are both present or both absent'
if (present(Aineq)):
assert Aineq.shape == (mineq, num_vars), f'{srname} SIZE(Aineq) == [Mineq, N]'
assert present(Aeq) == present(beq), \
f'{srname} Aeq and Beq are both present or both absent'
if (present(Aeq)):
assert Aeq.shape == (meq, num_vars), f'{srname} SIZE(Aeq) == [Meq, N]'
if (present(xl)):
assert len(xl) == num_vars, f'{srname} SIZE(XL) == N'
if (present(xu)):
assert len(xu) == num_vars, f'{srname} SIZE(XU) == N'
# N.B.: If NLCONSTR0 is present, then F0 must be present, and we assume that
# F(X0) = F0 even if F0 is NaN; if NLCONSTR0 is absent, then F0 must be either
# absent or NaN, both of which will be interpreted as F(X0) is not provided.
if present(nlconstr0):
assert present(f0), f'{srname} If NLCONSTR0 is present, then F0 is present'
if present(f0):
assert np.isnan(f0) or present(nlconstr0), \
f'{srname} If F0 is present and not NaN, then NLCONSTR0 is present'
# Exit if the size of NLCONSTR0 is inconsistent with M_NLCON.
if present(nlconstr0):
assert np.size(nlconstr0) == m_nlcon
# Read the inputs.
if xl is not None:
xl = copy(xl)
xl[np.isnan(xl)] = -BOUNDMAX
xl[xl < -BOUNDMAX] = -BOUNDMAX
if xu is not None:
xu = copy(xu)
xu[np.isnan(xu)] = BOUNDMAX
xu[xu > BOUNDMAX] = BOUNDMAX
# Wrap the linear and bound constraints into a single constraint: AMAT@X <= BVEC.
amat, bvec = get_lincon(Aeq, Aineq, beq, bineq, xl, xu)
# Create constraint vector
constr = np.zeros(mmm)
# Set [F_LOC, CONSTR_LOC] to [F(X0), CONSTR(X0)] after evaluating the latter if
# needed. In this way, COBYLB only needs one interface.
# N.B.: Due to the preconditions above, there are two possibilities for F0 and
# NLCONSTR0.
# If NLCONSTR0 is present, then F0 must be present, and we assume that F(X0) = F0
# even if F0 is NaN.
# If NLCONSTR0 is absent, then F0 must be either absent or NaN, both of which will
# be interpreted as F(X0) is not provided and we have to evaluate F(X0) and
# NLCONSTR(X0) now.
if (present(f0) and present(nlconstr0) and all(np.isfinite(x))):
f = moderatef(f0)
if amat is not None:
constr[:mmm - m_nlcon] = moderatec(matprod(amat, x) - bvec)
constr[mmm - m_nlcon:] = moderatec(nlconstr0)
else:
x = moderatex(x)
f, constr = evaluate(calcfc, x, m_nlcon, amat, bvec)
constr[:mmm - m_nlcon] = moderatec(constr[:mmm - m_nlcon])
# N.B.: Do NOT call FMSG, SAVEHIST, or SAVEFILT for the function/constraint evaluation at X0.
# They will be called during the initialization, which will read the function/constraint at X0.
cstrv = max(np.append(0, constr))
# If RHOBEG is present, use it; otherwise, RHOBEG takes the default value for
# RHOBEG, taking the value of RHOEND into account. Note that RHOEND is considered
# only if it is present and it is VALID (i.e., finite and positive). The other
# inputs are read similarly.
if present(rhobeg):
rhobeg = rhobeg
elif present(rhoend) and np.isfinite(rhoend) and rhoend > 0:
rhobeg = max(10 * rhoend, RHOBEG_DEFAULT)
else:
rhobeg = RHOBEG_DEFAULT
if present(rhoend):
rhoend = rhoend
elif rhobeg > 0:
rhoend = max(EPS, min(RHOEND_DEFAULT/RHOBEG_DEFAULT * rhobeg, RHOEND_DEFAULT))
else:
rhoend = RHOEND_DEFAULT
maxfun = maxfun if present(maxfun) else MAXFUN_DIM_DEFAULT * num_vars
if present(eta1):
eta1 = eta1
elif present(eta2) and 0 < eta2 < 1:
eta1 = max(EPS, eta2 / 7)
else:
eta1 = ETA1_DEFAULT
if present(eta2):
eta2 = eta2
elif 0 < eta1 < 1:
eta2 = (eta1 + 2) / 3
else:
eta2 = ETA2_DEFAULT
maxhist = (
maxhist
if present(maxhist)
else max(maxfun, num_vars + 2, MAXFUN_DIM_DEFAULT * num_vars)
)
# Preprocess the inputs in case some of them are invalid. It does nothing if all
# inputs are valid.
(
iprint,
maxfun,
maxhist,
ftarget,
rhobeg,
rhoend,
npt, # Unused in COBYLA
maxfilt,
ctol,
cweight,
eta1,
eta2,
gamma1,
gamma2,
_x0, # Unused in COBYLA
) = preproc(
solver,
num_vars,
iprint,
maxfun,
maxhist,
ftarget,
rhobeg,
rhoend,
num_constraints=mmm,
maxfilt=maxfilt,
ctol=ctol,
cweight=cweight,
eta1=eta1,
eta2=eta2,
gamma1=gamma1,
gamma2=gamma2,
is_constrained=(mmm > 0),
)
# Further revise MAXHIST according to MAXHISTMEM, and allocate memory for the history.
# In MATLAB/Python/Julia/R implementation, we should simply set MAXHIST = MAXFUN and initialize
# CHIST = NaN(1, MAXFUN), CONHIST = NaN(M, MAXFUN), FHIST = NaN(1, MAXFUN), XHIST = NaN(N, MAXFUN)
# if they are requested; replace MAXFUN with 0 for the history that is not requested.
# prehist(maxhist, num_vars, present(xhist), xhist_loc, present(fhist), fhist_loc, &
# & present(chist), chist_loc, m, present(conhist), conhist_loc)
# call cobylb, which performs the real calculations
x, f, constr, cstrv, nf, xhist, fhist, chist, conhist, info = cobylb(
calcfc,
iprint,
maxfilt,
maxfun,
amat,
bvec,
ctol,
cweight,
eta1,
eta2,
ftarget,
gamma1,
gamma2,
rhobeg,
rhoend,
constr,
f,
x,
maxhist,
callback
)
return COBYLAResult(x, f, constr, cstrv, nf, xhist, fhist, chist, conhist, info)
def get_lincon(Aeq=None, Aineq=None, beq=None, bineq=None, xl=None, xu=None):
"""
This subroutine wraps the linear and bound constraints into a single constraint:
AMAT*X <= BVEC.
N.B.:
LINCOA normalizes the linear constraints so that each constraint has a gradient
of norm 1. However, COBYLA does not do this.
"""
# Sizes
if Aeq is not None:
num_vars = Aeq.shape[1]
elif Aineq is not None:
num_vars = Aineq.shape[1]
elif xl is not None:
num_vars = len(xl)
elif xu is not None:
num_vars = len(xu)
else:
return None, None
# Preconditions
if DEBUGGING:
assert Aineq is None or Aineq.shape == (len(bineq), num_vars)
assert Aeq is None or Aeq.shape == (len(beq), num_vars)
assert (xl is None or xu is None) or len(xl) == len(xu) == num_vars
#====================#
# Calculation starts #
#====================#
# Define the indices of the nontrivial bound constraints.
ixl = np.where(xl > -BOUNDMAX)[0] if xl is not None else None
ixu = np.where(xu < BOUNDMAX)[0] if xu is not None else None
# Wrap the linear constraints.
# The bound constraint XL <= X <= XU is handled as two constraints:
# -X <= -XL, X <= XU.
# The equality constraint Aeq*X = Beq is handled as two constraints:
# -Aeq*X <= -Beq, Aeq*X <= Beq.
# N.B.:
# 1. The treatment of the equality constraints is naive. One may choose to
# eliminate them instead.
idmat = np.eye(num_vars)
amat = np.vstack([
-idmat[ixl, :] if ixl is not None else np.empty((0, num_vars)),
idmat[ixu, :] if ixu is not None else np.empty((0, num_vars)),
-Aeq if Aeq is not None else np.empty((0, num_vars)),
Aeq if Aeq is not None else np.empty((0, num_vars)),
Aineq if Aineq is not None else np.empty((0, num_vars))
])
bvec = np.hstack([
-xl[ixl] if ixl is not None else np.empty(0),
xu[ixu] if ixu is not None else np.empty(0),
-beq if beq is not None else np.empty(0),
beq if beq is not None else np.empty(0),
bineq if bineq is not None else np.empty(0)
])
amat = amat if amat.shape[0] > 0 else None
bvec = bvec if bvec.shape[0] > 0 else None
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert (amat is None and bvec is None) or amat.shape == (len(bvec), num_vars)
return amat, bvec
venv/lib/python3.11/site-packages/scipy/_lib/pyprima/cobyla/cobylb.py
+714
View File
@@ -0,0 +1,714 @@
'''
This module performs the major calculations of COBYLA.
Translated from Zaikun Zhang's modern-Fortran reference implementation in PRIMA.
Dedicated to late Professor M. J. D. Powell FRS (1936--2015).
Python translation by Nickolai Belakovski.
'''
import numpy as np
from ..common.checkbreak import checkbreak_con
from ..common.consts import REALMAX, EPS, DEBUGGING, MIN_MAXFILT
from ..common.infos import INFO_DEFAULT, MAXTR_REACHED, DAMAGING_ROUNDING, \
SMALL_TR_RADIUS, CALLBACK_TERMINATE
from ..common.evaluate import evaluate
from ..common.history import savehist
from ..common.linalg import isinv, matprod, inprod, norm, primasum, primapow2
from ..common.message import fmsg, retmsg, rhomsg
from ..common.ratio import redrat
from ..common.redrho import redrho
from ..common.selectx import savefilt, selectx
from .update import updatepole, findpole, updatexfc
from .geometry import setdrop_tr, geostep
from .trustregion import trstlp, trrad
from .initialize import initxfc, initfilt
def cobylb(calcfc, iprint, maxfilt, maxfun, amat, bvec, ctol, cweight, eta1, eta2,
ftarget, gamma1, gamma2, rhobeg, rhoend, constr, f, x, maxhist, callback):
'''
This subroutine performs the actual computations of COBYLA.
'''
# Outputs
xhist = []
fhist = []
chist = []
conhist = []
# Local variables
solver = 'COBYLA'
A = np.zeros((np.size(x), np.size(constr))) # A contains the approximate gradient for the constraints
distsq = np.zeros(np.size(x) + 1)
# CPENMIN is the minimum of the penalty parameter CPEN for the L-infinity
# constraint violation in the merit function. Note that CPENMIN = 0 in Powell's
# implementation, which allows CPEN to be 0. Here, we take CPENMIN > 0 so that CPEN
# is always positive. This avoids the situation where PREREM becomes 0 when
# PREREF = 0 = CPEN. It brings two advantages as follows.
# 1. If the trust-region subproblem solver works correctly and the trust-region
# center is not optimal for the subproblem, then PREREM > 0 is guaranteed. This
# is because, in theory, PREREC >= 0 and MAX(PREREC, PREREF) > 0, and the
# definition of CPEN in GETCPEN ensures that PREREM > 0.
# 2. There is no need to revise ACTREM and PREREM when CPEN = 0 and F = FVAL(N+1)
# as in lines 312--314 of Powell's cobylb.f code. Powell's code revises ACTREM
# to CVAL(N + 1) - CSTRV and PREREM to PREREC in this case, which is crucial for
# feasibility problems.
cpenmin = EPS
# Sizes
m_lcon = np.size(bvec) if bvec is not None else 0
num_constraints = np.size(constr)
m_nlcon = num_constraints - m_lcon
num_vars = np.size(x)
# Preconditions
if DEBUGGING:
assert abs(iprint) <= 3
assert num_constraints >= m_lcon and m_lcon >= 0
assert num_vars >= 1
assert maxfun >= num_vars + 2
assert rhobeg >= rhoend and rhoend > 0
assert all(np.isfinite(x))
assert 0 <= eta1 <= eta2 < 1
assert 0 < gamma1 < 1 < gamma2
assert 0 <= ctol
assert 0 <= cweight
assert 0 <= maxhist <= maxfun
assert amat is None or np.shape(amat) == (m_lcon, num_vars)
assert min(MIN_MAXFILT, maxfun) <= maxfilt <= maxfun
#====================#
# Calculation starts #
#====================#
# Initialize SIM, FVAL, CONMAT, and CVAL, together with the history.
# After the initialization, SIM[:, NUM_VARS] holds the vertex of the initial
# simplex with the smallest function value (regardless of the constraint
# violation), and SIM[:, :NUM_VARS] holds the displacements from the other vertices
# to SIM[:, NUM_VARS]. FVAL, CONMAT, and CVAL hold the function values, constraint
# values, and constraint violations on the vertices in the order corresponding to
# SIM.
evaluated, conmat, cval, sim, simi, fval, nf, subinfo = initxfc(calcfc, iprint,
maxfun, constr, amat, bvec, ctol, f, ftarget, rhobeg, x,
xhist, fhist, chist, conhist, maxhist)
# Initialize the filter, including xfilt, ffilt, confilt, cfilt, and nfilt.
# N.B.: The filter is used only when selecting which iterate to return. It does not
# interfere with the iterations. COBYLA is NOT a filter method but a trust-region
# method based on an L-infinity merit function. Powell's implementation does not
# use a filter to select the iterate, possibly returning a suboptimal iterate.
cfilt = np.zeros(np.minimum(np.maximum(maxfilt, 1), maxfun))
confilt = np.zeros((np.size(constr), np.size(cfilt)))
ffilt = np.zeros(np.size(cfilt))
xfilt = np.zeros((np.size(x), np.size(cfilt)))
nfilt = initfilt(conmat, ctol, cweight, cval, fval, sim, evaluated, cfilt, confilt,
ffilt, xfilt)
# Check whether to return due to abnormal cases that may occur during the initialization.
if subinfo != INFO_DEFAULT:
info = subinfo
# Return the best calculated values of the variables
# N.B: Selectx and findpole choose X by different standards, one cannot replace the other
kopt = selectx(ffilt[:nfilt], cfilt[:nfilt], cweight, ctol)
x = xfilt[:, kopt]
f = ffilt[kopt]
constr = confilt[:, kopt]
cstrv = cfilt[kopt]
# print a return message according to IPRINT.
retmsg(solver, info, iprint, nf, f, x, cstrv, constr)
# Postconditions
if DEBUGGING:
assert nf <= maxfun
assert np.size(x) == num_vars and not any(np.isnan(x))
assert not (np.isnan(f) or np.isposinf(f))
# assert np.size(xhist, 0) == n and np.size(xhist, 1) == maxxhist
# assert not any(np.isnan(xhist(:, 1:min(nf, maxxhist))))
# The last calculated X can be Inf (finite + finite can be Inf numerically).
# assert np.size(fhist) == maxfhist
# assert not any(np.isnan(fhist(1:min(nf, maxfhist))) or np.isposinf(fhist(1:min(nf, maxfhist))))
# assert np.size(conhist, 0) == m and np.size(conhist, 1) == maxconhist
# assert not any(np.isnan(conhist(:, 1:min(nf, maxconhist))) or np.isneginf(conhist(:, 1:min(nf, maxconhist))))
# assert np.size(chist) == maxchist
# assert not any(chist(1:min(nf, maxchist)) < 0 or np.isnan(chist(1:min(nf, maxchist))) or np.isposinf(chist(1:min(nf, maxchist))))
# nhist = minval([nf, maxfhist, maxchist])
# assert not any(isbetter(fhist(1:nhist), chist(1:nhist), f, cstrv, ctol))
return x, f, constr, cstrv, nf, xhist, fhist, chist, conhist, info
# Set some more initial values.
# We must initialize shortd, ratio, and jdrop_tr because these get defined on
# branches that are not guaranteed to be executed, but their values are used later.
# Our initialization of CPEN differs from Powell's in two ways. First, we use the
# ratio defined in (13) of Powell's COBYLA paper to initialize CPEN. Second, we
# impose CPEN >= CPENMIN > 0. Powell's code simply initializes CPEN to 0.
rho = rhobeg
delta = rhobeg
cpen = np.maximum(cpenmin, np.minimum(1.0E3, fcratio(conmat, fval))) # Powell's code: CPEN = ZERO
shortd = False
ratio = -1
jdrop_tr = 0
# If DELTA <= GAMMA3*RHO after an update, we set DELTA to RHO. GAMMA3 must be less
# than GAMMA2. The reason is as follows. Imagine a very successful step with
# DNORM = the un-updated DELTA = RHO. The TRRAD will update DELTA to GAMMA2*RHO.
# If GAMMA3 >= GAMMA2, then DELTA will be reset to RHO, which is not reasonable as
# D is very successful. See paragraph two of Sec 5.2.5 in T. M. Ragonneau's thesis:
# "Model-Based Derivative-Free Optimization Methods and Software." According to
# test on 20230613, for COBYLA, this Powellful updating scheme of DELTA works
# slightly better than setting directly DELTA = max(NEW_DELTA, RHO).
gamma3 = np.maximum(1, np.minimum(0.75 * gamma2, 1.5))
# MAXTR is the maximal number of trust region iterations. Each trust-region
# iteration takes 1 or 2 function evaluations unless the trust-region step is short
# or the trust-region subproblem solver fails but the geometry step is not invoked.
# Thus the following MAXTR is unlikely to be reached.
maxtr = 10 * maxfun
info = MAXTR_REACHED
# Begin the iterative procedure
# After solving a trust-region subproblem, we use three boolean variables to
# control the workflow.
# SHORTD - Is the trust-region trial step too short to invoke # a function
# evaluation?
# IMPROVE_GEO - Will we improve the model after the trust-region iteration? If yes,
# a geometry step will be taken, corresponding to the "Branch (Delta)"
# in the COBYLA paper.
# REDUCE_RHO - Will we reduce rho after the trust-region iteration?
# COBYLA never sets IMPROVE_GEO and REDUCE_RHO to True simultaneously.
for tr in range(maxtr):
# Increase the penalty parameter CPEN, if needed, so that
# PREREM = PREREF + CPEN * PREREC > 0.
# This is the first (out of two) update of CPEN, where CPEN increases or
# remains the same.
# N.B.: CPEN and the merit function PHI = FVAL + CPEN*CVAL are used in three
# places only.
# 1. In FINDPOLE/UPDATEPOLE, deciding the optimal vertex of the current simplex.
# 2. After the trust-region trial step, calculating the reduction ratio.
# 3. In GEOSTEP, deciding the direction of the geometry step.
# They do not appear explicitly in the trust-region subproblem, though the
# trust-region center (i.e. the current optimal vertex) is defined by them.
cpen = getcpen(amat, bvec, conmat, cpen, cval, delta, fval, rho, sim, simi)
# Switch the best vertex of the current simplex to SIM[:, NUM_VARS].
conmat, cval, fval, sim, simi, subinfo = updatepole(cpen, conmat, cval, fval,
sim, simi)
# Check whether to exit due to damaging rounding in UPDATEPOLE.
if subinfo == DAMAGING_ROUNDING:
info = subinfo
break # Better action to take? Geometry step, or simply continue?
# Does the interpolation set have adequate geometry? It affects improve_geo and
# reduce_rho.
adequate_geo = all(primasum(primapow2(sim[:, :num_vars]), axis=0) <= 4 * primapow2(delta))
# Calculate the linear approximations to the objective and constraint functions.
# N.B.: TRSTLP accesses A mostly by columns, so it is more reasonable to save A
# instead of A^T.
# Zaikun 2023108: According to a test on 2023108, calculating G and
# A(:, M_LCON+1:M) by solving the linear systems SIM^T*G = FVAL(1:N)-FVAL(N+1)
# and SIM^T*A = CONMAT(:, 1:N)-CONMAT(:, N+1) does not seem to improve or worsen
# the performance of COBYLA in terms of the number of function evaluations. The
# system was solved by SOLVE in LINALG_MOD based on a QR factorization of SIM
# (not necessarily a good algorithm). No preconditioning or scaling was used.
g = matprod((fval[:num_vars] - fval[num_vars]), simi)
A[:, :m_lcon] = amat.T if amat is not None else amat
A[:, m_lcon:] = matprod((conmat[m_lcon:, :num_vars] -
np.tile(conmat[m_lcon:, num_vars], (num_vars, 1)).T), simi).T
# Calculate the trust-region trial step d. Note that d does NOT depend on cpen.
d = trstlp(A, -conmat[:, num_vars], delta, g)
dnorm = min(delta, norm(d))
# Is the trust-region trial step short? N.B.: we compare DNORM with RHO, not
# DELTA. Powell's code especially defines SHORTD by SHORTD = (DNORM < 0.5 *
# RHO). In our tests 1/10 seems to work better than 1/2 or 1/4, especially for
# linearly constrained problems. Note that LINCOA has a slightly more
# sophisticated way of defining SHORTD, taking into account whether D causes a
# change to the active set. Should we try the same here?
shortd = (dnorm <= 0.1 * rho)
# Predict the change to F (PREREF) and to the constraint violation (PREREC) due
# to D. We have the following in precise arithmetic. They may fail to hold due
# to rounding errors.
# 1. B[:NUM_CONSTRAINTS] = -CONMAT[:, NUM_VARS] and hence
# np.max(np.append(B[:NUM_CONSTRAINTS] - D@A[:, :NUM_CONSTRAINTS], 0)) is the
# L-infinity violation of the linearized constraints corresponding to D. When
# D=0, the violation is np.max(np.append(B[:NUM_CONSTRAINTS], 0)) =
# CVAL[NUM_VARS]. PREREC is the reduction of this violation achieved by D,
# which is nonnegative in theory; PREREC = 0 iff B[:NUM_CONSTRAINTS] <= 0, i.e.
# the trust-region center satisfies the linearized constraints.
# 2. PREREF may be negative or 0, but it is positive when PREREC = 0 and shortd
# is False
# 3. Due to 2, in theory, max(PREREC, PREREF) > 0 if shortd is False.
preref = -inprod(d, g) # Can be negative
prerec = cval[num_vars] - np.max(np.append(0, conmat[:, num_vars] + matprod(d, A)))
# Evaluate PREREM, which is the predicted reduction in the merit function.
# In theory, PREREM >= 0 and it is 0 iff CPEN = 0 = PREREF. This may not be true
# numerically.
prerem = preref + cpen * prerec
trfail = not (prerem > 1.0E-6 * min(cpen, 1) * rho)
if shortd or trfail:
# Reduce DELTA if D is short or if D fails to render PREREM > 0. The latter
# can only happen due to rounding errors. This seems quite important for
# performance
delta *= 0.1
if delta <= gamma3 * rho:
delta = rho # set delta to rho when it is close to or below
else:
# Calculate the next value of the objective and constraint functions.
# If X is close to one of the points in the interpolation set, then we do
# not evaluate the objective and constraints at X, assuming them to have
# the values at the closest point.
# N.B.: If this happens, do NOT include X into the filter, as F and CONSTR
# are inaccurate.
x = sim[:, num_vars] + d
distsq[num_vars] = primasum(primapow2(x - sim[:, num_vars]))
distsq[:num_vars] = primasum(primapow2(x.reshape(num_vars, 1) -
(sim[:, num_vars].reshape(num_vars, 1) + sim[:, :num_vars])), axis=0)
j = np.argmin(distsq)
if distsq[j] <= primapow2(1e-4 * rhoend):
f = fval[j]
constr = conmat[:, j]
cstrv = cval[j]
else:
# Evaluate the objective and constraints at X, taking care of possible
# inf/nan values.
f, constr = evaluate(calcfc, x, m_nlcon, amat, bvec)
cstrv = np.max(np.append(0, constr))
nf += 1
# Save X, F, CONSTR, CSTRV into the history.
savehist(maxhist, x, xhist, f, fhist, cstrv, chist, constr, conhist)
# Save X, F, CONSTR, CSTRV into the filter.
nfilt, cfilt, ffilt, xfilt, confilt = savefilt(cstrv, ctol, cweight, f,
x, nfilt, cfilt, ffilt,
xfilt, constr, confilt)
# Print a message about the function/constraint evaluation according to
# iprint
fmsg(solver, 'Trust region', iprint, nf, delta, f, x, cstrv, constr)
# Evaluate ACTREM, which is the actual reduction in the merit function
actrem = (fval[num_vars] + cpen * cval[num_vars]) - (f + cpen * cstrv)
# Calculate the reduction ratio by redrat, which hands inf/nan carefully
ratio = redrat(actrem, prerem, eta1)
# Update DELTA. After this, DELTA < DNORM may hold.
# N.B.:
# 1. Powell's code uses RHO as the trust-region radius and updates it as
# follows.
# Reduce RHO to GAMMA1*RHO if ADEQUATE_GEO is TRUE and either SHORTD is
# TRUE or RATIO < ETA1, and then revise RHO to RHOEND if its new value is
# not more than GAMMA3*RHOEND; RHO remains unchanged in all other cases;
# in particular, RHO is never increased.
# 2. Our implementation uses DELTA as the trust-region radius, while using
# RHO as a lower bound for DELTA. DELTA is updated in a way that is
# typical for trust-region methods, and it is revised to RHO if its new
# value is not more than GAMMA3*RHO. RHO reflects the current resolution
# of the algorithm; its update is essentially the same as the update of
# RHO in Powell's code (see the definition of REDUCE_RHO below). Our
# implementation aligns with UOBYQA/NEWUOA/BOBYQA/LINCOA and improves the
# performance of COBYLA.
# 3. The same as Powell's code, we do not reduce RHO unless ADEQUATE_GEO is
# TRUE. This is also how Powell updated RHO in
# UOBYQA/NEWUOA/BOBYQA/LINCOA. What about we also use ADEQUATE_GEO ==
# TRUE as a prerequisite for reducing DELTA? The argument would be that
# the bad (small) value of RATIO may be because of a bad geometry (and
# hence a bad model) rather than an improperly large DELTA, and it might
# be good to try improving the geometry first without reducing DELTA.
# However, according to a test on 20230206, it does not improve the
# performance if we skip the update of DELTA when ADEQUATE_GEO is FALSE
# and RATIO < 0.1. Therefore, we choose to update DELTA without checking
# ADEQUATE_GEO.
delta = trrad(delta, dnorm, eta1, eta2, gamma1, gamma2, ratio)
if delta <= gamma3*rho:
delta = rho # Set delta to rho when it is close to or below.
# Is the newly generated X better than the current best point?
ximproved = actrem > 0 # If ACTREM is NaN, then XIMPROVED should and will be False
# Set JDROP_TR to the index of the vertex to be replaced with X. JDROP_TR = 0 means there
# is no good point to replace, and X will not be included into the simplex; in this case,
# the geometry of the simplex likely needs improvement, which will be handled below.
jdrop_tr = setdrop_tr(ximproved, d, delta, rho, sim, simi)
# Update SIM, SIMI, FVAL, CONMAT, and CVAL so that SIM[:, JDROP_TR] is replaced with D.
# UPDATEXFC does nothing if JDROP_TR is None, as the algorithm decides to discard X.
sim, simi, fval, conmat, cval, subinfo = updatexfc(jdrop_tr, constr, cpen, cstrv, d, f, conmat, cval, fval, sim, simi)
# Check whether to break due to damaging rounding in UPDATEXFC
if subinfo == DAMAGING_ROUNDING:
info = subinfo
break # Better action to take? Geometry step, or a RESCUE as in BOBYQA?
# Check whether to break due to maxfun, ftarget, etc.
subinfo = checkbreak_con(maxfun, nf, cstrv, ctol, f, ftarget, x)
if subinfo != INFO_DEFAULT:
info = subinfo
break
# End of if SHORTD or TRFAIL. The normal trust-region calculation ends.
# Before the next trust-region iteration, we possibly improve the geometry of the simplex or
# reduce RHO according to IMPROVE_GEO and REDUCE_RHO. Now we decide these indicators.
# N.B.: We must ensure that the algorithm does not set IMPROVE_GEO = True at infinitely many
# consecutive iterations without moving SIM[:, NUM_VARS] or reducing RHO. Otherwise, the algorithm
# will get stuck in repetitive invocations of GEOSTEP. This is ensured by the following facts:
# 1. If an iteration sets IMPROVE_GEO to True, it must also reduce DELTA or set DELTA to RHO.
# 2. If SIM[:, NUM_VARS] and RHO remain unchanged, then ADEQUATE_GEO will become True after at
# most NUM_VARS invocations of GEOSTEP.
# BAD_TRSTEP: Is the last trust-region step bad?
bad_trstep = shortd or trfail or ratio <= 0 or jdrop_tr is None
# IMPROVE_GEO: Should we take a geometry step to improve the geometry of the interpolation set?
improve_geo = bad_trstep and not adequate_geo
# REDUCE_RHO: Should we enhance the resolution by reducing rho?
reduce_rho = bad_trstep and adequate_geo and max(delta, dnorm) <= rho
# COBYLA never sets IMPROVE_GEO and REDUCE_RHO to True simultaneously.
# assert not (IMPROVE_GEO and REDUCE_RHO), 'IMPROVE_GEO or REDUCE_RHO are not both TRUE, COBYLA'
# If SHORTD or TRFAIL is True, then either IMPROVE_GEO or REDUCE_RHO is True unless ADEQUATE_GEO
# is True and max(DELTA, DNORM) > RHO.
# assert not (shortd or trfail) or (improve_geo or reduce_rho or (adequate_geo and max(delta, dnorm) > rho)), \
# 'If SHORTD or TRFAIL is TRUE, then either IMPROVE_GEO or REDUCE_RHO is TRUE unless ADEQUATE_GEO is TRUE and MAX(DELTA, DNORM) > RHO'
# Comments on BAD_TRSTEP:
# 1. Powell's definition of BAD_TRSTEP is as follows. The one used above seems to work better,
# especially for linearly constrained problems due to the factor TENTH (= ETA1).
# !bad_trstep = (shortd .or. actrem <= 0 .or. actrem < TENTH * prerem .or. jdrop_tr == 0)
# Besides, Powell did not check PREREM > 0 in BAD_TRSTEP, which is reasonable to do but has
# little impact upon the performance.
# 2. NEWUOA/BOBYQA/LINCOA would define BAD_TRSTEP, IMPROVE_GEO, and REDUCE_RHO as follows. Two
# different thresholds are used in BAD_TRSTEP. It outperforms Powell's version.
# !bad_trstep = (shortd .or. trfail .or. ratio <= eta1 .or. jdrop_tr == 0)
# !improve_geo = bad_trstep .and. .not. adequate_geo
# !bad_trstep = (shortd .or. trfail .or. ratio <= 0 .or. jdrop_tr == 0)
# !reduce_rho = bad_trstep .and. adequate_geo .and. max(delta, dnorm) <= rho
# 3. Theoretically, JDROP_TR > 0 when ACTREM > 0 (guaranteed by RATIO > 0). However, in Powell's
# implementation, JDROP_TR may be 0 even RATIO > 0 due to NaN. The modernized code has rectified
# this in the function SETDROP_TR. After this rectification, we can indeed simplify the
# definition of BAD_TRSTEP by removing the condition JDROP_TR == 0. We retain it for robustness.
# Comments on REDUCE_RHO:
# When SHORTD is TRUE, UOBYQA/NEWUOA/BOBYQA/LINCOA all set REDUCE_RHO to TRUE if the recent
# models are sufficiently accurate according to certain criteria. See the paragraph around (37)
# in the UOBYQA paper and the discussions about Box 14 in the NEWUOA paper. This strategy is
# crucial for the performance of the solvers. However, as of 20221111, we have not managed to
# make it work in COBYLA. As in NEWUOA, we recorded the errors of the recent models, and set
# REDUCE_RHO to true if they are small (e.g., ALL(ABS(MODERR_REC) <= 0.1 * MAXVAL(ABS(A))*RHO) or
# ALL(ABS(MODERR_REC) <= RHO**2)) when SHORTD is TRUE. It made little impact on the performance.
# Since COBYLA never sets IMPROVE_GEO and REDUCE_RHO to TRUE simultaneously, the following
# two blocks are exchangeable: IF (IMPROVE_GEO) ... END IF and IF (REDUCE_RHO) ... END IF.
# Improve the geometry of the simplex by removing a point and adding a new one.
# If the current interpolation set has acceptable geometry, then we skip the geometry step.
# The code has a small difference from Powell's original code here: If the current geometry
# is acceptable, then we will continue with a new trust-region iteration; however, at the
# beginning of the iteration, CPEN may be updated, which may alter the pole point SIM(:, N+1)
# by UPDATEPOLE; the quality of the interpolation point depends on SIM(:, N + 1), meaning
# that the same interpolation set may have good or bad geometry with respect to different
# "poles"; if the geometry turns out bad with the new pole, the original COBYLA code will
# take a geometry step, but our code here will NOT do it but continue to take a trust-region
# step. The argument is this: even if the geometry step is not skipped in the first place, the
# geometry may turn out bad again after the pole is altered due to an update to CPEN; should
# we take another geometry step in that case? If no, why should we do it here? Indeed, this
# distinction makes no practical difference for CUTEst problems with at most 100 variables
# and 5000 constraints, while the algorithm framework is simplified.
if improve_geo and not all(primasum(primapow2(sim[:, :num_vars]), axis=0) <= 4 * primapow2(delta)):
# Before the geometry step, updatepole has been called either implicitly by UPDATEXFC or
# explicitly after CPEN is updated, so that SIM[:, :NUM_VARS] is the optimal vertex.
# Decide a vertex to drop from the simplex. It will be replaced with SIM[:, NUM_VARS] + D to
# improve the geometry of the simplex.
# N.B.:
# 1. COBYLA never sets JDROP_GEO = num_vars.
# 2. The following JDROP_GEO comes from UOBYQA/NEWUOA/BOBYQA/LINCOA.
# 3. In Powell's original algorithm, the geometry of the simplex is considered acceptable
# iff the distance between any vertex and the pole is at most 2.1*DELTA, and the distance
# between any vertex and the opposite face of the simplex is at least 0.25*DELTA, as
# specified in (14) of the COBYLA paper. Correspondingly, JDROP_GEO is set to the index of
# the vertex with the largest distance to the pole provided that the distance is larger than
# 2.1*DELTA, or the vertex with the smallest distance to the opposite face of the simplex,
# in which case the distance must be less than 0.25*DELTA, as the current simplex does not
# have acceptable geometry (see (15)--(16) of the COBYLA paper). Once JDROP_GEO is set, the
# algorithm replaces SIM(:, JDROP_GEO) with D specified in (17) of the COBYLA paper, which
# is orthogonal to the face opposite to SIM(:, JDROP_GEO) and has a length of 0.5*DELTA,
# intending to improve the geometry of the simplex as per (14).
# 4. Powell's geometry-improving procedure outlined above has an intrinsic flaw: it may lead
# to infinite cycling, as was observed in a test on 20240320. In this test, the geometry-
# improving point introduced in the previous iteration was replaced with the trust-region
# trial point in the current iteration, which was then replaced with the same geometry-
# improving point in the next iteration, and so on. In this process, the simplex alternated
# between two configurations, neither of which had acceptable geometry. Thus RHO was never
# reduced, leading to infinite cycling. (N.B.: Our implementation uses DELTA as the trust
# region radius, with RHO being its lower bound. When the infinite cycling occurred in this
# test, DELTA = RHO and it could not be reduced due to the requirement that DELTA >= RHO.)
jdrop_geo = np.argmax(primasum(primapow2(sim[:, :num_vars]), axis=0), axis=0)
# Calculate the geometry step D.
delbar = delta/2
d = geostep(jdrop_geo, amat, bvec, conmat, cpen, cval, delbar, fval, simi)
# Calculate the next value of the objective and constraint functions.
# If X is close to one of the points in the interpolation set, then we do not evaluate the
# objective and constraints at X, assuming them to have the values at the closest point.
# N.B.:
# 1. If this happens, do NOT include X into the filter, as F and CONSTR are inaccurate.
# 2. In precise arithmetic, the geometry improving step ensures that the distance between X
# and any interpolation point is at least DELBAR, yet X may be close to them due to
# rounding. In an experiment with single precision on 20240317, X = SIM(:, N+1) occurred.
x = sim[:, num_vars] + d
distsq[num_vars] = primasum(primapow2(x - sim[:, num_vars]))
distsq[:num_vars] = primasum(primapow2(x.reshape(num_vars, 1) -
(sim[:, num_vars].reshape(num_vars, 1) + sim[:, :num_vars])), axis=0)
j = np.argmin(distsq)
if distsq[j] <= primapow2(1e-4 * rhoend):
f = fval[j]
constr = conmat[:, j]
cstrv = cval[j]
else:
# Evaluate the objective and constraints at X, taking care of possible
# inf/nan values.
f, constr = evaluate(calcfc, x, m_nlcon, amat, bvec)
cstrv = np.max(np.append(0, constr))
nf += 1
# Save X, F, CONSTR, CSTRV into the history.
savehist(maxhist, x, xhist, f, fhist, cstrv, chist, constr, conhist)
# Save X, F, CONSTR, CSTRV into the filter.
nfilt, cfilt, ffilt, xfilt, confilt = savefilt(cstrv, ctol, cweight, f,
x, nfilt, cfilt, ffilt,
xfilt, constr, confilt)
# Print a message about the function/constraint evaluation according to iprint
fmsg(solver, 'Geometry', iprint, nf, delta, f, x, cstrv, constr)
# Update SIM, SIMI, FVAL, CONMAT, and CVAL so that SIM(:, JDROP_GEO) is replaced with D.
sim, simi, fval, conmat, cval, subinfo = updatexfc(jdrop_geo, constr, cpen, cstrv, d, f, conmat, cval, fval, sim, simi)
# Check whether to break due to damaging rounding in UPDATEXFC
if subinfo == DAMAGING_ROUNDING:
info = subinfo
break # Better action to take? Geometry step, or simply continue?
# Check whether to break due to maxfun, ftarget, etc.
subinfo = checkbreak_con(maxfun, nf, cstrv, ctol, f, ftarget, x)
if subinfo != INFO_DEFAULT:
info = subinfo
break
# end of if improve_geo. The procedure of improving the geometry ends.
# The calculations with the current RHO are complete. Enhance the resolution of the algorithm
# by reducing RHO; update DELTA and CPEN at the same time.
if reduce_rho:
if rho <= rhoend:
info = SMALL_TR_RADIUS
break
delta = max(0.5 * rho, redrho(rho, rhoend))
rho = redrho(rho, rhoend)
# THe second (out of two) updates of CPEN, where CPEN decreases or remains the same.
# Powell's code: cpen = min(cpen, fcratio(fval, conmat)), which may set CPEN to 0.
cpen = np.maximum(cpenmin, np.minimum(cpen, fcratio(conmat, fval)))
# Print a message about the reduction of rho according to iprint
rhomsg(solver, iprint, nf, fval[num_vars], rho, sim[:, num_vars], cval[num_vars], conmat[:, num_vars], cpen)
conmat, cval, fval, sim, simi, subinfo = updatepole(cpen, conmat, cval, fval, sim, simi)
# Check whether to break due to damaging rounding detected in updatepole
if subinfo == DAMAGING_ROUNDING:
info = subinfo
break # Better action to take? Geometry step, or simply continue?
# End of if reduce_rho. The procedure of reducing RHO ends.
# Report the current best value, and check if user asks for early termination.
if callback:
terminate = callback(sim[:, num_vars], fval[num_vars], nf, tr, cval[num_vars], conmat[:, num_vars])
if terminate:
info = CALLBACK_TERMINATE
break
# End of for loop. The iterative procedure ends
# Return from the calculation, after trying the last trust-region step if it has not been tried yet.
# Ensure that D has not been updated after SHORTD == TRUE occurred, or the code below is incorrect.
x = sim[:, num_vars] + d
if (info == SMALL_TR_RADIUS and
shortd and
norm(x - sim[:, num_vars]) > 1.0E-3 * rhoend and
nf < maxfun):
# Zaikun 20230615: UPDATEXFC or UPDATEPOLE is not called since the last trust-region step. Hence
# SIM[:, NUM_VARS] remains unchanged. Otherwise SIM[:, NUM_VARS] + D would not make sense.
f, constr = evaluate(calcfc, x, m_nlcon, amat, bvec)
cstrv = np.max(np.append(0, constr))
nf += 1
savehist(maxhist, x, xhist, f, fhist, cstrv, chist, constr, conhist)
nfilt, cfilt, ffilt, xfilt, confilt = savefilt(cstrv, ctol, cweight, f, x, nfilt, cfilt, ffilt, xfilt, constr, confilt)
# Zaikun 20230512: DELTA has been updated. RHO is only indicative here. TO BE IMPROVED.
fmsg(solver, 'Trust region', iprint, nf, rho, f, x, cstrv, constr)
# Return the best calculated values of the variables
# N.B.: SELECTX and FINDPOLE choose X by different standards, one cannot replace the other.
kopt = selectx(ffilt[:nfilt], cfilt[:nfilt], max(cpen, cweight), ctol)
x = xfilt[:, kopt]
f = ffilt[kopt]
constr = confilt[:, kopt]
cstrv = cfilt[kopt]
# Print a return message according to IPRINT.
retmsg(solver, info, iprint, nf, f, x, cstrv, constr)
return x, f, constr, cstrv, nf, xhist, fhist, chist, conhist, info
def getcpen(amat, bvec, conmat, cpen, cval, delta, fval, rho, sim, simi):
'''
This function gets the penalty parameter CPEN so that PREREM = PREREF + CPEN * PREREC > 0.
See the discussions around equation (9) of the COBYLA paper.
'''
# Even after nearly all of the pycutest problems were showing nearly bit for bit
# identical results between Python and the Fortran bindings, HS102 was still off by
# more than machine epsilon. It turned out to be due to the fact that getcpen was
# modifying fval, among other. It just goes to show that even when you're nearly
# perfect, you can still have non trivial bugs.
conmat = conmat.copy()
cval = cval.copy()
fval = fval.copy()
sim = sim.copy()
simi = simi.copy()
# Intermediate variables
A = np.zeros((np.size(sim, 0), np.size(conmat, 0)))
itol = 1
# Sizes
m_lcon = np.size(bvec) if bvec is not None else 0
num_constraints = np.size(conmat, 0)
num_vars = np.size(sim, 0)
# Preconditions
if DEBUGGING:
assert num_constraints >= 0
assert num_vars >= 1
assert cpen > 0
assert np.size(conmat, 0) == num_constraints and np.size(conmat, 1) == num_vars + 1
assert not (np.isnan(conmat) | np.isneginf(conmat)).any()
assert np.size(cval) == num_vars + 1 and \
not any(cval < 0 | np.isnan(cval) | np.isposinf(cval))
assert np.size(fval) == num_vars + 1 and not any(np.isnan(fval) | np.isposinf(fval))
assert np.size(sim, 0) == num_vars and np.size(sim, 1) == num_vars + 1
assert np.isfinite(sim).all()
assert all(np.max(abs(sim[:, :num_vars]), axis=0) > 0)
assert np.size(simi, 0) == num_vars and np.size(simi, 1) == num_vars
assert np.isfinite(simi).all()
assert isinv(sim[:, :num_vars], simi, itol)
assert delta >= rho and rho > 0
#====================#
# Calculation starts #
#====================#
# Initialize INFO which is needed in the postconditions
info = INFO_DEFAULT
# Increase CPEN if necessary to ensure PREREM > 0. Branch back for the next loop
# if this change alters the optimal vertex of the current simplex.
# Note the following:
# 1. In each loop, CPEN is changed only if PREREC > 0 > PREREF, in which case
# PREREM is guaranteed positive after the update. Note that PREREC >= 0 and
# max(PREREC, PREREF) > 0 in theory. If this holds numerically as well then CPEN
# is not changed only if PREREC = 0 or PREREF >= 0, in which case PREREM is
# currently positive, explaining why CPEN needs no update.
# 2. Even without an upper bound for the loop counter, the loop can occur at most
# NUM_VARS+1 times. This is because the update of CPEN does not decrease CPEN,
# and hence it can make vertex J (J <= NUM_VARS) become the new optimal vertex
# only if CVAL[J] is less than CVAL[NUM_VARS], which can happen at most NUM_VARS
# times. See the paragraph below (9) in the COBYLA paper. After the "correct"
# optimal vertex is found, one more loop is needed to calculate CPEN, and hence
# the loop can occur at most NUM_VARS+1 times.
for iter in range(num_vars + 1):
# Switch the best vertex of the current simplex to SIM[:, NUM_VARS]
conmat, cval, fval, sim, simi, info = updatepole(cpen, conmat, cval, fval, sim,
simi)
# Check whether to exit due to damaging rounding in UPDATEPOLE
if info == DAMAGING_ROUNDING:
break
# Calculate the linear approximations to the objective and constraint functions.
g = matprod(fval[:num_vars] - fval[num_vars], simi)
A[:, :m_lcon] = amat.T if amat is not None else amat
A[:, m_lcon:] = matprod((conmat[m_lcon:, :num_vars] -
np.tile(conmat[m_lcon:, num_vars], (num_vars, 1)).T), simi).T
# Calculate the trust-region trial step D. Note that D does NOT depend on CPEN.
d = trstlp(A, -conmat[:, num_vars], delta, g)
# Predict the change to F (PREREF) and to the constraint violation (PREREC) due
# to D.
preref = -inprod(d, g) # Can be negative
prerec = cval[num_vars] - np.max(np.append(0, conmat[:, num_vars] + matprod(d, A)))
# PREREC <= 0 or PREREF >=0 or either is NaN
if not (prerec > 0 and preref < 0):
break
# Powell's code defines BARMU = -PREREF / PREREC, and CPEN is increased to
# 2*BARMU if and only if it is currently less than 1.5*BARMU, a very
# "Powellful" scheme. In our implementation, however, we set CPEN directly to
# the maximum between its current value and 2*BARMU while handling possible
# overflow. The simplifies the scheme without worsening the performance of
# COBYLA.
cpen = max(cpen, min(-2 * preref / prerec, REALMAX))
if findpole(cpen, cval, fval) == num_vars:
break
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert cpen >= cpen and cpen > 0
assert preref + cpen * prerec > 0 or info == DAMAGING_ROUNDING or \
not (prerec >= 0 and np.maximum(prerec, preref) > 0) or not np.isfinite(preref)
return cpen
def fcratio(conmat, fval):
'''
This function calculates the ratio between the "typical change" of F and that of CONSTR.
See equations (12)-(13) in Section 3 of the COBYLA paper for the definition of the ratio.
'''
# Preconditions
if DEBUGGING:
assert np.size(fval) >= 1
assert np.size(conmat, 1) == np.size(fval)
#====================#
# Calculation starts #
#====================#
cmin = np.min(-conmat, axis=1)
cmax = np.max(-conmat, axis=1)
fmin = min(fval)
fmax = max(fval)
if any(cmin < 0.5 * cmax) and fmin < fmax:
denom = np.min(np.maximum(cmax, 0) - cmin, where=cmin < 0.5 * cmax, initial=np.inf)
# Powell mentioned the following alternative in section 4 of his COBYLA paper. According to a test
# on 20230610, it does not make much difference to the performance.
# denom = np.max(max(*cmax, 0) - cmin, mask=(cmin < 0.5 * cmax))
r = (fmax - fmin) / denom
else:
r = 0
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert r >= 0
return r
venv/lib/python3.11/site-packages/scipy/_lib/pyprima/cobyla/geometry.py
+226
View File
@@ -0,0 +1,226 @@
'''
This module contains subroutines concerning the geometry-improving of the interpolation set.
Translated from Zaikun Zhang's modern-Fortran reference implementation in PRIMA.
Dedicated to late Professor M. J. D. Powell FRS (1936--2015).
Python translation by Nickolai Belakovski.
'''
from ..common.consts import DEBUGGING
from ..common.linalg import isinv, matprod, inprod, norm, primasum, primapow2
import numpy as np
def setdrop_tr(ximproved, d, delta, rho, sim, simi):
'''
This function finds (the index) of a current interpolation point to be replaced with
the trust-region trial point. See (19)-(22) of the COBYLA paper.
N.B.:
1. If XIMPROVED == True, then JDROP > 0 so that D is included into XPT. Otherwise,
it is a bug.
2. COBYLA never sets JDROP = NUM_VARS
TODO: Check whether it improves the performance if JDROP = NUM_VARS is allowed when
XIMPROVED is True. Note that UPDATEXFC should be revised accordingly.
'''
# Local variables
itol = 0.1
# Sizes
num_vars = np.size(sim, 0)
# Preconditions
if DEBUGGING:
assert num_vars >= 1
assert np.size(d) == num_vars and all(np.isfinite(d))
assert delta >= rho and rho > 0
assert np.size(sim, 0) == num_vars and np.size(sim, 1) == num_vars + 1
assert np.isfinite(sim).all()
assert all(np.max(abs(sim[:, :num_vars]), axis=0) > 0)
assert np.size(simi, 0) == num_vars and np.size(simi, 1) == num_vars
assert np.isfinite(simi).all()
assert isinv(sim[:, :num_vars], simi, itol)
#====================#
# Calculation starts #
#====================#
# -------------------------------------------------------------------------------------------------- #
# The following code is Powell's scheme for defining JDROP.
# -------------------------------------------------------------------------------------------------- #
# ! JDROP = 0 by default. It cannot be removed, as JDROP may not be set below in some cases (e.g.,
# ! when XIMPROVED == FALSE, MAXVAL(ABS(SIMID)) <= 1, and MAXVAL(VETA) <= EDGMAX).
# jdrop = 0
#
# ! SIMID(J) is the value of the J-th Lagrange function at D. It is the counterpart of VLAG in UOBYQA
# ! and DEN in NEWUOA/BOBYQA/LINCOA, but it excludes the value of the (N+1)-th Lagrange function.
# simid = matprod(simi, d)
# if (any(abs(simid) > 1) .or. (ximproved .and. any(.not. is_nan(simid)))) then
# jdrop = int(maxloc(abs(simid), mask=(.not. is_nan(simid)), dim=1), kind(jdrop))
# !!MATLAB: [~, jdrop] = max(simid, [], 'omitnan');
# end if
#
# ! VETA(J) is the distance from the J-th vertex of the simplex to the best vertex, taking the trial
# ! point SIM(:, N+1) + D into account.
# if (ximproved) then
# veta = sqrt(sum((sim(:, 1:n) - spread(d, dim=2, ncopies=n))**2, dim=1))
# !!MATLAB: veta = sqrt(sum((sim(:, 1:n) - d).^2)); % d should be a column! Implicit expansion
# else
# veta = sqrt(sum(sim(:, 1:n)**2, dim=1))
# end if
#
# ! VSIG(J) (J=1, .., N) is the Euclidean distance from vertex J to the opposite face of the simplex.
# vsig = ONE / sqrt(sum(simi**2, dim=2))
# sigbar = abs(simid) * vsig
#
# ! The following JDROP will overwrite the previous one if its premise holds.
# mask = (veta > factor_delta * delta .and. (sigbar >= factor_alpha * delta .or. sigbar >= vsig))
# if (any(mask)) then
# jdrop = int(maxloc(veta, mask=mask, dim=1), kind(jdrop))
# !!MATLAB: etamax = max(veta(mask)); jdrop = find(mask & ~(veta < etamax), 1, 'first');
# end if
#
# ! Powell's code does not include the following instructions. With Powell's code, if SIMID consists
# ! of only NaN, then JDROP can be 0 even when XIMPROVED == TRUE (i.e., D reduces the merit function).
# ! With the following code, JDROP cannot be 0 when XIMPROVED == TRUE, unless VETA is all NaN, which
# ! should not happen if X0 does not contain NaN, the trust-region/geometry steps never contain NaN,
# ! and we exit once encountering an iterate containing Inf (due to overflow).
# if (ximproved .and. jdrop <= 0) then ! Write JDROP <= 0 instead of JDROP == 0 for robustness.
# jdrop = int(maxloc(veta, mask=(.not. is_nan(veta)), dim=1), kind(jdrop))
# !!MATLAB: [~, jdrop] = max(veta, [], 'omitnan');
# end if
# -------------------------------------------------------------------------------------------------- #
# Powell's scheme ends here.
# -------------------------------------------------------------------------------------------------- #
# The following definition of JDROP is inspired by SETDROP_TR in UOBYQA/NEWUOA/BOBYQA/LINCOA.
# It is simpler and works better than Powell's scheme. Note that we allow JDROP to be NUM_VARS+1 if
# XIMPROVED is True, whereas Powell's code does not.
# See also (4.1) of Scheinberg-Toint-2010: Self-Correcting Geometry in Model-Based Algorithms for
# Derivative-Free Unconstrained Optimization, which refers to the strategy here as the "combined
# distance/poisedness criteria".
# DISTSQ[j] is the square of the distance from the jth vertex of the simplex to get "best" point so
# far, taking the trial point SIM[:, NUM_VARS] + D into account.
distsq = np.zeros(np.size(sim, 1))
if ximproved:
distsq[:num_vars] = primasum(primapow2(sim[:, :num_vars] - np.tile(d, (num_vars, 1)).T), axis=0)
distsq[num_vars] = primasum(d*d)
else:
distsq[:num_vars] = primasum(primapow2(sim[:, :num_vars]), axis=0)
distsq[num_vars] = 0
weight = np.maximum(1, distsq / primapow2(np.maximum(rho, delta/10))) # Similar to Powell's NEWUOA code.
# Other possible definitions of weight. They work almost the same as the one above.
# weight = distsq # Similar to Powell's LINCOA code, but WRONG. See comments in LINCOA/geometry.f90.
# weight = max(1, max(25 * distsq / delta**2)) # Similar to Powell's BOBYQA code, works well.
# weight = max(1, max(10 * distsq / delta**2))
# weight = max(1, max(1e2 * distsq / delta**2))
# weight = max(1, max(distsq / rho**2)) ! Similar to Powell's UOBYQA
# If 0 <= j < NUM_VARS, SIMID[j] is the value of the jth Lagrange function at D; the value of the
# (NUM_VARS+1)th Lagrange function is 1 - sum(SIMID). [SIMID, 1 - sum(SIMID)] is the counterpart of
# VLAG in UOBYQA and DEN in NEWUOA/BOBYQA/LINCOA.
simid = matprod(simi, d)
score = weight * abs(np.array([*simid, 1 - primasum(simid)]))
# If XIMPROVED = False (D does not render a better X), set SCORE[NUM_VARS] = -1 to avoid JDROP = NUM_VARS.
if not ximproved:
score[num_vars] = -1
# score[j] is NaN implies SIMID[j] is NaN, but we want abs(SIMID) to be big. So we
# exclude such j.
score[np.isnan(score)] = -1
jdrop = None
# The following if statement works a bit better than
# `if any(score > 1) or (any(score > 0) and ximproved)` from Powell's UOBYQA and
# NEWUOA code.
if any(score > 0): # Powell's BOBYQA and LINCOA code.
jdrop = np.argmax(score)
if (ximproved and jdrop is None):
jdrop = np.argmax(distsq)
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert jdrop is None or (0 <= jdrop < num_vars + 1)
assert jdrop <= num_vars or ximproved
assert jdrop >= 0 or not ximproved
# JDROP >= 1 when XIMPROVED = TRUE unless NaN occurs in DISTSQ, which should not happen if the
# starting point does not contain NaN and the trust-region/geometry steps never contain NaN.
return jdrop
def geostep(jdrop, amat, bvec, conmat, cpen, cval, delbar, fval, simi):
'''
This function calculates a geometry step so that the geometry of the interpolation set is improved
when SIM[: JDROP_GEO] is replaced with SIM[:, NUM_VARS] + D. See (15)--(17) of the COBYLA paper.
'''
# Sizes
m_lcon = np.size(bvec, 0) if bvec is not None else 0
num_constraints = np.size(conmat, 0)
num_vars = np.size(simi, 0)
# Preconditions
if DEBUGGING:
assert num_constraints >= m_lcon >= 0
assert num_vars >= 1
assert delbar > 0
assert cpen > 0
assert np.size(simi, 0) == num_vars and np.size(simi, 1) == num_vars
assert np.isfinite(simi).all()
assert np.size(fval) == num_vars + 1 and not any(np.isnan(fval) | np.isposinf(fval))
assert np.size(conmat, 0) == num_constraints and np.size(conmat, 1) == num_vars + 1
assert not np.any(np.isnan(conmat) | np.isposinf(conmat))
assert np.size(cval) == num_vars + 1 and not any(cval < 0 | np.isnan(cval) | np.isposinf(cval))
assert 0 <= jdrop < num_vars
#====================#
# Calculation starts #
#====================#
# SIMI[JDROP, :] is a vector perpendicular to the face of the simplex to the opposite of vertex
# JDROP. Set D to the vector in this direction and with length DELBAR.
d = simi[jdrop, :]
d = delbar * (d / norm(d))
# The code below chooses the direction of D according to an approximation of the merit function.
# See (17) of the COBYLA paper and line 225 of Powell's cobylb.f.
# Calculate the coefficients of the linear approximations to the objective and constraint functions.
# N.B.: CONMAT and SIMI have been updated after the last trust-region step, but G and A have not.
# So we cannot pass G and A from outside.
g = matprod(fval[:num_vars] - fval[num_vars], simi)
A = np.zeros((num_vars, num_constraints))
A[:, :m_lcon] = amat.T if amat is not None else amat
A[:, m_lcon:] = matprod((conmat[m_lcon:, :num_vars] -
np.tile(conmat[m_lcon:, num_vars], (num_vars, 1)).T), simi).T
# CVPD and CVND are the predicted constraint violation of D and -D by the linear models.
cvpd = np.max(np.append(0, conmat[:, num_vars] + matprod(d, A)))
cvnd = np.max(np.append(0, conmat[:, num_vars] - matprod(d, A)))
if -inprod(d, g) + cpen * cvnd < inprod(d, g) + cpen * cvpd:
d *= -1
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert np.size(d) == num_vars and all(np.isfinite(d))
# In theory, ||S|| == DELBAR, which may be false due to rounding, but not too far.
# It is crucial to ensure that the geometry step is nonzero, which holds in theory.
assert 0.9 * delbar < np.linalg.norm(d) <= 1.1 * delbar
return d
venv/lib/python3.11/site-packages/scipy/_lib/pyprima/cobyla/initialize.py
+215
View File
@@ -0,0 +1,215 @@
'''
This module contains subroutines for initialization.
Translated from Zaikun Zhang's modern-Fortran reference implementation in PRIMA.
Dedicated to late Professor M. J. D. Powell FRS (1936--2015).
Python translation by Nickolai Belakovski.
'''
from ..common.checkbreak import checkbreak_con
from ..common.consts import DEBUGGING, REALMAX
from ..common.infos import INFO_DEFAULT
from ..common.evaluate import evaluate
from ..common.history import savehist
from ..common.linalg import inv
from ..common.message import fmsg
from ..common.selectx import savefilt
import numpy as np
def initxfc(calcfc, iprint, maxfun, constr0, amat, bvec, ctol, f0, ftarget, rhobeg, x0,
xhist, fhist, chist, conhist, maxhist):
'''
This subroutine does the initialization concerning X, function values, and
constraints.
'''
# Local variables
solver = 'COBYLA'
srname = "INITIALIZE"
# Sizes
num_constraints = np.size(constr0)
m_lcon = np.size(bvec) if bvec is not None else 0
m_nlcon = num_constraints - m_lcon
num_vars = np.size(x0)
# Preconditions
if DEBUGGING:
assert num_constraints >= 0, f'M >= 0 {srname}'
assert num_vars >= 1, f'N >= 1 {srname}'
assert abs(iprint) <= 3, f'IPRINT is 0, 1, -1, 2, -2, 3, or -3 {srname}'
# assert conmat.shape == (num_constraints , num_vars + 1), f'CONMAT.shape = [M, N+1] {srname}'
# assert cval.size == num_vars + 1, f'CVAL.size == N+1 {srname}'
# assert maxchist * (maxchist - maxhist) == 0, f'CHIST.shape == 0 or MAXHIST {srname}'
# assert conhist.shape[0] == num_constraints and maxconhist * (maxconhist - maxhist) == 0, 'CONHIST.shape[0] == num_constraints, SIZE(CONHIST, 2) == 0 or MAXHIST {srname)}'
# assert maxfhist * (maxfhist - maxhist) == 0, f'FHIST.shape == 0 or MAXHIST {srname}'
# assert xhist.shape[0] == num_vars and maxxhist * (maxxhist - maxhist) == 0, 'XHIST.shape[0] == N, SIZE(XHIST, 2) == 0 or MAXHIST {srname)}'
assert all(np.isfinite(x0)), f'X0 is finite {srname}'
assert rhobeg > 0, f'RHOBEG > 0 {srname}'
#====================#
# Calculation starts #
#====================#
# Initialize info to the default value. At return, a value different from this
# value will indicate an abnormal return
info = INFO_DEFAULT
# Initialize the simplex. It will be revised during the initialization.
sim = np.eye(num_vars, num_vars+1) * rhobeg
sim[:, num_vars] = x0
# Initialize the matrix simi. In most cases simi is overwritten, but not always.
simi = np.eye(num_vars) / rhobeg
# evaluated[j] = True iff the function/constraint of SIM[:, j] has been evaluated.
evaluated = np.zeros(num_vars+1, dtype=bool)
# Initialize fval
fval = np.zeros(num_vars+1) + REALMAX
cval = np.zeros(num_vars+1) + REALMAX
conmat = np.zeros((num_constraints, num_vars+1)) + REALMAX
for k in range(num_vars + 1):
x = sim[:, num_vars].copy()
# We will evaluate F corresponding to SIM(:, J).
if k == 0:
j = num_vars
f = f0
constr = constr0
else:
j = k - 1
x[j] += rhobeg
f, constr = evaluate(calcfc, x, m_nlcon, amat, bvec)
cstrv = np.max(np.append(0, constr))
# Print a message about the function/constraint evaluation according to IPRINT.
fmsg(solver, 'Initialization', iprint, k, rhobeg, f, x, cstrv, constr)
# Save X, F, CONSTR, CSTRV into the history.
savehist(maxhist, x, xhist, f, fhist, cstrv, chist, constr, conhist)
# Save F, CONSTR, and CSTRV to FVAL, CONMAT, and CVAL respectively.
evaluated[j] = True
fval[j] = f
conmat[:, j] = constr
cval[j] = cstrv
# Check whether to exit.
subinfo = checkbreak_con(maxfun, k, cstrv, ctol, f, ftarget, x)
if subinfo != INFO_DEFAULT:
info = subinfo
break
# Exchange the new vertex of the initial simplex with the optimal vertex if necessary.
# This is the ONLY part that is essentially non-parallel.
if j < num_vars and fval[j] < fval[num_vars]:
fval[j], fval[num_vars] = fval[num_vars], fval[j]
cval[j], cval[num_vars] = cval[num_vars], cval[j]
conmat[:, [j, num_vars]] = conmat[:, [num_vars, j]]
sim[:, num_vars] = x
sim[j, :j+1] = -rhobeg # SIM[:, :j+1] is lower triangular
nf = np.count_nonzero(evaluated)
if evaluated.all():
# Initialize SIMI to the inverse of SIM[:, :num_vars]
simi = inv(sim[:, :num_vars])
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert nf <= maxfun, f'NF <= MAXFUN {srname}'
assert evaluated.size == num_vars + 1, f'EVALUATED.size == Num_vars + 1 {srname}'
# assert chist.size == maxchist, f'CHIST.size == MAXCHIST {srname}'
# assert conhist.shape== (num_constraints, maxconhist), f'CONHIST.shape == [M, MAXCONHIST] {srname}'
assert conmat.shape == (num_constraints, num_vars + 1), f'CONMAT.shape = [M, N+1] {srname}'
assert not (np.isnan(conmat).any() or np.isneginf(conmat).any()), f'CONMAT does not contain NaN/-Inf {srname}'
assert cval.size == num_vars + 1 and not (any(cval < 0) or any(np.isnan(cval)) or any(np.isposinf(cval))), f'CVAL.shape == Num_vars+1 and CVAL does not contain negative values or NaN/+Inf {srname}'
# assert fhist.shape == maxfhist, f'FHIST.shape == MAXFHIST {srname}'
# assert maxfhist * (maxfhist - maxhist) == 0, f'FHIST.shape == 0 or MAXHIST {srname}'
assert fval.size == num_vars + 1 and not (any(np.isnan(fval)) or any(np.isposinf(fval))), f'FVAL.shape == Num_vars+1 and FVAL is not NaN/+Inf {srname}'
# assert xhist.shape == (num_vars, maxxhist), f'XHIST.shape == [N, MAXXHIST] {srname}'
assert sim.shape == (num_vars, num_vars + 1), f'SIM.shape == [N, N+1] {srname}'
assert np.isfinite(sim).all(), f'SIM is finite {srname}'
assert all(np.max(abs(sim[:, :num_vars]), axis=0) > 0), f'SIM(:, 1:N) has no zero column {srname}'
assert simi.shape == (num_vars, num_vars), f'SIMI.shape == [N, N] {srname}'
assert np.isfinite(simi).all(), f'SIMI is finite {srname}'
assert np.allclose(sim[:, :num_vars] @ simi, np.eye(num_vars), rtol=0.1, atol=0.1) or not all(evaluated), f'SIMI = SIM(:, 1:N)^{-1} {srname}'
return evaluated, conmat, cval, sim, simi, fval, nf, info
def initfilt(conmat, ctol, cweight, cval, fval, sim, evaluated, cfilt, confilt, ffilt, xfilt):
'''
This function initializes the filter (XFILT, etc) that will be used when selecting
x at the end of the solver.
N.B.:
1. Why not initialize the filters using XHIST, etc? Because the history is empty if
the user chooses not to output it.
2. We decouple INITXFC and INITFILT so that it is easier to parallelize the former
if needed.
'''
# Sizes
num_constraints = conmat.shape[0]
num_vars = sim.shape[0]
maxfilt = len(ffilt)
# Preconditions
if DEBUGGING:
assert num_constraints >= 0
assert num_vars >= 1
assert maxfilt >= 1
assert np.size(confilt, 0) == num_constraints and np.size(confilt, 1) == maxfilt
assert np.size(cfilt) == maxfilt
assert np.size(xfilt, 0) == num_vars and np.size(xfilt, 1) == maxfilt
assert np.size(ffilt) == maxfilt
assert np.size(conmat, 0) == num_constraints and np.size(conmat, 1) == num_vars + 1
assert not (np.isnan(conmat) | np.isneginf(conmat)).any()
assert np.size(cval) == num_vars + 1 and not any(cval < 0 | np.isnan(cval) | np.isposinf(cval))
assert np.size(fval) == num_vars + 1 and not any(np.isnan(fval) | np.isposinf(fval))
assert np.size(sim, 0) == num_vars and np.size(sim, 1) == num_vars + 1
assert np.isfinite(sim).all()
assert all(np.max(abs(sim[:, :num_vars]), axis=0) > 0)
assert np.size(evaluated) == num_vars + 1
#====================#
# Calculation starts #
#====================#
nfilt = 0
for i in range(num_vars+1):
if evaluated[i]:
if i < num_vars:
x = sim[:, i] + sim[:, num_vars]
else:
x = sim[:, i] # i == num_vars, i.e. the last column
nfilt, cfilt, ffilt, xfilt, confilt = savefilt(cval[i], ctol, cweight, fval[i], x, nfilt, cfilt, ffilt, xfilt, conmat[:, i], confilt)
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert nfilt <= maxfilt
assert np.size(confilt, 0) == num_constraints and np.size(confilt, 1) == maxfilt
assert not (np.isnan(confilt[:, :nfilt]) | np.isneginf(confilt[:, :nfilt])).any()
assert np.size(cfilt) == maxfilt
assert not any(cfilt[:nfilt] < 0 | np.isnan(cfilt[:nfilt]) | np.isposinf(cfilt[:nfilt]))
assert np.size(xfilt, 0) == num_vars and np.size(xfilt, 1) == maxfilt
assert not (np.isnan(xfilt[:, :nfilt])).any()
# The last calculated X can be Inf (finite + finite can be Inf numerically).
assert np.size(ffilt) == maxfilt
assert not any(np.isnan(ffilt[:nfilt]) | np.isposinf(ffilt[:nfilt]))
return nfilt
venv/lib/python3.11/site-packages/scipy/_lib/pyprima/cobyla/trustregion.py
+492
View File
@@ -0,0 +1,492 @@
'''
This module provides subroutines concerning the trust-region calculations of COBYLA.
Translated from Zaikun Zhang's modern-Fortran reference implementation in PRIMA.
Dedicated to late Professor M. J. D. Powell FRS (1936--2015).
Python translation by Nickolai Belakovski.
'''
import numpy as np
import numpy.typing as npt
from ..common.consts import DEBUGGING, REALMIN, REALMAX, EPS
from ..common.powalg import qradd_Rdiag, qrexc_Rdiag
from ..common.linalg import isminor, matprod, inprod, lsqr, primasum
def trstlp(A, b, delta, g):
'''
This function calculated an n-component vector d by the following two stages. In the first
stage, d is set to the shortest vector that minimizes the greatest violation of the constraints
A.T @ D <= B, K = 1, 2, 3, ..., M,
subject to the Euclidean length of d being at most delta. If its length is strictly less than
delta, then the second stage uses the resultant freedom in d to minimize the objective function
G.T @ D
subject to no increase in any greatest constraint violation.
It is possible but rare that a degeneracy may prevent d from attaining the target length delta.
cviol is the largest constraint violation of the current d: max(max(A.T@D - b), 0)
icon is the index of a most violated constraint if cviol is positive.
nact is the number of constraints in the active set and iact[0], ..., iact[nact-1] are their indices,
while the remainder of the iact contains a permutation of the remaining constraint indicies.
N.B.: nact <= min(num_constraints, num_vars). Obviously nact <= num_constraints. In addition, the constraints
in iact[0, ..., nact-1] have linearly independent gradients (see the comments above the instruction
that delete a constraint from the active set to make room for the new active constraint with index iact[icon]);
it can also be seen from the update of nact: starting from 0, nact is incremented only if nact < n.
Further, Z is an orthogonal matrix whose first nact columns can be regarded as the result of
Gram-Schmidt applied to the active constraint gradients. For j = 0, 1, ..., nact-1, the number
zdota[j] is the scalar product of the jth column of Z with the gradient of the jth active
constraint. d is the current vector of variables and here the residuals of the active constraints
should be zero. Further, the active constraints have nonnegative Lagrange multipliers that are
held at the beginning of vmultc. The remainder of this vector holds the residuals of the inactive
constraints at d, the ordering of the components of vmultc being in agreement with the permutation
of the indices of the constraints that is in iact. All these residuals are nonnegative, which is
achieved by the shift cviol that makes the least residual zero.
N.B.:
0. In Powell's implementation, the constraints are A.T @ D >= B. In other words, the A and B in
our implementation are the negative of those in Powell's implementation.
1. The algorithm was NOT documented in the COBYLA paper. A note should be written to introduce it!
2. As a major part of the algorithm (see trstlp_sub), the code maintains and updates the QR
factorization of A[iact[:nact]], i.e. the gradients of all the active (linear) constraints. The
matrix Z is indeed Q, and the vector zdota is the diagonal of R. The factorization is updated by
Givens rotations when an index is added in or removed from iact.
3. There are probably better algorithms available for the trust-region linear programming problem.
'''
# Sizes
num_constraints = A.shape[1]
num_vars = A.shape[0]
# Preconditions
if DEBUGGING:
assert num_vars >= 1
assert num_constraints >= 0
assert np.size(g) == num_vars
assert np.size(b) == num_constraints
assert delta > 0
vmultc = np.zeros(num_constraints + 1)
iact = np.zeros(num_constraints + 1, dtype=int)
nact = 0
d = np.zeros(num_vars)
z = np.zeros((num_vars, num_vars))
# ==================
# Calculation starts
# ==================
# Form A_aug and B_aug. This allows the gradient of the objective function to be regarded as the
# gradient of a constraint in the second stage.
A_aug = np.hstack([A, g.reshape((num_vars, 1))])
b_aug = np.hstack([b, 0])
# Scale the problem if A contains large values. Otherwise floating point exceptions may occur.
# Note that the trust-region step is scale invariant.
for i in range(num_constraints+1): # Note that A_aug.shape[1] == num_constraints+1
if (maxval:=max(abs(A_aug[:, i]))) > 1e12:
modscal = max(2*REALMIN, 1/maxval)
A_aug[:, i] *= modscal
b_aug[i] *= modscal
# Stage 1: minimize the 1+infinity constraint violation of the linearized constraints.
iact[:num_constraints], nact, d, vmultc[:num_constraints], z = trstlp_sub(iact[:num_constraints], nact, 1, A_aug[:, :num_constraints], b_aug[:num_constraints], delta, d, vmultc[:num_constraints], z)
# Stage 2: minimize the linearized objective without increasing the 1_infinity constraint violation.
iact, nact, d, vmultc, z = trstlp_sub(iact, nact, 2, A_aug, b_aug, delta, d, vmultc, z)
# ================
# Calculation ends
# ================
# Postconditions
if DEBUGGING:
assert all(np.isfinite(d))
# Due to rounding, it may happen that ||D|| > DELTA, but ||D|| > 2*DELTA is highly improbable.
assert np.linalg.norm(d) <= 2 * delta
return d
def trstlp_sub(iact: npt.NDArray, nact: int, stage, A, b, delta, d, vmultc, z):
'''
This subroutine does the real calculations for trstlp, both stage 1 and stage 2.
Major differences between stage 1 and stage 2:
1. Initialization. Stage 2 inherits the values of some variables from stage 1, so they are
initialized in stage 1 but not in stage 2.
2. cviol. cviol is updated after at iteration in stage 1, while it remains a constant in stage2.
3. sdirn. See the definition of sdirn in the code for details.
4. optnew. The two stages have different objectives, so optnew is updated differently.
5. step. step <= cviol in stage 1.
'''
zdasav = np.zeros(z.shape[1])
vmultd = np.zeros(np.size(vmultc))
zdota = np.zeros(np.size(z, 1))
# Sizes
mcon = np.size(A, 1)
num_vars = np.size(A, 0)
# Preconditions
if DEBUGGING:
assert num_vars >= 1
assert stage == 1 or stage == 2
assert (mcon >= 0 and stage == 1) or (mcon >= 1 and stage == 2)
assert np.size(b) == mcon
assert np.size(iact) == mcon
assert np.size(vmultc) == mcon
assert np.size(d) == num_vars
assert np.size(z, 0) == num_vars and np.size(z, 1) == num_vars
assert delta > 0
if stage == 2:
assert all(np.isfinite(d)) and np.linalg.norm(d) <= 2 * delta
assert nact >= 0 and nact <= np.minimum(mcon, num_vars)
assert all(vmultc[:mcon]) >= 0
# N.B.: Stage 1 defines only VMULTC(1:M); VMULTC(M+1) is undefined!
# Initialize according to stage
if stage == 1:
iact = np.linspace(0, mcon-1, mcon, dtype=int)
nact = 0
d = np.zeros(num_vars)
cviol = np.max(np.append(0, -b))
vmultc = cviol + b
z = np.eye(num_vars)
if mcon == 0 or cviol <= 0:
# Check whether a quick return is possible. Make sure the in-outputs have been initialized.
return iact, nact, d, vmultc, z
if all(np.isnan(b)):
return iact, nact, d, vmultc, z
else:
icon = np.nanargmax(-b)
num_constraints = mcon
sdirn = np.zeros(len(d))
else:
if inprod(d, d) >= delta*delta:
# Check whether a quick return is possible.
return iact, nact, d, vmultc, z
iact[mcon-1] = mcon-1
vmultc[mcon-1] = 0
num_constraints = mcon - 1
icon = mcon - 1
# In Powell's code, stage 2 uses the zdota and cviol calculated by stage1. Here we recalculate
# them so that they need not be passed from stage 1 to 2, and hence the coupling is reduced.
cviol = np.max(np.append(0, matprod(d, A[:, :num_constraints]) - b[:num_constraints]))
zdota[:nact] = [inprod(z[:, k], A[:, iact[k]]) for k in range(nact)]
# More initialization
optold = REALMAX
nactold = nact
nfail = 0
# Zaikun 20211011: vmultd is computed from scratch at each iteration, but vmultc is inherited
# Powell's code can encounter infinite cycling, which did happen when testing the following CUTEst
# problems: DANWOODLS, GAUSS1LS, GAUSS2LS, GAUSS3LS, KOEBHELB, TAX13322, TAXR13322. Indeed, in all
# these cases, Inf/NaN appear in d due to extremely large values in A (up to 10^219). To resolve
# this, we set the maximal number of iterations to maxiter, and terminate if Inf/NaN occurs in d.
maxiter = np.minimum(10000, 100*max(num_constraints, num_vars))
for iter in range(maxiter):
if DEBUGGING:
assert all(vmultc >= 0)
if stage == 1:
optnew = cviol
else:
optnew = inprod(d, A[:, mcon-1])
# End the current stage of the calculation if 3 consecutive iterations have either failed to
# reduce the best calculated value of the objective function or to increase the number of active
# constraints since the best value was calculated. This strategy prevents cycling, but there is
# a remote possibility that it will cause premature termination.
if optnew < optold or nact > nactold:
nactold = nact
nfail = 0
else:
nfail += 1
optold = np.minimum(optold, optnew)
if nfail == 3:
break
# If icon exceeds nact, then we add the constraint with index iact[icon] to the active set.
if icon >= nact: # In Python this needs to be >= since Python is 0-indexed (in Fortran we have 1 > 0, in Python we need 0 >= 0)
zdasav[:nact] = zdota[:nact]
nactsav = nact
z, zdota, nact = qradd_Rdiag(A[:, iact[icon]], z, zdota, nact) # May update nact to nact+1
# Indeed it suffices to pass zdota[:min(num_vars, nact+1)] to qradd as follows:
# qradd(A[:, iact[icon]], z, zdota[:min(num_vars, nact+1)], nact)
if nact == nactsav + 1:
# N.B.: It is possible to index arrays using [nact, icon] when nact == icon.
# Zaikun 20211012: Why should vmultc[nact] = 0?
if nact != (icon + 1): # Need to add 1 to Python for 0 indexing
vmultc[[icon, nact-1]] = vmultc[nact-1], 0
iact[[icon, nact-1]] = iact[[nact-1, icon]]
else:
vmultc[nact-1] = 0
else:
# Zaikun 20211011:
# 1. VMULTD is calculated from scratch for the first time (out of 2) in one iteration.
# 2. Note that IACT has not been updated to replace IACT[NACT] with IACT[ICON]. Thus
# A[:, IACT[:NACT]] is the UNUPDATED version before QRADD (note Z[:, :NACT] remains the
# same before and after QRADD). Therefore if we supply ZDOTA to LSQR (as Rdiag) as
# Powell did, we should use the UNUPDATED version, namely ZDASAV.
# vmultd[:nact] = lsqr(A[:, iact[:nact]], A[:, iact[icon]], z[:, :nact], zdasav[:nact])
vmultd[:nact] = lsqr(A[:, iact[:nact]], A[:, iact[icon]], z[:, :nact], zdasav[:nact])
if not any(np.logical_and(vmultd[:nact] > 0, iact[:nact] <= num_constraints)):
# N.B.: This can be triggered by NACT == 0 (among other possibilities)! This is
# important, because NACT will be used as an index in the sequel.
break
# vmultd[NACT+1:mcon] is not used, but we have to initialize it in Fortran, or compilers
# complain about the where construct below (another solution: restrict where to 1:NACT).
vmultd[nact:mcon] = -1 # len(vmultd) == mcon
# Revise the Lagrange multipliers. The revision is not applicable to vmultc[nact:num_constraints].
fracmult = [vmultc[i]/vmultd[i] if vmultd[i] > 0 and iact[i] <= num_constraints else REALMAX for i in range(nact)]
# Only the places with vmultd > 0 and iact <= m is relevant below, if any.
frac = min(fracmult[:nact]) # fracmult[nact:mcon] may contain garbage
vmultc[:nact] = np.maximum(np.zeros(len(vmultc[:nact])), vmultc[:nact] - frac*vmultd[:nact])
# Reorder the active constraints so that the one to be replaced is at the end of the list.
# Exit if the new value of zdota[nact] is not acceptable. Powell's condition for the
# following If: not abs(zdota[nact]) > 0. Note that it is different from
# 'abs(zdota[nact]) <=0)' as zdota[nact] can be NaN.
# N.B.: We cannot arrive here with nact == 0, which should have triggered a break above
if np.isnan(zdota[nact - 1]) or abs(zdota[nact - 1]) <= EPS**2:
break
vmultc[[icon, nact - 1]] = 0, frac # vmultc[[icon, nact]] is valid as icon > nact
iact[[icon, nact - 1]] = iact[[nact - 1, icon]]
# end if nact == nactsav + 1
# In stage 2, ensure that the objective continues to be treated as the last active constraint.
# Zaikun 20211011, 20211111: Is it guaranteed for stage 2 that iact[nact-1] = mcon when
# iact[nact] != mcon??? If not, then how does the following procedure ensure that mcon is
# the last of iact[:nact]?
if stage == 2 and iact[nact - 1] != (mcon - 1):
if nact <= 1:
# We must exit, as nact-2 is used as an index below. Powell's code does not have this.
break
z, zdota[:nact] = qrexc_Rdiag(A[:, iact[:nact]], z, zdota[:nact], nact - 2) # We pass nact-2 in Python instead of nact-1
# Indeed, it suffices to pass Z[:, :nact] to qrexc as follows:
# z[:, :nact], zdota[:nact] = qrexc(A[:, iact[:nact]], z[:, :nact], zdota[:nact], nact - 1)
iact[[nact-2, nact-1]] = iact[[nact-1, nact-2]]
vmultc[[nact-2, nact-1]] = vmultc[[nact-1, nact-2]]
# Zaikun 20211117: It turns out that the last few lines do not guarantee iact[nact] == num_vars in
# stage 2; the following test cannot be passed. IS THIS A BUG?!
# assert iact[nact] == mcon or stage == 1, 'iact[nact] must == mcon in stage 2'
# Powell's code does not have the following. It avoids subsequent floating points exceptions.
if np.isnan(zdota[nact-1]) or abs(zdota[nact-1]) <= EPS**2:
break
# Set sdirn to the direction of the next change to the current vector of variables
# Usually during stage 1 the vector sdirn gives a search direction that reduces all the
# active constraint violations by one simultaneously.
if stage == 1:
sdirn -= ((inprod(sdirn, A[:, iact[nact-1]]) + 1)/zdota[nact-1])*z[:, nact-1]
else:
sdirn = -1/zdota[nact-1]*z[:, nact-1]
else: # icon < nact
# Delete the constraint with the index iact[icon] from the active set, which is done by
# reordering iact[icon:nact] into [iact[icon+1:nact], iact[icon]] and then reduce nact to
# nact - 1. In theory, icon > 0.
# assert icon > 0, "icon > 0 is required" # For Python I think this is irrelevant
z, zdota[:nact] = qrexc_Rdiag(A[:, iact[:nact]], z, zdota[:nact], icon) # qrexc does nothing if icon == nact
# Indeed, it suffices to pass Z[:, :nact] to qrexc as follows:
# z[:, :nact], zdota[:nact] = qrexc(A[:, iact[:nact]], z[:, :nact], zdota[:nact], icon)
iact[icon:nact] = [*iact[icon+1:nact], iact[icon]]
vmultc[icon:nact] = [*vmultc[icon+1:nact], vmultc[icon]]
nact -= 1
# Powell's code does not have the following. It avoids subsequent exceptions.
# Zaikun 20221212: In theory, nact > 0 in stage 2, as the objective function should always
# be considered as an "active constraint" --- more precisely, iact[nact] = mcon. However,
# looking at the code, I cannot see why in stage 2 nact must be positive after the reduction
# above. It did happen in stage 1 that nact became 0 after the reduction --- this is
# extremely rare, and it was never observed until 20221212, after almost one year of
# random tests. Maybe nact is theoretically positive even in stage 1?
if stage == 2 and nact < 0:
break # If this case ever occurs, we have to break, as nact is used as an index below.
if nact > 0:
if np.isnan(zdota[nact-1]) or abs(zdota[nact-1]) <= EPS**2:
break
# Set sdirn to the direction of the next change to the current vector of variables.
if stage == 1:
sdirn -= inprod(sdirn, z[:, nact]) * z[:, nact]
# sdirn is orthogonal to z[:, nact+1]
else:
sdirn = -1/zdota[nact-1] * z[:, nact-1]
# end if icon > nact
# Calculate the step to the trust region boundary or take the step that reduces cviol to 0.
# ----------------------------------------------------------------------------------------- #
# The following calculation of step is adopted from NEWUOA/BOBYQA/LINCOA. It seems to improve
# the performance of COBYLA. We also found that removing the precaution about underflows is
# beneficial to the overall performance of COBYLA --- the underflows are harmless anyway.
dd = delta*delta - inprod(d, d)
ss = inprod(sdirn, sdirn)
sd = inprod(sdirn, d)
if dd <= 0 or ss <= EPS * delta*delta or np.isnan(sd):
break
# sqrtd: square root of a discriminant. The max avoids sqrtd < abs(sd) due to underflow
sqrtd = max(np.sqrt(ss*dd + sd*sd), abs(sd), np.sqrt(ss * dd))
if sd > 0:
step = dd / (sqrtd + sd)
else:
step = (sqrtd - sd) / ss
# step < 0 should not happen. Step can be 0 or NaN when, e.g., sd or ss becomes inf
if step <= 0 or not np.isfinite(step):
break
# Powell's approach and comments are as follows.
# -------------------------------------------------- #
# The two statements below that include the factor eps prevent
# some harmless underflows that occurred in a test calculation
# (Zaikun: here, eps is the machine epsilon; Powell's original
# code used 1.0e-6, and Powell's code was written in single
# precision). Further, we skip the step if it could be 0 within
# a reasonable tolerance for computer rounding errors.
# !dd = delta*delta - sum(d**2, mask=(abs(d) >= EPS * delta))
# !ss = inprod(sdirn, sdirn)
# !if (dd <= 0) then
# ! exit
# !end if
# !sd = inprod(sdirn, d)
# !if (abs(sd) >= EPS * sqrt(ss * dd)) then
# ! step = dd / (sqrt(ss * dd + sd*sd) + sd)
# !else
# ! step = dd / (sqrt(ss * dd) + sd)
# !end if
# -------------------------------------------------- #
if stage == 1:
if isminor(cviol, step):
break
step = min(step, cviol)
# Set dnew to the new variables if step is the steplength, and reduce cviol to the corresponding
# maximum residual if stage 1 is being done
dnew = d + step * sdirn
if stage == 1:
cviol = np.max(np.append(0, matprod(dnew, A[:, iact[:nact]]) - b[iact[:nact]]))
# N.B.: cviol will be used when calculating vmultd[nact+1:mcon].
# Zaikun 20211011:
# 1. vmultd is computed from scratch for the second (out of 2) time in one iteration.
# 2. vmultd[:nact] and vmultd[nact:mcon] are calculated separately with no coupling.
# 3. vmultd will be calculated from scratch again in the next iteration.
# Set vmultd to the vmultc vector that would occur if d became dnew. A device is included to
# force vmultd[k] = 0 if deviations from this value can be attributed to computer rounding
# errors. First calculate the new Lagrange multipliers.
vmultd[:nact] = -lsqr(A[:, iact[:nact]], dnew, z[:, :nact], zdota[:nact])
if stage == 2:
vmultd[nact-1] = max(0, vmultd[nact-1]) # This seems never activated.
# Complete vmultd by finding the new constraint residuals. (Powell wrote "Complete vmultc ...")
cvshift = cviol - (matprod(dnew, A[:, iact]) - b[iact]) # Only cvshift[nact+1:mcon] is needed
cvsabs = matprod(abs(dnew), abs(A[:, iact])) + abs(b[iact]) + cviol
cvshift[isminor(cvshift, cvsabs)] = 0
vmultd[nact:mcon] = cvshift[nact:mcon]
# Calculate the fraction of the step from d to dnew that will be taken
fracmult = [vmultc[i]/(vmultc[i] - vmultd[i]) if vmultd[i] < 0 else REALMAX for i in range(len(vmultd))]
# Only the places with vmultd < 0 are relevant below, if any.
icon = np.argmin(np.append(1, fracmult)) - 1
frac = min(np.append(1, fracmult))
# Update d, vmultc, and cviol
dold = d
d = (1 - frac)*d + frac * dnew
vmultc = np.maximum(0, (1 - frac)*vmultc + frac*vmultd)
# Break in the case of inf/nan in d or vmultc.
if not (np.isfinite(primasum(abs(d))) and np.isfinite(primasum(abs(vmultc)))):
d = dold # Should we restore also iact, nact, vmultc, and z?
break
if stage == 1:
# cviol = (1 - frac) * cvold + frac * cviol # Powell's version
# In theory, cviol = np.max(np.append(d@A - b, 0)), yet the
# cviol updated as above can be quite different from this value if A has huge entries (e.g., > 1e20)
cviol = np.max(np.append(0, matprod(d, A) - b))
if icon < 0 or icon >= mcon:
# In Powell's code, the condition is icon == 0. Indeed, icon < 0 cannot hold unless
# fracmult contains only nan, which should not happen; icon >= mcon should never occur.
break
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert np.size(iact) == mcon
assert np.size(vmultc) == mcon
assert all(vmultc >= 0)
assert np.size(d) == num_vars
assert all(np.isfinite(d))
assert np.linalg.norm(d) <= 2 * delta
assert np.size(z, 0) == num_vars and np.size(z, 1) == num_vars
assert nact >= 0 and nact <= np.minimum(mcon, num_vars)
return iact, nact, d, vmultc, z
def trrad(delta_in, dnorm, eta1, eta2, gamma1, gamma2, ratio):
'''
This function updates the trust region radius according to RATIO and DNORM.
'''
# Preconditions
if DEBUGGING:
assert delta_in >= dnorm > 0
assert 0 <= eta1 <= eta2 < 1
assert 0 < gamma1 < 1 < gamma2
# By the definition of RATIO in ratio.f90, RATIO cannot be NaN unless the
# actual reduction is NaN, which should NOT happen due to the moderated extreme
# barrier.
assert not np.isnan(ratio)
#====================#
# Calculation starts #
#====================#
if ratio <= eta1:
delta = gamma1 * dnorm # Powell's UOBYQA/NEWUOA
# delta = gamma1 * delta_in # Powell's COBYLA/LINCOA
# delta = min(gamma1 * delta_in, dnorm) # Powell's BOBYQA
elif ratio <= eta2:
delta = max(gamma1 * delta_in, dnorm) # Powell's UOBYQA/NEWUOA/BOBYQA/LINCOA
else:
delta = max(gamma1 * delta_in, gamma2 * dnorm) # Powell's NEWUOA/BOBYQA
# delta = max(delta_in, gamma2 * dnorm) # Modified version. Works well for UOBYQA
# For noise-free CUTEst problems of <= 100 variables, Powell's version works slightly better
# than the modified one.
# delta = max(delta_in, 1.25*dnorm, dnorm + rho) # Powell's UOBYQA
# delta = min(max(gamma1 * delta_in, gamma2 * dnorm), gamma3 * delta_in) # Powell's LINCOA, gamma3 = np.sqrt(2)
# For noisy problems, the following may work better.
# if ratio <= eta1:
# delta = gamma1 * dnorm
# elseif ratio <= eta2: # Ensure DELTA >= DELTA_IN
# delta = delta_in
# else: # Ensure DELTA > DELTA_IN with a constant factor
# delta = max(delta_in * (1 + gamma2) / 2, gamma2 * dnorm)
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert delta > 0
return delta
venv/lib/python3.11/site-packages/scipy/_lib/pyprima/cobyla/update.py
+289
View File
@@ -0,0 +1,289 @@
'''
This module contains subroutines concerning the update of the interpolation set.
Translated from Zaikun Zhang's modern-Fortran reference implementation in PRIMA.
Dedicated to late Professor M. J. D. Powell FRS (1936--2015).
Python translation by Nickolai Belakovski.
'''
from ..common.consts import DEBUGGING
from ..common.infos import DAMAGING_ROUNDING, INFO_DEFAULT
from ..common.linalg import isinv, matprod, outprod, inprod, inv, primasum
import numpy as np
def updatexfc(jdrop, constr, cpen, cstrv, d, f, conmat, cval, fval, sim, simi):
'''
This function revises the simplex by updating the elements of SIM, SIMI, FVAL, CONMAT, and CVAL
'''
# Local variables
itol = 1
# Sizes
num_constraints = np.size(constr)
num_vars = np.size(sim, 0)
# Preconditions
if DEBUGGING:
assert num_constraints >= 0
assert num_vars >= 1
assert jdrop >= 0 and jdrop <= num_vars + 1
assert not any(np.isnan(constr) | np.isneginf(constr))
assert not (np.isnan(cstrv) | np.isposinf(cstrv))
assert np.size(d) == num_vars and all(np.isfinite(d))
assert not (np.isnan(f) | np.isposinf(f))
assert np.size(conmat, 0) == num_constraints and np.size(conmat, 1) == num_vars + 1
assert not (np.isnan(conmat) | np.isneginf(conmat)).any()
assert np.size(cval) == num_vars + 1 and not any(cval < 0 | np.isnan(cval) | np.isposinf(cval))
assert np.size(fval) == num_vars + 1 and not any(np.isnan(fval) | np.isposinf(fval))
assert np.size(sim, 0) == num_vars and np.size(sim, 1) == num_vars + 1
assert np.isfinite(sim).all()
assert all(primasum(abs(sim[:, :num_vars]), axis=0) > 0)
assert np.size(simi, 0) == num_vars and np.size(simi, 1) == num_vars
assert np.isfinite(simi).all()
assert isinv(sim[:, :num_vars], simi, itol)
#====================#
# Calculation starts #
#====================#
# Do nothing when JDROP is None. This can only happen after a trust-region step.
if jdrop is None: # JDROP is None is impossible if the input is correct.
return conmat, cval, fval, sim, simi, INFO_DEFAULT
sim_old = sim
simi_old = simi
if jdrop < num_vars:
sim[:, jdrop] = d
simi_jdrop = simi[jdrop, :] / inprod(simi[jdrop, :], d)
simi -= outprod(matprod(simi, d), simi_jdrop)
simi[jdrop, :] = simi_jdrop
else: # jdrop == num_vars
sim[:, num_vars] += d
sim[:, :num_vars] -= np.tile(d, (num_vars, 1)).T
simid = matprod(simi, d)
sum_simi = primasum(simi, axis=0)
simi += outprod(simid, sum_simi / (1 - sum(simid)))
# Check whether SIMI is a poor approximation to the inverse of SIM[:, :NUM_VARS]
# Calculate SIMI from scratch if the current one is damaged by rounding errors.
itol = 1
erri = np.max(abs(matprod(simi, sim[:, :num_vars]) - np.eye(num_vars))) # np.max returns NaN if any input is NaN
if erri > 0.1 * itol or np.isnan(erri):
simi_test = inv(sim[:, :num_vars])
erri_test = np.max(abs(matprod(simi_test, sim[:, :num_vars]) - np.eye(num_vars)))
if erri_test < erri or (np.isnan(erri) and not np.isnan(erri_test)):
simi = simi_test
erri = erri_test
# If SIMI is satisfactory, then update FVAL, CONMAT, CVAL, and the pole position. Otherwise restore
# SIM and SIMI, and return with INFO = DAMAGING_ROUNDING.
if erri <= itol:
fval[jdrop] = f
conmat[:, jdrop] = constr
cval[jdrop] = cstrv
# Switch the best vertex to the pole position SIM[:, NUM_VARS] if it is not there already
conmat, cval, fval, sim, simi, info = updatepole(cpen, conmat, cval, fval, sim, simi)
else:
info = DAMAGING_ROUNDING
sim = sim_old
simi = simi_old
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert np.size(conmat, 0) == num_constraints and np.size(conmat, 1) == num_vars + 1
assert not (np.isnan(conmat) | np.isneginf(conmat)).any()
assert np.size(cval) == num_vars + 1 and not any(cval < 0 | np.isnan(cval) | np.isposinf(cval))
assert np.size(fval) == num_vars + 1 and not any(np.isnan(fval) | np.isposinf(fval))
assert np.size(sim, 0) == num_vars and np.size(sim, 1) == num_vars + 1
assert np.isfinite(sim).all()
assert all(primasum(abs(sim[:, :num_vars]), axis=0) > 0)
assert np.size(simi, 0) == num_vars and np.size(simi, 1) == num_vars
assert np.isfinite(simi).all()
assert isinv(sim[:, :num_vars], simi, itol) or info == DAMAGING_ROUNDING
return sim, simi, fval, conmat, cval, info
def findpole(cpen, cval, fval):
'''
This subroutine identifies the best vertex of the current simplex with respect to the merit
function PHI = F + CPEN * CSTRV.
'''
# Size
num_vars = np.size(fval) - 1
# Preconditions
if DEBUGGING:
assert cpen > 0
assert np.size(cval) == num_vars + 1 and not any(cval < 0 | np.isnan(cval) | np.isposinf(cval))
assert np.size(fval) == num_vars + 1 and not any(np.isnan(fval) | np.isposinf(fval))
#====================#
# Calculation starts #
#====================#
# Identify the optimal vertex of the current simplex
jopt = np.size(fval) - 1
phi = fval + cpen * cval
phimin = min(phi)
# Essentially jopt = np.argmin(phi). However, we keep jopt = num_vars unless there
# is a strictly better choice. When there are multiple choices, we choose the jopt
# with the smallest value of cval.
if phimin < phi[jopt] or any((cval < cval[jopt]) & (phi <= phi[jopt])):
# While we could use argmin(phi), there may be two places where phi achieves
# phimin, and in that case we should choose the one with the smallest cval.
jopt = np.ma.array(cval, mask=(phi > phimin)).argmin()
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert jopt >= 0 and jopt < num_vars + 1
assert jopt == num_vars or phi[jopt] < phi[num_vars] or (phi[jopt] <= phi[num_vars] and cval[jopt] < cval[num_vars])
return jopt
def updatepole(cpen, conmat, cval, fval, sim, simi):
#--------------------------------------------------------------------------------------------------!
# This subroutine identifies the best vertex of the current simplex with respect to the merit
# function PHI = F + CPEN * CSTRV, and then switch this vertex to SIM[:, NUM_VARS], which Powell called
# the "pole position" in his comments. CONMAT, CVAL, FVAL, and SIMI are updated accordingly.
#
# N.B. 1: In precise arithmetic, the following two procedures produce the same results:
# 1) apply UPDATEPOLE to SIM twice, first with CPEN = CPEN1 and then with CPEN = CPEN2;
# 2) apply UPDATEPOLE to SIM with CPEN = CPEN2.
# In finite-precision arithmetic, however, they may produce different results unless CPEN1 = CPEN2.
#
# N.B. 2: When JOPT == N+1, the best vertex is already at the pole position, so there is nothing to
# switch. However, as in Powell's code, the code below will check whether SIMI is good enough to
# work as the inverse of SIM(:, 1:N) or not. If not, Powell's code would invoke an error return of
# COBYLB; our implementation, however, will try calculating SIMI from scratch; if the recalculated
# SIMI is still of poor quality, then UPDATEPOLE will return with INFO = DAMAGING_ROUNDING,
# informing COBYLB that SIMI is poor due to damaging rounding errors.
#
# N.B. 3: UPDATEPOLE should be called when and only when FINDPOLE can potentially returns a value
# other than N+1. The value of FINDPOLE is determined by CPEN, CVAL, and FVAL, the latter two being
# decided by SIM. Thus UPDATEPOLE should be called after CPEN or SIM changes. COBYLA updates CPEN at
# only two places: the beginning of each trust-region iteration, and when REDRHO is called;
# SIM is updated only by UPDATEXFC, which itself calls UPDATEPOLE internally. Therefore, we only
# need to call UPDATEPOLE after updating CPEN at the beginning of each trust-region iteration and
# after each invocation of REDRHO.
# Local variables
itol = 1
# Sizes
num_constraints = conmat.shape[0]
num_vars = sim.shape[0]
# Preconditions
if DEBUGGING:
assert num_constraints >= 0
assert num_vars >= 1
assert cpen > 0
assert np.size(conmat, 0) == num_constraints and np.size(conmat, 1) == num_vars + 1
assert not (np.isnan(conmat) | np.isneginf(conmat)).any()
assert np.size(cval) == num_vars + 1 and not any(cval < 0 | np.isnan(cval) | np.isposinf(cval))
assert np.size(fval) == num_vars + 1 and not any(np.isnan(fval) | np.isposinf(fval))
assert np.size(sim, 0) == num_vars and np.size(sim, 1) == num_vars + 1
assert np.isfinite(sim).all()
assert all(primasum(abs(sim[:, :num_vars]), axis=0) > 0)
assert np.size(simi, 0) == num_vars and np.size(simi, 1) == num_vars
assert np.isfinite(simi).all()
assert isinv(sim[:, :num_vars], simi, itol)
#====================#
# Calculation starts #
#====================#
# INFO must be set, as it is an output.
info = INFO_DEFAULT
# Identify the optimal vertex of the current simplex.
jopt = findpole(cpen, cval, fval)
# Switch the best vertex to the pole position SIM[:, NUM_VARS] if it is not there already and update
# SIMI. Before the update, save a copy of SIM and SIMI. If the update is unsuccessful due to
# damaging rounding errors, we restore them and return with INFO = DAMAGING_ROUNDING.
sim_old = sim.copy()
simi_old = simi.copy()
if 0 <= jopt < num_vars:
# Unless there is a bug in FINDPOLE it is guaranteed that JOPT >= 0
# When JOPT == NUM_VARS, there is nothing to switch; in addition SIMI[JOPT, :] will be illegal.
# fval[[jopt, -1]] = fval[[-1, jopt]]
# conmat[:, [jopt, -1]] = conmat[:, [-1, jopt]] # Exchange CONMAT[:, JOPT] AND CONMAT[:, -1]
# cval[[jopt, -1]] = cval[[-1, jopt]]
sim[:, num_vars] += sim[:, jopt]
sim_jopt = sim[:, jopt].copy()
sim[:, jopt] = 0 # np.zeros(num_constraints)?
sim[:, :num_vars] -= np.tile(sim_jopt, (num_vars, 1)).T
# The above update is equivalent to multiplying SIM[:, :NUM_VARS] from the right side by a matrix whose
# JOPT-th row is [-1, -1, ..., -1], while all the other rows are the same as those of the
# identity matrix. It is easy to check that the inverse of this matrix is itself. Therefore,
# SIMI should be updated by a multiplication with this matrix (i.e. its inverse) from the left
# side, as is done in the following line. The JOPT-th row of the updated SIMI is minus the sum
# of all rows of the original SIMI, whereas all the other rows remain unchanged.
# NDB 20250114: In testing the cutest problem 'SYNTHES2' between the Python implementation and
# the Fortran bindings, I saw a difference between the following for loop and the
# np.sum command. The differences were small, on the order of 1e-16, i.e. epsilon.
# According to numpy documentation, np.sum sometimes uses partial pairwise summation,
# depending on the memory layout of the array and the axis specified.
# for i in range(simi.shape[1]):
# simi[jopt, i] = -sum(simi[:, i])
simi[jopt, :] = -primasum(simi, axis=0)
# Check whether SIMI is a poor approximation to the inverse of SIM[:, :NUM_VARS]
# Calculate SIMI from scratch if the current one is damaged by rounding errors.
erri = np.max(abs(matprod(simi, sim[:, :num_vars]) - np.eye(num_vars))) # np.max returns NaN if any input is NaN
itol = 1
if erri > 0.1 * itol or np.isnan(erri):
simi_test = inv(sim[:, :num_vars])
erri_test = np.max(abs(matprod(simi_test, sim[:, :num_vars]) - np.eye(num_vars)))
if erri_test < erri or (np.isnan(erri) and not np.isnan(erri_test)):
simi = simi_test
erri = erri_test
# If SIMI is satisfactory, then update FVAL, CONMAT, and CVAL. Otherwise restore SIM and SIMI, and
# return with INFO = DAMAGING_ROUNDING.
if erri <= itol:
if 0 <= jopt < num_vars:
fval[[jopt, num_vars]] = fval[[num_vars, jopt]]
conmat[:, [jopt, num_vars]] = conmat[:, [num_vars, jopt]]
cval[[jopt, num_vars]] = cval[[num_vars, jopt]]
else: # erri > itol or erri is NaN
info = DAMAGING_ROUNDING
sim = sim_old
simi = simi_old
#==================#
# Calculation ends #
#==================#
# Postconditions
if DEBUGGING:
assert findpole(cpen, cval, fval) == num_vars or info == DAMAGING_ROUNDING
assert np.size(conmat, 0) == num_constraints and np.size(conmat, 1) == num_vars + 1
assert not (np.isnan(conmat) | np.isneginf(conmat)).any()
assert np.size(cval) == num_vars + 1 and not any(cval < 0 | np.isnan(cval) | np.isposinf(cval))
assert np.size(fval) == num_vars + 1 and not any(np.isnan(fval) | np.isposinf(fval))
assert np.size(sim, 0) == num_vars and np.size(sim, 1) == num_vars + 1
assert np.isfinite(sim).all()
assert all(primasum(abs(sim[:, :num_vars]), axis=0) > 0)
assert np.size(simi, 0) == num_vars and np.size(simi, 1) == num_vars
assert np.isfinite(simi).all()
# Do not check SIMI = SIM[:, :num_vars]^{-1}, as it may not be true due to damaging rounding.
assert isinv(sim[:, :num_vars], simi, itol) or info == DAMAGING_ROUNDING
return conmat, cval, fval, sim, simi, info