import numpy as np
import fnmatch
from abc import ABCMeta
from .base import Base
from .function_list import OptFunctionList
from .memory import Memory
from iwopy.utils import RegularDiscretizationGrid
[docs]class Problem(Base, metaclass=ABCMeta):
"""
Abstract base class for optimization problems.
Attributes
----------
objs: iwopy.core.OptFunctionList
The objective functions
cons: iwopy.core.OptFunctionList
The constraints
memory: iwopy.core.Memory
The memory, or None
:group: core
"""
INT_INF = RegularDiscretizationGrid.INT_INF
[docs] def __init__(self, name, mem_size=None, mem_keyf=None):
"""
Constructor
Parameters
----------
name: str
The problem's name
mem_size: int, optional
The memory size, default no memory
mem_keyf: Function, optional
The memory key function. Parameters:
(vars_int, vars_float), returns key Object
"""
super().__init__(name)
self.objs = OptFunctionList(self, "objs")
self.cons = OptFunctionList(self, "cons")
self.memory = None
self._mem_size = mem_size
self._mem_keyf = mem_keyf
self._cons_mi = None
self._cons_ma = None
self._cons_tol = None
[docs] def var_names_int(self):
"""
The names of integer variables.
Returns
-------
names: list of str
The names of the integer variables
"""
return []
[docs] def initial_values_int(self):
"""
The initial values of the integer variables.
Returns
-------
values: numpy.ndarray
Initial int values, shape: (n_vars_int,)
"""
return 0
[docs] def min_values_int(self):
"""
The minimal values of the integer variables.
Use -self.INT_INF for unbounded.
Returns
-------
values: numpy.ndarray
Minimal int values, shape: (n_vars_int,)
"""
return -self.INT_INF
[docs] def max_values_int(self):
"""
The maximal values of the integer variables.
Use self.INT_INF for unbounded.
Returns
-------
values: numpy.ndarray
Maximal int values, shape: (n_vars_int,)
"""
return self.INT_INF
@property
def n_vars_int(self):
"""
The number of int variables
Returns
-------
n: int
The number of int variables
"""
return len(self.var_names_int())
[docs] def var_names_float(self):
"""
The names of float variables.
Returns
-------
names: list of str
The names of the float variables
"""
return []
[docs] def initial_values_float(self):
"""
The initial values of the float variables.
Returns
-------
values: numpy.ndarray
Initial float values, shape: (n_vars_float,)
"""
return None
[docs] def min_values_float(self):
"""
The minimal values of the float variables.
Use -numpy.inf for unbounded.
Returns
-------
values: numpy.ndarray
Minimal float values, shape: (n_vars_float,)
"""
return -np.inf
[docs] def max_values_float(self):
"""
The maximal values of the float variables.
Use numpy.inf for unbounded.
Returns
-------
values: numpy.ndarray
Maximal float values, shape: (n_vars_float,)
"""
return np.inf
@property
def n_vars_float(self):
"""
The number of float variables
Returns
-------
n: int
The number of float variables
"""
return len(self.var_names_float())
def _apply_varmap(self, vtype, f, ftype, varmap):
"""
Helper function for mapping function variables
to problem variables
"""
if varmap is None:
return
pnms = (
list(self.var_names_int())
if vtype == "int"
else list(self.var_names_float())
)
vmap = {}
for fv, pv in varmap.items():
if isinstance(pv, str):
pvl = fnmatch.filter(pnms, pv)
if len(pvl) == 0:
raise ValueError(
f"Problem '{self.name}': {vtype} varmap rule '{fv} --> {pv}' failed for {ftype} '{f.name}', pattern '{pv}' not found among problem {vtype} variables {pnms}"
)
elif len(pvl) > 1:
raise ValueError(
f"Problem '{self.name}': Require unique match of {vtype} variable '{fv}' of {ftype} '{f.name}' to problem variables, found {pvl} for pattern '{pv}'"
)
else:
vmap[fv] = pvl[0]
elif np.issubdtype(type(pv), np.integer):
if pv < 0 or pv >= len(pnms):
raise ValueError(
f"Problem '{self.name}': varmap rule '{fv} --> {pv}' cannot be applied for {len(pnms)} {vtype} variables {pnms}"
)
vmap[fv] = pnms[pv]
else:
raise ValueError(
f"Problem '{self.name}': varmap_{vtype} target variable in '{fv} --> {pv}' of {ftype} '{f.name}' is neither str nor int"
)
if vtype == "int":
f.rename_vars_int(vmap)
else:
f.rename_vars_float(vmap)
[docs] def add_objective(
self,
objective,
varmap_int=None,
varmap_float=None,
verbosity=0,
):
"""
Add an objective to the problem.
Parameters
----------
objective: iwopy.Objective
The objective
varmap_int: dict, optional
Mapping from objective variables to
problem variables. Key: str or int,
value: str or int
varmap_float: dict, optional
Mapping from objective variables to
problem variables. Key: str or int,
value: str or int
verbosity: int
The verbosity level, 0 = silent
"""
if not objective.initialized:
objective.initialize(verbosity)
self._apply_varmap("int", objective, "objective", varmap_int)
self._apply_varmap("float", objective, "objective", varmap_float)
self.objs.append(objective)
[docs] def add_constraint(
self,
constraint,
varmap_int=None,
varmap_float=None,
verbosity=0,
):
"""
Add a constraint to the problem.
Parameters
----------
constraint: iwopy.Constraint
The constraint
varmap_int: dict, optional
Mapping from objective variables to
problem variables. Key: str or int,
value: str or int
varmap_float: dict, optional
Mapping from objective variables to
problem variables. Key: str or int,
value: str or int
verbosity: int
The verbosity level, 0 = silent
"""
if not constraint.initialized:
constraint.initialize(verbosity)
self._apply_varmap("int", constraint, "constraint", varmap_int)
self._apply_varmap("float", constraint, "constraint", varmap_float)
self.cons.append(constraint)
cmi, cma = constraint.get_bounds()
ctol = np.zeros(constraint.n_components(), dtype=np.float64)
ctol[:] = constraint.tol
if self._cons_mi is None:
self._cons_mi = cmi
self._cons_ma = cma
self._cons_tol = ctol
else:
self._cons_mi = np.append(self._cons_mi, cmi, axis=0)
self._cons_ma = np.append(self._cons_ma, cma, axis=0)
self._cons_tol = np.append(self._cons_tol, ctol, axis=0)
@property
def min_values_constraints(self):
"""
Gets the minimal values of constraints
Returns
-------
cmi: numpy.ndarray
The minimal constraint values, shape: (n_constraints,)
"""
return self._cons_mi
@property
def max_values_constraints(self):
"""
Gets the maximal values of constraints
Returns
-------
cma: numpy.ndarray
The maximal constraint values, shape: (n_constraints,)
"""
return self._cons_ma
@property
def constraints_tol(self):
"""
Gets the tolerance values of constraints
Returns
-------
ctol: numpy.ndarray
The constraint tolerance values, shape: (n_constraints,)
"""
return self._cons_tol
@property
def n_objectives(self):
"""
The total number of objectives,
i.e., the sum of all components
Returns
-------
n_obj: int
The total number of objective
functions
"""
return self.objs.n_components()
@property
def n_constraints(self):
"""
The total number of constraints,
i.e., the sum of all components
Returns
-------
n_con: int
The total number of constraint
functions
"""
return self.cons.n_components()
def _find_vars(self, vars_int, vars_float, func, ret_inds=False):
"""
Helper function for reducing problem variables
to function variables
"""
vnmsi = list(self.var_names_int())
vnmsf = list(self.var_names_float())
ivars = []
for v in func.var_names_int:
if v not in vnmsi:
raise ValueError(
f"Problem '{self.name}': int variable '{v}' of function '{func.name}' not among int problem variables {vnmsi}"
)
ivars.append(vnmsi.index(v))
fvars = []
for v in func.var_names_float:
if v not in vnmsf:
raise ValueError(
f"Problem '{self.name}': float variable '{v}' of function '{func.name}' not among float problem variables {vnmsf}"
)
fvars.append(vnmsf.index(v))
if len(vars_float.shape) == 1:
varsi = vars_int[ivars] if len(vars_int) else np.array([], dtype=np.float64)
varsf = (
vars_float[fvars] if len(vars_float) else np.array([], dtype=np.float64)
)
else:
n_pop = vars_float.shape[0]
varsi = (
vars_int[:, ivars]
if len(vars_int)
else np.zeros((n_pop, 0), dtype=np.float64)
)
varsf = (
vars_float[:, fvars]
if len(vars_float)
else np.zeros((n_pop, 0), dtype=np.float64)
)
if ret_inds:
return ivars, fvars
else:
return varsi, varsf
[docs] def calc_gradients(
self,
vars_int,
vars_float,
func,
components,
ivars,
fvars,
vrs,
pop=False,
verbosity=0,
):
"""
The actual gradient calculation, not to be called directly
(call `get_gradients` instead).
Can be overloaded in derived classes, the base class only considers
analytic derivatives.
Parameters
----------
vars_int: np.array
The integer variable values, shape: (n_vars_int,)
vars_float: np.array
The float variable values, shape: (n_vars_float,)
func: iwopy.core.OptFunctionList, optional
The functions to be differentiated, or None
for a list of all objectives and all constraints
(in that order)
components: list of int, optional
The function's component selection, or None for all
ivars: list of int
The indices of the function int variables in the problem
fvars: list of int
The indices of the function float variables in the problem
vrs: list of int
The function float variable indices wrt which the
derivatives are to be calculated
pop: bool
Flag for vectorizing calculations via population
verbosity: int
The verbosity level, 0 = silent
Returns
-------
gradients: numpy.ndarray
The gradients of the functions, shape:
(n_components, n_vrs)
"""
n_vars = len(vrs)
n_cmpnts = func.n_components() if components is None else len(components)
varsi = vars_int[ivars] if len(vars_int) else np.array([])
varsf = vars_float[fvars] if len(vars_float) else np.array([])
gradients = np.full((n_cmpnts, n_vars), np.nan, dtype=np.float64)
for vi, v in enumerate(vrs):
if v in fvars:
gradients[:, vi] = func.ana_deriv(
varsi, varsf, fvars.index(v), components
)
else:
gradients[:, vi] = 0
return gradients
[docs] def get_gradients(
self,
vars_int,
vars_float,
func=None,
components=None,
vars=None,
pop=False,
verbosity=0,
):
"""
Obtain gradients of a function that is linked to the
problem.
The func object typically is a `iwopy.core.OptFunctionList`
object that contains a selection of objectives and/or constraints
that were previously added to this problem. By default all
objectives and constraints (and all their components) are
being considered, cf. class `ProblemDefaultFunc`.
Parameters
----------
vars_int: np.array
The integer variable values, shape: (n_vars_int,)
vars_float: np.array
The float variable values, shape: (n_vars_float,)
func: iwopy.core.OptFunctionList, optional
The functions to be differentiated, or None
for a list of all objectives and all constraints
(in that order)
components: list of int, optional
The function's component selection, or None for all
vars: list of int or str, optional
The float variables wrt which the
derivatives are to be calculated, or
None for all
verbosity: int
The verbosity level, 0 = silent
pop: bool
Flag for vectorizing calculations via population
Returns
-------
gradients: numpy.ndarray
The gradients of the functions, shape:
(n_components, n_vars)
"""
# set and check func:
if func is None:
func = ProblemDefaultFunc(self)
if func.problem is not self:
raise ValueError(
f"Problem '{self.name}': Attempt to calculate gradient for function '{func.name}' which is linked to different problem '{func.problem.name}'"
)
if not func.initialized:
func.initialize(verbosity=(0 if verbosity < 2 else verbosity - 1))
# find function variables:
ivars, fvars = self._find_vars(vars_int, vars_float, func, ret_inds=True)
# find names of differentiation variables:
vnmsf = list(self.var_names_float())
if vars is None:
vars = vnmsf
else:
tvars = []
for v in vars:
if np.issubdtype(type(v), np.integer):
if v < 0 or v > len(vnmsf):
raise ValueError(
f"Problem '{self.name}': Variable index {v} exceeds problem float variables, count = {len(vnmsf)}"
)
tvars.append(vnmsf[v])
elif isinstance(v, str):
vl = fnmatch.filter(vnmsf, v)
if not len(vl):
raise ValueError(
f"Problem '{self.name}': No match for variable pattern '{v}' among problem float variable {vnmsf}"
)
tvars += vl
else:
raise TypeError(
f"Problem '{self.name}': Illegal type '{type(v)}', expecting str or int"
)
vars = tvars
# find variable indices among function float variables:
vrs = []
hvnmsf = np.array(vnmsf)[fvars].tolist()
for v in vars:
if v not in hvnmsf:
raise ValueError(
f"Problem '{self.name}': Selected gradient variable '{v}' not in function variables '{hvnmsf}' for function '{func.name}'"
)
vrs.append(hvnmsf.index(v))
# calculate gradients:
gradients = self.calc_gradients(
vars_int,
vars_float,
func,
components,
ivars,
fvars,
vrs,
pop=pop,
verbosity=verbosity,
)
# check success:
nog = np.where(np.isnan(gradients))[1]
if len(nog):
nvrs = np.unique(np.array(vars)[nog]).tolist()
raise ValueError(
f"Problem '{self.name}': Failed to calculate derivatives for variables {nvrs}. Maybe wrap this problem into DiscretizeRegGrid?"
)
return gradients
[docs] def initialize(self, verbosity=1):
"""
Initialize the problem.
Parameters
----------
verbosity: int
The verbosity level, 0 = silent
"""
if not self.objs.initialized:
self.objs.initialize(verbosity)
if not self.cons.initialized:
self.cons.initialize(verbosity)
if verbosity:
s = f"Problem '{self.name}' ({type(self).__name__}): Initializing"
print(s)
self._hline = "-" * len(s)
print(self._hline)
if self._mem_size is not None:
self.memory = Memory(self._mem_size, self._mem_keyf)
if verbosity:
print(f" Memory size : {self.memory.size}")
print(self._hline)
n_int = self.n_vars_int
n_float = self.n_vars_float
if verbosity:
print(f" n_vars_int : {n_int}")
print(f" n_vars_float: {n_float}")
print(self._hline)
if verbosity:
print(f" n_objectives: {self.objs.n_functions}")
print(f" n_obj_cmptns: {self.n_objectives}")
print(self._hline)
print(f" n_constraints: {self.cons.n_functions}")
print(f" n_con_cmptns: {self.n_constraints}")
print(self._hline)
if self.n_objectives == 0:
raise ValueError("Problem initialized without added objectives.")
self._maximize = np.zeros(self.n_objectives, dtype=bool)
i0 = 0
for f in self.objs.functions:
i1 = i0 + f.n_components()
self._maximize[i0:i1] = f.maximize()
i0 = i1
super().initialize(verbosity)
@property
def maximize_objs(self):
"""
Flags for objective maximization
Returns
-------
maximize: numpy.ndarray
Boolean flag for maximization of objective,
shape: (n_objectives,)
"""
return self._maximize
[docs] def apply_individual(self, vars_int, vars_float):
"""
Apply new variables to the problem.
Parameters
----------
vars_int: np.array
The integer variable values, shape: (n_vars_int,)
vars_float: np.array
The float variable values, shape: (n_vars_float,)
Returns
-------
problem_results: Any
The results of the variable application
to the problem
"""
return None
[docs] def apply_population(self, vars_int, vars_float):
"""
Apply new variables to the problem,
for a whole population.
Parameters
----------
vars_int: np.array
The integer variable values, shape: (n_pop, n_vars_int)
vars_float: np.array
The float variable values, shape: (n_pop, n_vars_float)
Returns
-------
problem_results: Any
The results of the variable application
to the problem
"""
return None
[docs] def evaluate_individual(self, vars_int, vars_float, ret_prob_res=False):
"""
Evaluate a single individual of the problem.
Parameters
----------
vars_int: np.array
The integer variable values, shape: (n_vars_int,)
vars_float: np.array
The float variable values, shape: (n_vars_float,)
ret_prob_res: bool
Flag for additionally returning of problem results
Returns
-------
objs: np.array
The objective function values, shape: (n_objectives,)
con: np.array
The constraints values, shape: (n_constraints,)
prob_res: object, optional
The problem results
"""
objs, cons = None, None
if not ret_prob_res and self.memory is not None:
memres = self.memory.lookup_individual(vars_int, vars_float)
if memres is not None:
objs, cons = memres
results = None
if objs is None:
results = self.apply_individual(vars_int, vars_float)
varsi, varsf = self._find_vars(vars_int, vars_float, self.objs)
objs = self.objs.calc_individual(varsi, varsf, results)
varsi, varsf = self._find_vars(vars_int, vars_float, self.cons)
cons = self.cons.calc_individual(varsi, varsf, results)
if self.memory is not None:
self.memory.store_individual(vars_int, vars_float, objs, cons)
if ret_prob_res:
return objs, cons, results
else:
return objs, cons
[docs] def evaluate_population(self, vars_int, vars_float, ret_prob_res=False):
"""
Evaluate all individuals of a population.
Parameters
----------
vars_int: np.array
The integer variable values, shape: (n_pop, n_vars_int)
vars_float: np.array
The float variable values, shape: (n_pop, n_vars_float)
ret_prob_res: bool
Flag for additionally returning of problem results
Returns
-------
objs: np.array
The objective function values, shape: (n_pop, n_objectives)
cons: np.array
The constraints values, shape: (n_pop, n_constraints)
prob_res: object, optional
The problem results
"""
from_mem = False
if not ret_prob_res and self.memory is not None:
memres = self.memory.lookup_population(vars_int, vars_float)
if memres is not None:
todo = np.any(np.isnan(memres), axis=1)
from_mem = not np.all(todo)
if from_mem:
objs = memres[:,: self.n_objectives]
cons = memres[:, self.n_objectives :]
del memres
if np.any(todo):
vals_int = vars_int[todo]
vals_float = vars_float[todo]
results = self.apply_population(vals_int, vals_float)
varsi, varsf = self._find_vars(vals_int, vals_float, self.objs)
ores = self.objs.calc_population(varsi, varsf, results)
objs[todo] = ores
varsi, varsf = self._find_vars(vals_int, vals_float, self.cons)
cres = self.cons.calc_population(varsi, varsf, results)
cons[todo] = cres
self.memory.store_population(vals_int, vals_float, ores, cres)
else:
results = self.apply_population(vars_int, vars_float)
varsi, varsf = self._find_vars(vars_int, vars_float, self.objs)
objs = self.objs.calc_population(varsi, varsf, results)
varsi, varsf = self._find_vars(vars_int, vars_float, self.cons)
cons = self.cons.calc_population(varsi, varsf, results)
if self.memory is not None:
self.memory.store_population(vars_int, vars_float, objs, cons)
if ret_prob_res:
return objs, cons, results
else:
return objs, cons
[docs] def check_constraints_individual(self, constraint_values, verbosity=0):
"""
Check if the constraints are fullfilled for the
given individual.
Parameters
----------
constraint_values: np.array
The constraint values, shape: (n_components,)
verbosity: int
The verbosity level, 0 = silent
Returns
-------
values: np.array
The boolean result, shape: (n_components,)
"""
val = constraint_values
out = np.zeros(self.n_constraints, dtype=bool)
i0 = 0
for c in self.cons.functions:
i1 = i0 + c.n_components()
out[i0:i1] = c.check_individual(val[i0:i1], verbosity)
i0 = i1
return out
[docs] def check_constraints_population(self, constraint_values, verbosity=0):
"""
Check if the constraints are fullfilled for the
given population.
Parameters
----------
constraint_values: np.array
The constraint values, shape: (n_pop, n_components)
verbosity: int
The verbosity level, 0 = silent
Returns
-------
values: np.array
The boolean result, shape: (n_pop, n_components)
"""
val = constraint_values
n_pop = val.shape[0]
out = np.zeros((n_pop, self.n_constraints), dtype=bool)
i0 = 0
for c in self.cons.functions:
i1 = i0 + c.n_components()
out[:, i0:i1] = c.check_population(val[:, i0:i1], verbosity)
i0 = i1
return out
[docs] def finalize_individual(self, vars_int, vars_float, verbosity=1):
"""
Finalization, given the champion data.
Parameters
----------
vars_int: np.array
The optimal integer variable values, shape: (n_vars_int,)
vars_float: np.array
The optimal float variable values, shape: (n_vars_float,)
verbosity: int
The verbosity level, 0 = silent
Returns
-------
problem_results: Any
The results of the variable application
to the problem
objs: np.array
The objective function values, shape: (n_objectives,)
cons: np.array
The constraints values, shape: (n_constraints,)
"""
results = self.apply_individual(vars_int, vars_float)
varsi, varsf = self._find_vars(vars_int, vars_float, self.objs)
objs = self.objs.finalize_individual(varsi, varsf, results, verbosity)
varsi, varsf = self._find_vars(vars_int, vars_float, self.cons)
cons = self.cons.finalize_individual(varsi, varsf, results, verbosity)
return results, objs, cons
[docs] def finalize_population(self, vars_int, vars_float, verbosity=0):
"""
Finalization, given the final population data.
Parameters
----------
vars_int: np.array
The integer variable values of the final
generation, shape: (n_pop, n_vars_int)
vars_float: np.array
The float variable values of the final
generation, shape: (n_pop, n_vars_float)
verbosity: int
The verbosity level, 0 = silent
Returns
-------
problem_results: Any
The results of the variable application
to the problem
objs: np.array
The final objective function values, shape: (n_pop, n_components)
cons: np.array
The final constraint values, shape: (n_pop, n_constraints)
"""
results = self.apply_population(vars_int, vars_float)
varsi, varsf = self._find_vars(vars_int, vars_float, self.objs)
objs = self.objs.finalize_population(varsi, varsf, results, verbosity)
varsi, varsf = self._find_vars(vars_int, vars_float, self.cons)
cons = self.cons.finalize_population(varsi, varsf, results, verbosity)
return results, objs, cons
[docs] def prob_res_einsum_individual(self, prob_res_list, coeffs):
"""
Calculate the einsum of problem results
Parameters
----------
prob_res_list: list
The problem results
coeffs: numpy.ndarray
The coefficients
Returns
-------
prob_res: object
The weighted sum of problem results
"""
if not len(prob_res_list) or prob_res_list[0] is None:
return None
raise NotImplementedError(
f"Problem '{self.name}': Einsum not implemented for problem results type '{type(prob_res_list[0]).__name__}'"
)
[docs] def prob_res_einsum_population(self, prob_res_list, coeffs):
"""
Calculate the einsum of problem results
Parameters
----------
prob_res_list: list
The problem results
coeffs: numpy.ndarray
The coefficients
Returns
-------
prob_res: object
The weighted sum of problem results
"""
if not len(prob_res_list) or prob_res_list[0] is None:
return None
raise NotImplementedError(
f"Problem '{self.name}': Einsum not implemented for problem results type '{type(prob_res_list[0]).__name__}'"
)
[docs]class ProblemDefaultFunc(OptFunctionList):
"""
The default function of a problem
for gradient calculations.
:group: core
"""
[docs] def __init__(self, problem):
"""
Constructor
Parameters
----------
problem: iwopy.core.Problem
The problem
"""
super().__init__(problem, "objs_cons")
for f in problem.objs.functions:
self.append(f)
for f in problem.cons.functions:
self.append(f)