IT++ Logo
newton_search.cpp
Go to the documentation of this file.
1
30#include <itpp/base/specmat.h>
31#include <itpp/stat/misc_stat.h>
32
33
34namespace itpp
35{
36
37
39{
40 method = BFGS;
41
42 initial_stepsize = 1.0;
43 stop_epsilon_1 = 1e-4;
44 stop_epsilon_2 = 1e-8;
45 max_evaluations = 100;
46
47 f = NULL;
48 df_dx = NULL;
49
50 no_feval = 0;
51 init = false;
52 finished = false;
53 trace = false;
54}
55
56void Newton_Search::set_function(double(*function)(const vec&))
57{
58 // Add checks to see that function is OK???
59 f = function;
60}
61
63{
64 // Add checks to see that function is OK???
65 df_dx = gradient;
66}
67
68void Newton_Search::set_start_point(const vec &x, const mat &D)
69{
70 // check that parameters are valid???
71 x_start = x;
72 n = x.size();
73 D_start = D;
74
75 finished = false;
76 init = true;
77}
78
80{
81 // check that parameters are valid???
82 x_start = x;
83 n = x.size();
84 D_start = eye(n);
85
86 finished = false;
87 init = true;
88}
89
91{
92 // Check parameters and function call ???
93 // check that x_start is a valid point, not a NaN and that norm(x0) is not inf
94
95 it_assert(f != NULL, "Newton_Search: Function pointer is not set");
96 it_assert(df_dx != NULL, "Newton_Search: Gradient function pointer is not set");
97
98 it_assert(init, "Newton_Search: Starting point is not set");
99
100
101 F = f(x_start); // function initial value
102 vec g = df_dx(x_start); // gradient initial value
103 vec x = x_start;
104 no_feval++;
105
106 finished = false;
107
108 // Initial inverse Hessian, D
109 mat D = D_start;
110
111
112 bool fst = true; // what is this???
113
114 bool stop = false;
115
116 // Finish initialization
117 no_iter = 0;
118 ng = max(abs(g)); // norm(g,inf)
119
120 double Delta = initial_stepsize;
121 nh = 0; // what is this???
122 vec h;
123
124 if (trace) { // prepare structures to store trace data
125 x_values.set_size(max_evaluations);
126 F_values.set_size(max_evaluations);
127 ng_values.set_size(max_evaluations);
128 Delta_values.set_size(max_evaluations);
129 }
130
132 ls.set_functions(f, df_dx);
133
134 if (ng <= stop_epsilon_1)
135 stop = true;
136 else {
137 h = zeros(n);
138 nh = 0;
139 ls.set_stop_values(0.05, 0.99);
140 ls.set_max_iterations(5);
141 ls.set_max_stepsize(2);
142 }
143
144 bool more = true; //???
145
146 while (!stop && more) {
147 vec h, w, y, v;
148 double yh, yv, a;
149
150 // Previous values
151 vec xp = x, gp = g;
152 // double Fp = F; ### 2006-02-03 by ediap: Unused variable!
153 double nx = norm(x);
154
155 h = D * (-g);
156 nh = norm(h);
157 bool red = false;
158
159 if (nh <= stop_epsilon_2*(stop_epsilon_2 + nx)) // stop criterion
160 stop = true;
161 else {
162 if (fst || nh > Delta) { // Scale to ||h|| = Delta
163 h = (Delta / nh) * h;
164 nh = Delta;
165 fst = false;
166 red = true;
167 }
168 // Line search
169 ls.set_start_point(x, F, g, h);
170 more = ls.search(x, F, g);
171 no_feval = no_feval + ls.get_no_function_evaluations();
172
173 if (more == false) { // something wrong in linesearch?
174 x_end = x;
175 return false;
176 }
177 else {
178 if (ls.get_alpha() < 1) // Reduce Delta
179 Delta = .35 * Delta;
180 else if (red && (ls.get_slope_ratio() > .7)) // Increase Delta
181 Delta = 3 * Delta;
182
183 // Update ||g||
184 ng = max(abs(g)); // norm(g,inf);
185
186 if (trace) { // store trace
187 x_values(no_iter) = x;
188 F_values(no_iter) = F;
189 ng_values(no_iter) = ng;
190 Delta_values(no_iter) = Delta;
191 }
192
193 no_iter++;
194 h = x - xp;
195 nh = norm(h);
196
197 //if (nh == 0)
198 // found = 4;
199 //else {
200 y = g - gp;
201 yh = dot(y, h);
202 if (yh > std::sqrt(eps) * nh * norm(y)) {
203 // Update D
204 v = D * y;
205 yv = dot(y, v);
206 a = (1 + yv / yh) / yh;
207 w = (a / 2) * h - v / yh;
208 D += outer_product(w, h) + outer_product(h, w); //D = D + w*h' + h*w';
209 } // update D
210 // Check stopping criteria
211 double thrx = stop_epsilon_2 * (stop_epsilon_2 + norm(x));
212 if (ng <= stop_epsilon_1)
213 stop = true; // stop = 1, stop by small gradient
214 else if (nh <= thrx)
215 stop = true; // stop = 2, stop by small x-step
216 else if (no_feval >= max_evaluations)
217 stop = true; // stop = 3, number of function evaluations exeeded
218 else
219 Delta = std::max(Delta, 2 * thrx);
220 //} found =4
221 } // Nonzero h
222 } // nofail
223 } // iteration
224
225 // Set return values
226 x_end = x;
227 finished = true;
228
229 if (trace) { // trim size of trace output
230 x_values.set_size(no_iter, true);
231 F_values.set_size(no_iter, true);
232 ng_values.set_size(no_iter, true);
233 Delta_values.set_size(no_iter, true);
234 }
235
236 return true;
237}
238
240{
241 bool state = search();
242 xn = get_solution();
243 return state;
244}
245
246bool Newton_Search::search(const vec &x0, vec &xn)
247{
249 bool state = search();
250 xn = get_solution();
251 return state;
252}
253
255{
256 it_assert(finished, "Newton_Search: search is not run yet");
257 return x_end;
258}
259
261{
262 if (finished)
263 return F;
264 else
265 it_warning("Newton_Search::get_function_value, search has not been run");
266
267 return 0.0;
268}
269
271{
272 if (finished)
273 return ng;
274 else
275 it_warning("Newton_Search::get_stop_1, search has not been run");
276
277 return 0.0;
278}
279
281{
282 if (finished)
283 return nh;
284 else
285 it_warning("Newton_Search::get_stop_2, search has not been run");
286
287 return 0.0;
288}
289
291{
292 if (finished)
293 return no_iter;
294 else
295 it_warning("Newton_Search::get_no_iterations, search has not been run");
296
297 return 0;
298}
299
301{
302 if (finished)
303 return no_feval;
304 else
305 it_warning("Newton_Search::get_no_function_evaluations, search has not been run");
306
307 return 0;
308}
309
310
312{
313 if (finished) {
314 if (trace) { // trim size of trace output
315 xvalues = x_values;
316 Fvalues = F_values;
317 ngvalues = ng_values;
318 dvalues = Delta_values;
319 }
320 else
321 it_warning("Newton_Search::get_trace, trace is not enabled");
322 }
323 else
324 it_warning("Newton_Search::get_trace, search has not been run");
325}
326
327//================================== Line_Search =============================================
328
330{
331 method = Soft;
332
333 if (method == Soft) {
334 stop_rho = 1e-3;
335 stop_beta = 0.99;
336 }
337
338 max_iterations = 10;
339 max_stepsize = 10;
340
341 f = NULL;
342 df_dx = NULL;
343 no_feval = 0;
344 init = false;
345 finished = false;
346 trace = false;
347}
348
349void Line_Search::set_function(double(*function)(const vec&))
350{
351 // Add checks to see that function is OK???
352 f = function;
353}
354
355void Line_Search::set_gradient(vec(*gradient)(const vec&))
356{
357 // Add checks to see that function is OK???
358 df_dx = gradient;
359}
360
361
362void Line_Search::set_stop_values(double rho, double beta)
363{
364 // test input values???
365 stop_rho = rho;
366 stop_beta = beta;
367}
368
369
370void Line_Search::set_start_point(const vec &x, double F, const vec &g, const vec &h)
371{
372 // check values ???
373 x_start = x;
374 F_start = F;
375 g_start = g;
376 h_start = h;
377 n = x.size();
378
379 finished = false;
380 init = true;
381}
382
383void Line_Search::get_solution(vec &xn, double &Fn, vec &gn)
384{
385 it_assert(finished, "Line_Search: search is not run yet");
386
387 xn = x_end;
388 Fn = F_end;
389 gn = g_end;
390}
391
393{
394 it_assert(f != NULL, "Line_Search: Function pointer is not set");
395 it_assert(df_dx != NULL, "Line_Search: Gradient function pointer is not set");
396
397 it_assert(init, "Line_search: Starting point is not set");
398
399 // Default return values and simple checks
400 x_end = x_start;
401 F_end = F_start;
402 g_end = g_start;
403
404 // add some checks???
405 finished = false;
406
407 vec g;
408
409 // return parameters
410 no_feval = 0;
411 slope_ratio = 1;
412
413
414
415 // Check descent condition
416 double dF0 = dot(h_start, g_end);
417
418 if (trace) { // prepare structures to store trace data
419 alpha_values.set_size(max_iterations);
420 F_values.set_size(max_iterations);
421 dF_values.set_size(max_iterations);
422 alpha_values(0) = 0;
423 F_values(0) = F_end;
424 dF_values(0) = dF0;
425 }
426
427
428 if (dF0 >= -10*eps*norm(h_start)*norm(g_end)) { // not significantly downhill
429 if (trace) { // store trace
430 alpha_values.set_size(1, true);
431 F_values.set_size(1, true);
432 dF_values.set_size(1, true);
433 }
434 return false;
435 }
436
437 // Finish initialization
438 double F0 = F_start, slope0, slopethr;
439
440 if (method == Soft) {
441 slope0 = stop_rho * dF0;
442 slopethr = stop_beta * dF0;
443 }
444 else { // exact line search
445 slope0 = 0;
446 slopethr = stop_rho * std::abs(dF0);
447 }
448
449 // Get an initial interval for am
450 double a = 0, Fa = F_end, dFa = dF0;
451 bool stop = false;
452 double b = std::min(1.0, max_stepsize), Fb = 0, dFb = 0;
453
454
455 while (!stop) {
456 Fb = f(x_start + b * h_start);
457 g = df_dx(x_start + b * h_start);
458 // check if these values are OK if not return false???
459 no_feval++;
460
461 dFb = dot(g, h_start);
462 if (trace) { // store trace
463 alpha_values(no_feval) = b;
464 F_values(no_feval) = Fb;
465 dF_values(no_feval) = dFb;
466 }
467
468 if (Fb < F0 + slope0*b) { // new lower bound
469 alpha = b;
470 slope_ratio = dFb / dF0; // info(2);
471
472 if (method == Soft) {
473 a = b;
474 Fa = Fb;
475 dFa = dFb;
476 }
477
478 x_end = x_start + b * h_start;
479 F_end = Fb;
480 g_end = g;
481
482 if ((dFb < std::min(slopethr, 0.0)) && (no_feval < max_iterations) && (b < max_stepsize)) {
483 // Augment right hand end
484 if (method == Exact) {
485 a = b;
486 Fa = Fb;
487 dFa = dFb;
488 }
489 if (2.5*b >= max_stepsize)
490 b = max_stepsize;
491 else
492 b = 2 * b;
493 }
494 else
495 stop = true;
496 }
497 else
498 stop = true;
499 } // phase 1: expand interval
500
501
502
503 if (stop) // OK so far. Check stopping criteria
504 stop = (no_feval >= max_iterations)
505 || (b >= max_stepsize && dFb < slopethr)
506 || (a > 0 && dFb >= slopethr);
507 // Commented by ediap 2006-07-17: redundant check
508 // || ( (method == Soft) && (a > 0 & dFb >= slopethr) ); // OK
509
510
511 if (stop && trace) {
512 alpha_values.set_size(no_feval, true);
513 F_values.set_size(no_feval, true);
514 dF_values.set_size(no_feval, true);
515 }
516
517 // Refine interval
518 while (!stop) {
519
520 double c, Fc, dFc;
521
522 //c = interpolate(xfd,n);
523 double C = Fb - Fa - (b - a) * dFa;
524 if (C >= 5*n*eps*b) {
525 double A = a - 0.5 * dFa * (sqr(b - a) / C);
526 c = std::min(std::max(a + 0.1 * (b - a), A), b - 0.1 * (b - a)); // % Ensure significant resuction
527 }
528 else
529 c = (a + b) / 2;
530
531 Fc = f(x_start + c * h_start);
532 g = df_dx(x_start + c * h_start);
533 dFc = dot(g, h_start);
534 // check these values???
535 no_feval++;
536
537 if (trace) { // store trace
538 alpha_values(no_feval) = c;
539 F_values(no_feval) = Fc;
540 dF_values(no_feval) = dFc;
541 }
542
543 if (method == Soft) {
544 // soft line method
545 if (Fc < F0 + slope0*c) { // new lower bound
546 alpha = c;
547 slope_ratio = dFc / dF0;
548
549 x_end = x_start + c * h_start;
550 F_end = Fc;
551 g_end = g;
552 a = c;
553 Fa = Fc;
554 dFa = dFc; // xfd(:,1) = xfd(:,3);
555 stop = (dFc > slopethr);
556 }
557 else { // new upper bound
558 b = c;
559 Fb = Fc;
560 dFb = dFc; // xfd(:,2) = xfd(:,3);
561 }
562
563 }
564 else { // Exact line search
565 if (Fc < F_end) { // better approximant
566 alpha = c;
567 slope_ratio = dFc / dF0;
568 x_end = x_start + c * h_start;
569 F_end = Fc;
570 g_end = g;
571 }
572 if (dFc < 0) { // new lower bound
573 a = c;
574 Fa = Fc;
575 dFa = dFc; // xfd(:,1) = xfd(:,3);
576 }
577 else { //new upper bound
578 b = c;
579 Fb = Fc;
580 dFb = dFc; // xfd(:,2) = xfd(:,3);
581 }
582 stop = (std::abs(dFc) <= slopethr) | ((b - a) < stop_beta * b);
583 }
584
585 stop = (stop | (no_feval >= max_iterations));
586 } // refine
587
588 finished = true;
589
590 if (trace) { // store trace
591 alpha_values.set_size(no_feval + 1, true);
592 F_values.set_size(no_feval + 1, true);
593 dF_values.set_size(no_feval + 1, true);
594 }
595
596 return true;
597}
598
599bool Line_Search::search(vec &xn, double &Fn, vec &gn)
600{
601 bool state = search();
602 get_solution(xn, Fn, gn);
603 return state;
604}
605
606bool Line_Search::search(const vec &x, double F, const vec &g, const vec &h,
607 vec &xn, double &Fn, vec &gn)
608{
609 set_start_point(x, F, g, h);
610 bool state = search();
611 get_solution(xn, Fn, gn);
612 return state;
613}
614
615
617{
618 if (finished)
619 return alpha;
620 else
621 it_warning("Line_Search::get_alpha, search has not been run");
622
623 return 0.0;
624}
625
627{
628 if (finished)
629 return slope_ratio;
630 else
631 it_warning("Line_Search::get_slope_raio, search has not been run");
632
633 return 0.0;
634}
635
637{
638 if (finished)
639 return no_feval;
640 else
641 it_warning("Line_Search::get_no_function_evaluations, search has not been run");
642
643 return 0;
644}
645
646
648{
649 it_assert(value > 0, "Line_Search, max iterations must be > 0");
650 max_iterations = value;
651}
652
654{
655 it_assert(value > 0, "Line_Search, max stepsize must be > 0");
656 max_stepsize = value;
657}
658
660{
661 method = search_method;
662
663 if (method == Soft) {
664 stop_rho = 1e-3;
665 stop_beta = 0.99;
666 }
667 else { // exact line search
668 method = Exact;
669 stop_rho = 1e-3;
670 stop_beta = 1e-3;
671 }
672}
673
674
676{
677 if (finished) {
678 if (trace) { // trim size of trace output
679 alphavalues = alpha_values;
680 Fvalues = F_values;
681 dFvalues = dF_values;
682 }
683 else
684 it_warning("Line_Search::get_trace, trace is not enabled");
685 }
686 else
687 it_warning("Line_Search::get_trace, search has not been run");
688}
689
690// =========================== functions ==============================================
691
692vec fminunc(double(*function)(const vec&), vec(*gradient)(const vec&), const vec &x0)
693{
695 newton.set_functions(function, gradient);
696
697 vec xn;
698 newton.search(x0, xn);
699
700 return xn;
701}
702
703
704
705} // namespace itpp
General array class.
Definition array.h:105
int size() const
Returns the number of data elements in the array object.
Definition array.h:155
void set_size(int n, bool copy=false)
Resizing an Array<T>.
Definition array.h:257
void set_function(double(*function)(const vec &))
Set function pointer.
void set_method(const Line_Search_Method &method)
Set Line search method.
void set_stop_values(double rho, double beta)
Set stop criterion values.
void set_gradient(vec(*gradient)(const vec &))
Set gradient function pointer.
void set_max_stepsize(double value)
Set max stepsize.
void get_trace(vec &alphavalues, vec &Fvalues, vec &dFvalues)
bool search()
Do the line search.
void get_solution(vec &xn, double &Fn, vec &gn)
Get solution, function value and gradient at solution point.
Line_Search()
Default constructor.
int get_no_function_evaluations()
return number of function evaluations used in search
void set_max_iterations(int value)
Set max number of iterations.
double get_alpha()
return alpha at solution point, xn = x + alpha h
double get_slope_ratio()
return the slope ratio at solution poin, xn
void set_start_point(const vec &x, double F, const vec &g, const vec &h)
Set start point for search.
Newton Search.
vec get_solution()
Get solution, function value and gradient at solution point.
double get_stop_1()
get value of stop criterion 1 at solution point
double get_stop_2()
get value of stop criterion 2 at solution point
void get_trace(Array< vec > &xvalues, vec &Fvalues, vec &ngvalues, vec &dvalues)
void set_gradient(vec(*gradient)(const vec &))
Set gradient function pointer.
bool search()
Do the line search.
void set_function(double(*function)(const vec &))
Set function pointer.
void set_start_point(const vec &x, const mat &D)
Set start point x for search and approx inverse Hessian at x.
double get_function_value()
get function value at solution point
int get_no_iterations()
get number of iterations used to reach solution
int get_no_function_evaluations()
get number of function evaluations used to reach solution
Newton_Search()
Default constructor.
#define it_warning(s)
Display a warning message.
Definition itassert.h:173
#define it_assert(t, s)
Abort if t is not true.
Definition itassert.h:94
T max(const Vec< T > &v)
Maximum value of vector.
Definition min_max.h:45
vec sqr(const cvec &data)
Absolute square of elements.
Definition elem_math.cpp:36
ITPP_EXPORT vec zeros(int size)
A Double vector of zeros.
double norm(const cvec &v)
Calculate the 2-norm: norm(v)=sqrt(sum(abs(v).^2))
Definition misc_stat.cpp:77
Miscellaneous statistics functions and classes - header file.
itpp namespace
Definition itmex.h:37
Line_Search_Method
Line Search method.
vec fminunc(double(*function)(const vec &), vec(*gradient)(const vec &), const vec &x0)
Unconstrained minimization.
template void eye(int, mat &)
Template instantiation of eye.
Num_T dot(const Vec< Num_T > &v1, const Vec< Num_T > &v2)
Inner (dot) product of two vectors v1 and v2.
Definition vec.h:1005
const double eps
Constant eps.
Definition misc.h:109
bin abs(const bin &inbin)
absolute value of bin
Definition binary.h:174
Mat< Num_T > outer_product(const Vec< Num_T > &v1, const Vec< Num_T > &v2, bool hermitian=false)
Outer product of two vectors v1 and v2.
Definition vec.h:1021
Newton Search optimization algorithms - header file.
Definitions of special vectors and matrices.
SourceForge Logo

Generated on Mon Jun 10 2024 11:49:23 for IT++ by Doxygen 1.9.8