Combining Conjugate Direction Methods with Stochastic Approximation of Gradients

@inproceedings{citeulike:2644133, abstract = {The method of conjugate directions provides a very effective way to optimize large, deterministic systems by gradient descent. In its standard form, however, it is not amenable to stochastic approximation of the gradient. Here we explore ideas from conjugate gradient in the stochastic (online) setting, using fast Hessian-gradient products to set up low-dimensional Krylov subspaces within individual mini-batches. In our benchmark experiments the resulting online learning algorithms converge orders of magnitude faster than ordinary stochastic gradient descent. Numerical experiments are carried out for both the linear, realisable as well as the non-linear, non-realisable case.}, author = {Schraudolph, Nicol and Graepel, Thore }, citeulike-article-id = {2644133}, editor = {Christopher}, journal = {Proceedings of the Ninth International Workshop on Artificial Intelligence and Statistics}, keywords = {optimization}, posted-at = {2008-06-09 20:52:55}, priority = {2}, title = {Combining Conjugate Direction Methods with Stochastic Approximation of Gradients}, year = {2003} }

TY - CONF ID - citeulike:2644133 L3 - citeulike-article-id:2644133 TI - Combining Conjugate Direction Methods with Stochastic Approximation of Gradients JF - Proceedings of the Ninth International Workshop on Artificial Intelligence and Statistics N2 - The method of conjugate directions provides a very effective way to optimize large, deterministic systems by gradient descent. In its standard form, however, it is not amenable to stochastic approximation of the gradient. Here we explore ideas from conjugate gradient in the stochastic (online) setting, using fast Hessian-gradient products to set up low-dimensional Krylov subspaces within individual mini-batches. In our benchmark experiments the resulting online learning algorithms converge orders of magnitude faster than ordinary stochastic gradient descent. Numerical experiments are carried out for both the linear, realisable as well as the non-linear, non-realisable case. KW - optimization AU - Schraudolph, Nicol AU - Graepel, Thore A2 - Christopher PY - 2003/// ER -