From 4e6d6c0fe56d77cc8f7600763ac539e0ab250c23 Mon Sep 17 00:00:00 2001 From: Neel Ghoshal <72409908+NeelGhoshal@users.noreply.github.com> Date: Tue, 31 Aug 2021 09:25:21 +0530 Subject: [PATCH 1/6] added backtracking params --- .../python/optimizer/linesearch/BUILD | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tensorflow_probability/python/optimizer/linesearch/BUILD b/tensorflow_probability/python/optimizer/linesearch/BUILD index 72bb8fc04a..a4d6f2aac6 100644 --- a/tensorflow_probability/python/optimizer/linesearch/BUILD +++ b/tensorflow_probability/python/optimizer/linesearch/BUILD @@ -29,6 +29,7 @@ multi_substrate_py_library( srcs = ["__init__.py"], deps = [ ":hager_zhang", + ":backtracking", "//tensorflow_probability/python/internal:all_util", "//tensorflow_probability/python/optimizer/linesearch/internal", ], @@ -45,6 +46,16 @@ multi_substrate_py_library( ], ) +multi_substrate_py_library( + name = "backtracking", + srcs = ["backtracking.py"], + srcs_version = "PY3", + deps = [ + # tensorflow dep, + "//tensorflow_probability/python/internal:dtype_util", + ], +) + multi_substrate_py_test( name = "hager_zhang_test", size = "medium", @@ -59,3 +70,18 @@ multi_substrate_py_test( "//tensorflow_probability/python/internal:test_util", ], ) + +multi_substrate_py_test( + name = "backtracking_test", + size = "small", + srcs = ["backtracking_test.py"], + numpy_tags = ["notap"], + shard_count = 5, + deps = [ + # absl/testing:parameterized dep, + # numpy dep, + # tensorflow dep, + "//tensorflow_probability", + "//tensorflow_probability/python/internal:test_util", + ], +) From 802da6dcac2a2108bafd63b38d0558299055ba4d Mon Sep 17 00:00:00 2001 From: Neel Ghoshal <72409908+NeelGhoshal@users.noreply.github.com> Date: Tue, 31 Aug 2021 09:26:37 +0530 Subject: [PATCH 2/6] added backtracking line search --- .../optimizer/linesearch/backtracking.py | 47 +++++++++++++++++++ .../optimizer/linesearch/backtracking_test.py | 20 ++++++++ 2 files changed, 67 insertions(+) create mode 100644 tensorflow_probability/python/optimizer/linesearch/backtracking.py create mode 100644 tensorflow_probability/python/optimizer/linesearch/backtracking_test.py diff --git a/tensorflow_probability/python/optimizer/linesearch/backtracking.py b/tensorflow_probability/python/optimizer/linesearch/backtracking.py new file mode 100644 index 0000000000..14a2defecb --- /dev/null +++ b/tensorflow_probability/python/optimizer/linesearch/backtracking.py @@ -0,0 +1,47 @@ +# Copyright 2018 The TensorFlow Probability Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Implements the Backtracking line search algorithm. +Line searches are a central component for many optimization algorithms (e.g. +BFGS, conjugate gradient, ISTA, FISTA etc). Sophisticated line search methods +aim to find the appropriate step length. +This module implements the Backtracking Line Search Algorithm. +""" + +function = lambda x: x**2 +3*x +differentiation = lambda x: 2*x + 3 +value = 11 + + + + +def backtracking ( function, + differentiation, + value, + beta = 0.707, + alpha = 1): + + while function(value-(alpha*differentiation(value)))>function(value) -(alpha/2)*((differentiation(value))**2): + alpha *= beta + return alpha + +backtracking(function,differentiation,value) + + + + + + + + diff --git a/tensorflow_probability/python/optimizer/linesearch/backtracking_test.py b/tensorflow_probability/python/optimizer/linesearch/backtracking_test.py new file mode 100644 index 0000000000..9d9c570c23 --- /dev/null +++ b/tensorflow_probability/python/optimizer/linesearch/backtracking_test.py @@ -0,0 +1,20 @@ +import unittest +import Backtracking + + +class TestBacktracking(unittest.TestCase): + + + + def test_ndegree(self): + self.assertEqual(Backtracking.backtracking + (lambda x: x**2 +3*x, lambda x: 2*x + 3,11), 0.49984899999999993) + + self.assertEqual(Backtracking.backtracking + (lambda x: x**10 +3*x, lambda x: 10*(x**9) + 3,2), 6.07776055631376e-05) + + self.assertEqual(Backtracking.backtracking + (lambda x: x**5 - 3*x, lambda x: 5*(x**4) - 3,2),1) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 46ce72750712b0984c21f282ca9f3e13e5d7be45 Mon Sep 17 00:00:00 2001 From: Neel Ghoshal <72409908+NeelGhoshal@users.noreply.github.com> Date: Tue, 31 Aug 2021 09:30:17 +0530 Subject: [PATCH 3/6] updated backtracking_test --- .../optimizer/linesearch/backtracking_test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tensorflow_probability/python/optimizer/linesearch/backtracking_test.py b/tensorflow_probability/python/optimizer/linesearch/backtracking_test.py index 9d9c570c23..31bd408f44 100644 --- a/tensorflow_probability/python/optimizer/linesearch/backtracking_test.py +++ b/tensorflow_probability/python/optimizer/linesearch/backtracking_test.py @@ -1,20 +1,23 @@ import unittest -import Backtracking - +import backtracking +from tensorflow_probability.python.internal import test_util class TestBacktracking(unittest.TestCase): def test_ndegree(self): - self.assertEqual(Backtracking.backtracking + self.assertEqual(backtracking.backtracking (lambda x: x**2 +3*x, lambda x: 2*x + 3,11), 0.49984899999999993) - self.assertEqual(Backtracking.backtracking + self.assertEqual(backtracking.backtracking (lambda x: x**10 +3*x, lambda x: 10*(x**9) + 3,2), 6.07776055631376e-05) - self.assertEqual(Backtracking.backtracking + self.assertEqual(backtracking.backtracking (lambda x: x**5 - 3*x, lambda x: 5*(x**4) - 3,2),1) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() + +if __name__ == '__main__': + test_util.main() From 4f4e5e46ff300c50d44b8cd67e05cd0cc4b469bc Mon Sep 17 00:00:00 2001 From: Neel Ghoshal <72409908+NeelGhoshal@users.noreply.github.com> Date: Tue, 31 Aug 2021 09:31:35 +0530 Subject: [PATCH 4/6] updated init --- tensorflow_probability/python/optimizer/linesearch/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_probability/python/optimizer/linesearch/__init__.py b/tensorflow_probability/python/optimizer/linesearch/__init__.py index 2e46361b39..5a07baed07 100644 --- a/tensorflow_probability/python/optimizer/linesearch/__init__.py +++ b/tensorflow_probability/python/optimizer/linesearch/__init__.py @@ -20,10 +20,12 @@ from tensorflow_probability.python.internal import all_util from tensorflow_probability.python.optimizer.linesearch.hager_zhang import hager_zhang +from tensorflow_probability.python.optimizer.linesearch.backtracking import backtracking _allowed_symbols = [ 'hager_zhang', + 'backtracking', ] all_util.remove_undocumented(__name__, _allowed_symbols) From 07fafc047f1d057ae4f2c0d4a44606de4ac4ed50 Mon Sep 17 00:00:00 2001 From: Neel Ghoshal <72409908+NeelGhoshal@users.noreply.github.com> Date: Tue, 31 Aug 2021 09:34:04 +0530 Subject: [PATCH 5/6] updated backtracking line search --- .../python/optimizer/linesearch/backtracking.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/tensorflow_probability/python/optimizer/linesearch/backtracking.py b/tensorflow_probability/python/optimizer/linesearch/backtracking.py index 14a2defecb..e9f44502c7 100644 --- a/tensorflow_probability/python/optimizer/linesearch/backtracking.py +++ b/tensorflow_probability/python/optimizer/linesearch/backtracking.py @@ -32,16 +32,7 @@ def backtracking ( function, beta = 0.707, alpha = 1): - while function(value-(alpha*differentiation(value)))>function(value) -(alpha/2)*((differentiation(value))**2): + while function(value-(alpha*differentiation(value)))>function(value) - + (alpha/2)*((differentiation(value))**2): alpha *= beta return alpha - -backtracking(function,differentiation,value) - - - - - - - - From 45ed841e3cff6cdc7cd1b2d96dd874d9070318f7 Mon Sep 17 00:00:00 2001 From: Neel Ghoshal <72409908+NeelGhoshal@users.noreply.github.com> Date: Tue, 31 Aug 2021 09:48:47 +0530 Subject: [PATCH 6/6] update backtracking line search --- .../python/optimizer/linesearch/backtracking.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tensorflow_probability/python/optimizer/linesearch/backtracking.py b/tensorflow_probability/python/optimizer/linesearch/backtracking.py index e9f44502c7..c1d249e114 100644 --- a/tensorflow_probability/python/optimizer/linesearch/backtracking.py +++ b/tensorflow_probability/python/optimizer/linesearch/backtracking.py @@ -18,13 +18,9 @@ aim to find the appropriate step length. This module implements the Backtracking Line Search Algorithm. """ - -function = lambda x: x**2 +3*x -differentiation = lambda x: 2*x + 3 -value = 11 - - - +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function def backtracking ( function, differentiation,