From 502fc50f5ad53df1187ff53099c1fc4ad69eb71d Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Tue, 25 Oct 2022 15:47:09 -0400 Subject: [PATCH 1/7] closes #320 - real time output for tool deploy scripts --- .gitignore | 3 ++ scripts/plugins/deploy_plugins.py | 56 ++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 2b34c0ba..fdfaec34 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,6 @@ cython_debug/ # Developers Notes.md Dockerfile.local + +# Secrets +secrets diff --git a/scripts/plugins/deploy_plugins.py b/scripts/plugins/deploy_plugins.py index d93c59cf..cb1a465e 100644 --- a/scripts/plugins/deploy_plugins.py +++ b/scripts/plugins/deploy_plugins.py @@ -270,28 +270,60 @@ def Deploy_Plugins(): ) try: - result = subprocess.run( - [ - plugin_deploy_language, - plugin_deploy_script_path, - stack_action, - ], - universal_newlines=True, - capture_output=True, + # error_file = "stderr_{}.log".format(plugin_name) + # with open(error_file,"wb") as err: + process = subprocess.Popen([ + plugin_deploy_language, + plugin_deploy_script_path, + stack_action, + ], + # stdout=subprocess.PIPE, + # TODO: adding any form of stderr to the above call causes the below polling to not work properly for real time output + # instead, it causes the script to wait until everything is complete before outputting anything + # tried: stderr=err, stderr=subprocess.PIPE, stderr=subprocess.STDOUT + # for stderr=err, err is a file: with open(error_file,"wb") as err: + # stderr=err, + # stderr=subprocess.STDOUT, + + # TODO: oddly, if you set only stderr=subprocess.PIPE and NOT stdout, + # the process.poll() below being focused on stderr does seem to output both stdout and stderr in real time + stderr=subprocess.PIPE, + universal_newlines=True ) + while True: + # output_stdout = process.stdout.readline() + output_stderr = process.stderr.readline() + + # if we have a return code, exit the while loop + if process.poll() is not None: + break + + # if output_stdout: + # # TODO: parse output for secrets + # # TODO: specify plugin and output tight output (no extra newlines) + # # logger.info(output_stdout) + # print(output_stdout) + + if output_stderr: + # TODO: parse output for secrets + # TODO: specify plugin and output tight output (no extra newlines) + # logger.info(plugin_name + " " + output_stderr) (can we modify a specific handler to add handler.terminator = "") + print(output_stderr) + + rc = process.poll() + except Exception as exc: logger.error(exc) if BITOPS_fast_fail_mode: quit(101) - if result.returncode == 0: + if process.returncode == 0: logger.info( "\n~#~#~#~DEPLOYING OPS REPO [{deployment}] SUCCESSFULLY COMPLETED~#~#~#~".format( deployment=deployment ) ) - logger.debug(result.stdout) # TODO: DEEP DEBUG # logger.debug("\n\tSTDERR: [{stderr}]\n\tRESULTS: [{result}]".format(stdout=result.stdout, stderr=result.stderr, result=result)) else: @@ -300,11 +332,11 @@ def Deploy_Plugins(): deployment=deployment ) ) - logger.debug(result.stdout) - logger.error(result.stderr) + # TODO: DEEP DEBUG # logger.debug("\n\tSTDOUT:[{stdout}]\n\tSTDERR: [{stderr}]\n\tRESULTS: [{result}]".format(stdout=result.stdout, stderr=result.stderr, result=result)) + # ~#~#~#~#~#~# STAGE 5 - AFTER HOOKS #~#~#~#~#~#~# if plugin_deploy_after_hook_scripts_flag: hooks_folder = opsrepo_environment_dir + "/bitops.after-deploy.d" From 38c994199e272632290dc2ca314ee719816e24a1 Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Thu, 27 Oct 2022 09:39:02 -0400 Subject: [PATCH 2/7] fix formatting for black and add comments for another live output test based on PR comment --- scripts/plugins/deploy_plugins.py | 54 ++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/scripts/plugins/deploy_plugins.py b/scripts/plugins/deploy_plugins.py index cb1a465e..2da9f848 100644 --- a/scripts/plugins/deploy_plugins.py +++ b/scripts/plugins/deploy_plugins.py @@ -270,25 +270,44 @@ def Deploy_Plugins(): ) try: + # Tried this approach, and the output was not live output + # tested by adding the following in mounted plugin code + # sleep 3; echo "hello"; sleep 3; echo "hello 2"; + # expected to see "hello" and then a pause, and then "hello 2" + # result: no output was shown until after the plugin deploy script finished + # process = subprocess.Popen( + # [ + # plugin_deploy_language, + # plugin_deploy_script_path, + # stack_action, + # ], + # stdout=subprocess.PIPE, + # stderr=subprocess.STDOUT + # ) + + # for combined_output in process.stdout: + # print(combined_output) + # error_file = "stderr_{}.log".format(plugin_name) # with open(error_file,"wb") as err: - process = subprocess.Popen([ - plugin_deploy_language, - plugin_deploy_script_path, - stack_action, - ], - # stdout=subprocess.PIPE, - # TODO: adding any form of stderr to the above call causes the below polling to not work properly for real time output - # instead, it causes the script to wait until everything is complete before outputting anything - # tried: stderr=err, stderr=subprocess.PIPE, stderr=subprocess.STDOUT - # for stderr=err, err is a file: with open(error_file,"wb") as err: - # stderr=err, - # stderr=subprocess.STDOUT, - - # TODO: oddly, if you set only stderr=subprocess.PIPE and NOT stdout, - # the process.poll() below being focused on stderr does seem to output both stdout and stderr in real time - stderr=subprocess.PIPE, - universal_newlines=True + + process = subprocess.Popen( + [ + plugin_deploy_language, + plugin_deploy_script_path, + stack_action, + ], + # stdout=subprocess.PIPE, + # TODO: adding any form of stderr to the above call causes the below polling to not work properly for real time output + # instead, it causes the script to wait until everything is complete before outputting anything + # tried: stderr=err, stderr=subprocess.PIPE, stderr=subprocess.STDOUT + # for stderr=err, err is a file: with open(error_file,"wb") as err: + # stderr=err, + # stderr=subprocess.STDOUT, + # TODO: oddly, if you set only stderr=subprocess.PIPE and NOT stdout, + # the process.poll() below being focused on stderr does seem to output both stdout and stderr in real time + stderr=subprocess.PIPE, + universal_newlines=True, ) while True: @@ -336,7 +355,6 @@ def Deploy_Plugins(): # TODO: DEEP DEBUG # logger.debug("\n\tSTDOUT:[{stdout}]\n\tSTDERR: [{stderr}]\n\tRESULTS: [{result}]".format(stdout=result.stdout, stderr=result.stderr, result=result)) - # ~#~#~#~#~#~# STAGE 5 - AFTER HOOKS #~#~#~#~#~#~# if plugin_deploy_after_hook_scripts_flag: hooks_folder = opsrepo_environment_dir + "/bitops.after-deploy.d" From b7d6f9a436ff08b91c6b5367c517add1f51b954e Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Thu, 27 Oct 2022 10:01:46 -0400 Subject: [PATCH 3/7] adjust to more clearly show various approaches --- scripts/plugins/deploy_plugins.py | 128 +++++++++++++++++++++--------- 1 file changed, 91 insertions(+), 37 deletions(-) diff --git a/scripts/plugins/deploy_plugins.py b/scripts/plugins/deploy_plugins.py index 2da9f848..8f3ef0d2 100644 --- a/scripts/plugins/deploy_plugins.py +++ b/scripts/plugins/deploy_plugins.py @@ -270,60 +270,34 @@ def Deploy_Plugins(): ) try: - # Tried this approach, and the output was not live output - # tested by adding the following in mounted plugin code - # sleep 3; echo "hello"; sleep 3; echo "hello 2"; - # expected to see "hello" and then a pause, and then "hello 2" - # result: no output was shown until after the plugin deploy script finished - # process = subprocess.Popen( - # [ - # plugin_deploy_language, - # plugin_deploy_script_path, - # stack_action, - # ], - # stdout=subprocess.PIPE, - # stderr=subprocess.STDOUT - # ) - - # for combined_output in process.stdout: - # print(combined_output) - - # error_file = "stderr_{}.log".format(plugin_name) - # with open(error_file,"wb") as err: - + # TROUBLESHOOTING DETAILS + # adding any form of stderr to the above call causes the below polling to not work properly for real time output + # instead, it causes the script to wait until everything is complete before outputting anything + # tried: stderr=err, stderr=subprocess.PIPE, stderr=subprocess.STDOUT + # for stderr=err, err is a file: with open(error_file,"wb") as err: + # END TROUBLESHOOTING DETAILS + + ################ APPROACH 0 ################ + # The following works + # TODO: oddly, if you set only stderr=subprocess.PIPE and NOT stdout, + # the process.poll() below being focused on stderr does seem to output both stdout and stderr in real time process = subprocess.Popen( [ plugin_deploy_language, plugin_deploy_script_path, stack_action, ], - # stdout=subprocess.PIPE, - # TODO: adding any form of stderr to the above call causes the below polling to not work properly for real time output - # instead, it causes the script to wait until everything is complete before outputting anything - # tried: stderr=err, stderr=subprocess.PIPE, stderr=subprocess.STDOUT - # for stderr=err, err is a file: with open(error_file,"wb") as err: - # stderr=err, - # stderr=subprocess.STDOUT, - # TODO: oddly, if you set only stderr=subprocess.PIPE and NOT stdout, - # the process.poll() below being focused on stderr does seem to output both stdout and stderr in real time stderr=subprocess.PIPE, universal_newlines=True, ) while True: - # output_stdout = process.stdout.readline() output_stderr = process.stderr.readline() # if we have a return code, exit the while loop if process.poll() is not None: break - # if output_stdout: - # # TODO: parse output for secrets - # # TODO: specify plugin and output tight output (no extra newlines) - # # logger.info(output_stdout) - # print(output_stdout) - if output_stderr: # TODO: parse output for secrets # TODO: specify plugin and output tight output (no extra newlines) @@ -331,6 +305,86 @@ def Deploy_Plugins(): print(output_stderr) rc = process.poll() + ################ END APPROACH 0 ################ + + ################ APPROACH 1 ################ + # Tried this approach, and the output was not live output + # tested by adding the following in mounted plugin code + # sleep 3; echo "hello"; sleep 3; echo "hello 2"; + # expected to see "hello" and then a pause, and then "hello 2" + # result: no output was shown until after the plugin deploy script finished + # process = subprocess.Popen( + # [ + # plugin_deploy_language, + # plugin_deploy_script_path, + # stack_action, + # ], + # stdout=subprocess.PIPE, + # stderr=subprocess.STDOUT + # ) + + # for combined_output in process.stdout: + # print(combined_output) + ################ END APPROACH 1 ################ + + ################ APPROACH 2 ################ + # did not work - no real time + # process = subprocess.Popen( + # [ + # plugin_deploy_language, + # plugin_deploy_script_path, + # stack_action, + # ], + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE, + # universal_newlines=True + # ) + + # while True: + # output_stdout = process.stdout.readline() + # output_stderr = process.stderr.readline() + + # # if we have a return code, exit the while loop + # if process.poll() is not None: + # break + + # if output_stdout: + # print(output_stdout) + + # if output_stderr: + # print(output_stderr) + + # rc = process.poll() + ################ END APPROACH 2 ################ + + ################ APPROACH 3 ################ + # did not work - no real time + # process = subprocess.Popen( + # [ + # plugin_deploy_language, + # plugin_deploy_script_path, + # stack_action, + # ], + # stdout=subprocess.PIPE, + # stderr=subprocess.STDOUT, + # universal_newlines=True + # ) + + # while True: + # output_stdout = process.stdout.readline() + + # # if we have a return code, exit the while loop + # if process.poll() is not None: + # break + + # if output_stdout: + # # TODO: parse output for secrets + # # TODO: specify plugin and output tight output (no extra newlines) + # # logger.info(output_stdout) + # print(output_stdout) + + # rc = process.poll() + ################ END APPROACH 3 ################ except Exception as exc: logger.error(exc) From e11707e1e2b78b238b1bbe302f9a1c12e891365b Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 2 Nov 2022 13:52:25 -0400 Subject: [PATCH 4/7] add PYTHONUNBUFFERED to docker image --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 10805f18..36d2f4f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ ## Using alpine fails on awscli install FROM python:3.8.6-alpine ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 RUN apk add --no-cache bash RUN apk update From e4e0b5404602bfb5a1e51b275994780bae9cec79 Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 2 Nov 2022 13:52:39 -0400 Subject: [PATCH 5/7] clean up troubleshooting comments --- scripts/plugins/deploy_plugins.py | 115 +++--------------------------- 1 file changed, 8 insertions(+), 107 deletions(-) diff --git a/scripts/plugins/deploy_plugins.py b/scripts/plugins/deploy_plugins.py index 8f3ef0d2..5a3b411b 100644 --- a/scripts/plugins/deploy_plugins.py +++ b/scripts/plugins/deploy_plugins.py @@ -270,121 +270,22 @@ def Deploy_Plugins(): ) try: - # TROUBLESHOOTING DETAILS - # adding any form of stderr to the above call causes the below polling to not work properly for real time output - # instead, it causes the script to wait until everything is complete before outputting anything - # tried: stderr=err, stderr=subprocess.PIPE, stderr=subprocess.STDOUT - # for stderr=err, err is a file: with open(error_file,"wb") as err: - # END TROUBLESHOOTING DETAILS - - ################ APPROACH 0 ################ - # The following works - # TODO: oddly, if you set only stderr=subprocess.PIPE and NOT stdout, - # the process.poll() below being focused on stderr does seem to output both stdout and stderr in real time process = subprocess.Popen( [ plugin_deploy_language, plugin_deploy_script_path, stack_action, ], - stderr=subprocess.PIPE, - universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True ) - while True: - output_stderr = process.stderr.readline() - - # if we have a return code, exit the while loop - if process.poll() is not None: - break - - if output_stderr: - # TODO: parse output for secrets - # TODO: specify plugin and output tight output (no extra newlines) - # logger.info(plugin_name + " " + output_stderr) (can we modify a specific handler to add handler.terminator = "") - print(output_stderr) - - rc = process.poll() - ################ END APPROACH 0 ################ - - ################ APPROACH 1 ################ - # Tried this approach, and the output was not live output - # tested by adding the following in mounted plugin code - # sleep 3; echo "hello"; sleep 3; echo "hello 2"; - # expected to see "hello" and then a pause, and then "hello 2" - # result: no output was shown until after the plugin deploy script finished - # process = subprocess.Popen( - # [ - # plugin_deploy_language, - # plugin_deploy_script_path, - # stack_action, - # ], - # stdout=subprocess.PIPE, - # stderr=subprocess.STDOUT - # ) - - # for combined_output in process.stdout: - # print(combined_output) - ################ END APPROACH 1 ################ - - ################ APPROACH 2 ################ - # did not work - no real time - # process = subprocess.Popen( - # [ - # plugin_deploy_language, - # plugin_deploy_script_path, - # stack_action, - # ], - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE, - # universal_newlines=True - # ) - - # while True: - # output_stdout = process.stdout.readline() - # output_stderr = process.stderr.readline() - - # # if we have a return code, exit the while loop - # if process.poll() is not None: - # break - - # if output_stdout: - # print(output_stdout) - - # if output_stderr: - # print(output_stderr) - - # rc = process.poll() - ################ END APPROACH 2 ################ - - ################ APPROACH 3 ################ - # did not work - no real time - # process = subprocess.Popen( - # [ - # plugin_deploy_language, - # plugin_deploy_script_path, - # stack_action, - # ], - # stdout=subprocess.PIPE, - # stderr=subprocess.STDOUT, - # universal_newlines=True - # ) - - # while True: - # output_stdout = process.stdout.readline() - - # # if we have a return code, exit the while loop - # if process.poll() is not None: - # break - - # if output_stdout: - # # TODO: parse output for secrets - # # TODO: specify plugin and output tight output (no extra newlines) - # # logger.info(output_stdout) - # print(output_stdout) - - # rc = process.poll() - ################ END APPROACH 3 ################ + for combined_output in process.stdout: + # TODO: parse output for secrets + # TODO: specify plugin and output tight output (no extra newlines) + # logger.info(plugin_name + " " + output_stderr) (can we modify a specific handler to add handler.terminator = "") + sys.stdout.write(combined_output) except Exception as exc: logger.error(exc) From 9b369996c918a6da0dd51b4e9bbf1e146b9172bd Mon Sep 17 00:00:00 2001 From: Mick McGrath Date: Wed, 2 Nov 2022 13:53:13 -0400 Subject: [PATCH 6/7] black formatting --- scripts/plugins/deploy_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plugins/deploy_plugins.py b/scripts/plugins/deploy_plugins.py index 5a3b411b..f7fe9578 100644 --- a/scripts/plugins/deploy_plugins.py +++ b/scripts/plugins/deploy_plugins.py @@ -278,7 +278,7 @@ def Deploy_Plugins(): ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True + universal_newlines=True, ) for combined_output in process.stdout: From caa91554889314adb8d9a8086e9bf16a4c9c0765 Mon Sep 17 00:00:00 2001 From: PhillypHenning Date: Wed, 2 Nov 2022 15:29:17 -0400 Subject: [PATCH 7/7] removing comment bloat --- scripts/plugins/deploy_plugins.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/plugins/deploy_plugins.py b/scripts/plugins/deploy_plugins.py index f7fe9578..1d19e1d9 100644 --- a/scripts/plugins/deploy_plugins.py +++ b/scripts/plugins/deploy_plugins.py @@ -284,7 +284,7 @@ def Deploy_Plugins(): for combined_output in process.stdout: # TODO: parse output for secrets # TODO: specify plugin and output tight output (no extra newlines) - # logger.info(plugin_name + " " + output_stderr) (can we modify a specific handler to add handler.terminator = "") + # TODO: can we modify a specific handler to add handler.terminator = "" ? sys.stdout.write(combined_output) except Exception as exc: @@ -298,8 +298,6 @@ def Deploy_Plugins(): deployment=deployment ) ) - # TODO: DEEP DEBUG - # logger.debug("\n\tSTDERR: [{stderr}]\n\tRESULTS: [{result}]".format(stdout=result.stdout, stderr=result.stderr, result=result)) else: logger.warning( "\n~#~#~#~DEPLOYING OPS REPO [{deployment}] FAILED~#~#~#~".format( @@ -307,9 +305,6 @@ def Deploy_Plugins(): ) ) - # TODO: DEEP DEBUG - # logger.debug("\n\tSTDOUT:[{stdout}]\n\tSTDERR: [{stderr}]\n\tRESULTS: [{result}]".format(stdout=result.stdout, stderr=result.stderr, result=result)) - # ~#~#~#~#~#~# STAGE 5 - AFTER HOOKS #~#~#~#~#~#~# if plugin_deploy_after_hook_scripts_flag: hooks_folder = opsrepo_environment_dir + "/bitops.after-deploy.d"