From cbe21b0baa128d908f16f20136c7e5941577241f Mon Sep 17 00:00:00 2001 From: risemeup1 <515586620@qq.com> Date: Sun, 12 Mar 2023 07:53:29 +0000 Subject: [PATCH 1/7] optimizing setup.py develop command --- python/paddle/.gitignore | 4 +++ setup.py | 61 ++++++++++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/python/paddle/.gitignore b/python/paddle/.gitignore index 98527864664d32..71d9d11b4960e6 100644 --- a/python/paddle/.gitignore +++ b/python/paddle/.gitignore @@ -1 +1,5 @@ version.py +python/paddle/distributed/fleet/proto/ +python/paddle/fluid/proto +python/paddle/version/ +python/paddle/libs/*.so* diff --git a/setup.py b/setup.py index 75969c5f41cbba..1c33b298002492 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ from subprocess import CalledProcessError from setuptools import Command, Extension, setup +from setuptools.command.develop import develop as DevelopCommandBase from setuptools.command.egg_info import egg_info from setuptools.command.install import install as InstallCommandBase from setuptools.command.install_lib import install_lib @@ -222,6 +223,51 @@ def finalize_options(self): return ret +class DevelopCommand(DevelopCommandBase): + def run(self): + # copy proto and .so to python_source_dir + fluid_proto_binary_path = ( + paddle_binary_dir + '/python/paddle/fluid/proto' + ) + fluid_proto_source_path = ( + paddle_source_dir + '/python/paddle/fluid/proto' + ) + distributed_proto_binary_path = ( + paddle_binary_dir + '/python/paddle/distributed/fleet/proto' + ) + distributed_proto_source_path = ( + paddle_source_dir + '/python/paddle/distributed/fleet/proto' + ) + if not os.path.exists(fluid_proto_source_path): + shutil.copytree(fluid_proto_binary_path, fluid_proto_source_path) + if not os.path.exists(distributed_proto_source_path): + shutil.copytree( + distributed_proto_binary_path, distributed_proto_source_path + ) + dynamic_library_binary_path = paddle_binary_dir + '/python/paddle/libs/' + dynamic_library_source_path = paddle_source_dir + '/python/paddle/libs/' + for lib_so in os.listdir(dynamic_library_binary_path): + shutil.copy( + dynamic_library_binary_path + lib_so, + dynamic_library_source_path, + ) + # write version.py and cuda_env_config_py to python_source_dir + write_version_py( + filename='{}/python/paddle/version/__init__.py'.format( + paddle_source_dir + ) + ) + write_cuda_env_config_py( + filename='{}/python/paddle/cuda_env.py'.format(paddle_source_dir) + ) + write_parameter_server_version_py( + filename='{}/python/paddle/incubate/distributed/fleet/parameter_server/version.py'.format( + paddle_source_dir + ) + ) + DevelopCommandBase.run(self) + + class EggInfo(egg_info): """Copy license file into `.dist-info` folder.""" @@ -828,18 +874,7 @@ def get_package_data_and_package_dir(): paddle_binary_dir + '/python/paddle/cost_model/static_op_benchmark.json' ] if 'develop' in sys.argv: - package_dir = { - '': paddle_binary_dir.split('/')[-1] + '/python', - # '':'build/python', - # The paddle.fluid.proto will be generated while compiling. - # So that package points to other directory. - 'paddle.fluid.proto.profiler': paddle_binary_dir.split('/')[-1] - + '/paddle/fluid/platform', - 'paddle.fluid.proto': paddle_binary_dir.split('/')[-1] - + '/paddle/fluid/framework', - 'paddle.fluid': paddle_binary_dir.split('/')[-1] - + '/python/paddle/fluid', - } + package_dir = {'': 'python'} else: package_dir = { '': env_dict.get("PADDLE_BINARY_DIR") + '/python', @@ -1470,6 +1505,7 @@ def main(): # preparing parameters for setup() paddle_version = env_dict.get("PADDLE_VERSION") package_name = env_dict.get("PACKAGE_NAME") + write_version_py( filename='{}/python/paddle/version/__init__.py'.format( paddle_binary_dir @@ -1534,6 +1570,7 @@ def main(): 'install': InstallCommand, 'egg_info': EggInfo, 'install_lib': InstallLib, + 'develop': DevelopCommand, }, entry_points={ 'console_scripts': [ From 829f7854f9a5478377d63ae253afda6fa21861a8 Mon Sep 17 00:00:00 2001 From: risemeup1 <515586620@qq.com> Date: Sun, 12 Mar 2023 13:13:51 +0000 Subject: [PATCH 2/7] add libpaddle.so --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 1c33b298002492..5b4a0c8785ec7c 100644 --- a/setup.py +++ b/setup.py @@ -244,6 +244,10 @@ def run(self): shutil.copytree( distributed_proto_binary_path, distributed_proto_source_path ) + shutil.copy( + paddle_binary_dir + '/python/paddle/fluid/libpaddle.so', + paddle_source_dir + '/python/paddle/fluid/', + ) dynamic_library_binary_path = paddle_binary_dir + '/python/paddle/libs/' dynamic_library_source_path = paddle_source_dir + '/python/paddle/libs/' for lib_so in os.listdir(dynamic_library_binary_path): From 91a83f283efe88863cc93cc8d21c81439ffb7c55 Mon Sep 17 00:00:00 2001 From: risemeup1 <515586620@qq.com> Date: Sun, 12 Mar 2023 16:00:21 +0000 Subject: [PATCH 3/7] modify setup.py --- setup.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 5b4a0c8785ec7c..c4137d4222747d 100644 --- a/setup.py +++ b/setup.py @@ -227,23 +227,23 @@ class DevelopCommand(DevelopCommandBase): def run(self): # copy proto and .so to python_source_dir fluid_proto_binary_path = ( - paddle_binary_dir + '/python/paddle/fluid/proto' + paddle_binary_dir + '/python/paddle/fluid/proto/' ) fluid_proto_source_path = ( - paddle_source_dir + '/python/paddle/fluid/proto' + paddle_source_dir + '/python/paddle/fluid/proto/' ) distributed_proto_binary_path = ( - paddle_binary_dir + '/python/paddle/distributed/fleet/proto' + paddle_binary_dir + '/python/paddle/distributed/fleet/proto/' ) distributed_proto_source_path = ( - paddle_source_dir + '/python/paddle/distributed/fleet/proto' + paddle_source_dir + '/python/paddle/distributed/fleet/proto/' + ) + os.system("rm -rf {}".format(fluid_proto_source_path)) + shutil.copytree(fluid_proto_binary_path, fluid_proto_source_path) + os.system("rm -rf {}".format(distributed_proto_source_path)) + shutil.copytree( + distributed_proto_binary_path, distributed_proto_source_path ) - if not os.path.exists(fluid_proto_source_path): - shutil.copytree(fluid_proto_binary_path, fluid_proto_source_path) - if not os.path.exists(distributed_proto_source_path): - shutil.copytree( - distributed_proto_binary_path, distributed_proto_source_path - ) shutil.copy( paddle_binary_dir + '/python/paddle/fluid/libpaddle.so', paddle_source_dir + '/python/paddle/fluid/', From 8c316a5a29de057392097010001566b31b97e017 Mon Sep 17 00:00:00 2001 From: risemeup1 <515586620@qq.com> Date: Sun, 12 Mar 2023 16:17:48 +0000 Subject: [PATCH 4/7] add python/paddle/distributed/fleet/.gitignore --- python/paddle/distributed/fleet/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 python/paddle/distributed/fleet/.gitignore diff --git a/python/paddle/distributed/fleet/.gitignore b/python/paddle/distributed/fleet/.gitignore new file mode 100644 index 00000000000000..2ff540d5764b76 --- /dev/null +++ b/python/paddle/distributed/fleet/.gitignore @@ -0,0 +1 @@ +proto From a8dc19c8dd087063fa00afeb652218a3562a65be Mon Sep 17 00:00:00 2001 From: risemeup1 <515586620@qq.com> Date: Sun, 12 Mar 2023 16:19:13 +0000 Subject: [PATCH 5/7] add libpaddle.so to .gitignore --- python/paddle/fluid/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/fluid/.gitignore b/python/paddle/fluid/.gitignore index 80c1cf3fcb86aa..0745bad8c6791b 100644 --- a/python/paddle/fluid/.gitignore +++ b/python/paddle/fluid/.gitignore @@ -1,2 +1,3 @@ proto core.so +*.so From 19f9b3b015bd6b047ebad4a8904aef95f8134889 Mon Sep 17 00:00:00 2001 From: risemeup1 <515586620@qq.com> Date: Sun, 12 Mar 2023 16:22:29 +0000 Subject: [PATCH 6/7] add *.so to python/paddle/libs/.gitignore --- python/paddle/libs/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 python/paddle/libs/.gitignore diff --git a/python/paddle/libs/.gitignore b/python/paddle/libs/.gitignore new file mode 100644 index 00000000000000..8d977535b81865 --- /dev/null +++ b/python/paddle/libs/.gitignore @@ -0,0 +1 @@ +*.so* From 1e09a5f2e36525770d66ac70f1ac29204fbf8509 Mon Sep 17 00:00:00 2001 From: risemeup1 <515586620@qq.com> Date: Sun, 12 Mar 2023 16:23:26 +0000 Subject: [PATCH 7/7] add new gitignore --- python/.gitignore | 1 + python/paddle/.gitignore | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/.gitignore b/python/.gitignore index 53a2b7a76b0dd2..d0c7300f8c443f 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -7,3 +7,4 @@ paddlepaddle_gpu.egg-info .idea paddle/proto/*.py paddle/proto/*.pyc +_foo.* diff --git a/python/paddle/.gitignore b/python/paddle/.gitignore index 71d9d11b4960e6..b405a11db9a8df 100644 --- a/python/paddle/.gitignore +++ b/python/paddle/.gitignore @@ -1,5 +1,3 @@ version.py -python/paddle/distributed/fleet/proto/ -python/paddle/fluid/proto -python/paddle/version/ -python/paddle/libs/*.so* +cuda_env.py +version