Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

optimizing setup.py develop command #51528

Merged
merged 8 commits into from
Mar 15, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions python/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ paddlepaddle_gpu.egg-info
.idea
paddle/proto/*.py
paddle/proto/*.pyc
_foo.*
2 changes: 2 additions & 0 deletions python/paddle/.gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
version.py
cuda_env.py
version
1 change: 1 addition & 0 deletions python/paddle/distributed/fleet/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
proto
1 change: 1 addition & 0 deletions python/paddle/fluid/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
proto
core.so
*.so
1 change: 1 addition & 0 deletions python/paddle/libs/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*.so*
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

建议规范一下编译中间产物的位置

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

好的

65 changes: 53 additions & 12 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from subprocess import CalledProcessError

from setuptools import Command, Extension, setup
from setuptools.command.develop import develop as DevelopCommandBase
from setuptools.command.egg_info import egg_info
from setuptools.command.install import install as InstallCommandBase
from setuptools.command.install_lib import install_lib
Expand Down Expand Up @@ -222,6 +223,55 @@ def finalize_options(self):
return ret


class DevelopCommand(DevelopCommandBase):
def run(self):
# copy proto and .so to python_source_dir
fluid_proto_binary_path = (
paddle_binary_dir + '/python/paddle/fluid/proto/'
)
fluid_proto_source_path = (
paddle_source_dir + '/python/paddle/fluid/proto/'
)
distributed_proto_binary_path = (
paddle_binary_dir + '/python/paddle/distributed/fleet/proto/'
)
distributed_proto_source_path = (
paddle_source_dir + '/python/paddle/distributed/fleet/proto/'
)
os.system("rm -rf {}".format(fluid_proto_source_path))
shutil.copytree(fluid_proto_binary_path, fluid_proto_source_path)
os.system("rm -rf {}".format(distributed_proto_source_path))
shutil.copytree(
distributed_proto_binary_path, distributed_proto_source_path
)
shutil.copy(
paddle_binary_dir + '/python/paddle/fluid/libpaddle.so',
paddle_source_dir + '/python/paddle/fluid/',
)
dynamic_library_binary_path = paddle_binary_dir + '/python/paddle/libs/'
dynamic_library_source_path = paddle_source_dir + '/python/paddle/libs/'
for lib_so in os.listdir(dynamic_library_binary_path):
shutil.copy(
dynamic_library_binary_path + lib_so,
dynamic_library_source_path,
)
# write version.py and cuda_env_config_py to python_source_dir
write_version_py(
filename='{}/python/paddle/version/__init__.py'.format(
paddle_source_dir
)
)
write_cuda_env_config_py(
filename='{}/python/paddle/cuda_env.py'.format(paddle_source_dir)
)
write_parameter_server_version_py(
filename='{}/python/paddle/incubate/distributed/fleet/parameter_server/version.py'.format(
paddle_source_dir
)
)
DevelopCommandBase.run(self)


class EggInfo(egg_info):
"""Copy license file into `.dist-info` folder."""

Expand Down Expand Up @@ -828,18 +878,7 @@ def get_package_data_and_package_dir():
paddle_binary_dir + '/python/paddle/cost_model/static_op_benchmark.json'
]
if 'develop' in sys.argv:
package_dir = {
'': paddle_binary_dir.split('/')[-1] + '/python',
# '':'build/python',
# The paddle.fluid.proto will be generated while compiling.
# So that package points to other directory.
'paddle.fluid.proto.profiler': paddle_binary_dir.split('/')[-1]
+ '/paddle/fluid/platform',
'paddle.fluid.proto': paddle_binary_dir.split('/')[-1]
+ '/paddle/fluid/framework',
'paddle.fluid': paddle_binary_dir.split('/')[-1]
+ '/python/paddle/fluid',
}
package_dir = {'': 'python'}
else:
package_dir = {
'': env_dict.get("PADDLE_BINARY_DIR") + '/python',
Expand Down Expand Up @@ -1470,6 +1509,7 @@ def main():
# preparing parameters for setup()
paddle_version = env_dict.get("PADDLE_VERSION")
package_name = env_dict.get("PACKAGE_NAME")

write_version_py(
filename='{}/python/paddle/version/__init__.py'.format(
paddle_binary_dir
Expand Down Expand Up @@ -1534,6 +1574,7 @@ def main():
'install': InstallCommand,
'egg_info': EggInfo,
'install_lib': InstallLib,
'develop': DevelopCommand,
},
entry_points={
'console_scripts': [
Expand Down