From 2244d7bec687fe7f72c3a8565dacdefd47deba0d Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Sat, 19 Sep 2020 00:31:21 -0700 Subject: [PATCH] Support building sonic-utilities as a Python wheel package instead of a Debian package (#1122) Support building sonic-utilities as a Python wheel package rather than a Debian package. Notes: - Wheel packaging restricts the installation of files to within the dist-packages directory. Thus, we cannot install data files to system directories. Therefore, I am building a separate 'sonic-utilities-data' package for installing the data files (bash_completion, templates) - Wheel packages install scripts/entrypoints to the /usr/local/bin directory, whereas Debian packages install to /usr/bin --- .gitignore | 31 ++++++++++++++++--- config/main.py | 4 +-- scripts/fast-reboot | 14 ++++----- scripts/generate_dump | 2 +- scripts/sonic-kdump-config | 8 ++--- setup.py | 24 +++++--------- sonic-utilities-data/LICENSE | 15 +++++++++ sonic-utilities-data/MAINTAINERS | 7 +++++ sonic-utilities-data/README.md | 19 ++++++++++++ .../bash_completion.d/acl-loader | 0 .../bash_completion.d/config | 0 .../bash_completion.d/connect | 0 .../bash_completion.d/consutil | 0 .../bash_completion.d/counterpoll | 0 .../bash_completion.d/crm | 0 .../bash_completion.d/debug | 0 .../bash_completion.d/fwutil | 0 .../bash_completion.d/pddf_fanutil | 0 .../bash_completion.d/pddf_ledutil | 0 .../bash_completion.d/pddf_psuutil | 0 .../bash_completion.d/pddf_thermalutil | 0 .../bash_completion.d/pfc | 0 .../bash_completion.d/pfcwd | 0 .../bash_completion.d/psuutil | 0 .../bash_completion.d/sfputil | 0 .../bash_completion.d/show | 0 .../bash_completion.d/sonic-clear | 0 .../bash_completion.d/sonic-installer | 0 .../bash_completion.d/sonic_installer | 0 .../bash_completion.d/undebug | 0 sonic-utilities-data/debian/changelog | 5 +++ sonic-utilities-data/debian/compat | 1 + sonic-utilities-data/debian/control | 11 +++++++ sonic-utilities-data/debian/copyright | 0 sonic-utilities-data/debian/install | 2 ++ sonic-utilities-data/debian/rules | 6 ++++ .../templates/sonic-environment.j2 | 0 37 files changed, 114 insertions(+), 35 deletions(-) create mode 100644 sonic-utilities-data/LICENSE create mode 100644 sonic-utilities-data/MAINTAINERS create mode 100644 sonic-utilities-data/README.md rename {data/etc => sonic-utilities-data}/bash_completion.d/acl-loader (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/config (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/connect (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/consutil (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/counterpoll (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/crm (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/debug (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/fwutil (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/pddf_fanutil (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/pddf_ledutil (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/pddf_psuutil (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/pddf_thermalutil (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/pfc (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/pfcwd (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/psuutil (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/sfputil (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/show (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/sonic-clear (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/sonic-installer (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/sonic_installer (100%) rename {data/etc => sonic-utilities-data}/bash_completion.d/undebug (100%) create mode 100644 sonic-utilities-data/debian/changelog create mode 100644 sonic-utilities-data/debian/compat create mode 100644 sonic-utilities-data/debian/control create mode 100644 sonic-utilities-data/debian/copyright create mode 100644 sonic-utilities-data/debian/install create mode 100755 sonic-utilities-data/debian/rules rename {sonic_installer => sonic-utilities-data}/templates/sonic-environment.j2 (100%) diff --git a/.gitignore b/.gitignore index 2a9d7b934883..439e053c6422 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,28 @@ +# Compiled Python files +*.pyc +scripts/aclshowc +scripts/neighbor_advertiserc +scripts/port2aliasc + +# Generated by packaging +*.egg-info/ +.eggs/ build/ -deb_dist/ dist/ -*.egg-info/ -*.pyc -.cache -*.tar.gz + +# Unit test coverage +.coverage +coverage.xml +htmlcov/ + +# Ignores for sonic-utilities-data +sonic-utilities-data/debian/* +!sonic-utilities-data/debian/changelog +!sonic-utilities-data/debian/compat +!sonic-utilities-data/debian/control +!sonic-utilities-data/debian/copyright +!sonic-utilities-data/debian/install +!sonic-utilities-data/debian/rules +sonic-utilities-data_*.deb +sonic-utilities-data_*.buildinfo +sonic-utilities-data_*.changes diff --git a/config/main.py b/config/main.py index 32501bdd4dcb..0dc0d71b8c8a 100755 --- a/config/main.py +++ b/config/main.py @@ -1101,7 +1101,7 @@ def reload(db, filename, yes, load_sysinfo, no_service_restart): client.set(config_db.INIT_INDICATOR, 1) # Migrate DB contents to latest version - db_migrator='/usr/bin/db_migrator.py' + db_migrator='/usr/local/bin/db_migrator.py' if os.path.isfile(db_migrator) and os.access(db_migrator, os.X_OK): if namespace is None: command = "{} -o migrate".format(db_migrator) @@ -1200,7 +1200,7 @@ def load_minigraph(db, no_service_restart): clicommon.run_command("config qos reload", display_cmd=True) # Write latest db version string into db - db_migrator='/usr/bin/db_migrator.py' + db_migrator='/usr/local/bin/db_migrator.py' if os.path.isfile(db_migrator) and os.access(db_migrator, os.X_OK): for namespace in namespace_list: if namespace is DEFAULT_NAMESPACE: diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 87bb3a3e9401..dc8eb0c5d866 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -12,12 +12,12 @@ FORCE=no STRICT=no REBOOT_METHOD="/sbin/kexec -e" ASSISTANT_IP_LIST="" -ASSISTANT_SCRIPT="/usr/bin/neighbor_advertiser" -WATCHDOG_UTIL="/usr/bin/watchdogutil" +ASSISTANT_SCRIPT="/usr/local/bin/neighbor_advertiser" +WATCHDOG_UTIL="/usr/local/bin/watchdogutil" DEVPATH="/usr/share/sonic/device" PLATFORM=$(sonic-cfggen -H -v DEVICE_METADATA.localhost.platform) PLATFORM_PLUGIN="${REBOOT_TYPE}_plugin" -LOG_SSD_HEALTH="/usr/bin/log_ssd_health" +LOG_SSD_HEALTH="/usr/local/bin/log_ssd_health" SSD_FW_UPDATE="ssd-fw-upgrade" # Require 100M available on the hard drive for warm reboot temp files, # Size is in 1K blocks: @@ -324,7 +324,7 @@ function reboot_pre_check() fi # Make sure ASIC configuration has not changed between images - ASIC_CONFIG_CHECK_SCRIPT="/usr/bin/asic_config_check" + ASIC_CONFIG_CHECK_SCRIPT="/usr/local/bin/asic_config_check" ASIC_CONFIG_CHECK_SUCCESS=0 if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then ASIC_CONFIG_CHECK_EXIT_CODE=0 @@ -447,7 +447,7 @@ if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then CONFIG_DB_FILE=/etc/sonic/config_db.json mkdir -p $DUMP_DIR FAST_REBOOT_DUMP_RC=0 - /usr/bin/fast-reboot-dump.py -t $DUMP_DIR || FAST_REBOOT_DUMP_RC=$? + /usr/local/bin/fast-reboot-dump.py -t $DUMP_DIR || FAST_REBOOT_DUMP_RC=$? if [[ FAST_REBOOT_DUMP_RC -ne 0 ]]; then error "Failed to run fast-reboot-dump.py. Exit code: $FAST_REBOOT_DUMP_RC" unload_kernel @@ -456,7 +456,7 @@ if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then FILTER_FDB_ENTRIES_RC=0 # Filter FDB entries using MAC addresses from ARP table - /usr/bin/filter_fdb_entries -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$? + /usr/local/bin/filter_fdb_entries -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$? if [[ FILTER_FDB_ENTRIES_RC -ne 0 ]]; then error "Failed to filter FDb entries. Exit code: $FILTER_FDB_ENTRIES_RC" unload_kernel @@ -497,7 +497,7 @@ fi # Kill nat docker after saving the conntrack table debug "Stopping nat ..." -/usr/bin/dump_nat_entries.py +/usr/local/bin/dump_nat_entries.py docker kill nat > /dev/null || true systemctl stop nat debug "Stopped nat ..." diff --git a/scripts/generate_dump b/scripts/generate_dump index 01e45f801a7b..ba69f02a4f6b 100755 --- a/scripts/generate_dump +++ b/scripts/generate_dump @@ -342,7 +342,7 @@ main() { $MKDIR $V -p $TARDIR # Start with this script so its obvious what code is responsible - $LN $V -s /usr/bin/generate_dump $TARDIR + $LN $V -s /usr/local/bin/generate_dump $TARDIR $TAR $V -chf $TARFILE -C $DUMPDIR $BASE $RM $V -f $TARDIR/sonic_dump diff --git a/scripts/sonic-kdump-config b/scripts/sonic-kdump-config index 696a87adc818..c19b4dea16bc 100755 --- a/scripts/sonic-kdump-config +++ b/scripts/sonic-kdump-config @@ -167,7 +167,7 @@ def get_kdump_administrative_mode(): # # @return The current memory string used for kdump (read from running configuration) def get_kdump_memory(): - (rc, lines, err_str) = run_command("/usr/bin/show kdump memory", use_shell=False) + (rc, lines, err_str) = run_command("show kdump memory", use_shell=False) try: if rc == 0 and len(lines) == 1: p = lines[0].find(': ') @@ -185,7 +185,7 @@ def get_kdump_memory(): # @return The maximum number of kernel dump files stored locally # (read from running configuration) def get_kdump_num_dumps(): - (rc, lines, err_str) = run_command("/usr/bin/show kdump num_dumps", use_shell=False) + (rc, lines, err_str) = run_command("show kdump num_dumps", use_shell=False) try: if rc == 0 and len(lines) == 1: p = lines[0].find(': ') @@ -386,7 +386,7 @@ def cmd_kdump_disable(verbose, image=get_current_image()): # If None, display current value read from running configuration def cmd_kdump_memory(verbose, memory): if memory is None: - (rc, lines, err_str) = run_command("/usr/bin/show kdump memory", use_shell=False); + (rc, lines, err_str) = run_command("show kdump memory", use_shell=False); print('\n'.join(lines)) else: use_kdump_in_cfg = read_use_kdump() @@ -403,7 +403,7 @@ def cmd_kdump_memory(verbose, memory): # If None, display current value read from running configuration def cmd_kdump_num_dumps(verbose, num_dumps): if num_dumps is None: - (rc, lines, err_str) = run_command("/usr/bin/show kdump num_dumps", use_shell=False); + (rc, lines, err_str) = run_command("show kdump num_dumps", use_shell=False); print('\n'.join(lines)) else: write_num_dumps(num_dumps) diff --git a/setup.py b/setup.py index 23a07f7e2df6..cb6df6b3c94c 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ pkg_resources.require('fastentrypoints') import fastentrypoints -import glob from setuptools import setup setup( @@ -118,10 +117,6 @@ 'scripts/watermarkcfg', 'scripts/sonic-kdump-config' ], - data_files=[ - ('/etc/bash_completion.d', glob.glob('data/etc/bash_completion.d/*')), - ('/usr/share/sonic/templates', ['sonic_installer/templates/sonic-environment.j2']), - ], entry_points={ 'console_scripts': [ 'acl-loader = acl_loader.main:cli', @@ -151,21 +146,18 @@ 'watchdogutil = watchdogutil.main:watchdogutil', ] }, - # NOTE: sonic-utilities also depends on other packages that are either only - # available as .whl files or the latest available Debian packages are - # out-of-date and we must install newer versions via pip. These - # dependencies cannot be listed here, as this package is built as a .deb, - # therefore all dependencies will be assumed to also be available as .debs. - # These unlistable dependencies are as follows: - # - sonic-config-engine - # - sonic-py-common - # - sonic-py-swsssdk - # - tabulate install_requires=[ 'click', 'ipaddress', + 'jsondiff==1.2.0', + 'm2crypto', 'natsort', - 'm2crypto' + 'pexpect', + 'sonic-config-engine', + 'sonic-py-common', + 'swsssdk>=2.0.1', + 'tabulate==0.8.2', + 'xmltodict==0.12.0' ], setup_requires= [ 'pytest-runner' diff --git a/sonic-utilities-data/LICENSE b/sonic-utilities-data/LICENSE new file mode 100644 index 000000000000..6e2e81a75fda --- /dev/null +++ b/sonic-utilities-data/LICENSE @@ -0,0 +1,15 @@ +Copyright (C) 2020 Microsoft + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. diff --git a/sonic-utilities-data/MAINTAINERS b/sonic-utilities-data/MAINTAINERS new file mode 100644 index 000000000000..fde1eede3214 --- /dev/null +++ b/sonic-utilities-data/MAINTAINERS @@ -0,0 +1,7 @@ +# This file describes the maintainers for sonic-device-data +# See the SONiC project governance document for more information + +Name = "Joe LeVeque" +Email = "jolevequ@microsoft.com" +Github = jleveque +Mailinglist = sonicproject@googlegroups.com diff --git a/sonic-utilities-data/README.md b/sonic-utilities-data/README.md new file mode 100644 index 000000000000..8abc021cb1f2 --- /dev/null +++ b/sonic-utilities-data/README.md @@ -0,0 +1,19 @@ +# sonic-utilities-data +Data files required for SONiC command line utilities + + +## To build + +``` +dpkg-buildpackage -rfakeroot -b -us -uc +``` + +## To clean + +``` +dpkg-buildpackage -rfakeroot -Tclean +``` + +--- + +See the [SONiC Website](http://azure.github.io/SONiC/) for more information about the SONiC project. diff --git a/data/etc/bash_completion.d/acl-loader b/sonic-utilities-data/bash_completion.d/acl-loader similarity index 100% rename from data/etc/bash_completion.d/acl-loader rename to sonic-utilities-data/bash_completion.d/acl-loader diff --git a/data/etc/bash_completion.d/config b/sonic-utilities-data/bash_completion.d/config similarity index 100% rename from data/etc/bash_completion.d/config rename to sonic-utilities-data/bash_completion.d/config diff --git a/data/etc/bash_completion.d/connect b/sonic-utilities-data/bash_completion.d/connect similarity index 100% rename from data/etc/bash_completion.d/connect rename to sonic-utilities-data/bash_completion.d/connect diff --git a/data/etc/bash_completion.d/consutil b/sonic-utilities-data/bash_completion.d/consutil similarity index 100% rename from data/etc/bash_completion.d/consutil rename to sonic-utilities-data/bash_completion.d/consutil diff --git a/data/etc/bash_completion.d/counterpoll b/sonic-utilities-data/bash_completion.d/counterpoll similarity index 100% rename from data/etc/bash_completion.d/counterpoll rename to sonic-utilities-data/bash_completion.d/counterpoll diff --git a/data/etc/bash_completion.d/crm b/sonic-utilities-data/bash_completion.d/crm similarity index 100% rename from data/etc/bash_completion.d/crm rename to sonic-utilities-data/bash_completion.d/crm diff --git a/data/etc/bash_completion.d/debug b/sonic-utilities-data/bash_completion.d/debug similarity index 100% rename from data/etc/bash_completion.d/debug rename to sonic-utilities-data/bash_completion.d/debug diff --git a/data/etc/bash_completion.d/fwutil b/sonic-utilities-data/bash_completion.d/fwutil similarity index 100% rename from data/etc/bash_completion.d/fwutil rename to sonic-utilities-data/bash_completion.d/fwutil diff --git a/data/etc/bash_completion.d/pddf_fanutil b/sonic-utilities-data/bash_completion.d/pddf_fanutil similarity index 100% rename from data/etc/bash_completion.d/pddf_fanutil rename to sonic-utilities-data/bash_completion.d/pddf_fanutil diff --git a/data/etc/bash_completion.d/pddf_ledutil b/sonic-utilities-data/bash_completion.d/pddf_ledutil similarity index 100% rename from data/etc/bash_completion.d/pddf_ledutil rename to sonic-utilities-data/bash_completion.d/pddf_ledutil diff --git a/data/etc/bash_completion.d/pddf_psuutil b/sonic-utilities-data/bash_completion.d/pddf_psuutil similarity index 100% rename from data/etc/bash_completion.d/pddf_psuutil rename to sonic-utilities-data/bash_completion.d/pddf_psuutil diff --git a/data/etc/bash_completion.d/pddf_thermalutil b/sonic-utilities-data/bash_completion.d/pddf_thermalutil similarity index 100% rename from data/etc/bash_completion.d/pddf_thermalutil rename to sonic-utilities-data/bash_completion.d/pddf_thermalutil diff --git a/data/etc/bash_completion.d/pfc b/sonic-utilities-data/bash_completion.d/pfc similarity index 100% rename from data/etc/bash_completion.d/pfc rename to sonic-utilities-data/bash_completion.d/pfc diff --git a/data/etc/bash_completion.d/pfcwd b/sonic-utilities-data/bash_completion.d/pfcwd similarity index 100% rename from data/etc/bash_completion.d/pfcwd rename to sonic-utilities-data/bash_completion.d/pfcwd diff --git a/data/etc/bash_completion.d/psuutil b/sonic-utilities-data/bash_completion.d/psuutil similarity index 100% rename from data/etc/bash_completion.d/psuutil rename to sonic-utilities-data/bash_completion.d/psuutil diff --git a/data/etc/bash_completion.d/sfputil b/sonic-utilities-data/bash_completion.d/sfputil similarity index 100% rename from data/etc/bash_completion.d/sfputil rename to sonic-utilities-data/bash_completion.d/sfputil diff --git a/data/etc/bash_completion.d/show b/sonic-utilities-data/bash_completion.d/show similarity index 100% rename from data/etc/bash_completion.d/show rename to sonic-utilities-data/bash_completion.d/show diff --git a/data/etc/bash_completion.d/sonic-clear b/sonic-utilities-data/bash_completion.d/sonic-clear similarity index 100% rename from data/etc/bash_completion.d/sonic-clear rename to sonic-utilities-data/bash_completion.d/sonic-clear diff --git a/data/etc/bash_completion.d/sonic-installer b/sonic-utilities-data/bash_completion.d/sonic-installer similarity index 100% rename from data/etc/bash_completion.d/sonic-installer rename to sonic-utilities-data/bash_completion.d/sonic-installer diff --git a/data/etc/bash_completion.d/sonic_installer b/sonic-utilities-data/bash_completion.d/sonic_installer similarity index 100% rename from data/etc/bash_completion.d/sonic_installer rename to sonic-utilities-data/bash_completion.d/sonic_installer diff --git a/data/etc/bash_completion.d/undebug b/sonic-utilities-data/bash_completion.d/undebug similarity index 100% rename from data/etc/bash_completion.d/undebug rename to sonic-utilities-data/bash_completion.d/undebug diff --git a/sonic-utilities-data/debian/changelog b/sonic-utilities-data/debian/changelog new file mode 100644 index 000000000000..dba0fada8298 --- /dev/null +++ b/sonic-utilities-data/debian/changelog @@ -0,0 +1,5 @@ +sonic-utilities-data (1.0-1) UNRELEASED; urgency=low + + * Initial release + + -- Joe LeVeque Thu, 17 Sep 2020 01:48:49 +0000 diff --git a/sonic-utilities-data/debian/compat b/sonic-utilities-data/debian/compat new file mode 100644 index 000000000000..ec635144f600 --- /dev/null +++ b/sonic-utilities-data/debian/compat @@ -0,0 +1 @@ +9 diff --git a/sonic-utilities-data/debian/control b/sonic-utilities-data/debian/control new file mode 100644 index 000000000000..b5de2089a329 --- /dev/null +++ b/sonic-utilities-data/debian/control @@ -0,0 +1,11 @@ +Source: sonic-utilities-data +Maintainer: Joe LeVeque +Section: misc +Priority: optional +Standards-Version: 0.1 +Build-Depends: debhelper (>=9) + +Package: sonic-utilities-data +Architecture: all +Depends: ${misc:Depends} bash-completion +Description: Data files required for SONiC command line utilities diff --git a/sonic-utilities-data/debian/copyright b/sonic-utilities-data/debian/copyright new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sonic-utilities-data/debian/install b/sonic-utilities-data/debian/install new file mode 100644 index 000000000000..82d087d54d3b --- /dev/null +++ b/sonic-utilities-data/debian/install @@ -0,0 +1,2 @@ +bash_completion.d/ /etc/ +templates/*.j2 /usr/share/sonic/templates/ diff --git a/sonic-utilities-data/debian/rules b/sonic-utilities-data/debian/rules new file mode 100755 index 000000000000..a5e2f5acfc1f --- /dev/null +++ b/sonic-utilities-data/debian/rules @@ -0,0 +1,6 @@ +#!/usr/bin/make -f + +build: + +%: + dh $@ diff --git a/sonic_installer/templates/sonic-environment.j2 b/sonic-utilities-data/templates/sonic-environment.j2 similarity index 100% rename from sonic_installer/templates/sonic-environment.j2 rename to sonic-utilities-data/templates/sonic-environment.j2