diff --git a/.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml b/.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml new file mode 100644 index 000000000000..697c87b3c9ea --- /dev/null +++ b/.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml @@ -0,0 +1,55 @@ +# C/C++ with GCC +# Build your C/C++ project with GCC using make. +# Add steps that publish test results, save build artifacts, deploy, and more: +# https://docs.microsoft.com/azure/devops/pipelines/apps/c-cpp/gcc +pr: none + +trigger: + batch: true + branches: + include: + - master + +stages: +- stage: Build + + jobs: + - job: + displayName: "amd64/ubuntu-20.04" + pool: + vmImage: 'ubuntu-20.04' + + steps: + - checkout: self + submodules: true + - script: | + sudo apt-get update + sudo apt-get install -y make wget libtool m4 autoconf dh-exec libdebhelper-perl=13.6ubuntu1~bpo20.04.1 debhelper=13.6ubuntu1~bpo20.04.1 \ + cmake pkg-config python3-pip python cmake libgtest-dev libgmock-dev libyang-dev \ + debhelper-compat dh-elpa dh-sequence-python3 python3-all \ + libpython3-all-dev python3-six xmlto unzip rake-compiler gem2deb pkg-php-tools \ + ant default-jdk maven-repo-helper libguava-java + wget http://ftp.us.debian.org/debian/pool/main/libg/libgoogle-gson-java/libgoogle-gson-java_2.8.6-1+deb11u1_all.deb + sudo dpkg -i libgoogle-gson-java_2.8.6-1+deb11u1_all.deb + mkdir -p /tmp/artifacts + displayName: "Install dependencies" + - script: | + SONIC_CONFIG_MAKE_JOBS=$(nproc) CONFIGURED_ARCH=amd64 DEST=/tmp/artifacts make -f ../rules/protobuf.mk -f protobuf/Makefile + workingDirectory: src + displayName: "Build protobuf" + - script: | + sudo dpkg -i protobuf-compiler_3.21.12-3_amd64.deb libprotoc32_3.21.12-3_amd64.deb \ + libprotobuf32_3.21.12-3_amd64.deb libprotobuf-dev_3.21.12-3_amd64.deb \ + libprotobuf-lite32_3.21.12-3_amd64.deb + workingDirectory: /tmp/artifacts + displayName: "Install protobuf" + - script: | + dpkg-buildpackage -rfakeroot -b -us -uc + workingDirectory: src/sonic-dash-api + displayName: "Build sonic-dash-api" + - script: | + cp *.deb /tmp/artifacts + workingDirectory: src + - publish: /tmp/artifacts + artifact: sonic-buildimage.amd64.ubuntu20_04 + displayName: "Archive sonic-buildimage debian packages for ubuntu20.04" diff --git a/.gitmodules b/.gitmodules index 06930dab4139..602efaebd03a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -124,3 +124,6 @@ [submodule "src/dhcpmon"] path = src/dhcpmon url = https://github.com/sonic-net/sonic-dhcpmon.git +[submodule "src/sonic-dash-api/sonic-dash-api"] + path = src/sonic-dash-api/sonic-dash-api + url = https://github.com/sonic-net/sonic-dash-api.git diff --git a/dockers/docker-sonic-mgmt/Dockerfile.j2 b/dockers/docker-sonic-mgmt/Dockerfile.j2 index f3b95c0cb226..3540ea21722a 100755 --- a/dockers/docker-sonic-mgmt/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt/Dockerfile.j2 @@ -145,6 +145,27 @@ RUN dpkg -i \ debs/{{ deb }}{{' '}} {%- endfor %} +# Install protobuf 3.21.12 +RUN mkdir -p /tmp/protobuf \ + cd /tmp/protobuf \ + && wget https://sonicstorage.blob.core.windows.net/public/ubuntu/20.04/libprotobuf-dev_3.21.12-3_amd64.deb \ + && wget https://sonicstorage.blob.core.windows.net/public/ubuntu/20.04/libprotobuf-lite32_3.21.12-3_amd64.deb \ + && wget https://sonicstorage.blob.core.windows.net/public/ubuntu/20.04/libprotobuf32_3.21.12-3_amd64.deb \ + && wget https://sonicstorage.blob.core.windows.net/public/ubuntu/20.04/libprotoc-dev_3.21.12-3_amd64.deb \ + && wget https://sonicstorage.blob.core.windows.net/public/ubuntu/20.04/libprotoc32_3.21.12-3_amd64.deb \ + && wget https://sonicstorage.blob.core.windows.net/public/ubuntu/20.04/protobuf-compiler_3.21.12-3_amd64.deb \ + && wget https://sonicstorage.blob.core.windows.net/public/ubuntu/20.04/python3-protobuf_3.21.12-3_amd64.deb \ + && dpkg -i *.deb \ + && rm -rf /tmp/protobuf + +# Install dash-api +RUN cd /tmp \ + && mkdir -p /usr/lib/python3/dist-packages/dash_api \ + && wget https://raw.githubusercontent.com/sonic-net/sonic-buildimage/master/src/sonic-dash-api/pypkg/__init__.py -O /usr/lib/python3/dist-packages/dash_api/__init__.py \ + && git clone https://github.com/sonic-net/sonic-dash-api.git \ + && protoc -I=sonic-dash-api/proto --python_out=/usr/lib/python3/dist-packages/dash_api sonic-dash-api/proto/*.proto \ + && rm -rf /tmp/sonic-dash-api + RUN mkdir /var/run/sshd EXPOSE 22 diff --git a/rules/protobuf.dep b/rules/protobuf.dep new file mode 100644 index 000000000000..8761659deeed --- /dev/null +++ b/rules/protobuf.dep @@ -0,0 +1,9 @@ + +SPATH := $($(PROTOBUF)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/protobuf.mk rules/protobuf.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +DEP_FILES += $(shell git ls-files $(SPATH)) + +$(PROTOBUF)_CACHE_MODE := GIT_CONTENT_SHA +$(PROTOBUF)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(PROTOBUF)_DEP_FILES := $(DEP_FILES) diff --git a/rules/protobuf.mk b/rules/protobuf.mk new file mode 100644 index 000000000000..8822d1357498 --- /dev/null +++ b/rules/protobuf.mk @@ -0,0 +1,36 @@ +# protobuf package + +PROTOBUF_VERSION = 3.21.12 +PROTOBUF_VERSION_FULL = $(PROTOBUF_VERSION)-3 + +export PROTOBUF_VERSION +export PROTOBUF_VERSION_FULL + +PROTOBUF = libprotobuf32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(PROTOBUF)_SRC_PATH = $(SRC_PATH)/protobuf +SONIC_MAKE_DEBS += $(PROTOBUF) + +PROTOBUF_DEV = libprotobuf-dev_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(PROTOBUF_DEV)_DEPENDS = $(PROTOBUF) $(PROTOBUF_LITE) +$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_DEV))) + +PROTOBUF_LITE = libprotobuf-lite32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_LITE))) + +PROTOC = libprotoc_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(PROTOC)_RDEPENDS = $(PROTOBUF) $(PROTOBUF_LITE) +$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOC))) + +PROTOC32 = libprotoc32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(PROTOC32)_RDEPENDS = $(PROTOBUF) $(PROTOBUF_LITE) +$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOC32))) + +PROTOBUF_COMPILER = protobuf-compiler_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(PROTOBUF_COMPILER)_DEPENDS = $(PROTOC32) +$(PROTOBUF_COMPILER)_RDEPENDS = $(PROTOC) +$(eval $(call add_derived_package,$(PROTOBUF),$(PROTOBUF_COMPILER))) + +PYTHON3_PROTOBUF = python3-protobuf_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb +$(PYTHON3_PROTOBUF)_DEPENDS = $(PROTOBUF_DEV) $(PROTOBUF) +$(PYTHON3_PROTOBUF)_RDEPENDS = $(PROTOBUF) +$(eval $(call add_derived_package,$(PROTOBUF),$(PYTHON3_PROTOBUF))) diff --git a/rules/sonic-dash-api.dep b/rules/sonic-dash-api.dep new file mode 100644 index 000000000000..6d4ffcad847f --- /dev/null +++ b/rules/sonic-dash-api.dep @@ -0,0 +1,13 @@ + +SPATH := $($(LIB_SONIC_DASH_API)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-dash-api.mk rules/sonic-dash-api.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +DEP_FILES += $(shell git ls-files $(SPATH) | grep -v sonic-dash-api) +SMDEP_FILES := $(addprefix $(SPATH)/sonic-dash-api/,$(shell cd $(SPATH)/sonic-dash-api && git ls-files)) + +$(LIB_SONIC_DASH_API)_CACHE_MODE := GIT_CONTENT_SHA +$(LIB_SONIC_DASH_API)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(LIB_SONIC_DASH_API)_DEP_FILES := $(DEP_FILES) +$(LIB_SONIC_DASH_API)_SMDEP_FILES := $(SMDEP_FILES) +$(LIB_SONIC_DASH_API)_SMDEP_PATHS := $(SPATH) + diff --git a/rules/sonic-dash-api.mk b/rules/sonic-dash-api.mk new file mode 100644 index 000000000000..e6d9e34c0395 --- /dev/null +++ b/rules/sonic-dash-api.mk @@ -0,0 +1,20 @@ +# libdashsai package + +LIB_SONIC_DASH_API_VERSION = 1.0.0 + +LIB_SONIC_DASH_API = libdashapi_$(LIB_SONIC_DASH_API_VERSION)_$(CONFIGURED_ARCH).deb +$(LIB_SONIC_DASH_API)_SRC_PATH = $(SRC_PATH)/sonic-dash-api + +$(LIB_SONIC_DASH_API)_DEPENDS += $(PROTOBUF) $(PROTOBUF_LITE) $(PROTOBUF_DEV) $(PROTOBUF_COMPILER) +$(LIB_SONIC_DASH_API)_RDEPENDS += $(PROTOBUF) $(PROTOBUF_LITE) $(PYTHON3_PROTOBUF) + +SONIC_DPKG_DEBS += $(LIB_SONIC_DASH_API) + +LIB_SONIC_DASH_API_DBG = libdashapi-dbgsym_$(LIB_SONIC_DASH_API_VERSION)_$(CONFIGURED_ARCH).deb +$(eval $(call add_derived_package,$(LIB_SONIC_DASH_API),$(LIB_SONIC_DASH_API_DBG))) + +# The .c, .cpp, .h & .hpp files under src/{$DBG_SRC_ARCHIVE list} +# are archived into debug one image to facilitate debugging. +# +DBG_SRC_ARCHIVE += sonic-dash-api + diff --git a/rules/swss.mk b/rules/swss.mk index e4d18cf2eb3d..df12ade5a479 100644 --- a/rules/swss.mk +++ b/rules/swss.mk @@ -4,11 +4,13 @@ SWSS = swss_1.0.0_$(CONFIGURED_ARCH).deb $(SWSS)_SRC_PATH = $(SRC_PATH)/sonic-swss $(SWSS)_DEPENDS += $(LIBSAIREDIS_DEV) $(LIBSAIMETADATA_DEV) $(LIBTEAM_DEV) \ $(LIBTEAMDCTL) $(LIBTEAM_UTILS) $(LIBSWSSCOMMON_DEV) \ - $(LIBSAIVS) $(LIBSAIVS_DEV) + $(LIBSAIVS) $(LIBSAIVS_DEV) \ + $(PROTOBUF) $(PROTOBUF_LITE) $(PROTOBUF_DEV) $(LIB_SONIC_DASH_API) $(SWSS)_UNINSTALLS = $(LIBSAIVS_DEV) $(SWSS)_RDEPENDS += $(LIBSAIREDIS) $(LIBSAIMETADATA) $(LIBTEAM) \ - $(LIBTEAMDCTL) $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON) + $(LIBTEAMDCTL) $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON) \ + $(PROTOBUF) $(PROTOBUF_LITE) $(PYTHON3_PROTOBUF) $(LIB_SONIC_DASH_API) SONIC_DPKG_DEBS += $(SWSS) SWSS_DBG = swss-dbg_1.0.0_$(CONFIGURED_ARCH).deb diff --git a/slave.mk b/slave.mk index 7974d8baebe1..560ceec95d69 100644 --- a/slave.mk +++ b/slave.mk @@ -1577,4 +1577,4 @@ jessie : $$(addprefix $(TARGET_PATH)/,$$(JESSIE_DOCKER_IMAGES)) \ ## To build some commonly used libs. Some submodules depend on these libs. ## It is used in component pipelines. For example: swss needs libnl, libyang -lib-packages: $(addprefix $(DEBS_PATH)/,$(LIBNL3) $(LIBYANG)) +lib-packages: $(addprefix $(DEBS_PATH)/,$(LIBNL3) $(LIBYANG) $(PROTOBUF) $(LIB_SONIC_DASH_API)) diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2 index fbe6e37bf9b8..5e70804af22b 100644 --- a/sonic-slave-bullseye/Dockerfile.j2 +++ b/sonic-slave-bullseye/Dockerfile.j2 @@ -344,8 +344,6 @@ RUN apt-get update && apt-get install -y \ libnetfilter-conntrack-dev \ libnftnl-dev \ # For SAI3.7 - protobuf-compiler \ - libprotobuf-dev \ xxd \ # For DHCP Monitor tool libexplain-dev \ @@ -442,7 +440,13 @@ RUN apt-get update && apt-get install -y \ net-tools \ # For audisp-tacplus libauparse-dev \ - auditd + auditd \ +# For protobuf + dh-elpa \ + xmlto \ + rake-compiler \ + default-jdk \ + libgoogle-gson-java {%- if CROSS_BUILD_ENVIRON == "y" %} # Arm vs. amd64 versions conflict - remove amd64 packages diff --git a/src/protobuf/Makefile b/src/protobuf/Makefile new file mode 100644 index 000000000000..fc208af76779 --- /dev/null +++ b/src/protobuf/Makefile @@ -0,0 +1,33 @@ +SHELL = /bin/bash +.ONESHELL: +.SHELLFLAGS += -e + +MAIN_TARGET = libprotobuf32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb +DERIVED_TARGETS = protobuf-compiler_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \ + libprotobuf-dev_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \ + libprotobuf-lite32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \ + libprotoc32_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb \ + python3-protobuf_$(PROTOBUF_VERSION_FULL)_$(CONFIGURED_ARCH).deb + +$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : + # Remove any stale files + rm -rf protobuf-$(PROTOBUF_VERSION) + + wget -O protobuf_$(PROTOBUF_VERSION).orig.tar.gz http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION).orig.tar.gz + wget -O protobuf_$(PROTOBUF_VERSION_FULL).dsc http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION_FULL).dsc + wget -O protobuf_$(PROTOBUF_VERSION_FULL).debian.tar.xz http://deb.debian.org/debian/pool/main/p/protobuf/protobuf_$(PROTOBUF_VERSION_FULL).debian.tar.xz + dpkg-source -x protobuf_$(PROTOBUF_VERSION_FULL).dsc + + pushd protobuf-$(PROTOBUF_VERSION) + +ifeq ($(CROSS_BUILD_ENVIRON), y) + dpkg-buildpackage -us -uc -b -a$(CONFIGURED_ARCH) -Pcross,nocheck -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) +else + dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) +endif + + popd + + mv $(DERIVED_TARGETS) $* $(DEST)/ + +$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET) diff --git a/src/sonic-dash-api/.gitignore b/src/sonic-dash-api/.gitignore new file mode 100644 index 000000000000..ff64e429bd09 --- /dev/null +++ b/src/sonic-dash-api/.gitignore @@ -0,0 +1,2 @@ +build +debian/sonic-dash-api diff --git a/src/sonic-dash-api/Makefile b/src/sonic-dash-api/Makefile new file mode 100644 index 000000000000..0031800cced7 --- /dev/null +++ b/src/sonic-dash-api/Makefile @@ -0,0 +1,47 @@ +.ONESHELL: +SHELL = /bin/bash +.SHELLFLAGS += -e + +RM := rm -rf +CP := cp -rf +MKDIR := mkdir +MV := mv +LIBDASHAPI := libdashapi.so +BUILD_DIR := build +PYPKG_DIR := pypkg +DESTDIR := +DASH_API_PROTO_DIR := sonic-dash-api/proto +INSTALLED_HEADER_DIR := $(DESTDIR)/usr/include/dash_api +INSTALLED_LIB_DIR := $(DESTDIR)/usr/lib +INSTALLED_PYTHON_DIR := $(DESTDIR)/usr/lib/python3/dist-packages/dash_api + +all: compile_cpp_proto dashapi.so compile_py_proto + +compile_cpp_proto: + $(MKDIR) -p $(BUILD_DIR) + protoc -I=$(DASH_API_PROTO_DIR) --cpp_out=$(BUILD_DIR) $(DASH_API_PROTO_DIR)/*.proto + +dashapi.so: compile_cpp_proto + g++ -std=c++14 -fPIC -shared -o $(BUILD_DIR)/$(LIBDASHAPI) $(wildcard $(BUILD_DIR)/*.pb.cc) -lprotobuf + +compile_py_proto: + protoc -I=$(DASH_API_PROTO_DIR) --python_out=$(PYPKG_DIR) $(DASH_API_PROTO_DIR)/*.proto + +clean: + $(RM) $(BUILD_DIR) + $(RM) $(PYPKG_DIR)/*_pb2.py + +install: + $(MKDIR) -p $(INSTALLED_HEADER_DIR) + $(CP) $(BUILD_DIR)/*.pb.h $(INSTALLED_HEADER_DIR) + $(MKDIR) -p $(INSTALLED_LIB_DIR) + $(CP) $(BUILD_DIR)/$(LIBDASHAPI) $(INSTALLED_LIB_DIR) + $(MKDIR) -p $(INSTALLED_PYTHON_DIR) + $(CP) $(PYPKG_DIR)/* $(INSTALLED_PYTHON_DIR) + +uninstall: + $(RM) $(INSTALLED_HEADER_DIR) + $(RM) $(INSTALLED_LIB_DIR)/$(LIBDASHAPI) + $(RM) $(INSTALLED_PYTHON_DIR) + +.PHONY: uninstall clean diff --git a/src/sonic-dash-api/debian/changelog b/src/sonic-dash-api/debian/changelog new file mode 100644 index 000000000000..e5cb52398c4c --- /dev/null +++ b/src/sonic-dash-api/debian/changelog @@ -0,0 +1,6 @@ +sonic (1.0.0) stable; urgency=medium + + * Initial release. + + -- Ze Gan Wed, 14 Jun 2023 12:00:00 -0800 + diff --git a/src/sonic-dash-api/debian/compat b/src/sonic-dash-api/debian/compat new file mode 100644 index 000000000000..9d607966b721 --- /dev/null +++ b/src/sonic-dash-api/debian/compat @@ -0,0 +1 @@ +11 \ No newline at end of file diff --git a/src/sonic-dash-api/debian/control b/src/sonic-dash-api/debian/control new file mode 100644 index 000000000000..545204fbe7da --- /dev/null +++ b/src/sonic-dash-api/debian/control @@ -0,0 +1,13 @@ +Source: sonic +Maintainer: Ze Gan +Section: net +Priority: optional +Build-Depends: dh-exec (>=0.3), debhelper (>= 12), autotools-dev, +Standards-Version: 1.0.0 + +Package: libdashapi +Architecture: any +Build-Depends: protobuf-compiler (>=3.21.12), libprotobuf-dev (>=3.21.12) +Depends: libprotobuf32 (>=3.21.12), libprotobuf-lite32 (>=3.21.12) +Section: libs +Description: DASH API definition for the SONiC project. diff --git a/src/sonic-dash-api/debian/rules b/src/sonic-dash-api/debian/rules new file mode 100755 index 000000000000..e661a7acd0bf --- /dev/null +++ b/src/sonic-dash-api/debian/rules @@ -0,0 +1,8 @@ +#!/usr/bin/make -f +#export DH_VERBOSE = 1 + +%: + dh $@ + +override_dh_auto_build: + make all diff --git a/src/sonic-dash-api/pypkg/__init__.py b/src/sonic-dash-api/pypkg/__init__.py new file mode 100644 index 000000000000..ade95c521abf --- /dev/null +++ b/src/sonic-dash-api/pypkg/__init__.py @@ -0,0 +1,5 @@ +import sys +import os + + +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) diff --git a/src/sonic-dash-api/sonic-dash-api b/src/sonic-dash-api/sonic-dash-api new file mode 160000 index 000000000000..30415cd41ef2 --- /dev/null +++ b/src/sonic-dash-api/sonic-dash-api @@ -0,0 +1 @@ +Subproject commit 30415cd41ef2ae62841417b4a6a9113129cd718d