Fix docker & Makefile perms (#202)

* Incremental work on docker permissions fixes. * More progress on docker and make permissions issues. * Trial remove -u root from CI file. * Makefile and CI file perms * Fix CI docker flags * Final purge of obs sudo commands to fix perms. * Test for veth's before deleting. Co-authored-by: Chris Sommers <chrispsommers@gmail.com>
sonic-net · Sep 11, 2022 · e31982c · e31982c
1 parent 6d1c50a
commit e31982c
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 71 deletions.
diff --git a/.github/workflows/dash-bmv2-ci.yml b/.github/workflows/dash-bmv2-ci.yml
@@ -40,8 +40,9 @@ jobs:
     name: Build and Test DASH Pipeline
     runs-on: ubuntu-20.04
     env:
-      docker_fg_flags: -u root --privileged
-      docker_bg_flags: -d -u root --privileged
+      docker_fg_flags: --privileged
+      docker_fg_root_flags: --privileged -u root
+      docker_bg_flags: -d --privileged
     defaults:
       run:
         working-directory: ./dash-pipeline
@@ -76,9 +77,9 @@ jobs:
     - name: Run saithrift server
       run:  DOCKER_FLAGS=$docker_bg_flags make run-saithrift-server
     - name: Run PTF Tests
-      run:  DOCKER_FLAGS=$docker_fg_flags make run-saithrift-ptftests
+      run:  DOCKER_FLAGS=$docker_fg_root_flags make run-saithrift-ptftests
     - name: Deploy ixia-c Traffic Generator
       run:  DOCKER_FLAGS=$docker_fg_flags make deploy-ixiac
     - name: Run Pytests
-      run:  DOCKER_FLAGS=$docker_fg_flags make run-saithrift-pytests
+      run:  DOCKER_FLAGS=$docker_fg_root_flags make run-saithrift-pytests
 
diff --git a/dash-pipeline/Makefile b/dash-pipeline/Makefile
@@ -1,45 +1,44 @@
 SHELL = /bin/bash
 
 # "All" type targets for convenience
-all:fix-perms p4 sai saithrift-server docker-saithrift-client test
+all:p4 sai saithrift-server docker-saithrift-client test
 
 run-all-tests:run-libsai-test deploy-ixiac run-saithrift-client-tests
 run-saithrift-client-tests: run-saithrift-ptftests run-saithrift-pytests
 run-saithrift-client-dev-tests: run-saithrift-dev-ptftests run-saithrift-dev-pytests
 
 .PHONY:clean
-clean: fix-perms kill-all p4-clean sai-clean test-clean network-clean saithrift-server-clean
-	rm -rf bmv2/dash_pipeline.bmv2
+clean: kill-all p4-clean sai-clean test-clean network-clean saithrift-server-clean
+	rm -rf $(P4_OUTDIR)
 
 kill-all: kill-saithrift-server kill-switch undeploy-ixiac
 
-
-.PHONY: fix-perms
-fix-perms:
-	@echo "### Applying Fix permissions workaround..."
-	sudo chmod -R o+rw SAI/ bmv2/ tests/
-
 PWD := $(shell pwd)
+DASH_USER ?=dashuser
+DASH_GROUP ?=dashusers
+DASH_UID ?=4321
+DASH_GUID?=4321
+DASH_HOST?=dash
 
 # Default Docker images, override from env var
 
 # Needed Libraries only
 DOCKER_GRPC_IMG ?=chrissommers/dash-grpc:1.43.2
 
-# Slimmed-down version bmv2 backend only 632MB - works:
-DOCKER_P4C_BMV2_IMG ?=chrissommers/dash-p4c-bmv2:220819
+# Slimmed-down version bmv2 backend only 632MB:
+DOCKER_P4C_BMV2_IMG ?=chrissommers/dash-p4c-bmv2:220825
 
-# Builds sai-P4rt clients to run inside bmvs process
-DOCKER_BMV2_BLDR_IMG ?=chrissommers/dash-bmv2-bldr:220630
+# Builds SAI-P4RT clients to run inside bmv2 process
+DOCKER_BMV2_BLDR_IMG ?=chrissommers/dash-bmv2-bldr:220819
 
 # Runs bmv2 process - consider slimmer if don't need sai/p4rt clients inside
 DOCKER_BMV2_RUN_IMG ?=$(DOCKER_BMV2_BLDR_IMG)
 
 # Compiles sai headers, libsai, saithrift server & client/server libs
-DOCKER_SAITHRIFT_BLDR_IMG ?=chrissommers/dash-saithrift-bldr:220719
+DOCKER_SAITHRIFT_BLDR_IMG ?=chrissommers/dash-saithrift-bldr:220819
 
 # Base image with test frameworks, DASH client libs not installed
-DOCKER_SAITHRIFT_CLIENT_BLDR_IMG ?=chrissommers/dash-saithrift-client-bldr:220723
+DOCKER_SAITHRIFT_CLIENT_BLDR_IMG ?=chrissommers/dash-saithrift-client-bldr:220819
 
 # Client image built locally with saithrift client libs + PTF & Pytest frameworks
 # TODO: consider some other tagging scheme
@@ -70,24 +69,26 @@ sai-submodule:
 # P4 Source code compile TARGETS
 ######################################
 
-P4_ARTIFACTS=bmv2/dash_pipeline.bmv2/dash_pipeline.json bmv2/dash_pipeline.bmv2/dash_pipeline_p4rt.txt
+P4_SRC=$(wildcard bmv2/*.p4)
+P4_MAIN=bmv2/dash_pipeline.p4
+P4_OUTDIR=bmv2/dash_pipeline.bmv2
+
+P4_ARTIFACTS=$(P4_OUTDIR)/dash_pipeline.json $(P4_OUTDIR)/dash_pipeline_p4rt.txt
 
-p4: fix-perms bmv2/dash_pipeline.bmv2/dash_pipeline.json
+p4: $(P4_OUTDIR)/dash_pipeline.json
 
 .PHONY:p4-clean
 p4-clean:
-	-rm -rf bmv2/dash_pipeline.bmv2
-
-P4_SRC=$(wildcard bmv2/*.p4)
-P4_MAIN=bmv2/dash_pipeline.p4
-P4_OUTDIR=bmv2/dash_pipeline.bmv2
+	-rm -rf $(P4_OUTDIR)
 # Compile P4 into bmv2 .json fle and P4info for SAI header autogeneration
 $(P4_ARTIFACTS): $(P4_SRC)
 	@echo "Compile P4 program $(P4_MAIN) ..."
 	mkdir -p $(P4_OUTDIR) && \
+	chmod o+w $(P4_OUTDIR) && \
 	docker run \
 	--rm \
 	--name dash-p4c-$(USER) \
+	-u $(DASH_USER) \
 	$(DOCKER_FLAGS) \
 	-v $(PWD)/bmv2:/bmv2 \
 	-w / \
@@ -114,20 +115,18 @@ DOCKER_RUN_SAITHRIFT_BLDR =\
 # TODO - create separate rules for headers, libsai.so
 .PHONY:sai
 
-sai: fix-perms sai-clean sai-headers sai-meta libsai
+sai: sai-clean sai-headers sai-meta libsai
 
-sai-headers: fix-perms p4 | SAI/SAI 
+sai-headers: p4 | SAI/SAI 
 	@echo "Generate SAI library headers and implementation..."
-	mkdir -p SAI/lib
+	mkdir -p SAI/lib && chmod -R o+w SAI && \
 	$(DOCKER_RUN) \
 		$(DOCKER_FLAGS) \
 		--name build_sai-$(USER) \
 		-w /SAI $(DOCKER_SAITHRIFT_BLDR_IMG) \
 	    ./generate_dash_api.sh
-	# Fixup perms from generate_dash_api using sudo to call python,so subsequent scripts can access
-	sudo chmod -R a+rw $(PWD)/SAI/SAI
 
-sai-meta: fix-perms 
+sai-meta: 
 	@echo "Generate SAI metadata..."
 	# hack - remove scripts which cause Git ownership failures in CI pipelines
 	# We don't need them, they're to check that SAI headers didn't experience enum changes etc.
@@ -140,10 +139,9 @@ sai-meta: fix-perms
 		-v $(PWD)/..:/dash -w /dash/dash-pipeline/SAI/SAI/meta \
 		$(DOCKER_SAITHRIFT_BLDR_IMG) \
 	    make
-	sudo chmod -R o+rw SAI/
 
 # TODO - add SAI header dependencies
-libsai: fix-perms 
+libsai: 
 	@echo "build libsai.so..."
 	$(DOCKER_RUN) \
 		$(DOCKER_FLAGS) \
@@ -152,11 +150,11 @@ libsai: fix-perms
 		$(DOCKER_BMV2_BLDR_IMG) \
 	    make
 
-libsai-clean: fix-perms 
-	-rm -rf SAI/lib/libsai.so
+libsai-clean: 
+	-rm -rf SAI/lib/*
 
 .PHONY:sai-clean
-sai-clean: fix-perms SAI/SAI libsai-clean
+sai-clean: SAI/SAI libsai-clean saithrift-server-clean
 	@echo "Restoring SAI subdirectories to baseline..."
 	rm -rf SAI/SAI/inc SAI/SAI/experimental SAI/SAI/meta
 	cd SAI/SAI && git checkout -- inc experimental meta
@@ -176,8 +174,8 @@ run-switch: network
 	$(DOCKER_RUN) \
 		--name simple_switch-$(USER) \
 		-u root \
-		-v $(PWD)/bmv2/dash_pipeline.bmv2/dash_pipeline.json:/etc/dash/dash_pipeline.json \
-		-v $(PWD)/bmv2/dash_pipeline.bmv2/dash_pipeline_p4rt.txt:/etc/dash/dash_pipeline_p4rt.txt \
+		-v $(PWD)/$(P4_OUTDIR)/dash_pipeline.json:/etc/dash/dash_pipeline.json \
+		-v $(PWD)/$(P4_OUTDIR)/dash_pipeline_p4rt.txt:/etc/dash/dash_pipeline_p4rt.txt \
 		$(DOCKER_BMV2_RUN_IMG) \
 		env LD_LIBRARY_PATH=/usr/local/lib \
 		simple_switch_grpc \
@@ -194,7 +192,7 @@ kill-switch:
 # SAI-THRIFT SERVER TARGETS
 ###############################
 
-saithrift-server:fix-perms
+saithrift-server:
 	$(DOCKER_RUN_SAITHRIFT_BLDR) \
 	    make $@
 
@@ -204,8 +202,8 @@ DOCKER_RUN_SAITHRIFT_SRVR =\
 	$(DOCKER_FLAGS) \
 	--net=host \
 	--name dash-saithrift-server-$(USER) \
-	-v $(PWD)/bmv2/dash_pipeline.bmv2/dash_pipeline.json:/etc/dash/dash_pipeline.json \
-	-v $(PWD)/bmv2/dash_pipeline.bmv2/dash_pipeline_p4rt.txt:/etc/dash/dash_pipeline_p4rt.txt \
+	-v $(PWD)/$(P4_OUTDIR)/dash_pipeline.json:/etc/dash/dash_pipeline.json \
+	-v $(PWD)/$(P4_OUTDIR)/dash_pipeline_p4rt.txt:/etc/dash/dash_pipeline_p4rt.txt \
 	-v $(PWD)/SAI:/SAI \
 	-v $(PWD)/SAI/SAI/meta:/meta \
 	-w /SAI/rpc/usr/sbin \
@@ -226,6 +224,7 @@ run-saithrift-server-bash:
 saithrift-server-clean:
 	$(DOCKER_RUN_SAITHRIFT_BLDR) \
 	    make $@
+	rm -rf SAI/rpc
 
 ###############################
 # SAI TEST TARGETS
@@ -240,6 +239,7 @@ test-clean:
 
 .PHONY:libsai-test
 libsai-test:SAI/SAI SAI/lib/libsai.so
+	chmod -R a+w tests/libsai
 	$(DOCKER_RUN) \
 	    --name dash-build-test-$(USER) \
 		-w /tests/libsai  $(DOCKER_BMV2_BLDR_IMG) \
@@ -286,10 +286,11 @@ docker-bmv2-bldr:
 	docker build \
 		-f dockerfiles/Dockerfile.bmv2-bldr \
 	    -t $(DOCKER_BMV2_BLDR_IMG) \
-	    --build-arg user=$(USER) \
-	    --build-arg uid=$(shell id -u) \
-	    --build-arg guid=$(shell id -g) \
-	    --build-arg hostname=$(shell echo $$HOSTNAME) \
+	    --build-arg user=$(DASH_USER) \
+	    --build-arg group=$(DASH_GROUP) \
+	    --build-arg uid=$(DASH_UID) \
+	    --build-arg guid=$(DASH_GUID) \
+	    --build-arg hostname=$(DASH_HOST) \
 	    --build-arg available_processors=$(shell nproc) \
 	    dockerfiles
 
@@ -303,10 +304,11 @@ docker-saithrift-bldr:
 	docker build \
 		-f dockerfiles/Dockerfile.saithrift-bldr \
 	    -t $(DOCKER_SAITHRIFT_BLDR_IMG) \
-	    --build-arg user=$(USER) \
-	    --build-arg uid=$(shell id -u) \
-	    --build-arg guid=$(shell id -g) \
-	    --build-arg hostname=$(shell echo $$HOSTNAME) \
+	    --build-arg user=$(DASH_USER) \
+	    --build-arg group=$(DASH_GROUP) \
+	    --build-arg uid=$(DASH_UID) \
+	    --build-arg guid=$(DASH_GUID) \
+	    --build-arg hostname=$(DASH_HOST) \
 	    --build-arg available_processors=$(shell nproc) \
 		dockerfiles
 
@@ -322,10 +324,11 @@ docker-saithrift-client-bldr:
 	docker build \
 		-f dockerfiles/Dockerfile.saithrift-client-bldr \
 	    -t $(DOCKER_SAITHRIFT_CLIENT_BLDR_IMG) \
-	    --build-arg user=$(USER) \
-	    --build-arg uid=$(shell id -u) \
-	    --build-arg guid=$(shell id -g) \
-	    --build-arg hostname=$(shell echo $$HOSTNAME) \
+	    --build-arg user=$(DASH_USER) \
+	    --build-arg group=$(DASH_GROUP) \
+	    --build-arg uid=$(DASH_UID) \
+	    --build-arg guid=$(DASH_GUID) \
+	    --build-arg hostname=$(DASH_HOST) \
 	    --build-arg available_processors=$(shell nproc) \
 		./tests
 
@@ -394,14 +397,26 @@ run-saithrift-client-bash:
 
 ###############################
 
+# docker-dash-p4c:
+# 	docker build \
+# 		-f dockerfiles/Dockerfile.p4c-bmv2 \
+# 	    -t $(DOCKER_P4C_BMV2_IMG) \
+# 	    --build-arg user=$(USER) \
+# 	    --build-arg uid=$(shell id -u) \
+# 	    --build-arg guid=$(shell id -g) \
+# 	    --build-arg hostname=$(shell echo $$HOSTNAME) \
+# 	    --build-arg available_processors=$(shell nproc) \
+# 		dockerfiles
+
 docker-dash-p4c:
 	docker build \
 		-f dockerfiles/Dockerfile.p4c-bmv2 \
 	    -t $(DOCKER_P4C_BMV2_IMG) \
-	    --build-arg user=$(USER) \
-	    --build-arg uid=$(shell id -u) \
-	    --build-arg guid=$(shell id -g) \
-	    --build-arg hostname=$(shell echo $$HOSTNAME) \
+	    --build-arg user=$(DASH_USER) \
+	    --build-arg group=$(DASH_GROUP) \
+	    --build-arg uid=$(DASH_UID) \
+	    --build-arg guid=$(DASH_GUID) \
+	    --build-arg hostname=$(DASH_HOST) \
 	    --build-arg available_processors=$(shell nproc) \
 		dockerfiles
 
@@ -416,10 +431,11 @@ docker-dash-grpc:
 	docker build \
 		-f dockerfiles/Dockerfile.grpc1.43.2 \
 	    -t $(DOCKER_GRPC_IMG) \
-	    --build-arg user=$(USER) \
-	    --build-arg uid=$(shell id -u) \
-	    --build-arg guid=$(shell id -g) \
-	    --build-arg hostname=$(shell echo $$HOSTNAME) \
+	    --build-arg user=$(DASH_USER) \
+	    --build-arg group=$(DASH_GROUP) \
+	    --build-arg uid=$(DASH_UID) \
+	    --build-arg guid=$(DASH_GUID) \
+	    --build-arg hostname=$(DASH_HOST) \
 	    --build-arg available_processors=$(shell nproc) \
 		dockerfiles
 
@@ -473,10 +489,10 @@ veth2: /sys/class/net/veth2
 	sudo sysctl net.ipv6.conf.veth3.accept_ra_pinfo=0
 	sudo sysctl net.ipv6.conf.veth3.router_solicitations=0
 
+# Delete veth's, test existence to avoid needless use of sudo
 network-clean:
-	# Note, deleting one member (e.g. veth0) deletes its peer (e.g. veth1)
-	-sudo ip link delete dev veth0
-	-sudo ip link delete dev veth2
+	@-([ -e /sys/class/net/veth0 ] && sudo ip link delete dev veth0 && echo "Deleted veth0/1") || echo "No veth0, not deleting"
+	@-([ -e /sys/class/net/veth2 ] && sudo ip link delete dev veth2 && echo "Deleted veth2/3") || echo "No veth2, not deleting"
 
 ###############################
 # IXIA-C TARGETS

diff --git a/dash-pipeline/SAI/generate_dash_api.sh b/dash-pipeline/SAI/generate_dash_api.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-sudo ./sai_api_gen.py \
+./sai_api_gen.py \
     /bmv2/dash_pipeline.bmv2/dash_pipeline_p4rt.json \
     --ignore-tables=appliance,eni_meter,slb_decap \
     dash
diff --git a/dash-pipeline/SAI/saithrift/Makefile b/dash-pipeline/SAI/saithrift/Makefile
@@ -27,7 +27,7 @@ SAIRPC_EXTRA_LIBS="\
 		-lssl \
 		-laddress_sorting"
 
-# Below based on: https://github.com/opencomputeproject/SAI/blob/088627dd90c3420daf96d294c661b4a152afb01e/ptf/SAI_PTF_user-guide.md
+# Below based on: https://github.com/opencomputeproject/SAI/blob/master/ptf/SAI_PTF_user-guide.md
 # Dependencies are assumed to be installed, e.g. in the docker container
 saithrift-server:
 	# Copy headers to /usr/include/sai
@@ -41,6 +41,7 @@ saithrift-server:
 	sudo cp $(LIB)/libsai.so /usr/lib
 
 	@echo "Build SAI thrift server and libraries..."
+	mkdir -p $(RPC_INST_DIR) && chmod a+w $(RPC_INST_DIR)
 
 	cd $(SAI) && export SAITHRIFTV2=y && \
 		export GEN_SAIRPC_OPTS="-ve" && \
@@ -57,6 +58,7 @@ saithrift-server:
 
 	# Copy thrift libs from builder image onto host
 	cp /usr/lib/libthrift*so* /usr/lib/thrift-0.11.0.tar.gz $(RPC_INST_DIR)
+	chmod -R a+w $(META)/generated $(META)/html $(META)/xml $(RPC_INST_DIR)
 
 saithrift-server-clean:
 	cd $(SAI) && export SAITHRIFTV2=y  && make clean

diff --git a/dash-pipeline/dockerfiles/Dockerfile.bmv2-bldr b/dash-pipeline/dockerfiles/Dockerfile.bmv2-bldr
@@ -63,13 +63,14 @@ COPY --from=grpc /usr/local/lib/lib*grpc*.so* \
 
 ARG user
 ARG uid
+ARG group
 ARG guid
 ARG hostname
 
 ENV BUILD_HOSTNAME $hostname
 ENV USER $user
 
-RUN groupadd -f -r -g $guid g$user
+RUN groupadd -f -r -g $guid $group
 
 RUN useradd $user -l -u $uid -g $guid -d /var/$user -m -s /bin/bash
 

diff --git a/dash-pipeline/dockerfiles/Dockerfile.p4c-bmv2 b/dash-pipeline/dockerfiles/Dockerfile.p4c-bmv2
@@ -45,6 +45,19 @@ COPY --from=p4lang-p4c \
         /usr/local/bin/p4c-bm2-ss \
         /usr/local/bin/
 
+
+ARG user
+ARG uid
+ARG group
+ARG guid
+ARG hostname
+
+ENV BUILD_HOSTNAME $hostname
+ENV USER $user
+
+RUN groupadd -f -r -g $guid $group
+
+RUN useradd $user -l -u $uid -g $guid -d /var/$user -m -s /bin/bash
 CMD bash
 
 # # Alternate approach - selective remove backends etc.