From bb0b47c0255718b9f7c8d01759b53057ecc0a438 Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 10:05:45 +0200 Subject: [PATCH 01/11] refactor<>: Change the workflows folder name. made the test to run on all pushes It should be in plural --- .github/{workflow => workflows}/check_comments.sh | 0 .github/{workflow => workflows}/comment-check.yml | 6 ++---- 2 files changed, 2 insertions(+), 4 deletions(-) rename .github/{workflow => workflows}/check_comments.sh (100%) rename .github/{workflow => workflows}/comment-check.yml (80%) diff --git a/.github/workflow/check_comments.sh b/.github/workflows/check_comments.sh similarity index 100% rename from .github/workflow/check_comments.sh rename to .github/workflows/check_comments.sh diff --git a/.github/workflow/comment-check.yml b/.github/workflows/comment-check.yml similarity index 80% rename from .github/workflow/comment-check.yml rename to .github/workflows/comment-check.yml index 0f21c782..777a0be4 100644 --- a/.github/workflow/comment-check.yml +++ b/.github/workflows/comment-check.yml @@ -1,12 +1,10 @@ -name: Comment Check +name: Check comments percentage on: push: branches: - main - pull_request: - branches: - - main + - '*' jobs: build: From e62ebdda009e45481421bc66f7c3fcda87524777 Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 10:28:16 +0200 Subject: [PATCH 02/11] refactor<>: fix "Permission denied" for running the unix script --- .github/workflows/comment-check.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/comment-check.yml b/.github/workflows/comment-check.yml index 777a0be4..4c89e659 100644 --- a/.github/workflows/comment-check.yml +++ b/.github/workflows/comment-check.yml @@ -14,6 +14,9 @@ jobs: - name: Checkout code uses: actions/checkout@v2 + - name: Give execute permission for script + run: chmod +x ./.github/workflows/check_comments.sh + - name: Check comment percentage run: | ./.github/workflows/check_comments.sh From 0a94a44745aa150643ec5808b06202ee32259ab3 Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 10:31:10 +0200 Subject: [PATCH 03/11] feat<>: add test for docker images Test if the docker would run good without problem in it --- .github/workflows/docker-compose-test.yml | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/docker-compose-test.yml diff --git a/.github/workflows/docker-compose-test.yml b/.github/workflows/docker-compose-test.yml new file mode 100644 index 00000000..9469c6e9 --- /dev/null +++ b/.github/workflows/docker-compose-test.yml @@ -0,0 +1,36 @@ +name: Docker Compose Test + +# Make it rin on any push to any branch +on: + push: + branches: + - main + - '*' + +jobs: + test_root_docker_compose: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v2 + # Give containers some time to start, adjust as needed + - name: Docker compose up (root) + run: | + docker-compose -f docker-compose.yml up -d + + sleep 30 + docker-compose -f docker-compose.yml down + + test_raspi_docker_compose: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v2 + + - name: Docker compose up (raspi) + run: | + docker-compose -f raspi/docker-compose.yml up -d + sleep 30 + docker-compose -f raspi/docker-compose.yml down From 75b498ae065beac40b621b3922184a2504b735df Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 11:33:34 +0200 Subject: [PATCH 04/11] refactor: remove build for inference service The service needs to have the onnx and ini file. These files do not exist on the repo. Another implementation would have been adding shell script to move the files if the only exist, I went with keeping the error so it is obvious when working offline with it --- .github/workflows/docker-compose-test.yml | 24 ++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-compose-test.yml b/.github/workflows/docker-compose-test.yml index 9469c6e9..ee1d0249 100644 --- a/.github/workflows/docker-compose-test.yml +++ b/.github/workflows/docker-compose-test.yml @@ -1,6 +1,5 @@ name: Docker Compose Test -# Make it rin on any push to any branch on: push: branches: @@ -8,29 +7,36 @@ on: - '*' jobs: - test_root_docker_compose: + Test_Nano_Docker: runs-on: ubuntu-latest steps: - name: Check out repository uses: actions/checkout@v2 - # Give containers some time to start, adjust as needed - - name: Docker compose up (root) - run: | - docker-compose -f docker-compose.yml up -d + # Exculding inference because it needs .oonx and .ini files, which are private + - name: Docker compose build (root without inference) + run: docker-compose -f docker-compose.yml build $(docker-compose -f docker-compose.yml config --services | grep -v inference) - sleep 30 + # Make sure it would run good for at least 30 seconds + - name: Docker compose up (root without inference) + run: | + docker-compose -f docker-compose.yml up $(docker-compose -f docker-compose.yml config --services | grep -v inference) -d + sleep 30 docker-compose -f docker-compose.yml down - test_raspi_docker_compose: + Test_Pi_Docker: runs-on: ubuntu-latest steps: - name: Check out repository uses: actions/checkout@v2 + # Build the Docker services: This validates the build process and catches issues related to the Dockerfile and dependencies. + - name: Docker compose build (raspi) + run: docker-compose -f raspi/docker-compose.yml build + # Run (up): Ensures that the application behaves correctly in a containerized environment. - name: Docker compose up (raspi) run: | docker-compose -f raspi/docker-compose.yml up -d - sleep 30 + sleep 30 docker-compose -f raspi/docker-compose.yml down From 0407fa6db32e3505f0be8c5dda2c4df1f6d86d7a Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 11:40:10 +0200 Subject: [PATCH 05/11] feat<>: add python lint It will check the code formatting with black, it won't fix the code but it would show the errors. This approach would force unified formatting across the workflow in the team --- .github/workflows/python-CI.yml | 28 ++++++++++ docker-compose.yml | 98 ++++++++++----------------------- 2 files changed, 57 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/python-CI.yml diff --git a/.github/workflows/python-CI.yml b/.github/workflows/python-CI.yml new file mode 100644 index 00000000..ef81b2a8 --- /dev/null +++ b/.github/workflows/python-CI.yml @@ -0,0 +1,28 @@ +name: Python CI + +on: + push: + branches: + - main + - '*' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9, 3.10] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Check code formatting with Black + run: | + black --check . diff --git a/docker-compose.yml b/docker-compose.yml index 1f3ed09f..e1b13e18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "2" +version: '2' volumes: volume_zerotier_config: volume_wireguard_config: @@ -10,12 +10,12 @@ volumes: volume_byodr_sessions: services: zerotier: - cpuset: "0" + cpuset: '0' image: zyclonite/zerotier:1.6.6 restart: always network_mode: host devices: - - "/dev/net/tun" + - '/dev/net/tun' cap_add: - SYS_ADMIN - NET_ADMIN @@ -23,13 +23,13 @@ services: volumes: - volume_zerotier_config:/var/lib/zerotier-one:rw wireguard: - cpuset: "0" + cpuset: '0' image: masipcat/wireguard-go container_name: wireguard restart: always network_mode: host devices: - - "/dev/net/tun" + - '/dev/net/tun' cap_add: - SYS_ADMIN - NET_ADMIN @@ -37,49 +37,49 @@ services: volumes: - volume_wireguard_config:/etc/wireguard:rw httpd: - cpuset: "0" + cpuset: '0' build: context: . dockerfile: httpd/Dockerfile restart: always network_mode: host - command: ["python", "wrap.py"] + command: ['python', 'wrap.py'] stop_signal: SIGKILL volumes: - volume_byodr_config:/config:rw ftpd: - cpuset: "0" + cpuset: '0' build: context: . dockerfile: ftpd/Dockerfile restart: always - command: ["python3", "wrap.py"] + command: ['python3', 'wrap.py'] stop_signal: SIGKILL ports: - - "21:21" - - "30000-30009:30000-30009" + - '21:21' + - '30000-30009:30000-30009' volumes: - volume_ftpd_config:/etc/pureftpd:rw - volume_byodr_sessions:/home/ftpuser:rw rosnode: - cpuset: "0" + cpuset: '0' build: context: . dockerfile: rosnode/Dockerfile restart: always - command: ["python3", "app.py", "--name", "rosnode"] + command: ['python3', 'app.py', '--name', 'rosnode'] network_mode: host stop_signal: SIGKILL volumes: - volume_byodr_sockets:/byodr:rw - volume_byodr_config:/config:ro mongodb: - cpuset: "0" + cpuset: '0' build: context: . dockerfile: mongodb/Dockerfile restart: always - command: ["python3", "wrap.py"] + command: ['python3', 'wrap.py'] network_mode: host stop_signal: SIGKILL environment: @@ -89,17 +89,12 @@ services: - volume_mongodb_config:/config:rw - volume_mongodb_data:/data/db:rw teleop: - cpuset: "0" + cpuset: '0' build: context: . dockerfile: teleop/Dockerfile restart: always - command: - [ - "sh", - "-c", - "python3 -m teleop.app --name teleop --routes /sessions/routes", - ] + command: ['sh', '-c', 'python3 -m teleop.app --name teleop --routes /sessions/routes'] network_mode: host environment: LD_PRELOAD: libgomp.so.1 @@ -108,12 +103,12 @@ services: - volume_byodr_config:/config:rw - volume_byodr_sessions:/sessions:rw vehicle: - cpuset: "1" + cpuset: '1' build: context: . dockerfile: vehicles/rover/rover_vehicle.dockerfile restart: always - command: ["python", "app.py", "--name", "vehicle"] + command: ['python', 'app.py', '--name', 'vehicle'] privileged: true # NvMedia device creation for omx decoder. network_mode: host environment: @@ -122,24 +117,15 @@ services: - volume_byodr_sockets:/byodr:rw - volume_byodr_config:/config:rw pilot: - cpuset: "2" + cpuset: '2' build: context: . dockerfile: pilot/Dockerfile restart: always privileged: true # Access to usb devices without udev rules. - command: - [ - "python3", - "-m", - "pilot.app", - "--name", - "pilot", - "--routes", - "/sessions/routes", - ] + command: ['python3', '-m', 'pilot.app', '--name', 'pilot', '--routes', '/sessions/routes'] ports: - - "8082:8082" + - '8082:8082' environment: LD_PRELOAD: libgomp.so.1 volumes: @@ -153,63 +139,37 @@ services: dockerfile: inference/runtime-cp36-jp441.dockerfile restart: always privileged: true - command: - [ - "python3", - "-m", - "inference.app", - "--user", - "/sessions/models", - "--routes", - "/sessions/routes", - ] + command: ['python3', '-m', 'inference.app', '--user', '/sessions/models', '--routes', '/sessions/routes'] environment: LD_PRELOAD: libgomp.so.1 - OMP_PLACES: "{3}" + OMP_PLACES: '{3}' volumes: - volume_byodr_sockets:/byodr:rw - volume_byodr_config:/config:ro - volume_byodr_sessions:/sessions:rw - communication: - cpuset: "3" + cpuset: '3' build: context: . dockerfile: communication/Dockerfile restart: always - command: - [ - "python3", - "-m", - "communication.app", - "--name", - "communication", - ] + command: ['python3', '-m', 'communication.app', '--name', 'communication'] ports: - - "1112:1112" + - '1112:1112' environment: LD_PRELOAD: libgomp.so.1 volumes: - volume_byodr_sockets:/byodr:rw - volume_byodr_config:/config:rw - following: - cpuset: "3" + cpuset: '3' build: context: . dockerfile: following/Dockerfile restart: always - command: - [ - "python3", - "-m", - "following.app", - "--name", - "following", - ] + command: ['python3', '-m', 'following.app', '--name', 'following'] environment: LD_PRELOAD: libgomp.so.1 volumes: - volume_byodr_sockets:/byodr:rw - volume_byodr_config:/config:rw - From e926b262dfd692b9bbad4134aa4e84b7a48dd0ac Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 12:09:58 +0200 Subject: [PATCH 06/11] refactor<>: position the detached flag to be behind specifying services --- .github/workflows/docker-compose-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-compose-test.yml b/.github/workflows/docker-compose-test.yml index ee1d0249..efe78a5d 100644 --- a/.github/workflows/docker-compose-test.yml +++ b/.github/workflows/docker-compose-test.yml @@ -20,7 +20,7 @@ jobs: # Make sure it would run good for at least 30 seconds - name: Docker compose up (root without inference) run: | - docker-compose -f docker-compose.yml up $(docker-compose -f docker-compose.yml config --services | grep -v inference) -d + docker-compose -f docker-compose.yml up -d $(docker-compose -f docker-compose.yml config --services | grep -v inference) sleep 30 docker-compose -f docker-compose.yml down From 5b4d519b7dfb49f8c37fb1d2c7d0db963d163c46 Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 13:28:29 +0200 Subject: [PATCH 07/11] feat<>: Add docker-compose.override.yml to overcome CPU higher than 1 in pilot service --- .github/workflows/docker-compose-test.yml | 7 ++++--- docker-compose.override.yml | 5 +++++ requirements.txt | 24 ----------------------- 3 files changed, 9 insertions(+), 27 deletions(-) create mode 100644 docker-compose.override.yml delete mode 100644 requirements.txt diff --git a/.github/workflows/docker-compose-test.yml b/.github/workflows/docker-compose-test.yml index efe78a5d..53b1405e 100644 --- a/.github/workflows/docker-compose-test.yml +++ b/.github/workflows/docker-compose-test.yml @@ -14,15 +14,16 @@ jobs: - name: Check out repository uses: actions/checkout@v2 # Exculding inference because it needs .oonx and .ini files, which are private + # Add docker-compose.override.yml to overcome CPU higher than 1 in pilot service - name: Docker compose build (root without inference) - run: docker-compose -f docker-compose.yml build $(docker-compose -f docker-compose.yml config --services | grep -v inference) + run: docker-compose -f docker-compose.yml -f docker-compose.override.yml build $(docker-compose -f docker-compose.yml -f docker-compose.override.yml config --services | grep -v inference) # Make sure it would run good for at least 30 seconds - name: Docker compose up (root without inference) run: | - docker-compose -f docker-compose.yml up -d $(docker-compose -f docker-compose.yml config --services | grep -v inference) + docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d $(docker-compose -f docker-compose.yml -f docker-compose.override. yml config --services | grep -v inference) sleep 30 - docker-compose -f docker-compose.yml down + docker-compose -f docker-compose.yml -f docker-compose.override.yml down Test_Pi_Docker: runs-on: ubuntu-latest diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 00000000..68fcda94 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,5 @@ +version: '3' # or whatever version you're using + +services: + pilot: + cpuset: '' # Clearing the CPU set constraint for testing diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b2d47c07..00000000 --- a/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -cachetools==5.3.0 -can==0.0.0 -carla==0.9.14 -ConfigParser==5.3.0 -Equation==1.2.01 -geographiclib==2.0 -gpiozero==1.6.2 -numpy==1.24.2 -onnxruntime==1.14.1 -opencv_python==4.7.0.72 -pandas==1.5.3 -pymodbus==3.2.2 -pymongo==4.3.3 -pyserial==3.5 -pyueye==4.96.952 -pyusb==1.2.1 -pyvesc==1.0.5 -pyzmq==25.0.2 -requests==2.28.2 -scikit_learn==1.2.2 -scipy==1.10.1 -simple_pid==1.0.1 -six==1.16.0 -tornado==6.2 From f12f1900df6bf160e79ad36fc6e4f22789d2665a Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 13:41:06 +0200 Subject: [PATCH 08/11] feat<>: remove accidental space was added when fixing indentations --- .github/workflows/docker-compose-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-compose-test.yml b/.github/workflows/docker-compose-test.yml index 53b1405e..a78d5787 100644 --- a/.github/workflows/docker-compose-test.yml +++ b/.github/workflows/docker-compose-test.yml @@ -21,9 +21,10 @@ jobs: # Make sure it would run good for at least 30 seconds - name: Docker compose up (root without inference) run: | - docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d $(docker-compose -f docker-compose.yml -f docker-compose.override. yml config --services | grep -v inference) + docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d $(docker-compose -f docker-compose.yml -f docker-compose.override.yml config --services | grep -v inference) sleep 30 docker-compose -f docker-compose.yml -f docker-compose.override.yml down + Test_Pi_Docker: runs-on: ubuntu-latest From c1f95a541a5a9c6c839c52963c89cad0b6d67e4c Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 13:42:59 +0200 Subject: [PATCH 09/11] refactor<>: remove the test for python 3.7 and remove installing dependencies There is extra test made in Py 3.1 even though I didn't write it --- .github/workflows/python-CI.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/python-CI.yml b/.github/workflows/python-CI.yml index ef81b2a8..9f8f0d0d 100644 --- a/.github/workflows/python-CI.yml +++ b/.github/workflows/python-CI.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: [3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 @@ -19,10 +19,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - name: Check code formatting with Black run: | black --check . From 20881bf05621958536f4d33dffe2f4f731515e52 Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 13:47:32 +0200 Subject: [PATCH 10/11] refactor<>: add pip to install black and check the format --- .github/workflows/python-CI.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-CI.yml b/.github/workflows/python-CI.yml index 9f8f0d0d..fcd0598c 100644 --- a/.github/workflows/python-CI.yml +++ b/.github/workflows/python-CI.yml @@ -19,6 +19,10 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install black - name: Check code formatting with Black run: | black --check . From d2497f018f6e4115213c4edaf69645b72a936f90 Mon Sep 17 00:00:00 2001 From: Ahmed Mahfouz Date: Thu, 26 Oct 2023 13:58:21 +0200 Subject: [PATCH 11/11] refactor<>: add the formatting to work only on the pushed files and not the whole repository --- .github/workflows/python-CI.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-CI.yml b/.github/workflows/python-CI.yml index fcd0598c..f032c560 100644 --- a/.github/workflows/python-CI.yml +++ b/.github/workflows/python-CI.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, 3.10] + python-version: [3.8, 3.9] steps: - uses: actions/checkout@v2 @@ -23,6 +23,12 @@ jobs: run: | python -m pip install --upgrade pip pip install black - - name: Check code formatting with Black + - name: Check code formatting with Black on changed files run: | - black --check . + FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep '\.py$' | xargs) + if [ ! -z "$FILES" ]; then + # Check the formatting of those files with black + black --check $FILES + else + echo "No Python files have changed." + fi