From 4e87d92ea9433b5ed945996d2d5422e121093f4e Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Mon, 15 Jan 2024 12:52:07 +0800 Subject: [PATCH 1/9] docs: polishing arrangement of badges --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9963a6a..7c4bdf9 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ # NebulaGraph Lite +[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://bit.ly/nebula-colab) +[![Jupyter](https://img.shields.io/badge/Jupyter-Supported-brightgreen)](https://github.com/jupyterlab/jupyterlab) [![for NebulaGraph](https://img.shields.io/badge/Toolchain-NebulaGraph-blue)](https://github.com/vesoft-inc/nebula) [![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm.fming.dev) -[![Jupyter](https://img.shields.io/badge/Jupyter-Supported-brightgreen)](https://github.com/jupyterlab/jupyterlab) -[![GitHub release (latest by date)](https://img.shields.io/github/v/release/wey-gu/nebulagraph-lite?label=Version)](https://github.com/wey-gu/nebulagraph-lite/releases) + + +[![GitHub release (latest by date)](https://img.shields.io/github/v/release/wey-gu/nebulagraph-lite?label=release)](https://github.com/wey-gu/nebulagraph-lite/releases) [![pypi-version](https://img.shields.io/pypi/v/nebulagraph-lite)](https://pypi.org/project/nebulagraph-lite/) [![python-version](https://img.shields.io/badge/python-3.8%20|%203.9%20|%203.10%20|%203.11%20|%203.12-blue)](https://www.python.org/) -[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://bit.ly/nebula-colab) Try NebulaGraph with `pip install`, on Linux/ WSL2 or even [Google Colab](https://bit.ly/nebula-colab)! From d4d79e1b3d3a23cdaf4caf149f126a0f3b5f802e Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Mon, 15 Jan 2024 12:57:43 +0800 Subject: [PATCH 2/9] ci: add PR checks --- .github/workflows/pr.yaml | 24 ++++++++++++++++++++++++ pyproject.toml | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pr.yaml diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 0000000..fb81baf --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,24 @@ +name: PR Workflow +on: + pull_request: + branches: + - master + +jobs: + build: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + - uses: pdm-project/setup-pdm@v3 + with: + python-version: '3.x' + cache: true + - name: Install dependencies + run: pdm install + + - name: Lint + run: pdm run lint + + - name: Dry run nebulagraph start + run: nebulagraph --debug start --cleanup diff --git a/pyproject.toml b/pyproject.toml index 39e7c18..1e48be6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,8 @@ dev = [ [tool.pdm.scripts] nebulagraph = {call = "nebulagraph_lite.cli:main"} -fmt = "black -l 84 ." +fmt = "black --line-length 84 ." +lint = "black --line-length 84 --check ." [project.scripts] nebulagraph = "nebulagraph_lite.cli:main" From 531c30ca49d157ccc283dbbd2822c2f189290788 Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Mon, 15 Jan 2024 12:58:35 +0800 Subject: [PATCH 3/9] chore: update main version --- src/nebulagraph_lite/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nebulagraph_lite/__init__.py b/src/nebulagraph_lite/__init__.py index f466a59..1e156a8 100644 --- a/src/nebulagraph_lite/__init__.py +++ b/src/nebulagraph_lite/__init__.py @@ -2,4 +2,4 @@ __all__ = ["nebulagraph_let"] -__version__ = "0.0.9" +__version__ = "0.1.0" From 21a913fdcdd8937d38599ff1ace9f7bae4915c2e Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Mon, 15 Jan 2024 12:59:08 +0800 Subject: [PATCH 4/9] fix: watch main branch --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fb81baf..0dee514 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,7 @@ name: PR Workflow on: pull_request: branches: - - master + - main jobs: build: From 7682764865bc1735c59dbafe3552d2202e935e08 Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Mon, 15 Jan 2024 13:00:35 +0800 Subject: [PATCH 5/9] chore: black format --- src/nebulagraph_lite/nebulagraph.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/nebulagraph_lite/nebulagraph.py b/src/nebulagraph_lite/nebulagraph.py index 8b0ff27..b53ff36 100644 --- a/src/nebulagraph_lite/nebulagraph.py +++ b/src/nebulagraph_lite/nebulagraph.py @@ -184,9 +184,7 @@ def _run_udocker(self, command: str): f"udocker command failed with return code {result.returncode}" ) if output and self._debug: - fancy_print( - f"Info: [DEBUG] udocker command output:\n{output.decode()}" - ) + fancy_print(f"Info: [DEBUG] udocker command output:\n{output.decode()}") return result def _run_udocker_ps_filter(self, filter: str): From 27a8cb24992e134c4f2e4c9a2552c4c4883de79b Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Mon, 15 Jan 2024 13:02:07 +0800 Subject: [PATCH 6/9] fix ci: use pdm script trigger in dryrun --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0dee514..23065ab 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,5 +20,5 @@ jobs: - name: Lint run: pdm run lint - - name: Dry run nebulagraph start - run: nebulagraph --debug start --cleanup + - name: Dry run `nebulagraph start` + run: pdm run nebulagraph --debug start --cleanup From 0a33af07f8b1cdf9d3206b0cc57ad6758d09debf Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Mon, 15 Jan 2024 13:04:05 +0800 Subject: [PATCH 7/9] fix: install before dryrun --- .github/workflows/pr.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 23065ab..56f89f2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,6 +19,9 @@ jobs: - name: Lint run: pdm run lint + + - name: Build and Install NebulaGraph-Lite + run: pip3 install . - name: Dry run `nebulagraph start` - run: pdm run nebulagraph --debug start --cleanup + run: nebulagraph --debug start --cleanup From 787faad7ff93ca91d51c155b3362659864f49502 Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Mon, 15 Jan 2024 13:14:13 +0800 Subject: [PATCH 8/9] feat: use port listening probe to replace udocker ps --- src/nebulagraph_lite/nebulagraph.py | 9 ++++++--- src/nebulagraph_lite/utils.py | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/nebulagraph_lite/nebulagraph.py b/src/nebulagraph_lite/nebulagraph.py index b53ff36..273e11b 100644 --- a/src/nebulagraph_lite/nebulagraph.py +++ b/src/nebulagraph_lite/nebulagraph.py @@ -11,6 +11,7 @@ BANNER_ASCII, get_pid_by_port, kill_process_by_pid, + process_listening_on_port, ) LOCALHOST_V4 = "127.0.0.1" @@ -277,7 +278,7 @@ def start_metad(self, shoot=False): self._run_udocker_background(udocker_command) time.sleep(10) if not self.on_colab: - self._run_udocker_ps_filter("metad") + process_listening_on_port(9559) def start_graphd(self): self._try_shoot_service("graphd") @@ -305,7 +306,8 @@ def start_graphd(self): self._run_udocker_background(udocker_command) time.sleep(10) if not self.on_colab: - self._run_udocker_ps_filter("graphd") + # self._run_udocker_ps_filter("graphd") + process_listening_on_port(self.port) def activate_storaged(self): udocker_command = ( @@ -390,7 +392,8 @@ def start_storaged(self, shoot=False): self._run_udocker_background(udocker_command) time.sleep(20) if not self.on_colab: - self._run_udocker_ps_filter("storaged") + # self._run_udocker_ps_filter("storaged") + pass def start(self, fresh=False): shoot = bool(fresh) diff --git a/src/nebulagraph_lite/utils.py b/src/nebulagraph_lite/utils.py index 303a4a7..dd34ded 100644 --- a/src/nebulagraph_lite/utils.py +++ b/src/nebulagraph_lite/utils.py @@ -116,3 +116,11 @@ def kill_process_by_pid(pid): print(f"Process with PID {pid} has been terminated.") except psutil.NoSuchProcess: print(f"No process with PID {pid} exists.") + + +@retry((Exception,), tries=3, delay=5, backoff=3) +def process_listening_on_port(port): + for conn in psutil.net_connections(): + if conn.laddr.port == port and conn.status == "LISTEN": + return True + return False From 611a334bf929fbb3e583f2f03b6f0c896002ff72 Mon Sep 17 00:00:00 2001 From: Wey Gu Date: Mon, 15 Jan 2024 13:32:28 +0800 Subject: [PATCH 9/9] feat: load dataset change to non-fatel --- src/nebulagraph_lite/nebulagraph.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/nebulagraph_lite/nebulagraph.py b/src/nebulagraph_lite/nebulagraph.py index 273e11b..2871d81 100644 --- a/src/nebulagraph_lite/nebulagraph.py +++ b/src/nebulagraph_lite/nebulagraph.py @@ -346,6 +346,7 @@ def load_basketballplayer_dataset(self): f"-addr {self.host} -port {self.port} -u root -p nebula -e ':play basketballplayer'" ) try: + time.sleep(10) self._run_udocker(udocker_command) except Exception as e: fancy_dict_print( @@ -355,8 +356,13 @@ def load_basketballplayer_dataset(self): "udocker_command": udocker_command, } ) - raise Exception("Failed to load basketballplayer dataset") - time.sleep(10) + fancy_dict_print( + { + "Info:": "Failed to load basketballplayer dataset, probably because the graphd is not ready yet or the cluster is not healthy, try this later from the console manually", + "command": f"udocker {udocker_command}", + "error": str(e), + } + ) def start_storaged(self, shoot=False): if shoot: