star-whale · goldenxinxing · Oct 8, 2022 · Sep 23, 2022 · Sep 27, 2022 · Sep 28, 2022
diff --git a/.gitignore b/.gitignore
@@ -46,6 +46,7 @@ example/mnist/runtime.yaml
 example/mnist/sw_output
 example/mnist/test/task_volume
 example/cifar10/data/
+example/cifar10/models/
 example/PennFudanPed/data/
 example/PennFudanPed/models/
 example/nmt/models/

diff --git a/client/setup.py b/client/setup.py
@@ -19,7 +19,7 @@
     "fs>=2.4.0",
     "typing-extensions>=4.0.0",
     "commonmark>=0.9.1",
-    "textual>=0.1.18",
+    "textual==0.1.18",
     "jsonlines==3.0.0",
     "boto3==1.21.0",
     "scikit-learn>=0.20.0",

diff --git a/client/starwhale/consts/__init__.py b/client/starwhale/consts/__init__.py
@@ -124,6 +124,8 @@ class SWDSSubFileType:
 DEFAULT_PAGE_SIZE = 20
 
 RECOVER_DIRNAME = ".recover"
+OBJECT_STORE_DIRNAME = ".objectstore"
+DATA_STORE_DIRNAME = ".datastore"
 DEFAULT_PYTHON_VERSION = "3.8"
 LATEST_TAG = "latest"
 

diff --git a/client/starwhale/core/eval/cli.py b/client/starwhale/core/eval/cli.py
@@ -2,6 +2,8 @@
 
 import click
 
+from starwhale.consts.env import SWEnv
+
 from .view import JobTermView, get_term_view, DEFAULT_PAGE_IDX, DEFAULT_PAGE_SIZE
 
 
@@ -38,15 +40,26 @@ def _list(
 
 
 @eval_job_cmd.command("run", help="Run job")
-@click.argument("project", default="")
-@click.option("--version", default=None, help="Evaluation job version")
+@click.option(
+    "-p",
+    "--project",
+    envvar=SWEnv.project,
+    default="",
+    help=f"project name, env is {SWEnv.project}",
+)
+@click.option(
+    "--version",
+    envvar=SWEnv.eval_version,
+    default=None,
+    help=f"Evaluation job version, env is {SWEnv.eval_version}",
+)
 @click.option("--model", required=True, help="model uri or model.yaml dir path")
 # TODO:support multi dataset
 @click.option(
     "--dataset",
     required=True,
-    # multiple=True,
-    help="dataset uri, one or more",
+    envvar=SWEnv.dataset_uri,
+    help=f"dataset uri, env is {SWEnv.dataset_uri}",
 )
 @click.option("--runtime", default="", help="runtime uri")
 @click.option("--name", default="default", help="job name")

diff --git a/client/starwhale/core/model/cli.py b/client/starwhale/core/model/cli.py
@@ -130,18 +130,27 @@ def _extract(model: str, force: bool, target_dir: str) -> None:
     help="Model yaml filename, default use ${MODEL_DIR}/model.yaml file",
 )
 @click.option(
+    "-p",
     "--project",
     envvar=SWEnv.project,
     default="",
     help=f"project name, env is {SWEnv.project}",
 )
 @click.option(
-    "--version", envvar=SWEnv.eval_version, default=None, help="Evaluation job version"
+    "--version",
+    envvar=SWEnv.eval_version,
+    default=None,
+    help=f"Evaluation job version, env is {SWEnv.eval_version}",
 )
 @click.option("--step", default="", help="Evaluation run step")
 @click.option("--task-index", default=-1, help="Index of tasks in the current step")
 @click.option("--runtime", default="", help="runtime uri")
-@click.option("--dataset", envvar=SWEnv.dataset_uri, help="dataset uri")
+@click.option(
+    "--dataset",
+    required=True,
+    envvar=SWEnv.dataset_uri,
+    help=f"dataset uri, env is {SWEnv.dataset_uri}",
+)
 def _eval(
     project: str,
     target: str,

diff --git a/client/starwhale/mngt/__init__.py b/client/starwhale/mngt/__init__.py
@@ -12,8 +12,10 @@
 from starwhale.consts import (
     RECOVER_DIRNAME,
     SW_TMP_DIR_NAME,
+    DATA_STORE_DIRNAME,
     VERSION_PREFIX_CNT,
     STANDALONE_INSTANCE,
+    OBJECT_STORE_DIRNAME,
     DEFAULT_MANIFEST_NAME,
 )
 from starwhale.utils.fs import empty_dir
@@ -27,7 +29,12 @@ def gc(dry_run: bool = False, yes: bool = False) -> None:
 
     for project_dir in sw.rootdir.iterdir():
         project_name = project_dir.name
-        if project_name in (RECOVER_DIRNAME, SW_TMP_DIR_NAME):
+        if project_name in (
+            RECOVER_DIRNAME,
+            SW_TMP_DIR_NAME,
+            OBJECT_STORE_DIRNAME,
+            DATA_STORE_DIRNAME,
+        ):
             continue
 
         if not yes and not click.confirm(

diff --git a/client/starwhale/utils/config.py b/client/starwhale/utils/config.py
@@ -12,9 +12,11 @@
     DEFAULT_PROJECT,
     DEFAULT_INSTANCE,
     ENV_SW_CLI_CONFIG,
+    DATA_STORE_DIRNAME,
     STANDALONE_INSTANCE,
     ENV_SW_LOCAL_STORAGE,
     LOCAL_CONFIG_VERSION,
+    OBJECT_STORE_DIRNAME,
     DEFAULT_SW_LOCAL_STORAGE,
 )
 from starwhale.consts.env import SWEnv
@@ -111,11 +113,11 @@ def rootdir(self) -> Path:
 
     @property
     def datastore_dir(self) -> Path:
-        return self.rootdir / ".datastore"
+        return self.rootdir / DATA_STORE_DIRNAME
 
     @property
     def object_store_dir(self) -> Path:
-        return self.rootdir / ".objectstore"
+        return self.rootdir / OBJECT_STORE_DIRNAME
 
     @property
     def sw_remote_addr(self) -> str:

diff --git a/client/tests/sdk/test_dataset.py b/client/tests/sdk/test_dataset.py
@@ -9,6 +9,7 @@
 from pyfakefs.fake_filesystem_unittest import TestCase
 
 from starwhale import Link, MIMEType, UserRawBuildExecutor
+from starwhale.consts import OBJECT_STORE_DIRNAME
 from starwhale.utils.fs import blake2b_file
 from starwhale.utils.error import NoSupportError, FieldTypeOrValueError
 from starwhale.core.dataset.type import (
@@ -94,7 +95,7 @@ def setUp(self) -> None:
         super().setUp()
 
         self.object_store_dir = os.path.join(
-            self.local_storage, ".objectstore", DatasetStorage.object_hash_algo
+            self.local_storage, OBJECT_STORE_DIRNAME, DatasetStorage.object_hash_algo
         )
         self.raw_data = os.path.join(self.local_storage, ".user", "data")
         self.workdir = os.path.join(self.local_storage, ".user", "workdir")

diff --git a/docs/docs/community/contribute.md b/docs/docs/community/contribute.md
@@ -0,0 +1,3 @@
+---
+title: Contribute to Starwhale
+---
diff --git a/docs/docs/community/dev.md b/docs/docs/community/dev.md
@@ -0,0 +1,4 @@
+---
+title: Development Tutorial
+---
+
diff --git a/docs/docs/community/roadmap.md b/docs/docs/community/roadmap.md
@@ -0,0 +1,3 @@
+---
+title: Roadmap
+---
diff --git a/docs/docs/cloud/installation.md → docs/docs/guides/cloud/overview.md b/docs/docs/cloud/installation.md → docs/docs/guides/cloud/overview.md
diff --git a/docs/docs/standalone/dataset.md → docs/docs/guides/dataset.md b/docs/docs/standalone/dataset.md → docs/docs/guides/dataset.md
@@ -1,5 +1,5 @@
 ---
-title: Starwhale Dataset
+title: Dataset
 ---
 
 ## dataset.yaml Definition

diff --git a/docs/docs/guides/evaluation.md b/docs/docs/guides/evaluation.md
@@ -0,0 +1,3 @@
+---
+title: Evaluation
+---
diff --git a/docs/docs/guides/faq.md b/docs/docs/guides/faq.md
@@ -0,0 +1,3 @@
+---
+title: FAQ
+---
diff --git a/docs/docs/cloud/helm-charts.md → docs/docs/guides/install/helm-charts.md b/docs/docs/cloud/helm-charts.md → docs/docs/guides/install/helm-charts.md
diff --git a/docs/docs/standalone/installation.md → docs/docs/guides/install/standalone.md b/docs/docs/standalone/installation.md → docs/docs/guides/install/standalone.md
diff --git a/docs/docs/standalone/model.md → docs/docs/guides/model.md b/docs/docs/standalone/model.md → docs/docs/guides/model.md
@@ -1,5 +1,5 @@
 ---
-title: Starwhale Model
+title: Model
 ---
 ## model.yaml Definition
 

diff --git a/docs/docs/standalone/runtime.md → docs/docs/guides/runtime.md b/docs/docs/standalone/runtime.md → docs/docs/guides/runtime.md
@@ -1,5 +1,5 @@
 ---
-title: Starwhale Runtime
+title: Runtime
 ---
 
 ## What is Starwhale Runtime?

diff --git a/docs/docs/standalone/overview.md → docs/docs/guides/standalone/overview.md b/docs/docs/standalone/overview.md → docs/docs/guides/standalone/overview.md
diff --git a/docs/docs/cloud/main.md → docs/docs/guides/standalone/swignore.md b/docs/docs/cloud/main.md → docs/docs/guides/standalone/swignore.md
diff --git a/docs/docs/img/board.gif b/docs/docs/img/board.gif
diff --git a/docs/docs/img/completion.gif b/docs/docs/img/completion.gif
diff --git a/docs/docs/img/dataset-diff.png b/docs/docs/img/dataset-diff.png
diff --git a/docs/docs/img/dataset-info.gif b/docs/docs/img/dataset-info.gif
diff --git a/docs/docs/img/dataset-recover.png b/docs/docs/img/dataset-recover.png
diff --git a/docs/docs/img/dataset-tags.png b/docs/docs/img/dataset-tags.png
diff --git a/docs/docs/img/eval-compare.png b/docs/docs/img/eval-compare.png
diff --git a/docs/docs/img/examples/ag_news-eval.png b/docs/docs/img/examples/ag_news-eval.png
diff --git a/docs/docs/img/examples/ag_news-ui.gif b/docs/docs/img/examples/ag_news-ui.gif
diff --git a/docs/docs/img/examples/cifar10-dataset.gif b/docs/docs/img/examples/cifar10-dataset.gif
diff --git a/docs/docs/img/examples/mnist-eval.png b/docs/docs/img/examples/mnist-eval.png
diff --git a/docs/docs/img/examples/mnist-viewer.gif b/docs/docs/img/examples/mnist-viewer.gif
diff --git a/docs/docs/img/examples/nmt-cloud-result.gif b/docs/docs/img/examples/nmt-cloud-result.gif
diff --git a/docs/docs/img/examples/nmt-eval-info.png b/docs/docs/img/examples/nmt-eval-info.png
diff --git a/docs/docs/img/examples/nmt-train.png b/docs/docs/img/examples/nmt-train.png
diff --git a/docs/docs/img/examples/pfp-cloud-result-diff.png b/docs/docs/img/examples/pfp-cloud-result-diff.png
diff --git a/docs/docs/img/examples/pfp-dataset.gif b/docs/docs/img/examples/pfp-dataset.gif
diff --git a/docs/docs/img/examples/pfp-eval-info.png b/docs/docs/img/examples/pfp-eval-info.png
diff --git a/docs/docs/img/examples/sc_dataset_build.png b/docs/docs/img/examples/sc_dataset_build.png
diff --git a/docs/docs/img/examples/sc_dataset_info.png b/docs/docs/img/examples/sc_dataset_info.png
diff --git a/docs/docs/img/examples/sc_dataset_viewer.png b/docs/docs/img/examples/sc_dataset_viewer.png
diff --git a/docs/docs/img/examples/sc_eval.png b/docs/docs/img/examples/sc_eval.png
diff --git a/docs/docs/img/examples/sc_model.png b/docs/docs/img/examples/sc_model.png
diff --git a/docs/docs/img/examples/sc_train.png b/docs/docs/img/examples/sc_train.png
diff --git a/docs/docs/img/gc.gif b/docs/docs/img/gc.gif
diff --git a/docs/docs/img/instance-info.png b/docs/docs/img/instance-info.png
diff --git a/docs/docs/img/instance-list.png b/docs/docs/img/instance-list.png
diff --git a/docs/docs/img/model-eval.gif b/docs/docs/img/model-eval.gif
diff --git a/docs/docs/img/open-ui.gif b/docs/docs/img/open-ui.gif
diff --git a/docs/docs/img/quickstart-shell.gif b/docs/docs/img/quickstart-shell.gif
diff --git a/docs/docs/img/runtime-build.gif b/docs/docs/img/runtime-build.gif
diff --git a/docs/docs/img/runtime-dockerize.png b/docs/docs/img/runtime-dockerize.png
diff --git a/docs/docs/img/runtime-lock.gif b/docs/docs/img/runtime-lock.gif
diff --git a/docs/docs/quickstart/on-premises.md b/docs/docs/quickstart/on-premises.md
@@ -14,7 +14,7 @@ Starwhale provides two ways to install an On-Premises instance in your private c
 
   - Standard Kubernetes Cluster: A pre-deployed Kubernetes cluster is required.
   - Minikube: You should have minikube and docker installed on your machine.
-  - For more deployment details of Kubernetes, you can refer to this [doc](../cloud/helm-charts.md).
+  - For more deployment details of Kubernetes, you can refer to this [doc](../guides/install/helm-charts.md).
 
 :::notes
 In this tutorial, minikube is used instead of the standard Kubernetes cluster

diff --git a/docs/docs/quickstart/standalone.md b/docs/docs/quickstart/standalone.md
@@ -18,7 +18,7 @@ The Starwhale client version is currently under alpha preview. Thus `--pre` is r
 Starwhale standalone requires Python 3.7 or above. Currently, Starwhale only supports Linux and macOS X. Windows is coming soon.
 :::
 
-At the installation stage, we strongly recommend you follow the [doc](../standalone/installation.md).
+At the installation stage, we strongly recommend you follow the [doc](../guides/install/standalone.md).
 
 ## Downloading Examples
 

diff --git a/docs/docs/reference/cli/main.md → docs/docs/reference/cli/basic.md b/docs/docs/reference/cli/main.md → docs/docs/reference/cli/basic.md
diff --git a/docs/docs/reference/cli/dataset.md b/docs/docs/reference/cli/dataset.md
@@ -21,17 +21,19 @@ swcli [GLOBAL OPTIONS] dataset [OPTIONS] COMMAND [ARGS]...
   |-------|----------|-----|
   |build|✅|❌|
   |copy|✅|✅|
+  |diff|✅|❌|
   |history|✅|✅|
   |info|✅|✅|
   |list|✅|✅|
   |recover|✅|✅|
   |remove|✅|✅|
+  |summary|✅|✅|
   |tag|✅|❌|
 
 ## Build a dataset
 
 ```bash
-swcli build [OPTIONS] WORKDIR
+swcli dataset build [OPTIONS] WORKDIR
 ```
 
 - This command builds a dataset within the specified working directory. The working dir must contain a `dataset.yaml`, which defines the metadata and process handler for dataset building.