From 15f249ab66526a81d95638b363e4b6427d0aaac8 Mon Sep 17 00:00:00 2001 From: sunwp Date: Wed, 30 Jun 2021 18:19:22 +0800 Subject: [PATCH] chore: add docs for dragonfly2.0 (#234) * cdn error definition Signed-off-by: santong * docs of dragonfly2.0 Signed-off-by: santong * docs of dragonfly2.0 Signed-off-by: santong * docs:modify user guide Signed-off-by: santong * docs:modify user guide Signed-off-by: santong * docs: add readme Signed-off-by: santong * docs: add user guide Signed-off-by: santong * docs: add user guide Signed-off-by: santong --- FAQ.md | 1 + README.md | 27 ++- ROADMAP.md | 1 + api/README.md | 4 +- docs/README.md | 2 +- docs/en/README.md | 38 +++-- .../dfget.1 => api-reference/api.md} | 0 docs/en/cli-reference/cdn.md | 25 +-- docs/en/cli-reference/dfget.md | 4 +- docs/en/cli-reference/manager.md | 28 +++ docs/en/cli-reference/scheduler.md | 59 ++----- docs/en/config/cdn-nginx.conf | 2 +- docs/en/config/scheduler.yaml | 2 +- docs/en/user-guide/README.md | 0 docs/en/user-guide/install/README.md | 5 + docs/en/user-guide/install/install-cdn.md | 161 ++++++++++++++++++ docs/en/user-guide/install/install-manager.md | 129 ++++++++++++++ .../user-guide/install/install-scheduler.md | 128 ++++++++++++++ docs/en/user-guide/proxy/docker.md | 160 ----------------- docs/en/user-guide/quick-start.md | 15 +- .../{containerd.md => cri-containerd.md} | 0 docs/zh-CN/cli-reference/cdn.md | 26 +++ docs/zh-CN/cli-reference/manager.md | 20 +++ docs/zh-CN/cli-reference/scheduler.md | 20 +++ docs/zh-CN/config/cdn.yaml | 81 +++++++++ 25 files changed, 685 insertions(+), 253 deletions(-) create mode 100644 FAQ.md create mode 100644 ROADMAP.md rename docs/en/{cli-reference/dfget.1 => api-reference/api.md} (100%) create mode 100644 docs/en/cli-reference/manager.md create mode 100644 docs/en/user-guide/README.md create mode 100644 docs/en/user-guide/install/README.md create mode 100644 docs/en/user-guide/install/install-cdn.md create mode 100644 docs/en/user-guide/install/install-manager.md create mode 100644 docs/en/user-guide/install/install-scheduler.md delete mode 100644 docs/en/user-guide/proxy/docker.md rename docs/en/user-guide/registry-mirror/{containerd.md => cri-containerd.md} (100%) create mode 100644 docs/zh-CN/cli-reference/cdn.md create mode 100644 docs/zh-CN/cli-reference/manager.md create mode 100644 docs/zh-CN/cli-reference/scheduler.md create mode 100644 docs/zh-CN/config/cdn.yaml diff --git a/FAQ.md b/FAQ.md new file mode 100644 index 00000000000..32914fe4ef7 --- /dev/null +++ b/FAQ.md @@ -0,0 +1 @@ +FAQ \ No newline at end of file diff --git a/README.md b/README.md index bd0e7518e37..3738fcecf15 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,17 @@ Provide efficient, stable, secure, low-cost file and image distribution services to be the best practice and standard solution in the related Cloud-Native area. +## Introduction + +Dragonfly is an open source intelligent P2P based image and file distribution system. Its goal is to tackle all distribution problems in cloud native scenarios. Currently Dragonfly focuses on being: + +- Simple: well-defined user-facing API (HTTP), non-invasive to all container engines; +- Efficient: CDN support, P2P based file distribution to save enterprise bandwidth; +- Intelligent: host level speed limit, intelligent flow control due to host detection; +- Secure: block transmission encryption, HTTPS connection support. + +Dragonfly is now hosted by the Cloud Native Computing Foundation (CNCF) as an Incubating Level Project. Originally it was born to solve all kinds of distribution at very large scales, such as application distribution, cache distribution, log distribution, image distribution, and so on. + ## Features - Implement P2P files distribution with various storage types (HDFS, storage services from various cloud vendors, Maven, Yum, etc.) through a unified back-to-source adapter layer. @@ -45,9 +56,8 @@ Provide efficient, stable, secure, low-cost file and image distribution services ## Getting Started -- [Introduction][introduction] -- [Installation][installation] - [Quick start][quickstart] +- [Installation][installation] ## Documentation You can find the Dragonfly documentation [on the website][website]. @@ -64,12 +74,11 @@ You should check out our [CONTRIBUTING][contributing] and develop the project to ## Code of Conduct Please refer to our [Code of Conduct][codeconduct]. -[contributing]: CONTRIBUTING.md -[codeconduct]: CODE_OF_CONDUCT.md -[introduction]: https://github.com/dragonflyoss/Dragonfly2 -[installation]: https://github.com/dragonflyoss/Dragonfly2 -[quickstart]: https://github.com/dragonflyoss/Dragonfly2/blob/main/docs/en/user-guide/quick-start.md +[arch]: docs/en/images/arch.png +[logo-linear]: docs/en/images/logo/dragonfly-linear.svg +[quickstart]: docs/en/user-guide/quick-start.md +[installation]: docs/en/user-guide/install/README.md [website]: https://d7y.io [discussion]: https://github.com/dragonflyoss/Dragonfly2/discussions -[logo-linear]: docs/en/images/logo/dragonfly-linear.svg -[arch]: docs/en/images/arch.png +[contributing]: CONTRIBUTING.md +[codeconduct]: CODE_OF_CONDUCT.md diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 00000000000..594457daf0b --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1 @@ +ROADMAP \ No newline at end of file diff --git a/api/README.md b/api/README.md index 2bbcdc260d8..b59dab13c19 100644 --- a/api/README.md +++ b/api/README.md @@ -3,6 +3,8 @@ We encourage users to experience Dragonfly in different ways. When doing this, there are three ways users could choose to interact with dragonfly mostly: * For end-users, command line tool `dfget`, `dfdaemon` is mostly used. -* For developers, Dragonfly manager's raw API is the original thing they would make use of. For more details about API docs, please refer to [api.md](../docs/api_reference/api.md). We should also keep it in mind that doc [apis.md](../docs/api_reference/api.md) is automatically generated by [swagger2markup](https://github.com/Swagger2Markup/swagger2markup). Please **DO NOT** edit [api.md](../docs/api_reference/api.md) directly. +* For developers, Dragonfly manager's raw API is the original thing they would make use of. For more details about API docs, please refer to + [api.md](/docs/en/api-reference/api.md). We should also keep it in mind that doc [apis.md](../docs/en/api-reference/api.md) is automatically generated by + [swagger2markup](https://github.com/Swagger2Markup/swagger2markup). Please **DO NOT** edit [api.md](../docs/en/api-reference/api.md) directly. Directory `/api` mainly describes the second part **Dragonfly Manager's Raw API**. diff --git a/docs/README.md b/docs/README.md index df2074923b9..e5721fb87f8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Dragonfly Document -[English](./en/README.md) +[English](en/README.md) [TODO 简体中文](./zh-CN/README.md) diff --git a/docs/en/README.md b/docs/en/README.md index a90566e13c1..e6828bfbb75 100644 --- a/docs/en/README.md +++ b/docs/en/README.md @@ -7,14 +7,14 @@ Organization of document is as following: * [Quick Start](#Quick-Start) * [User Guide](#User-Guide) * [CLI Reference](#CLI-Reference) - * [WIP dfget](./cli-reference/dfget.md) - * [WIP cdnsystem](./cli-reference/cdn.md) - * [WIP scheduler](./cli-reference/scheduler.md) - * [TODO manager](./cli-reference/manager.md) + * [WIP dfget](cli-reference/dfget.md) + * [WIP cdnsystem](cli-reference/cdn.md) + * [WIP scheduler](cli-reference/scheduler.md) + * [TODO manager](cli-reference/manager.md) * [TODO API Reference](#API-Reference) * [Ecosystem](#Ecosystem) - * [Kubernetes Integration](./ecosystem/Kubernetes-with-Dragonfly.md) - * [WIP Harbor Integration](./ecosystem/Harbor-with-Dragonfly.md) + * [Kubernetes Integration](ecosystem/Kubernetes-with-Dragonfly.md) + * [WIP Harbor Integration](ecosystem/Harbor-with-Dragonfly.md) * [Developer Guide](#Developer-Guide) * [Design Doc](#Design-Doc) * [Test Guide](#Test-Guide) @@ -25,27 +25,33 @@ Find `WIP` or `TODO` in this page and follow [CONTRIBUTING](../../CONTRIBUTING.m ## Quick Start -[Quick Started](./user-guide/quick-start.md) is exactly what you need if you would give Dragonfly a try. This document includes what are the prerequisites, how to install Dragonfly and how to experience Dragonfly's usage. +[Quick Started](user-guide/quick-start.md) is exactly what you need if you would give Dragonfly a try. This document includes what are the +prerequisites, +how to install Dragonfly and how to experience Dragonfly's usage. -## [WIP] User Guide +## User Guide -[User Guide](./user-guide) helps all kinds of guidance end users need to experience Dragonfly. Not only the very brief [Quick Start](./quick-start), but the detailed binary installation and configure illustration. In addition, any concept and function which help users understand Dragonfly better would be included as well. +[User Guide](user-guide/README.md) helps all kinds of guidance end users need to experience Dragonfly. Not only the very brief [Quick Start] +(user-guide/quick-start.md), but the detailed binary installation and configure illustration. In addition, any concept and function which help users +understand Dragonfly better would be included as well. -## [WIP] CLI Reference +## CLI Reference -For almost all users, commandline is the first reference you may need. Document in directory [CLI Reference](./cli-reference) is about command detailed usage of Dragonfly CLI including `dfget`, `cdnsystem`, `scheduler` and `manager`. You can get introductions, synopsis, examples, options about command. Last but not least, Dragonfly can guarantee commandline docs is strongly consistent with Dragonfly CLI's source code. What's more, all commandline docs are auto generated via source code. +For almost all users, commandline is the first reference you may need. Document in directory [CLI Reference](cli-reference) is about command detailed usage of Dragonfly CLI including `dfget`, `cdnsystem`, `scheduler` and `manager`. You can get introductions, synopsis, examples, options about command. Last but not least, Dragonfly can guarantee commandline docs is strongly consistent with Dragonfly CLI's source code. What's more, all commandline docs are auto generated via source code. ## [TODO] API Reference Commandline is the easiest way to experience Dragonfly's ability. API extension will bring more further experience of Dragonfly. Commandline is just one kind of combination usage of API, if you wish to hack or take more advantages of Dragonfly, please see [API Reference](./api-reference). Like command line document, all API docs are auto generated via source code. -## [WIP] Ecosystem +## Ecosystem -Ecosystem documents show connections between Dragonfly and popular tool or system in cloud native ecosystem. They guide end users how to experience cloud native systems with Dragonfly, such as other CNCF project Kubernetes and Harbor. +Ecosystem documents show connections between Dragonfly and popular tool or system in cloud native ecosystem. They guide end users how to experience cloud +native systems with Dragonfly, such as other CNCF project [Kubernetes](ecosystem/Kubernetes-with-Dragonfly.md) and [Harbor](ecosystem/Harbor-with-Dragonfly.md). -## [WIP] Developer Guide +## Developer Guide -Develop Guide helps (potential) developers/contributors to understand the theory inside Dragonfly rather than the interface it exposes. With better understanding of how Dragonfly is designed, developer could learn source code of Dragonfly much easier and know how to debug, test and hack. +[Develop Guide](development/local.md) helps (potential) developers/contributors to understand the theory inside Dragonfly rather than the interface it exposes. With +better understanding of how Dragonfly is designed, developer could learn source code of Dragonfly much easier and know how to debug, test and hack. ### [TODO] Design Doc @@ -66,7 +72,7 @@ For more details, please refer to [test](./test-guide). The folder `/docs` does not contain all the document about Dragonfly. There are still other really helpful documents in other path of this repo, like: -* [TODO FAQ.md](../FAQ.md) +* [TODO FAQ.md](../../FAQ.md) * [CHANGELOG.md](../../CHANGELOG.md) * [TODO ROADMAP.md](../../ROADMAP.md) * others. diff --git a/docs/en/cli-reference/dfget.1 b/docs/en/api-reference/api.md similarity index 100% rename from docs/en/cli-reference/dfget.1 rename to docs/en/api-reference/api.md diff --git a/docs/en/cli-reference/cdn.md b/docs/en/cli-reference/cdn.md index 01f905a5086..a7f006cd4c7 100644 --- a/docs/en/cli-reference/cdn.md +++ b/docs/en/cli-reference/cdn.md @@ -7,7 +7,7 @@ CDN is a long-running process which caches downloaded data from source to avoid go run cmd/cdnsystem/main.go [Option] ``` ## Log configuration -set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal +set environment variable console=true if you want to print logs to Terminal ## Runtime metrics monitoring ``` @@ -16,20 +16,11 @@ go run cmd/cdnsystem/main.go --profiler ### Options ``` - --advertise-ip string the cdn node ip is the ip we advertise to other peers in the p2p-network - --clean-ratio int CleanRatio is the ratio to clean the disk and it is based on 10. the value of CleanRatio should be [1-10] (default 1) - --config string the path of cdn configuration file (default "/etc/dragonfly/cdn.yaml") - --download-port int downloadPort is the port for download files from cdnNode (default 8001) - --fail-access-interval duration fail access interval is the interval time after failed to access the URL (default 3m0s) - --gc-disk-interval duration gc disk interval is the interval time to execute GC disk. (default 15s) - --gc-initial-delay duration gc initial delay is the delay time from the start to the first GC execution (default 6s) - --gc-meta-interval duration gc meta interval is the interval time to execute the GC meta (default 2m0s) - -h, --help help for cdn - --home-dir string homeDir is the working directory of cdnNode (default "/Users/su*__*nweipeng1/cdn-system") - --max-bandwidth rate network rate that cdnNode can use (default 200MB) - --port int listenPort is the port that cdn server listens on (default 8003) - --profiler profiler sets whether cdnNode HTTP server setups profiler - --system-bandwidth rate network rate reserved for system (default 20MB) - --task-expire-time duration task expire time is the time that a task is treated expired if the task is not accessed within the time (default 3m0s) - --young-gc-threshold file-size gc disk interval is the interval time to execute GC disk. (default 100GB) + --config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/cdn.yaml, it can also be + set by env var:CDN_CONFIG，The settings and uses of each configuration item can refer to cdn.yaml in config directory + --console whether logger output records to the stdout + -h, --help help for cdn + --jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces + --pprof-port int listen port for pprof, 0 represents random port (default -1) + --verbose whether logger use debug level ``` diff --git a/docs/en/cli-reference/dfget.md b/docs/en/cli-reference/dfget.md index a42e30798ea..c66b04f2b69 100644 --- a/docs/en/cli-reference/dfget.md +++ b/docs/en/cli-reference/dfget.md @@ -21,7 +21,7 @@ dfget --schedulers 127.0.0.1:8002 -o /path/to/output -u "http://example.com/obje ## Log configuration -set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal +set environment variable console=true if you want to print logs to Terminal ### Options @@ -64,7 +64,7 @@ set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Te ## Log configuration -set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal +set environment variable console=true if you want to print logs to Terminal ### Options diff --git a/docs/en/cli-reference/manager.md b/docs/en/cli-reference/manager.md new file mode 100644 index 00000000000..5163eff078c --- /dev/null +++ b/docs/en/cli-reference/manager.md @@ -0,0 +1,28 @@ +## Manager + +Manager is a process that runs in the background and plays the role of the brain of each subsystem cluster in Dragonfly. It is used to manage the dynamic +configuration of each system module and provide functions such as heartbeat keeping alive, monitoring the market, and product functions. + +## Try it +``` +go run cmd/manager/main.go [Option] +``` + +## Log configuration +set environment variable console=local if you want to print logs to Terminal + +## Runtime metrics monitoring +``` +go run cmd/scheduler/main.go --profiler +``` +### Options + +``` + --config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/scheduler.yaml, it can + also be set by env var:SCHEDULER_CONFIG，The settings and uses of each configuration item can refer to scheduler.yaml in config directory + --console whether logger output records to the stdout + -h, --help help for cdn + --jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces + --pprof-port int listen port for pprof, 0 represents random port (default -1) + --verbose whether logger use debug level +``` diff --git a/docs/en/cli-reference/scheduler.md b/docs/en/cli-reference/scheduler.md index 61ff35216d4..7ee366cd17b 100644 --- a/docs/en/cli-reference/scheduler.md +++ b/docs/en/cli-reference/scheduler.md @@ -1,51 +1,26 @@ -## 调度器 +## Scheduler -调度器生成并维护下载过程中的P2P网络 - -### 说明 - -调度器是一个常驻后台运行的进程，用于接收和管理客户端的下载任务，通知CDN进行回源， 在下载过程中生成维护P2P网络，给客户端推送适合的下载节点 - -### 用法 +Scheduler is a long-running process which receives and manages download tasks from the client, notify the CDN to return to the source, +generate and maintain a P2P network during the download process, and push suitable download nodes to the client +## Try it ``` -scheduler [flags] +go run cmd/scheduler/main.go [Option] ``` +## Log configuration +set environment variable console=true if you want to print logs to Terminal -### 可选参数 - +## Runtime metrics monitoring ``` - --config string the path of scheduler's configuration file (default "conf/scheduler.yaml") - -h, --help help for scheduler - --port int port is the port that scheduler server listens on (default 8002) - --sender-job-pool-size int sender-job-pool-size is used for scheduler and do not change it (default 10000) - --sender-num int sender-num is used for scheduler and do not change it (default 50) - --worker-job-pool-size int worker-job-pool-size is used for scheduler and do not change it (default 10000) - --worker-num int worker-num is used for scheduler and do not change it (default 12) +go run cmd/scheduler/main.go --profiler ``` - -### 使用示例 - -scheduler --config your-config-path/scheduler.yaml - -### 配置文件说明 +### Options ``` -server: - port: 8001 rpc 端口 - -scheduler: - -worker: - worker-num: 5 工作线程数 - worker-job-pool-size: 10000 工作队列长度 - sender-num: 10 发送消息线程数 - sender-job-pool-size: 10000 发送消息队列长度 - -cdn: - list: CDN列表 - - - - cdn-name : "cdn" CDN服务器的HostName - ip: "127.0.0.1" CDN服务器的IP地址 - rpcPort: 8003 CDN的RPC端口 - download-port: 8002 CDN的下载端口 + --config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/scheduler.yaml, it can + also be set by env var:SCHEDULER_CONFIG，The settings and uses of each configuration item can refer to scheduler.yaml in config directory + --console whether logger output records to the stdout + -h, --help help for cdn + --jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces + --pprof-port int listen port for pprof, 0 represents random port (default -1) + --verbose whether logger use debug level ``` diff --git a/docs/en/config/cdn-nginx.conf b/docs/en/config/cdn-nginx.conf index 4465a219625..d97657ca0f6 100644 --- a/docs/en/config/cdn-nginx.conf +++ b/docs/en/config/cdn-nginx.conf @@ -45,7 +45,7 @@ http { server { listen 8001; location / { - root /home/admin/cdn-system/repo; + root /home/admin/ftp; } } } diff --git a/docs/en/config/scheduler.yaml b/docs/en/config/scheduler.yaml index c4d3ac9d0a0..6551734d054 100644 --- a/docs/en/config/scheduler.yaml +++ b/docs/en/config/scheduler.yaml @@ -4,7 +4,7 @@ debug: false server: - # ListenPort is the ip and port supernode server listens on. + # ListenPort is the ip and port scheduler server listens on. # default: 8002 port: 8002 diff --git a/docs/en/user-guide/README.md b/docs/en/user-guide/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/en/user-guide/install/README.md b/docs/en/user-guide/install/README.md new file mode 100644 index 00000000000..77ed3e9f891 --- /dev/null +++ b/docs/en/user-guide/install/README.md @@ -0,0 +1,5 @@ +[install-client](./install-client.md) is the installation instructions of client +[install-manager](./install-manager.md) is the installation instructions of manager +[install-scheduler](./install-scheduler.md) is the installation instructions of scheduler +[install-cdn](./install-cdn.md) is the installation instructions of CDN + diff --git a/docs/en/user-guide/install/install-cdn.md b/docs/en/user-guide/install/install-cdn.md new file mode 100644 index 00000000000..4ff61df83c4 --- /dev/null +++ b/docs/en/user-guide/install/install-cdn.md @@ -0,0 +1,161 @@ +# Installing Dragonfly CDN Server + +This topic explains how to install the Dragonfly CDN server. + +## Context + +Install CDN in one of the following ways: + +- Deploying with Docker. +- Deploying with physical machines: Recommended for production usage. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x +Nginx|0.8+ + +## Procedure - When Deploying with Docker + +### Get cdn image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image of the cdn. + + ```sh + docker pull d7yio/cdn + ``` + +Or you can build your own cdn image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-cdn D7Y_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the cdn. + + ```sh + docker image ls | grep 'cdn' | awk '{print $3}' | head -n1 + ``` + +### Start cdn + +**NOTE:** Replace ${cdnDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/ftp:/home/admin/ftp ${cdnDockerImageId} +--download-port=8001 +``` + +## Procedure - When Deploying with Physical Machines + +### Get cdn executable file + +1. Download a binary package of the cdn. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. + com/dragonflyoss/Dragonfly2/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `cdn` to your `PATH` environment variable to make sure you can directly use `cdn` command. + +Or you can build your own cdn executable file. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Compile the source code. + + ```sh + make build-cdn && make install-cdn + ``` + +### Start cdn + +```sh +cdnHomeDir=/home/admin/cdn +cdnDownloadPort=8001 +cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort +``` + +### Start file server + +You can start a file server in any way. However, the following conditions must be met: + +- It must be rooted at `${cdnHomeDir}/ftp` which is defined in the previous step. +- It must listen on the port `cdnDownloadPort` which is defined in the previous step. + +Let's take nginx as an example. + +1. Add the following configuration items to the Nginx configuration file. + + ```conf + server { + # Must be ${cdnDownloadPort} + listen 8001; + location / { + # Must be ${cdnHomeDir}/repo + root /home/admin/cdn/repo; + } + } + ``` + +2. Start Nginx. + + ```sh + sudo nginx + ``` + +## After this Task + +- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. + + ```sh + telnet 127.0.0.1 8001 + telnet 127.0.0.1 8003 + ``` diff --git a/docs/en/user-guide/install/install-manager.md b/docs/en/user-guide/install/install-manager.md new file mode 100644 index 00000000000..620d5784ef5 --- /dev/null +++ b/docs/en/user-guide/install/install-manager.md @@ -0,0 +1,129 @@ +# Installing Dragonfly Manager Server + +This topic explains how to install the Dragonfly manager server. + +## Context + +Install manager in one of the following ways: + +- Deploying with Docker: Recommended for production usage. +- Deploying with physical machines. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x +Nginx|0.8+ + +## Procedure - When Deploying with Docker + +### Get manager image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image of the manager. + + ```sh + docker pull d7yio/manager + ``` + +Or you can build your own manager image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-manager D7Y_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the manager. + + ```sh + docker image ls | grep 'manager' | awk '{print $3}' | head -n1 + ``` + +### Start manager + +**NOTE:** Replace ${managerDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name manager --restart=always -p 8004:8004 ${managerDockerImageId} +``` + +## Procedure - When Deploying with Physical Machines + +### Get cdn executable file + +1. Download a binary package of the manager. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. + com/dragonflyoss/Dragonfly2/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `manager` to your `PATH` environment variable to make sure you can directly use `manager` command. + +Or you can build your own manager executable file. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Compile the source code. + + ```sh + make build-manaager && make install-manager + ``` + +### Start manager + +```sh +manager --options +``` + +## After this Task + +- After manager is installed, run the following commands to verify if **manager** is started, and if Port `8004` and `8003` is available. + + ```sh + telnet 127.0.0.1 8004 + ``` diff --git a/docs/en/user-guide/install/install-scheduler.md b/docs/en/user-guide/install/install-scheduler.md new file mode 100644 index 00000000000..c005b3da394 --- /dev/null +++ b/docs/en/user-guide/install/install-scheduler.md @@ -0,0 +1,128 @@ +# Installing Dragonfly Scheduler Server + +This topic explains how to install the Dragonfly scheduler server. + +## Context + +Install scheduler in one of the following ways: + +- Deploying with Docker: Recommended for production usage. +- Deploying with physical machines. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x +Nginx|0.8+ + +## Procedure - When Deploying with Docker + +### Get scheduler image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image of the scheduler. + + ```sh + docker pull d7yio/scheduler + ``` + +Or you can build your own scheduler image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-scheduler D7Y_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the scheduler. + + ```sh + docker image ls | grep 'scheduler' | awk '{print $3}' | head -n1 + ``` + +### Start scheduler + +**NOTE:** Replace ${schedulerDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name scheduler --restart=always -p 8002 ${schedulerDockerImageId} +``` + +## Procedure - When Deploying with Physical Machines + +### Get scheduler executable file + +1. Download a binary package of the scheduler. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. + com/dragonflyoss/Dragonfly2/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `scheduler` to your `PATH` environment variable to make sure you can directly use `scheduler` command. + +Or you can build your own scheduler executable file. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Compile the source code. + + ```sh + make build-scheduler && make install-scheduler + ``` + +### Start scheduler + +```sh +scheduler --options +``` +## After this Task + +- After scheduler is installed, run the following commands to verify if **scheduler** is started, and if Port `8002` is available. + + ```sh + telnet 127.0.0.1 8002 + ``` diff --git a/docs/en/user-guide/proxy/docker.md b/docs/en/user-guide/proxy/docker.md deleted file mode 100644 index 61a4de51aba..00000000000 --- a/docs/en/user-guide/proxy/docker.md +++ /dev/null @@ -1,160 +0,0 @@ -# Use dfget daemon as HTTP proxy for docker daemon - -Currently, docker doesn't support private registries with `registry-mirrors`, -in order to do so, we need to use HTTP proxy for docker daemon. - -## Quick Start - -### Step 1: Generate CA certificate for HTTP proxy - -Generate a CA certificate private key. - -```bash -openssl genrsa -out ca.key 2048 -``` - -Open openssl config file `openssl.conf`. Note set `basicConstraints` to true, that you can modify the values. - -```text -[ req ] -#default_bits = 2048 -#default_md = sha256 -#default_keyfile = privkey.pem -distinguished_name = req_distinguished_name -attributes = req_attributes -extensions = v3_ca -req_extensions = v3_ca - -[ req_distinguished_name ] -countryName = Country Name (2 letter code) -countryName_min = 2 -countryName_max = 2 -stateOrProvinceName = State or Province Name (full name) -localityName = Locality Name (eg, city) -0.organizationName = Organization Name (eg, company) -organizationalUnitName = Organizational Unit Name (eg, section) -commonName = Common Name (eg, fully qualified host name) -commonName_max = 64 -emailAddress = Email Address -emailAddress_max = 64 - -[ req_attributes ] -challengePassword = A challenge password -challengePassword_min = 4 -challengePassword_max = 20 - -[ v3_ca ] -basicConstraints = CA:TRUE -``` - -Generate the CA certificate. - -```bash -openssl req -new -key ca.key -nodes -out ca.csr -config openssl.conf -openssl x509 -req -days 36500 -extfile openssl.conf -extensions v3_ca -in ca.csr -signkey ca.key -out ca.crt -``` - -### Step 2: Configure dfget daemon - -To use dfget daemon as HTTP proxy, first you need to append a proxy rule in -`/var/log/dragonfly/dfget.yaml`, This will proxy `your.private.registry`'s requests for image layers: - -```yaml -proxy: - security: - insecure: true - tcp_listen: - listen: 0.0.0.0 - port: 65001 - proxies: - - regx: blobs/sha256.* - hijack_https: - # CA certificate's path used to hijack https requests - cert: ca.crt - key: ca.key - hosts: - - regx: your.private.registry -``` - -### Step 3: Configure Docker daemon - -Add your private registry to `insecure-registries` in -`/etc/docker/daemon.json`, in order to ignore the certificate error: - -```json -{ - "insecure-registries": ["your.private.registry"] -} -``` - -### Step 4: Configure Docker daemon - -Set dfdaemon as `HTTP_PROXY` and `HTTPS_PROXY` for docker daemon in -`/etc/systemd/system/docker.service.d/http-proxy.conf`: - -``` -[Service] -Environment="HTTP_PROXY=http://127.0.0.1:65001" -Environment="HTTPS_PROXY=http://127.0.0.1:65001" -``` - -### Step 5: Pull images with proxy - -Through the above steps, we can start to validate if Dragonfly works as expected. - -And you can pull the image as usual, for example: - -```bash -docker pull your.private.registry/namespace/image:latest -``` - -## Custom assets - -### Registry uses a self-signed certificate - -If your registry uses a self-signed certificate, you can either choose to -ignore the certificate error with: - -```yaml -proxy: - security: - insecure: true - tcp_listen: - listen: 0.0.0.0 - port: 65001 - proxies: - - regx: blobs/sha256.* - hijack_https: - # CA certificate's path used to hijack https requests - cert: ca.crt - key: ca.key - hosts: - - regx: your.private.registry - insecure: true -``` - -Or provide a certificate with: - -```yaml -proxy: - security: - insecure: true - tcp_listen: - listen: 0.0.0.0 - port: 65001 - proxies: - - regx: blobs/sha256.* - hijack_https: - # CA certificate's path used to hijack https requests - cert: ca.crt - key: ca.key - hosts: - - regx: your.private.registry - certs: ["server.crt"] -``` - -You can get the certificate of your server with: - -``` -openssl x509 -in <(openssl s_client -showcerts -servername xxx -connect xxx:443 -prexit 2>/dev/null) -``` diff --git a/docs/en/user-guide/quick-start.md b/docs/en/user-guide/quick-start.md index 307a05e32d9..6253719b3cd 100644 --- a/docs/en/user-guide/quick-start.md +++ b/docs/en/user-guide/quick-start.md @@ -9,10 +9,19 @@ This table describes some container runtimes version and documents. | Runtime | Version | Document | CRI Support | Pull Command | | --- | --- | --- | --- | --- | -| Docker | All | [Link](./proxy/docker.md) | No | docker pull docker.io/library/alpine | | Containerd without CRI | All | [Link](./proxy/containerd.md) | No | ctr image pull docker.io/library/alpine | -| Containerd with CRI | v1.1.0+ | [Link](./registry-mirror/containerd.md) | Yes | crictl pull docker.io/library/alpine:latest | +| Containerd with CRI | v1.1.0+ | [Link](./registry-mirror/cri-containerd.md) | Yes | crictl pull docker.io/library/alpine:latest | | CRI-O | All | [Link](./registry-mirror/cri-o.md) | Yes | crictl pull docker.io/library/alpine:latest | When using Dragonfly in Kubernetes, we recommend to use `Containerd with CRI` and `CRI-O`, deploying document can be -found in [Kubernetes-with-Dragonfly](../ecosystem/Kubernetes-with-Dragonfly.md). \ No newline at end of file +found in [Kubernetes-with-Dragonfly](../ecosystem/Kubernetes-with-Dragonfly.md). + +## SEE ALSO + +- [install manager](../user-guide/install/install-manager.md) - how to install the Dragonfly manager +- [install cdn](../user-guide/install/install-cdn.md) - how to install the Dragonfly cdn +- [install scheduler](../user-guide/install/install-scheduler.md) - how to install the Dragonfly scheduler +- [proxy](../user-guide/proxy/containerd.md) - make Dragonfly as an HTTP proxy for docker daemon +- Container Runtimes + - [cri-o mirror](../user-guide/registry-mirror/cri-o.md) - make Dragonfly as Registry Mirror for CRIO daemon + - [cri-containerd_mirror](../user-guide/registry-mirror/cri-containerd.md) - make Dragonfly as Registry Mirror for containerd daemon \ No newline at end of file diff --git a/docs/en/user-guide/registry-mirror/containerd.md b/docs/en/user-guide/registry-mirror/cri-containerd.md similarity index 100% rename from docs/en/user-guide/registry-mirror/containerd.md rename to docs/en/user-guide/registry-mirror/cri-containerd.md diff --git a/docs/zh-CN/cli-reference/cdn.md b/docs/zh-CN/cli-reference/cdn.md new file mode 100644 index 00000000000..d59e1fc7c59 --- /dev/null +++ b/docs/zh-CN/cli-reference/cdn.md @@ -0,0 +1,26 @@ +## CDN + +CDN 是一个长时间运行的服务进程，它缓存从源下载的数据，以避免重复从源下载相同的文件 + +## 用法 +``` +go run cmd/cdnsystem/main.go [Option] +``` +## 输出日志配置 +如果你期望把系统执行日志输出到命令终端，设置环境变量 console=true + +## Runtime metrics monitoring +``` +go run cmd/cdnsystem/main.go --profiler +``` +### 可选参数 + +``` + --config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/cdn.yaml, it can also be + set by env var:CDN_CONFIG，The settings and uses of each configuration item can refer to cdn.yaml in config directory + --console whether logger output records to the stdout + -h, --help help for cdn + --jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces + --pprof-port int listen port for pprof, 0 represents random port (default -1) + --verbose whether logger use debug level +``` \ No newline at end of file diff --git a/docs/zh-CN/cli-reference/manager.md b/docs/zh-CN/cli-reference/manager.md new file mode 100644 index 00000000000..35357d6b574 --- /dev/null +++ b/docs/zh-CN/cli-reference/manager.md @@ -0,0 +1,20 @@ +## Scheduler + +Manager 是一个常驻后台运行的进程，它在蜻蜓中扮演各个子系统集群大脑的角色， 用于管理各个系统模块依赖的动态配置，以及提供心跳保活、监控大盘和产品化的功能。 + +### 用法 +``` +go run cmd/manager/main.go [Option] +``` + +### 可选参数 + +``` + --config string the path of scheduler's configuration file (default "conf/scheduler.yaml") + -h, --help help for scheduler + --port int port is the port that scheduler server listens on (default 8002) + --sender-job-pool-size int sender-job-pool-size is used for scheduler and do not change it (default 10000) + --sender-num int sender-num is used for scheduler and do not change it (default 50) + --worker-job-pool-size int worker-job-pool-size is used for scheduler and do not change it (default 10000) + --worker-num int worker-num is used for scheduler and do not change it (default 12) +``` \ No newline at end of file diff --git a/docs/zh-CN/cli-reference/scheduler.md b/docs/zh-CN/cli-reference/scheduler.md new file mode 100644 index 00000000000..2d1f188f43b --- /dev/null +++ b/docs/zh-CN/cli-reference/scheduler.md @@ -0,0 +1,20 @@ +## Scheduler + +Scheduler 是一个常驻后台运行的进程，用于接收和管理客户端的下载任务，通知CDN进行回源， 在下载过程中生成维护P2P网络，给客户端推送适合的下载节点 + +### 用法 +``` +go run cmd/scheduler/main.go [Option] +``` + +### 可选参数 + +``` + --config string the path of scheduler's configuration file (default "conf/scheduler.yaml") + -h, --help help for scheduler + --port int port is the port that scheduler server listens on (default 8002) + --sender-job-pool-size int sender-job-pool-size is used for scheduler and do not change it (default 10000) + --sender-num int sender-num is used for scheduler and do not change it (default 50) + --worker-job-pool-size int worker-job-pool-size is used for scheduler and do not change it (default 10000) + --worker-num int worker-num is used for scheduler and do not change it (default 12) +``` \ No newline at end of file diff --git a/docs/zh-CN/config/cdn.yaml b/docs/zh-CN/config/cdn.yaml new file mode 100644 index 00000000000..06ef9f517d1 --- /dev/null +++ b/docs/zh-CN/config/cdn.yaml @@ -0,0 +1,81 @@ +# 这个文件是 CDN 系统的配置模版 +# 你可以通过修改这里的配置项来自定义你的 CDN 系统 +--- +base: # 基础配置项 + # listenPort cdn 服务的监听端口. + # default: 8003 + listenPort: 8003 + + # DownloadPort CDN提供文件下载的端口. + # 你需要先启动一个文件服务器，并且该文件服务器监听该下载端口。. + # default: 8001 + downloadPort: 8001 + + # SystemReservedBandwidth is the network bandwidth reserved for system software. + # default: 20 MB, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte. + systemReservedBandwidth: 20M + + # MaxBandwidth is the network bandwidth that cdn can use. + # default: 1G, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte. + maxBandwidth: 1G + + # FailAccessInterval is the interval time after failed to access the URL. + # If a task failed to be downloaded from the source, it will not be retried in the time since the last failure. + # default: 3m + failAccessInterval: 3m + + # GCInitialDelay is the delay time from the start to the first GC execution. + # default: 6s + gcInitialDelay: 6s + + # GCMetaInterval is the interval time to execute GC meta. + # default: 2m0s + gcMetaInterval: 2m + + # gcStorageInterval is the interval time to execute GC storage. + # default: 15s + gcStorageInterval: 15s + + # TaskExpireTime when a task is not accessed within the taskExpireTime, + # and it will be treated to be expired. + # default: 3m0s + taskExpireTime: 3m + + # StoragePattern is the pattern of storage policy, [disk/hybrid] + storagePattern: disk + +plugins: # 插件配置 + storage: # 存储插件配置 + - name: disk + enable: true + config: + baseDir: /tmp/cdnsystem2 + gcConfig: + youngGCThreshold: 100G + fullGCThreshold: 5G + cleanRatio: 1 + intervalThreshold: 2h + - name: memory + enable: true + config: + baseDir: /tmp/memory/dragonfly + gcConfig: + youngGCThreshold: 100G + fullGCThreshold: 5G + cleanRatio: 3 + intervalThreshold: 2h + +# Console shows log on console +# default: false +console: false + +# Whether to enable debug level logger and enable pprof +# default: false +verbose: false + +# listen port for pprof, only valid when the verbose option is true +# default is random port +pprofPort: 0 + +# the service address that provides the configuration item +configServer: 127.0.0.1:8004 \ No newline at end of file