From eefaccdc19d46b62621ad23422bf4ea690715c08 Mon Sep 17 00:00:00 2001 From: santong Date: Wed, 23 Jun 2021 01:13:31 +0800 Subject: [PATCH 1/8] cdn error definition Signed-off-by: santong --- cdnsystem/daemon/cdn/cache_detector.go | 29 ++-- cdnsystem/daemon/cdn/cache_detector_test.go | 6 +- cdnsystem/daemon/task/manager_util.go | 4 +- cdnsystem/errors/errors.go | 94 ++++++------ cdnsystem/errors/errors_test.go | 138 ++---------------- cdnsystem/storedriver/local/local_driver.go | 3 +- pkg/source/httpprotocol/http_source_client.go | 5 +- pkg/source/ossprotocol/oss_source_client.go | 8 +- 8 files changed, 95 insertions(+), 192 deletions(-) diff --git a/cdnsystem/daemon/cdn/cache_detector.go b/cdnsystem/daemon/cdn/cache_detector.go index c4b7258e4f8..aff489de59a 100644 --- a/cdnsystem/daemon/cdn/cache_detector.go +++ b/cdnsystem/daemon/cdn/cache_detector.go @@ -89,7 +89,7 @@ func (cd *cacheDetector) doDetect(task *types.SeedTask, fileMd5 hash.Hash) (resu return nil, errors.Wrapf(err, "read file meta data of task %s", task.TaskID) } if err := checkSameFile(task, fileMetaData); err != nil { - return nil, errors.Wrapf(err, "task does not match meta information of task file") + return nil, errors.Wrapf(err, "check same file") } ctx, expireCancel := context.WithTimeout(context.Background(), 4*time.Second) defer expireCancel() @@ -100,8 +100,7 @@ func (cd *cacheDetector) doDetect(task *types.SeedTask, fileMd5 hash.Hash) (resu } logger.WithTaskID(task.TaskID).Debugf("task expired result: %t", expired) if expired { - return nil, errors.Wrapf(cdnerrors.ErrResourceExpired, "url:%s, expireInfo:%+v", task.URL, - fileMetaData.ExpireInfo) + return nil, cdnerrors.ErrResourceExpired{URL: task.URL} } // not expired if fileMetaData.Finish { @@ -117,7 +116,7 @@ func (cd *cacheDetector) doDetect(task *types.SeedTask, fileMd5 hash.Hash) (resu return nil, errors.Wrapf(err, "check if url(%s) supports range request", task.URL) } if !supportRange { - return nil, errors.Wrapf(cdnerrors.ErrResourceNotSupportRangeRequest, "url:%s", task.URL) + return nil, cdnerrors.ErrResourceNotSupportRangeRequest{URL: task.URL} } return cd.parseByReadFile(task.TaskID, fileMetaData, fileMd5) } @@ -125,15 +124,15 @@ func (cd *cacheDetector) doDetect(task *types.SeedTask, fileMd5 hash.Hash) (resu // parseByReadMetaFile detect cache by read meta and pieceMeta files of task func (cd *cacheDetector) parseByReadMetaFile(taskID string, fileMetaData *storage.FileMetaData) (*cacheResult, error) { if !fileMetaData.Success { - return nil, errors.Wrapf(cdnerrors.ErrDownloadFail, "success flag of download is false") + return nil, fmt.Errorf("success flag of taskID %s is false", taskID) } pieceMetaRecords, err := cd.cacheDataManager.readAndCheckPieceMetaRecords(taskID, fileMetaData.PieceMd5Sign) if err != nil { return nil, errors.Wrapf(err, "check piece meta integrity") } if fileMetaData.TotalPieceCount > 0 && len(pieceMetaRecords) != int(fileMetaData.TotalPieceCount) { - return nil, errors.Wrapf(cdnerrors.ErrPieceCountNotEqual, "piece file piece count(%d), "+ - "meta file piece count(%d)", len(pieceMetaRecords), fileMetaData.TotalPieceCount) + err := cdnerrors.ErrInconsistentValues{Expected: fileMetaData.TotalPieceCount, Actual: len(pieceMetaRecords)} + return nil, errors.Wrapf(err, "compare file piece count") } storageInfo, err := cd.cacheDataManager.statDownloadFile(taskID) if err != nil { @@ -141,8 +140,11 @@ func (cd *cacheDetector) parseByReadMetaFile(taskID string, fileMetaData *storag } // check file data integrity by file size if fileMetaData.CdnFileLength != storageInfo.Size { - return nil, errors.Wrapf(cdnerrors.ErrFileLengthNotEqual, "meta size %d, storage size %d", - fileMetaData.CdnFileLength, storageInfo.Size) + err := cdnerrors.ErrInconsistentValues{ + Expected: fileMetaData.CdnFileLength, + Actual: storageInfo.Size, + } + return nil, errors.Wrapf(err, "compare file cdn file length") } return &cacheResult{ breakPoint: -1, @@ -249,13 +251,16 @@ func checkPieceContent(reader io.Reader, pieceRecord *storage.PieceMetaRecord, f pieceMd5 := md5.New() tee := io.TeeReader(io.TeeReader(io.LimitReader(reader, int64(pieceRecord.PieceLen)), pieceMd5), fileMd5) if n, err := io.Copy(ioutil.Discard, tee); n != int64(pieceRecord.PieceLen) || err != nil { - return fmt.Errorf("read piece content: %v", err) + return errors.Wrap(err, "read piece content") } realPieceMd5 := digestutils.ToHashString(pieceMd5) // check piece content if realPieceMd5 != pieceRecord.Md5 { - return errors.Wrapf(cdnerrors.ErrPieceMd5NotMatch, "realPieceMd5 md5 (%s), expected md5 (%s)", - realPieceMd5, pieceRecord.Md5) + err := cdnerrors.ErrInconsistentValues{ + Expected: pieceRecord.Md5, + Actual: realPieceMd5, + } + return errors.Wrap(err, "compare piece md5") } return nil } diff --git a/cdnsystem/daemon/cdn/cache_detector_test.go b/cdnsystem/daemon/cdn/cache_detector_test.go index 1f02878ae61..24ef6bcf851 100644 --- a/cdnsystem/daemon/cdn/cache_detector_test.go +++ b/cdnsystem/daemon/cdn/cache_detector_test.go @@ -59,7 +59,7 @@ func (suite *CacheDetectorTestSuite) SetupSuite() { storageMgr.EXPECT().ReadFileMetaData(fullNoExpiredCache.taskID).Return(fullNoExpiredCache.fileMeta, nil).AnyTimes() storageMgr.EXPECT().ReadFileMetaData(partialNotSupportRangeCache.taskID).Return(partialNotSupportRangeCache.fileMeta, nil).AnyTimes() storageMgr.EXPECT().ReadFileMetaData(partialSupportRangeCache.taskID).Return(partialSupportRangeCache.fileMeta, nil).AnyTimes() - storageMgr.EXPECT().ReadFileMetaData(noCache.taskID).Return(noCache.fileMeta, cdnerrors.ErrFileNotExist).AnyTimes() + storageMgr.EXPECT().ReadFileMetaData(noCache.taskID).Return(noCache.fileMeta, cdnerrors.ErrFileNotExist{}).AnyTimes() storageMgr.EXPECT().ReadDownloadFile(fullNoExpiredCache.taskID).DoAndReturn( func(taskID string) (io.ReadCloser, error) { content, err := ioutil.ReadFile("../../testdata/cdn/go.html") @@ -78,11 +78,11 @@ func (suite *CacheDetectorTestSuite) SetupSuite() { suite.Nil(err) return ioutil.NopCloser(strings.NewReader(string(content))), nil }).AnyTimes() - storageMgr.EXPECT().ReadDownloadFile(noCache.taskID).Return(nil, cdnerrors.ErrFileNotExist).AnyTimes() + storageMgr.EXPECT().ReadDownloadFile(noCache.taskID).Return(nil, cdnerrors.ErrFileNotExist{}).AnyTimes() storageMgr.EXPECT().ReadPieceMetaRecords(fullNoExpiredCache.taskID).Return(fullNoExpiredCache.pieces, nil).AnyTimes() storageMgr.EXPECT().ReadPieceMetaRecords(partialNotSupportRangeCache.taskID).Return(partialNotSupportRangeCache.pieces, nil).AnyTimes() storageMgr.EXPECT().ReadPieceMetaRecords(partialSupportRangeCache.taskID).Return(partialSupportRangeCache.pieces, nil).AnyTimes() - storageMgr.EXPECT().ReadPieceMetaRecords(noCache.taskID).Return(nil, cdnerrors.ErrFileNotExist).AnyTimes() + storageMgr.EXPECT().ReadPieceMetaRecords(noCache.taskID).Return(nil, cdnerrors.ErrFileNotExist{}).AnyTimes() storageMgr.EXPECT().StatDownloadFile(fullNoExpiredCache.taskID).Return(&storedriver.StorageInfo{ Path: "", Size: 9789, diff --git a/cdnsystem/daemon/task/manager_util.go b/cdnsystem/daemon/task/manager_util.go index 82debd94abe..a4a1395e082 100644 --- a/cdnsystem/daemon/task/manager_util.go +++ b/cdnsystem/daemon/task/manager_util.go @@ -48,8 +48,8 @@ func (tm *Manager) addOrUpdateTask(ctx context.Context, request *types.TaskRegis if key, err := tm.taskURLUnReachableStore.Get(taskID); err == nil { if unReachableStartTime, ok := key.(time.Time); ok && time.Since(unReachableStartTime) < tm.cfg.FailAccessInterval { - return nil, cdnerrors.ErrURLNotReachable{URL: request.URL, Cause: fmt.Errorf("task hit unReachable cache and interval less than %d, url: %s", - tm.cfg.FailAccessInterval, request.URL)} + return nil, errors.Wrapf(cdnerrors.ErrURLNotReachable{URL: request.URL}, "task hit unReachable cache and interval less than %d, "+ + "url: %s", tm.cfg.FailAccessInterval, request.URL) } tm.taskURLUnReachableStore.Delete(taskID) logger.Debugf("delete taskID:%s from url unReachable store", taskID) diff --git a/cdnsystem/errors/errors.go b/cdnsystem/errors/errors.go index fe08e282526..0de97f0e5f0 100644 --- a/cdnsystem/errors/errors.go +++ b/cdnsystem/errors/errors.go @@ -24,12 +24,11 @@ import ( // ErrURLNotReachable represents the url is a not reachable. type ErrURLNotReachable struct { - URL string - Cause error + URL string } func (e ErrURLNotReachable) Error() string { - return fmt.Sprintf("url %s not reachable: %v", e.URL, e.Cause) + return fmt.Sprintf("url %s not reachable", e.URL) } // ErrTaskIDDuplicate represents the task id is in conflict. @@ -42,42 +41,57 @@ func (e ErrTaskIDDuplicate) Error() string { return fmt.Sprintf("taskId %s conflict: %v", e.TaskID, e.Cause) } -var ( - // ErrSystemError represents the error is a system error. - ErrSystemError = errors.New("system error") +type ErrInconsistentValues struct { + Expected interface{} + Actual interface{} +} - // ErrPieceCountNotEqual represents the number of pieces downloaded does not match the amount of meta information - ErrPieceCountNotEqual = errors.New("inconsistent number of pieces") +func (e ErrInconsistentValues) Error() string { + return fmt.Sprintf("inconsistent number of pieces, expected %s, actual: %s", e.Expected, e.Actual) +} - // ErrFileLengthNotEqual represents the file length of downloaded dose not match the length of meta information - ErrFileLengthNotEqual = errors.New("inconsistent file length") +// ErrResourceExpired represents the downloaded resource has expired +type ErrResourceExpired struct { + URL string +} - // ErrDownloadFail represents an exception was encountered while downloading the file - ErrDownloadFail = errors.New("resource download failed") +func (e ErrResourceExpired) Error() string { + return fmt.Sprintf("url %s expired", e.URL) +} - // ErrResourceExpired represents the downloaded resource has expired - ErrResourceExpired = errors.New("resource expired") +// ErrResourceNotSupportRangeRequest represents the downloaded resource does not support Range downloads +type ErrResourceNotSupportRangeRequest struct { + URL string +} - // ErrResourceNotSupportRangeRequest represents the downloaded resource does not support Range downloads - ErrResourceNotSupportRangeRequest = errors.New("resource does not support range request") +func (e ErrResourceNotSupportRangeRequest) Error() string { + return fmt.Sprintf("url %s does not support range request", e.URL) +} + +// ErrFileNotExist represents the file is not exists +type ErrFileNotExist struct { + File string +} + +func (e ErrFileNotExist) Error() string { + return fmt.Sprintf("file or dir %s not exist", e.File) +} + +var ( + // ErrSystemError represents the error is a system error. + ErrSystemError = errors.New("system error") - // ErrPieceMd5NotMatch represents the MD5 value of the download file is inconsistent with the meta information - ErrPieceMd5NotMatch = errors.New("piece md5 check fail") + // ErrTaskDownloadFail represents an exception was encountered while downloading the file + ErrTaskDownloadFail = errors.New("resource download failed") // ErrDataNotFound represents the data cannot be found. ErrDataNotFound = errors.New("data not found") - // ErrFileNotExist represents the file is not exists - ErrFileNotExist = errors.New("file or directory not exist") - // ErrInvalidValue represents the value is invalid. ErrInvalidValue = errors.New("invalid value") // ErrConvertFailed represents failed to convert. ErrConvertFailed = errors.New("convert failed") - - // ErrRangeNotSatisfiable represents the length of file is insufficient. - ErrRangeNotSatisfiable = errors.New("range not satisfiable") ) // IsSystemError checks the error is a system error or not. @@ -99,28 +113,26 @@ func IsTaskIDDuplicate(err error) bool { return ok } -func IsPieceCountNotEqual(err error) bool { - return errors.Cause(err) == ErrPieceCountNotEqual -} - -func IsFileLengthNotEqual(err error) bool { - return errors.Cause(err) == ErrFileLengthNotEqual +func IsInconsistentValues(err error) bool { + err = errors.Cause(err) + _, ok := err.(ErrInconsistentValues) + return ok } func IsDownloadFail(err error) bool { - return errors.Cause(err) == ErrDownloadFail + return errors.Cause(err) == ErrTaskDownloadFail } func IsResourceExpired(err error) bool { - return errors.Cause(err) == ErrResourceExpired + err = errors.Cause(err) + _, ok := err.(ErrResourceExpired) + return ok } func IsResourceNotSupportRangeRequest(err error) bool { - return errors.Cause(err) == ErrResourceNotSupportRangeRequest -} - -func IsPieceMd5NotMatch(err error) bool { - return errors.Cause(err) == ErrPieceMd5NotMatch + err = errors.Cause(err) + _, ok := err.(ErrResourceNotSupportRangeRequest) + return ok } func IsDataNotFound(err error) bool { @@ -135,10 +147,8 @@ func IsConvertFailed(err error) bool { return errors.Cause(err) == ErrConvertFailed } -func IsRangeNotSatisfiable(err error) bool { - return errors.Cause(err) == ErrRangeNotSatisfiable -} - func IsFileNotExist(err error) bool { - return errors.Cause(err) == ErrFileNotExist + err = errors.Cause(err) + _, ok := err.(ErrFileNotExist) + return ok } diff --git a/cdnsystem/errors/errors_test.go b/cdnsystem/errors/errors_test.go index fb6a14b7c2a..aa888868efa 100644 --- a/cdnsystem/errors/errors_test.go +++ b/cdnsystem/errors/errors_test.go @@ -116,13 +116,13 @@ func (s *ErrorTestSuite) TestIsDownloadFail() { { name: "equal", args: args{ - err: ErrDownloadFail, + err: ErrTaskDownloadFail, }, want: true, }, { name: "wrap", args: args{ - err: errors.Wrapf(errors.Wrapf(ErrDownloadFail, "wrap err"), "wapp err"), + err: errors.Wrapf(errors.Wrapf(ErrTaskDownloadFail, "wrap err"), "wapp err"), }, want: true, }, { @@ -140,42 +140,6 @@ func (s *ErrorTestSuite) TestIsDownloadFail() { } } -func (s *ErrorTestSuite) TestIsFileLengthNotEqual() { - type args struct { - err error - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "equal", - args: args{ - err: ErrFileLengthNotEqual, - }, - want: true, - }, { - name: "wrap", - args: args{ - err: errors.Wrapf(errors.Wrapf(ErrFileLengthNotEqual, "wrap err"), "wapp err"), - }, - want: true, - }, { - name: "notEqual", - args: args{ - err: errors.Wrapf(ErrInvalidValue, "invaid"), - }, - want: false, - }, - } - for _, tt := range tests { - s.Run(tt.name, func() { - s.Equal(tt.want, IsFileLengthNotEqual(tt.args.err)) - }) - } -} - func (s *ErrorTestSuite) TestIsFileNotExist() { type args struct { err error @@ -188,13 +152,13 @@ func (s *ErrorTestSuite) TestIsFileNotExist() { { name: "equal", args: args{ - err: ErrFileNotExist, + err: ErrFileNotExist{}, }, want: true, }, { name: "wrap", args: args{ - err: errors.Wrapf(errors.Wrapf(ErrFileNotExist, "wrap err"), "wapp err"), + err: errors.Wrapf(errors.Wrapf(ErrFileNotExist{}, "wrap err"), "wapp err"), }, want: true, }, { @@ -248,79 +212,7 @@ func (s *ErrorTestSuite) TestIsInvalidValue() { } } -func (s *ErrorTestSuite) TestIsPieceCountNotEqual() { - type args struct { - err error - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "equal", - args: args{ - err: ErrPieceCountNotEqual, - }, - want: true, - }, { - name: "wrap", - args: args{ - err: errors.Wrapf(errors.Wrapf(ErrPieceCountNotEqual, "wrap err"), "wapp err"), - }, - want: true, - }, { - name: "notEqual", - args: args{ - err: errors.Wrapf(ErrInvalidValue, "invaid"), - }, - want: false, - }, - } - for _, tt := range tests { - s.Run(tt.name, func() { - s.Equal(tt.want, IsPieceCountNotEqual(tt.args.err)) - }) - } -} - -func (s *ErrorTestSuite) TestIsPieceMd5NotMatch() { - type args struct { - err error - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "equal", - args: args{ - err: ErrPieceMd5NotMatch, - }, - want: true, - }, { - name: "wrap", - args: args{ - err: errors.Wrapf(errors.Wrapf(ErrPieceMd5NotMatch, "wrap err"), "wapp err"), - }, - want: true, - }, { - name: "notEqual", - args: args{ - err: errors.Wrapf(ErrInvalidValue, "invaid"), - }, - want: false, - }, - } - for _, tt := range tests { - s.Run(tt.name, func() { - s.Equal(tt.want, IsPieceMd5NotMatch(tt.args.err)) - }) - } -} - -func (s *ErrorTestSuite) TestIsRangeNotSatisfiable() { +func (s *ErrorTestSuite) TestIsInconsistentValues() { type args struct { err error } @@ -332,13 +224,13 @@ func (s *ErrorTestSuite) TestIsRangeNotSatisfiable() { { name: "equal", args: args{ - err: ErrRangeNotSatisfiable, + err: ErrInconsistentValues{}, }, want: true, }, { name: "wrap", args: args{ - err: errors.Wrapf(errors.Wrapf(ErrRangeNotSatisfiable, "wrap err"), "wapp err"), + err: errors.Wrapf(errors.Wrapf(ErrInconsistentValues{}, "wrap err"), "wapp err"), }, want: true, }, { @@ -351,7 +243,7 @@ func (s *ErrorTestSuite) TestIsRangeNotSatisfiable() { } for _, tt := range tests { s.Run(tt.name, func() { - s.Equal(tt.want, IsRangeNotSatisfiable(tt.args.err)) + s.Equal(tt.want, IsInconsistentValues(tt.args.err)) }) } } @@ -368,13 +260,13 @@ func (s *ErrorTestSuite) TestIsResourceExpired() { { name: "equal", args: args{ - err: ErrResourceExpired, + err: ErrResourceExpired{}, }, want: true, }, { name: "wrap", args: args{ - err: errors.Wrapf(errors.Wrapf(ErrResourceExpired, "wrap err"), "wapp err"), + err: errors.Wrapf(errors.Wrapf(ErrResourceExpired{}, "wrap err"), "wapp err"), }, want: true, }, { @@ -404,13 +296,13 @@ func (s *ErrorTestSuite) TestIsResourceNotSupportRangeRequest() { { name: "equal", args: args{ - err: ErrResourceNotSupportRangeRequest, + err: ErrResourceNotSupportRangeRequest{}, }, want: true, }, { name: "wrap", args: args{ - err: errors.Wrapf(errors.Wrapf(ErrResourceNotSupportRangeRequest, "wrap err"), "wapp err"), + err: errors.Wrapf(errors.Wrapf(ErrResourceNotSupportRangeRequest{}, "wrap err"), "wapp err"), }, want: true, }, { @@ -518,8 +410,7 @@ func (s *ErrorTestSuite) TestIsURLNotReachable() { name: "equal", args: args{ err: ErrURLNotReachable{ - URL: "test", - Cause: fmt.Errorf("test"), + URL: "test", }, }, want: true, @@ -527,8 +418,7 @@ func (s *ErrorTestSuite) TestIsURLNotReachable() { name: "wrap", args: args{ err: errors.Wrapf(ErrURLNotReachable{ - URL: "test", - Cause: fmt.Errorf("test"), + URL: "test", }, "wapp err"), }, want: true, diff --git a/cdnsystem/storedriver/local/local_driver.go b/cdnsystem/storedriver/local/local_driver.go index 9955422d4f5..ea65930e24a 100644 --- a/cdnsystem/storedriver/local/local_driver.go +++ b/cdnsystem/storedriver/local/local_driver.go @@ -30,7 +30,6 @@ import ( "d7y.io/dragonfly/v2/pkg/unit" "d7y.io/dragonfly/v2/pkg/util/fileutils" "d7y.io/dragonfly/v2/pkg/util/statutils" - "github.com/pkg/errors" ) // Ensure driver implements the storedriver.Driver interface @@ -368,7 +367,7 @@ func (ds *driver) statPath(bucket, key string) (string, os.FileInfo, error) { f, err := os.Stat(filePath) if err != nil { if os.IsNotExist(err) { - return "", nil, errors.Wrapf(cdnerrors.ErrFileNotExist, "no such file or directory:%s exists", filePath) + return "", nil, cdnerrors.ErrFileNotExist{File: "filePath"} } return "", nil, err } diff --git a/pkg/source/httpprotocol/http_source_client.go b/pkg/source/httpprotocol/http_source_client.go index 7908999483d..0329ef39282 100644 --- a/pkg/source/httpprotocol/http_source_client.go +++ b/pkg/source/httpprotocol/http_source_client.go @@ -24,7 +24,6 @@ import ( "net/http" "time" - cdnerrors "d7y.io/dragonfly/v2/cdnsystem/errors" "d7y.io/dragonfly/v2/pkg/source" "d7y.io/dragonfly/v2/pkg/structure/maputils" "d7y.io/dragonfly/v2/pkg/util/stringutils" @@ -81,13 +80,13 @@ func WithHTTPClient(client *http.Client) HTTPSourceClientOption { func (client *httpSourceClient) GetContentLength(ctx context.Context, url string, header source.Header) (int64, error) { resp, err := client.doRequest(ctx, http.MethodGet, url, header) if err != nil { - return -1, cdnerrors.ErrURLNotReachable{URL: url, Cause: err} + return -1, err } resp.Body.Close() // todo Here if other status codes should be added to ErrURLNotReachable, if not, it will be downloaded frequently for 404 or 403 if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { // todo Whether this situation should be distinguished from the err situation, similar to proposing another error type to indicate that this error can interact with the URL, but the status code does not meet expectations - return -1, cdnerrors.ErrURLNotReachable{URL: url, Cause: fmt.Errorf("get http resource length failed, unexpected code: %d", resp.StatusCode)} + return -1, fmt.Errorf("get http resource length failed, unexpected code: %d", resp.StatusCode) } return resp.ContentLength, nil } diff --git a/pkg/source/ossprotocol/oss_source_client.go b/pkg/source/ossprotocol/oss_source_client.go index 85a33052189..6f963798568 100644 --- a/pkg/source/ossprotocol/oss_source_client.go +++ b/pkg/source/ossprotocol/oss_source_client.go @@ -178,19 +178,19 @@ func (osc *ossSourceClient) getMeta(ctx context.Context, url string, header map[ } ossObject, err := parseOssObject(url) if err != nil { - return nil, cdnerrors.ErrURLNotReachable{URL: url, Cause: fmt.Errorf("parse oss object: %v", err)} + return nil, errors.Wrapf(err, "parse oss object") } bucket, err := client.Bucket(ossObject.bucket) if err != nil { - return nil, cdnerrors.ErrURLNotReachable{URL: url, Cause: fmt.Errorf("get bucket:%s: %v", ossObject.bucket, err)} + return nil, errors.Wrapf(err, "get bucket:%s", ossObject.bucket) } isExist, err := bucket.IsObjectExist(ossObject.object) if err != nil { - return nil, cdnerrors.ErrURLNotReachable{URL: url, Cause: fmt.Errorf("prob object:%s exist: %v", ossObject.object, err)} + return nil, errors.Wrapf(err, "prob object:%s if exist", ossObject.object) } if !isExist { - return nil, cdnerrors.ErrURLNotReachable{URL: url, Cause: fmt.Errorf("oss object:%s does not exist", ossObject.object)} + return nil, fmt.Errorf("oss object:%s does not exist", ossObject.object) } return bucket.GetObjectMeta(ossObject.object, getOptions(header)...) } From 6b61266344a181c1404f3e9078ebea64e2d71962 Mon Sep 17 00:00:00 2001 From: santong Date: Tue, 18 May 2021 10:28:45 +0800 Subject: [PATCH 2/8] docs of dragonfly2.0 Signed-off-by: santong --- docs/README.md | 2 +- docs/en/README.md | 16 +- docs/en/quick_start/README.md | 117 ++++++++++++ docs/en/user-guide/download_files.md | 131 ++++++++++++++ docs/en/user-guide/install_cdn.md | 167 ++++++++++++++++++ docs/en/user-guide/install_client.md | 136 ++++++++++++++ docs/en/user-guide/install_manager.md | 167 ++++++++++++++++++ docs/en/user-guide/install_scheduler.md | 137 ++++++++++++++ .../user-guide/multi_machines_deployment.md | 118 +++++++++++++ docs/en/user-guide/proxy/docker.md | 160 ----------------- docs/en/user-guide/quick-start.md | 2 +- .../{containerd.md => cri-containerd.md} | 0 12 files changed, 983 insertions(+), 170 deletions(-) create mode 100644 docs/en/quick_start/README.md create mode 100644 docs/en/user-guide/download_files.md create mode 100644 docs/en/user-guide/install_cdn.md create mode 100644 docs/en/user-guide/install_client.md create mode 100644 docs/en/user-guide/install_manager.md create mode 100644 docs/en/user-guide/install_scheduler.md create mode 100644 docs/en/user-guide/multi_machines_deployment.md delete mode 100644 docs/en/user-guide/proxy/docker.md rename docs/en/user-guide/registry-mirror/{containerd.md => cri-containerd.md} (100%) diff --git a/docs/README.md b/docs/README.md index df2074923b9..e5721fb87f8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Dragonfly Document -[English](./en/README.md) +[English](en/README.md) [TODO 简体中文](./zh-CN/README.md) diff --git a/docs/en/README.md b/docs/en/README.md index a90566e13c1..85b0d09f95f 100644 --- a/docs/en/README.md +++ b/docs/en/README.md @@ -7,14 +7,14 @@ Organization of document is as following: * [Quick Start](#Quick-Start) * [User Guide](#User-Guide) * [CLI Reference](#CLI-Reference) - * [WIP dfget](./cli-reference/dfget.md) - * [WIP cdnsystem](./cli-reference/cdn.md) - * [WIP scheduler](./cli-reference/scheduler.md) + * [WIP dfget](cli-reference/dfget.md) + * [WIP cdnsystem](cli-reference/cdn.md) + * [WIP scheduler](cli-reference/scheduler.md) * [TODO manager](./cli-reference/manager.md) * [TODO API Reference](#API-Reference) * [Ecosystem](#Ecosystem) - * [Kubernetes Integration](./ecosystem/Kubernetes-with-Dragonfly.md) - * [WIP Harbor Integration](./ecosystem/Harbor-with-Dragonfly.md) + * [Kubernetes Integration](ecosystem/Kubernetes-with-Dragonfly.md) + * [WIP Harbor Integration](ecosystem/Harbor-with-Dragonfly.md) * [Developer Guide](#Developer-Guide) * [Design Doc](#Design-Doc) * [Test Guide](#Test-Guide) @@ -25,15 +25,15 @@ Find `WIP` or `TODO` in this page and follow [CONTRIBUTING](../../CONTRIBUTING.m ## Quick Start -[Quick Started](./user-guide/quick-start.md) is exactly what you need if you would give Dragonfly a try. This document includes what are the prerequisites, how to install Dragonfly and how to experience Dragonfly's usage. +[Quick Started](user-guide/quick-start.md) is exactly what you need if you would give Dragonfly a try. This document includes what are the prerequisites, how to install Dragonfly and how to experience Dragonfly's usage. ## [WIP] User Guide -[User Guide](./user-guide) helps all kinds of guidance end users need to experience Dragonfly. Not only the very brief [Quick Start](./quick-start), but the detailed binary installation and configure illustration. In addition, any concept and function which help users understand Dragonfly better would be included as well. +[User Guide]() helps all kinds of guidance end users need to experience Dragonfly. Not only the very brief [Quick Start](./quick-start), but the detailed binary installation and configure illustration. In addition, any concept and function which help users understand Dragonfly better would be included as well. ## [WIP] CLI Reference -For almost all users, commandline is the first reference you may need. Document in directory [CLI Reference](./cli-reference) is about command detailed usage of Dragonfly CLI including `dfget`, `cdnsystem`, `scheduler` and `manager`. You can get introductions, synopsis, examples, options about command. Last but not least, Dragonfly can guarantee commandline docs is strongly consistent with Dragonfly CLI's source code. What's more, all commandline docs are auto generated via source code. +For almost all users, commandline is the first reference you may need. Document in directory [CLI Reference](cli-reference) is about command detailed usage of Dragonfly CLI including `dfget`, `cdnsystem`, `scheduler` and `manager`. You can get introductions, synopsis, examples, options about command. Last but not least, Dragonfly can guarantee commandline docs is strongly consistent with Dragonfly CLI's source code. What's more, all commandline docs are auto generated via source code. ## [TODO] API Reference diff --git a/docs/en/quick_start/README.md b/docs/en/quick_start/README.md new file mode 100644 index 00000000000..1fc52874407 --- /dev/null +++ b/docs/en/quick_start/README.md @@ -0,0 +1,117 @@ +# Dragonfly Quick Start + +Dragonfly Quick Start document aims to help you to quick start Dragonfly journey. This experiment is quite easy and simplified. + +If you are using Dragonfly in your **production environment** to handle production image distribution, please refer to supernode and dfget's detailed production parameter configuration. + +## Prerequisites + +All steps in this document is doing on the same machine using the docker container, so make sure the docker container engine installed and started on your machine. You can also refer to the documentation: [multi-machine deployment](../user_guide/multi_machines_deployment.md) to experience Dragonfly. + +## Step 1: Deploy Dragonfly Manager Server + +```bash +docker run -d --name supernode \ + --restart=always \ + -p 8001:8001 \ + -p 8002:8002 \ + -v /home/admin/supernode:/home/admin/supernode \ + dragonflyoss/supernode:1.0.2 +``` + +## Step 2: Deploy Dragonfly CDN Server + +```bash +docker run -d --name supernode \ + --restart=always \ + -p 8001:8001 \ + -p 8002:8002 \ + -v /home/admin/supernode:/home/admin/supernode \ + dragonflyoss/supernode:1.0.2 +``` + +## Step 3: Deploy Dragonfly Scheduler Server + +```bash +docker run -d --name supernode \ + --restart=always \ + -p 8001:8001 \ + -p 8002:8002 \ + -v /home/admin/supernode:/home/admin/supernode \ + dragonflyoss/supernode:1.0.2 +``` + +## Step 2: Deploy Dragonfly Client + +```bash +SUPERNODE_IP=`docker inspect supernode -f '{{.NetworkSettings.Networks.bridge.IPAddress}}'` +docker run -d --name dfclient \ + --restart=always \ + -p 65001:65001 \ + -v $HOME/.small-dragonfly:/root/.small-dragonfly \ + dragonflyoss/dfclient:1.0.2 --registry https://index.docker.io --node $SUPERNODE_IP +``` + +**NOTE**: + +- The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the other **non-https image registries**. +- The `--node` parameter specifies the supernode's address in the format of **HOST:IP**. And the default value `8002` will be used if the port is not specified. Here we use `docker inspect` to get the ip of supernode container as the host value. Since the supernode container exposes its ports, you can specify this parameter to node ip address as well. + +## Step 3. Configure Docker Daemon + +We need to modify the Docker Daemon configuration to use the Dragonfly as a pull through registry. + +1. Add or update the configuration item `registry-mirrors` in the configuration file`/etc/docker/daemon.json`. + +```json +{ + "registry-mirrors": ["http://127.0.0.1:65001"] +} +``` + +**Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). + +2. Restart Docker Daemon. + +```bash +systemctl restart docker +``` + +## Step 4: Pull images with Dragonfly + +Through the above steps, we can start to validate if Dragonfly works as expected. + +And you can pull the image as usual, for example: + +```bash +docker pull nginx:latest +``` + +## Step 5: Validate Dragonfly + +You can execute the following command to check if the nginx image is distributed via Dragonfly. + +```bash +docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log +``` + +If the output of command above has content like + +``` +2019-03-29 15:49:53.913 INFO sign:96027-1553845785.119 : downloading piece:{"taskID":"00a0503ea12457638ebbef5d0bfae51f9e8e0a0a349312c211f26f53beb93cdc","superNode":"127.0.0.1","dstCid":"127.0.0.1-95953-1553845720.488","range":"67108864-71303167","result":503,"status":701,"pieceSize":4194304,"pieceNum":16} +``` + +then Dragonfly works successfully. + +## SEE ALSO + +- [multi machines deployment](../user-guide/multi_machines_deployment.md) - experience Dragonfly on multiple machines +- [install manager](../user-guide/install_manager.md) - how to install the Dragonfly manager +- [install cdn](../user-guide/install_cdn.md) - how to install the Dragonfly cdn +- [install scheduler](../user-guide/install_scheduler.md) - how to install the Dragonfly scheduler +- [install client](../user-guide/install_client.md) - how to install the Dragonfly client +- [proxy](../user_guide/proxy/docker.md) - make Dragonfly as an HTTP proxy for docker daemon +- [download files](../user-guide/download_files.md) - download files with Dragonfly +- Container Runtimes + - [cri-o mirror](../user_guide/registry/cri-o.md) - make Dragonfly as Registry Mirror for CRIO daemon + - [cri-containerd_mirror](../user_guide/registry/cri-containerd.md) - make Dragonfly as Registry Mirror for containerd daemon diff --git a/docs/en/user-guide/download_files.md b/docs/en/user-guide/download_files.md new file mode 100644 index 00000000000..e0c4c8b41e4 --- /dev/null +++ b/docs/en/user-guide/download_files.md @@ -0,0 +1,131 @@ +# Downloading Files with Dragonfly + +Things are done differently when you download container images and download general files with Dragonfly. + +## Prerequisites + +- You are using Linux operating system. +- The supernode service is started. + + **Tip:** For more information on the dfget command, see [dfget](../cli_reference/dfget.md). For more information on the installation of supernodes, see [Installing Server](./install_server.md). + +## Downloading container images + +1. Config the supernodes with the configuration file. + + ```shell + cat < /etc/dragonfly/dfget.yml + nodes: + - supernode01:port + - supernode02:port + - supernode03:port + EOD + ``` + +2. Start the dfget proxy (dfdaemon). + + ```sh + # Start dfdaemon and specify the image repo URL. The default port is `65001`. + dfdaemon --registry https://xxx.xx.x + # Review dfdaemon logs + tailf ~/.small-dragonfly/logs/dfdaemon.log + ``` + + **Tip:** To list all available parameters for dfdaemon, run `dfdaemon -h`. + +3. Configure the Docker daemon. + + a. Modify the configuration file `/etc/docker/daemon.json`. + + ```sh + vi /etc/docker/daemon.json + ``` + + **Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). + + b. Add or update the configuration item `registry-mirrors` in the configuration file. + + ```sh + "registry-mirrors": ["http://127.0.0.1:65001"] + ``` + + c. Restart Docker daemon. + + ```bash + systemctl restart docker + ``` + + d. Add authentication info for the private docker registry in `~/.docker/config.json` if the registry is configured with auth. + + ```json + { + "auths": { + "https://index.docker.io/v1/": { + "auth": "${auth_value}" + } + } + } + ``` + + The ${auth_value} is `base64("${username}:${password}")`. + + ```bash + echo "${username}:${password}" | base64 + ``` + +4. Download an image with Dragonfly. + + ```bash + docker pull {imageName} + ``` + + **Note:** Don't include the image repo URL in {imageName}, because the repo URL has been specified with the `registry` parameter when starting dfdaemon. + +## Downloading General Files + +1. Specify the supernodes in one of the following ways. + + - Specifying with the configuration file. + + ```sh + cat < /etc/dragonfly/dfget.yml + nodes: + - supernode01:port + - supernode02:port + - supernode03:port + EOD + ``` + + - Specifying with the parameter in the command line. + + ```sh + dfget -u "http://www.taobao.com" -o /tmp/test.html --node supernode01:port,supernode02:port,supernode03:port + ``` + + **Note:** When using this method, you must add the `node` parameter whenever you run the dfget command. And the parameter in the command line takes precedence over the configuration file. + +2. Download general files with Dragonfly in one of the following ways. + + - Download files with the default `/etc/dragonfly/dfget.yml` configuration. + + ```sh + dfget --url "http://xxx.xx.x" + ``` + + **Tip:** To list all available parameters for dfget, run `dfget -h`. + + - Download files with your specified supernodes. + + ```sh + dfget --url "http://xxx.xx.x" --node "127.0.0.1:8002" + ``` + + - Download files to your specified output file. + + ```sh + dfget --url "http://xxx.xx.x" -o a.txt + ``` + +## After this Task + +To review the downloading log, run `less ~/.small-dragonfly/logs/dfclient.log`. diff --git a/docs/en/user-guide/install_cdn.md b/docs/en/user-guide/install_cdn.md new file mode 100644 index 00000000000..4dc3ce54af0 --- /dev/null +++ b/docs/en/user-guide/install_cdn.md @@ -0,0 +1,167 @@ +# Installing Dragonfly CDN Server + +This topic explains how to install the Dragonfly cdn server. + +## Context + +Install CDN in one of the following ways: + +- Deploying with Docker. +- Deploying with physical machines: Recommended for production usage. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x +Nginx|0.8+ + +## Procedure - When Deploying with Docker + +### Get cdn image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image of the cdn. + + ```sh + docker pull d7yio/cdn + ``` + +Or you can build your own cdn image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-cdn D7Y_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the cdn. + + ```sh + docker image ls | grep 'cdn' | awk '{print $3}' | head -n1 + ``` + +### Start cdn + +**NOTE:** Replace ${cdnDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/cdn:/home/admin/cdn ${cdnDockerImageId} +--download-port=8001 +``` + +## Procedure - When Deploying with Physical Machines + +### Get cdn executable file + +1. Download a binary package of the cdn. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. + com/dragonflyoss/Dragonfly2/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `cdn` to your `PATH` environment variable to make sure you can directly use `cdn` command. + +Or you can build your own cdn executable file. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Compile the source code. + + ```sh + make build-cdn && make install-cdn + ``` + +### Start cdn + +```sh +cdnHomeDir=/home/admin/cdn +cdnDownloadPort=8001 +cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort +``` + +### Start file server + +You can start a file server in any way. However, the following conditions must be met: + +- It must be rooted at `${cdnHomeDir}/repo` which is defined in the previous step. +- It must listen on the port `cdnDownloadPort` which is defined in the previous step. + +Let's take nginx as an example. + +1. Add the following configuration items to the Nginx configuration file. + + ```conf + server { + # Must be ${cdnDownloadPort} + listen 8001; + location / { + # Must be ${cdnHomeDir}/repo + root /home/admin/cdn/repo; + } + } + ``` + +2. Start Nginx. + + ```sh + sudo nginx + ``` + +## After this Task + +- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. + + ```sh + telnet 127.0.0.1 8001 + telnet 127.0.0.1 8003 + ``` + +- [Install the Dragonfly client](install_client.md) and test if the downloading works. + + ```sh + dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" + ``` diff --git a/docs/en/user-guide/install_client.md b/docs/en/user-guide/install_client.md new file mode 100644 index 00000000000..08827f7b17c --- /dev/null +++ b/docs/en/user-guide/install_client.md @@ -0,0 +1,136 @@ +# Installing Dragonfly Client + +This topic explains how to install the Dragonfly `dfclient`. + +## Context + +Install the `dfclient` in one of the following ways: + +- Deploying with Docker. +- Deploying with physical machines. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x + +## Procedure - When Deploying with Docker + +### Get dfclient image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image ID of the SuperNode. + + ```sh + docker pull dragonflyoss/dfclient:1.0.0 + ``` + +Or you can build your own dfclient image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-client DF_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the `dfclient`. + + ```sh + docker image ls | grep 'dfclient' | awk '{print $3}' | head -n1 + ``` + +### Start the dfdaemon + +**NOTE:** You should prepare the [config files](../config) which should locate under `/etc/dragonfly` by default. + +```sh +version=1.0.0 +# Replace ${supernode} with your own supernode node with format `ip:port=weight`. +SUPERNODE=$supernode +docker run -d --name dfclient --restart=always -p 65001:65001 -v $HOME/.small-dragonfly:/root/.small-dragonfly -v /etc/dragonfly:/etc/dragonfly dragonflyoss/dfclient:$version --node $SUPERNODE +``` + +## Procedure - When Deploying with Physical Machines + +### Get dfclient executable file + +1. Download a binary package of the SuperNode. You can download one of the latest builds for Dragonfly on the [github releases page](https://github.com/dragonflyoss/Dragonfly/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly/releases/download/v$version/Dragonfly_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `dfget` and `dfdaemon` to your `PATH` environment variable to make sure you can directly use `dfget` and `dfdaemon` command. + +Or you can build your own dfclient executable files. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly + ``` + +3. Build `dfdaemon` and `dfget`. + + ```sh + make build-client && make install-client + ``` + +### Start the dfdaemon + +**NOTE:** You can ignore this step when using only dfget for file distribution . + +```sh +# Replace ${supernode} with your own supernode node with format `ip:port=weight`. +SUPERNODE=$supernode +dfdaemon --node $SUPERNODE +``` + +## After this Task + +Test if the downloading works. + +```sh +dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" +``` + +And test dfdaemon by [pulling an image](download_files.md). diff --git a/docs/en/user-guide/install_manager.md b/docs/en/user-guide/install_manager.md new file mode 100644 index 00000000000..4dc3ce54af0 --- /dev/null +++ b/docs/en/user-guide/install_manager.md @@ -0,0 +1,167 @@ +# Installing Dragonfly CDN Server + +This topic explains how to install the Dragonfly cdn server. + +## Context + +Install CDN in one of the following ways: + +- Deploying with Docker. +- Deploying with physical machines: Recommended for production usage. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x +Nginx|0.8+ + +## Procedure - When Deploying with Docker + +### Get cdn image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image of the cdn. + + ```sh + docker pull d7yio/cdn + ``` + +Or you can build your own cdn image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-cdn D7Y_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the cdn. + + ```sh + docker image ls | grep 'cdn' | awk '{print $3}' | head -n1 + ``` + +### Start cdn + +**NOTE:** Replace ${cdnDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/cdn:/home/admin/cdn ${cdnDockerImageId} +--download-port=8001 +``` + +## Procedure - When Deploying with Physical Machines + +### Get cdn executable file + +1. Download a binary package of the cdn. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. + com/dragonflyoss/Dragonfly2/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `cdn` to your `PATH` environment variable to make sure you can directly use `cdn` command. + +Or you can build your own cdn executable file. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Compile the source code. + + ```sh + make build-cdn && make install-cdn + ``` + +### Start cdn + +```sh +cdnHomeDir=/home/admin/cdn +cdnDownloadPort=8001 +cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort +``` + +### Start file server + +You can start a file server in any way. However, the following conditions must be met: + +- It must be rooted at `${cdnHomeDir}/repo` which is defined in the previous step. +- It must listen on the port `cdnDownloadPort` which is defined in the previous step. + +Let's take nginx as an example. + +1. Add the following configuration items to the Nginx configuration file. + + ```conf + server { + # Must be ${cdnDownloadPort} + listen 8001; + location / { + # Must be ${cdnHomeDir}/repo + root /home/admin/cdn/repo; + } + } + ``` + +2. Start Nginx. + + ```sh + sudo nginx + ``` + +## After this Task + +- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. + + ```sh + telnet 127.0.0.1 8001 + telnet 127.0.0.1 8003 + ``` + +- [Install the Dragonfly client](install_client.md) and test if the downloading works. + + ```sh + dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" + ``` diff --git a/docs/en/user-guide/install_scheduler.md b/docs/en/user-guide/install_scheduler.md new file mode 100644 index 00000000000..0216d9a9005 --- /dev/null +++ b/docs/en/user-guide/install_scheduler.md @@ -0,0 +1,137 @@ +# Installing Dragonfly Scheduler Server + +This topic explains how to install the Dragonfly scheduler server. + +## Context + +Install scheduler in one of the following ways: + +- Deploying with Docker. +- Deploying with physical machines. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x +Nginx|0.8+ + +## Procedure - When Deploying with Docker + +### Get scheduler image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image of the scheduler. + + ```sh + docker pull d7yio/scheduler + ``` + +Or you can build your own scheduler image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-scheduler D7Y_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the scheduler. + + ```sh + docker image ls | grep 'scheduler' | awk '{print $3}' | head -n1 + ``` + +### Start scheduler + +**NOTE:** Replace ${schedulerDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name scheduler --restart=always -p 8002 -v /home/admin/scheduler:/home/admin/scheduler ${schedulerDockerImageId} +--download-port=8001 +``` + +## Procedure - When Deploying with Physical Machines + +### Get scheduler executable file + +1. Download a binary package of the scheduler. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. + com/dragonflyoss/Dragonfly2/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `scheduler` to your `PATH` environment variable to make sure you can directly use `scheduler` command. + +Or you can build your own scheduler executable file. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Compile the source code. + + ```sh + make build-scheduler && make install-scheduler + ``` + +### Start scheduler + +```sh +schedulerHomeDir=/home/admin/scheduler +cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort +``` +## After this Task + +- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. + + ```sh + telnet 127.0.0.1 8001 + telnet 127.0.0.1 8003 + ``` + +- [Install the Dragonfly client](install_client.md) and test if the downloading works. + + ```sh + dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" + ``` diff --git a/docs/en/user-guide/multi_machines_deployment.md b/docs/en/user-guide/multi_machines_deployment.md new file mode 100644 index 00000000000..6c24ad43d3f --- /dev/null +++ b/docs/en/user-guide/multi_machines_deployment.md @@ -0,0 +1,118 @@ +# Dragonfly multi-machines deployment + +The multi-machines deployment documentation is designed to help you fully experience Dragonfly on multiple machines. + +If you are using Dragonfly in your production environment to handle production image distribution, please refer to supernode and dfget's detailed production parameter configuration. + +## Prerequisites + +Assuming that experiment requires us to prepare three host machines, one to play a role of supernode, and the other two for dfclient. Then the topology of the three nodes cluster is like the following: + +![quick start cluster topology](../images/quick-start-topo.png) + +Then, we must provide: + +1. three host nodes in a LAN, and we assume that 3 machine IPs are replaced by the following names. + + - **dfsupernode**: Dragonfly server + - **dfclient0**: Dragonfly client one + - **dfclient1**: Dragonfly client two + +2. every node has deployed docker daemon + +## Step 1: Deploy Dragonfly Server (SuperNode) + +Deploy the Dragonfly server (Supernode) on the machine `dfsupernode`. + +```bash +docker run -d --name supernode \ + --restart=always \ + -p 8001:8001 \ + -p 8002:8002 \ + -v /home/admin/supernode:/home/admin/supernode \ + dragonflyoss/supernode:1.0.2 --download-port=8001 +``` + +## Step 2: Deploy Dragonfly Client (dfclient) + +The following operations should be performed both on the client machine `dfclient0`, `dfclient1`. + +### Prepare the configuration file + +Dragonfly's configuration file is located in the `/etc/dragonfly` directory by default. When using the container to deploy the client, you need to mount the configuration file to the container. + +Configure the Dragonfly Supernode address for the client: + +```bash +cat < /etc/dragonfly/dfget.yml +nodes: + - dfsupernode +EOD +``` + +### Start Dragonfly Client + +```bash +docker run -d --name dfclient \ + --restart=always \ + -p 65001:65001 \ + -v /etc/dragonfly:/etc/dragonfly \ + -v $HOME/.small-dragonfly:/root/.small-dragonfly \ + dragonflyoss/dfclient:1.0.2 --registry https://index.docker.io +``` + +**NOTE**: The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the others. + +## Step 3. Configure Docker Daemon + +We need to modify the Docker Daemon configuration to use the Dragonfly as a pull through registry both on the client machine `dfclient0`, `dfclient1`. + +1. Add or update the configuration item `registry-mirrors` in the configuration file`/etc/docker/daemon.json`. + +```json +{ + "registry-mirrors": ["http://127.0.0.1:65001"] +} +``` + +**Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). + +2. Restart Docker Daemon. + +```bash +systemctl restart docker +``` + +## Step 4: Pull images with Dragonfly + +Through the above steps, we can start to validate if Dragonfly works as expected. + +And you can pull the image as usual on either `dfclient0` or `dfclient1`, for example: + +```bash +docker pull nginx:latest +``` + +## Step 5: Validate Dragonfly + +You can execute the following command to check if the nginx image is distributed via Dragonfly. + +```bash +docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log +``` + +If the output of command above has content like + +``` +2019-03-29 15:49:53.913 INFO sign:96027-1553845785.119 : downloading piece:{"taskID":"00a0503ea12457638ebbef5d0bfae51f9e8e0a0a349312c211f26f53beb93cdc","superNode":"127.0.0.1","dstCid":"127.0.0.1-95953-1553845720.488","range":"67108864-71303167","result":503,"status":701,"pieceSize":4194304,"pieceNum":16} +``` + +that means that the image download is done by Dragonfly. + +If you need to ensure that if the image is transferred through other peer nodes, you can execute the following command: + +```bash +docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log | grep -v cdnnode +``` + +If the above command does not output the result, the mirror does not complete the transmission through other peer nodes. Otherwise, the transmission is completed through other peer nodes. diff --git a/docs/en/user-guide/proxy/docker.md b/docs/en/user-guide/proxy/docker.md deleted file mode 100644 index 61a4de51aba..00000000000 --- a/docs/en/user-guide/proxy/docker.md +++ /dev/null @@ -1,160 +0,0 @@ -# Use dfget daemon as HTTP proxy for docker daemon - -Currently, docker doesn't support private registries with `registry-mirrors`, -in order to do so, we need to use HTTP proxy for docker daemon. - -## Quick Start - -### Step 1: Generate CA certificate for HTTP proxy - -Generate a CA certificate private key. - -```bash -openssl genrsa -out ca.key 2048 -``` - -Open openssl config file `openssl.conf`. Note set `basicConstraints` to true, that you can modify the values. - -```text -[ req ] -#default_bits = 2048 -#default_md = sha256 -#default_keyfile = privkey.pem -distinguished_name = req_distinguished_name -attributes = req_attributes -extensions = v3_ca -req_extensions = v3_ca - -[ req_distinguished_name ] -countryName = Country Name (2 letter code) -countryName_min = 2 -countryName_max = 2 -stateOrProvinceName = State or Province Name (full name) -localityName = Locality Name (eg, city) -0.organizationName = Organization Name (eg, company) -organizationalUnitName = Organizational Unit Name (eg, section) -commonName = Common Name (eg, fully qualified host name) -commonName_max = 64 -emailAddress = Email Address -emailAddress_max = 64 - -[ req_attributes ] -challengePassword = A challenge password -challengePassword_min = 4 -challengePassword_max = 20 - -[ v3_ca ] -basicConstraints = CA:TRUE -``` - -Generate the CA certificate. - -```bash -openssl req -new -key ca.key -nodes -out ca.csr -config openssl.conf -openssl x509 -req -days 36500 -extfile openssl.conf -extensions v3_ca -in ca.csr -signkey ca.key -out ca.crt -``` - -### Step 2: Configure dfget daemon - -To use dfget daemon as HTTP proxy, first you need to append a proxy rule in -`/var/log/dragonfly/dfget.yaml`, This will proxy `your.private.registry`'s requests for image layers: - -```yaml -proxy: - security: - insecure: true - tcp_listen: - listen: 0.0.0.0 - port: 65001 - proxies: - - regx: blobs/sha256.* - hijack_https: - # CA certificate's path used to hijack https requests - cert: ca.crt - key: ca.key - hosts: - - regx: your.private.registry -``` - -### Step 3: Configure Docker daemon - -Add your private registry to `insecure-registries` in -`/etc/docker/daemon.json`, in order to ignore the certificate error: - -```json -{ - "insecure-registries": ["your.private.registry"] -} -``` - -### Step 4: Configure Docker daemon - -Set dfdaemon as `HTTP_PROXY` and `HTTPS_PROXY` for docker daemon in -`/etc/systemd/system/docker.service.d/http-proxy.conf`: - -``` -[Service] -Environment="HTTP_PROXY=http://127.0.0.1:65001" -Environment="HTTPS_PROXY=http://127.0.0.1:65001" -``` - -### Step 5: Pull images with proxy - -Through the above steps, we can start to validate if Dragonfly works as expected. - -And you can pull the image as usual, for example: - -```bash -docker pull your.private.registry/namespace/image:latest -``` - -## Custom assets - -### Registry uses a self-signed certificate - -If your registry uses a self-signed certificate, you can either choose to -ignore the certificate error with: - -```yaml -proxy: - security: - insecure: true - tcp_listen: - listen: 0.0.0.0 - port: 65001 - proxies: - - regx: blobs/sha256.* - hijack_https: - # CA certificate's path used to hijack https requests - cert: ca.crt - key: ca.key - hosts: - - regx: your.private.registry - insecure: true -``` - -Or provide a certificate with: - -```yaml -proxy: - security: - insecure: true - tcp_listen: - listen: 0.0.0.0 - port: 65001 - proxies: - - regx: blobs/sha256.* - hijack_https: - # CA certificate's path used to hijack https requests - cert: ca.crt - key: ca.key - hosts: - - regx: your.private.registry - certs: ["server.crt"] -``` - -You can get the certificate of your server with: - -``` -openssl x509 -in <(openssl s_client -showcerts -servername xxx -connect xxx:443 -prexit 2>/dev/null) -``` diff --git a/docs/en/user-guide/quick-start.md b/docs/en/user-guide/quick-start.md index 307a05e32d9..b073b7d834a 100644 --- a/docs/en/user-guide/quick-start.md +++ b/docs/en/user-guide/quick-start.md @@ -11,7 +11,7 @@ This table describes some container runtimes version and documents. | --- | --- | --- | --- | --- | | Docker | All | [Link](./proxy/docker.md) | No | docker pull docker.io/library/alpine | | Containerd without CRI | All | [Link](./proxy/containerd.md) | No | ctr image pull docker.io/library/alpine | -| Containerd with CRI | v1.1.0+ | [Link](./registry-mirror/containerd.md) | Yes | crictl pull docker.io/library/alpine:latest | +| Containerd with CRI | v1.1.0+ | [Link](registry-mirror/cri-containerd.md) | Yes | crictl pull docker.io/library/alpine:latest | | CRI-O | All | [Link](./registry-mirror/cri-o.md) | Yes | crictl pull docker.io/library/alpine:latest | When using Dragonfly in Kubernetes, we recommend to use `Containerd with CRI` and `CRI-O`, deploying document can be diff --git a/docs/en/user-guide/registry-mirror/containerd.md b/docs/en/user-guide/registry-mirror/cri-containerd.md similarity index 100% rename from docs/en/user-guide/registry-mirror/containerd.md rename to docs/en/user-guide/registry-mirror/cri-containerd.md From 726a41c2d77963bfd8b90dedc3a3b4a560a3f2a9 Mon Sep 17 00:00:00 2001 From: santong Date: Tue, 18 May 2021 10:28:45 +0800 Subject: [PATCH 3/8] docs of dragonfly2.0 Signed-off-by: santong --- docs/en/user_guide/download_files.md | 131 ++++++++++++++ docs/en/user_guide/install_cdn.md | 167 ++++++++++++++++++ docs/en/user_guide/install_client.md | 136 ++++++++++++++ docs/en/user_guide/install_manager.md | 167 ++++++++++++++++++ docs/en/user_guide/install_scheduler.md | 137 ++++++++++++++ .../user_guide/multi_machines_deployment.md | 118 +++++++++++++ 6 files changed, 856 insertions(+) create mode 100644 docs/en/user_guide/download_files.md create mode 100644 docs/en/user_guide/install_cdn.md create mode 100644 docs/en/user_guide/install_client.md create mode 100644 docs/en/user_guide/install_manager.md create mode 100644 docs/en/user_guide/install_scheduler.md create mode 100644 docs/en/user_guide/multi_machines_deployment.md diff --git a/docs/en/user_guide/download_files.md b/docs/en/user_guide/download_files.md new file mode 100644 index 00000000000..e0c4c8b41e4 --- /dev/null +++ b/docs/en/user_guide/download_files.md @@ -0,0 +1,131 @@ +# Downloading Files with Dragonfly + +Things are done differently when you download container images and download general files with Dragonfly. + +## Prerequisites + +- You are using Linux operating system. +- The supernode service is started. + + **Tip:** For more information on the dfget command, see [dfget](../cli_reference/dfget.md). For more information on the installation of supernodes, see [Installing Server](./install_server.md). + +## Downloading container images + +1. Config the supernodes with the configuration file. + + ```shell + cat < /etc/dragonfly/dfget.yml + nodes: + - supernode01:port + - supernode02:port + - supernode03:port + EOD + ``` + +2. Start the dfget proxy (dfdaemon). + + ```sh + # Start dfdaemon and specify the image repo URL. The default port is `65001`. + dfdaemon --registry https://xxx.xx.x + # Review dfdaemon logs + tailf ~/.small-dragonfly/logs/dfdaemon.log + ``` + + **Tip:** To list all available parameters for dfdaemon, run `dfdaemon -h`. + +3. Configure the Docker daemon. + + a. Modify the configuration file `/etc/docker/daemon.json`. + + ```sh + vi /etc/docker/daemon.json + ``` + + **Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). + + b. Add or update the configuration item `registry-mirrors` in the configuration file. + + ```sh + "registry-mirrors": ["http://127.0.0.1:65001"] + ``` + + c. Restart Docker daemon. + + ```bash + systemctl restart docker + ``` + + d. Add authentication info for the private docker registry in `~/.docker/config.json` if the registry is configured with auth. + + ```json + { + "auths": { + "https://index.docker.io/v1/": { + "auth": "${auth_value}" + } + } + } + ``` + + The ${auth_value} is `base64("${username}:${password}")`. + + ```bash + echo "${username}:${password}" | base64 + ``` + +4. Download an image with Dragonfly. + + ```bash + docker pull {imageName} + ``` + + **Note:** Don't include the image repo URL in {imageName}, because the repo URL has been specified with the `registry` parameter when starting dfdaemon. + +## Downloading General Files + +1. Specify the supernodes in one of the following ways. + + - Specifying with the configuration file. + + ```sh + cat < /etc/dragonfly/dfget.yml + nodes: + - supernode01:port + - supernode02:port + - supernode03:port + EOD + ``` + + - Specifying with the parameter in the command line. + + ```sh + dfget -u "http://www.taobao.com" -o /tmp/test.html --node supernode01:port,supernode02:port,supernode03:port + ``` + + **Note:** When using this method, you must add the `node` parameter whenever you run the dfget command. And the parameter in the command line takes precedence over the configuration file. + +2. Download general files with Dragonfly in one of the following ways. + + - Download files with the default `/etc/dragonfly/dfget.yml` configuration. + + ```sh + dfget --url "http://xxx.xx.x" + ``` + + **Tip:** To list all available parameters for dfget, run `dfget -h`. + + - Download files with your specified supernodes. + + ```sh + dfget --url "http://xxx.xx.x" --node "127.0.0.1:8002" + ``` + + - Download files to your specified output file. + + ```sh + dfget --url "http://xxx.xx.x" -o a.txt + ``` + +## After this Task + +To review the downloading log, run `less ~/.small-dragonfly/logs/dfclient.log`. diff --git a/docs/en/user_guide/install_cdn.md b/docs/en/user_guide/install_cdn.md new file mode 100644 index 00000000000..db4cb1a0c5a --- /dev/null +++ b/docs/en/user_guide/install_cdn.md @@ -0,0 +1,167 @@ +# Installing Dragonfly CDN Server + +This topic explains how to install the Dragonfly cdn server. + +## Context + +Install CDN in one of the following ways: + +- Deploying with Docker. +- Deploying with physical machines: Recommended for production usage. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x +Nginx|0.8+ + +## Procedure - When Deploying with Docker + +### Get cdn image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image of the cdn. + + ```sh + docker pull d7yio/cdn + ``` + +Or you can build your own cdn image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-cdn D7Y_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the cdn. + + ```sh + docker image ls | grep 'cdn' | awk '{print $3}' | head -n1 + ``` + +### Start cdn + +**NOTE:** Replace ${cdnDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/cdn:/home/admin/cdn ${cdnDockerImageId} +--download-port=8001 +``` + +## Procedure - When Deploying with Physical Machines + +### Get cdn executable file + +1. Download a binary package of the cdn. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. + com/dragonflyoss/Dragonfly2/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `cdn` to your `PATH` environment variable to make sure you can directly use `cdn` command. + +Or you can build your own cdn executable file. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Compile the source code. + + ```sh + make build-cdn && make install-cdn + ``` + +### Start cdn + +```sh +cdnHomeDir=/home/admin/cdn +cdnDownloadPort=8001 +cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort +``` + +### Start file server + +You can start a file server in any way. However, the following conditions must be met: + +- It must be rooted at `${cdnHomeDir}/repo` which is defined in the previous step. +- It must listen on the port `cdnDownloadPort` which is defined in the previous step. + +Let's take nginx as an example. + +1. Add the following configuration items to the Nginx configuration file. + + ```conf + server { + # Must be ${cdnDownloadPort} + listen 8001; + location / { + # Must be ${cdnHomeDir}/repo + root /home/admin/cdn/repo; + } + } + ``` + +2. Start Nginx. + + ```sh + sudo nginx + ``` + +## After this Task + +- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. + + ```sh + telnet 127.0.0.1 8001 + telnet 127.0.0.1 8003 + ``` + +- [Install the Dragonfly client](./install_client.md) and test if the downloading works. + + ```sh + dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" + ``` diff --git a/docs/en/user_guide/install_client.md b/docs/en/user_guide/install_client.md new file mode 100644 index 00000000000..387434c7137 --- /dev/null +++ b/docs/en/user_guide/install_client.md @@ -0,0 +1,136 @@ +# Installing Dragonfly Client + +This topic explains how to install the Dragonfly `dfclient`. + +## Context + +Install the `dfclient` in one of the following ways: + +- Deploying with Docker. +- Deploying with physical machines. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x + +## Procedure - When Deploying with Docker + +### Get dfclient image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image ID of the SuperNode. + + ```sh + docker pull dragonflyoss/dfclient:1.0.0 + ``` + +Or you can build your own dfclient image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-client DF_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the `dfclient`. + + ```sh + docker image ls | grep 'dfclient' | awk '{print $3}' | head -n1 + ``` + +### Start the dfdaemon + +**NOTE:** You should prepare the [config files](../config) which should locate under `/etc/dragonfly` by default. + +```sh +version=1.0.0 +# Replace ${supernode} with your own supernode node with format `ip:port=weight`. +SUPERNODE=$supernode +docker run -d --name dfclient --restart=always -p 65001:65001 -v $HOME/.small-dragonfly:/root/.small-dragonfly -v /etc/dragonfly:/etc/dragonfly dragonflyoss/dfclient:$version --node $SUPERNODE +``` + +## Procedure - When Deploying with Physical Machines + +### Get dfclient executable file + +1. Download a binary package of the SuperNode. You can download one of the latest builds for Dragonfly on the [github releases page](https://github.com/dragonflyoss/Dragonfly/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly/releases/download/v$version/Dragonfly_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `dfget` and `dfdaemon` to your `PATH` environment variable to make sure you can directly use `dfget` and `dfdaemon` command. + +Or you can build your own dfclient executable files. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly + ``` + +3. Build `dfdaemon` and `dfget`. + + ```sh + make build-client && make install-client + ``` + +### Start the dfdaemon + +**NOTE:** You can ignore this step when using only dfget for file distribution . + +```sh +# Replace ${supernode} with your own supernode node with format `ip:port=weight`. +SUPERNODE=$supernode +dfdaemon --node $SUPERNODE +``` + +## After this Task + +Test if the downloading works. + +```sh +dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" +``` + +And test dfdaemon by [pulling an image](./download_files.md). diff --git a/docs/en/user_guide/install_manager.md b/docs/en/user_guide/install_manager.md new file mode 100644 index 00000000000..db4cb1a0c5a --- /dev/null +++ b/docs/en/user_guide/install_manager.md @@ -0,0 +1,167 @@ +# Installing Dragonfly CDN Server + +This topic explains how to install the Dragonfly cdn server. + +## Context + +Install CDN in one of the following ways: + +- Deploying with Docker. +- Deploying with physical machines: Recommended for production usage. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x +Nginx|0.8+ + +## Procedure - When Deploying with Docker + +### Get cdn image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image of the cdn. + + ```sh + docker pull d7yio/cdn + ``` + +Or you can build your own cdn image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-cdn D7Y_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the cdn. + + ```sh + docker image ls | grep 'cdn' | awk '{print $3}' | head -n1 + ``` + +### Start cdn + +**NOTE:** Replace ${cdnDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/cdn:/home/admin/cdn ${cdnDockerImageId} +--download-port=8001 +``` + +## Procedure - When Deploying with Physical Machines + +### Get cdn executable file + +1. Download a binary package of the cdn. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. + com/dragonflyoss/Dragonfly2/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `cdn` to your `PATH` environment variable to make sure you can directly use `cdn` command. + +Or you can build your own cdn executable file. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Compile the source code. + + ```sh + make build-cdn && make install-cdn + ``` + +### Start cdn + +```sh +cdnHomeDir=/home/admin/cdn +cdnDownloadPort=8001 +cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort +``` + +### Start file server + +You can start a file server in any way. However, the following conditions must be met: + +- It must be rooted at `${cdnHomeDir}/repo` which is defined in the previous step. +- It must listen on the port `cdnDownloadPort` which is defined in the previous step. + +Let's take nginx as an example. + +1. Add the following configuration items to the Nginx configuration file. + + ```conf + server { + # Must be ${cdnDownloadPort} + listen 8001; + location / { + # Must be ${cdnHomeDir}/repo + root /home/admin/cdn/repo; + } + } + ``` + +2. Start Nginx. + + ```sh + sudo nginx + ``` + +## After this Task + +- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. + + ```sh + telnet 127.0.0.1 8001 + telnet 127.0.0.1 8003 + ``` + +- [Install the Dragonfly client](./install_client.md) and test if the downloading works. + + ```sh + dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" + ``` diff --git a/docs/en/user_guide/install_scheduler.md b/docs/en/user_guide/install_scheduler.md new file mode 100644 index 00000000000..cac3edacbd7 --- /dev/null +++ b/docs/en/user_guide/install_scheduler.md @@ -0,0 +1,137 @@ +# Installing Dragonfly Scheduler Server + +This topic explains how to install the Dragonfly scheduler server. + +## Context + +Install scheduler in one of the following ways: + +- Deploying with Docker. +- Deploying with physical machines. + +## Prerequisites + +When deploying with Docker, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Docker|1.12.0+ + +When deploying with physical machines, the following conditions must be met. + +Required Software | Version Limit +---|--- +Git|1.9.1+ +Golang|1.12.x +Nginx|0.8+ + +## Procedure - When Deploying with Docker + +### Get scheduler image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image of the scheduler. + + ```sh + docker pull d7yio/scheduler + ``` + +Or you can build your own scheduler image. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Build the Docker image. + + ```sh + TAG="1.0.0" + make docker-build-scheduler D7Y_VERSION=$TAG + ``` + +4. Obtain the latest Docker image ID of the scheduler. + + ```sh + docker image ls | grep 'scheduler' | awk '{print $3}' | head -n1 + ``` + +### Start scheduler + +**NOTE:** Replace ${schedulerDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name scheduler --restart=always -p 8002 -v /home/admin/scheduler:/home/admin/scheduler ${schedulerDockerImageId} +--download-port=8001 +``` + +## Procedure - When Deploying with Physical Machines + +### Get scheduler executable file + +1. Download a binary package of the scheduler. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. + com/dragonflyoss/Dragonfly2/releases). + + ```sh + version=1.0.0 + wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz + ``` + +2. Unzip the package. + + ```bash + # Replace `xxx` with the installation directory. + tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx + ``` + +3. Move the `scheduler` to your `PATH` environment variable to make sure you can directly use `scheduler` command. + +Or you can build your own scheduler executable file. + +1. Obtain the source code of Dragonfly. + + ```sh + git clone https://github.com/dragonflyoss/Dragonfly2.git + ``` + +2. Enter the project directory. + + ```sh + cd Dragonfly2 + ``` + +3. Compile the source code. + + ```sh + make build-scheduler && make install-scheduler + ``` + +### Start scheduler + +```sh +schedulerHomeDir=/home/admin/scheduler +cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort +``` +## After this Task + +- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. + + ```sh + telnet 127.0.0.1 8001 + telnet 127.0.0.1 8003 + ``` + +- [Install the Dragonfly client](./install_client.md) and test if the downloading works. + + ```sh + dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" + ``` diff --git a/docs/en/user_guide/multi_machines_deployment.md b/docs/en/user_guide/multi_machines_deployment.md new file mode 100644 index 00000000000..6c24ad43d3f --- /dev/null +++ b/docs/en/user_guide/multi_machines_deployment.md @@ -0,0 +1,118 @@ +# Dragonfly multi-machines deployment + +The multi-machines deployment documentation is designed to help you fully experience Dragonfly on multiple machines. + +If you are using Dragonfly in your production environment to handle production image distribution, please refer to supernode and dfget's detailed production parameter configuration. + +## Prerequisites + +Assuming that experiment requires us to prepare three host machines, one to play a role of supernode, and the other two for dfclient. Then the topology of the three nodes cluster is like the following: + +![quick start cluster topology](../images/quick-start-topo.png) + +Then, we must provide: + +1. three host nodes in a LAN, and we assume that 3 machine IPs are replaced by the following names. + + - **dfsupernode**: Dragonfly server + - **dfclient0**: Dragonfly client one + - **dfclient1**: Dragonfly client two + +2. every node has deployed docker daemon + +## Step 1: Deploy Dragonfly Server (SuperNode) + +Deploy the Dragonfly server (Supernode) on the machine `dfsupernode`. + +```bash +docker run -d --name supernode \ + --restart=always \ + -p 8001:8001 \ + -p 8002:8002 \ + -v /home/admin/supernode:/home/admin/supernode \ + dragonflyoss/supernode:1.0.2 --download-port=8001 +``` + +## Step 2: Deploy Dragonfly Client (dfclient) + +The following operations should be performed both on the client machine `dfclient0`, `dfclient1`. + +### Prepare the configuration file + +Dragonfly's configuration file is located in the `/etc/dragonfly` directory by default. When using the container to deploy the client, you need to mount the configuration file to the container. + +Configure the Dragonfly Supernode address for the client: + +```bash +cat < /etc/dragonfly/dfget.yml +nodes: + - dfsupernode +EOD +``` + +### Start Dragonfly Client + +```bash +docker run -d --name dfclient \ + --restart=always \ + -p 65001:65001 \ + -v /etc/dragonfly:/etc/dragonfly \ + -v $HOME/.small-dragonfly:/root/.small-dragonfly \ + dragonflyoss/dfclient:1.0.2 --registry https://index.docker.io +``` + +**NOTE**: The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the others. + +## Step 3. Configure Docker Daemon + +We need to modify the Docker Daemon configuration to use the Dragonfly as a pull through registry both on the client machine `dfclient0`, `dfclient1`. + +1. Add or update the configuration item `registry-mirrors` in the configuration file`/etc/docker/daemon.json`. + +```json +{ + "registry-mirrors": ["http://127.0.0.1:65001"] +} +``` + +**Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). + +2. Restart Docker Daemon. + +```bash +systemctl restart docker +``` + +## Step 4: Pull images with Dragonfly + +Through the above steps, we can start to validate if Dragonfly works as expected. + +And you can pull the image as usual on either `dfclient0` or `dfclient1`, for example: + +```bash +docker pull nginx:latest +``` + +## Step 5: Validate Dragonfly + +You can execute the following command to check if the nginx image is distributed via Dragonfly. + +```bash +docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log +``` + +If the output of command above has content like + +``` +2019-03-29 15:49:53.913 INFO sign:96027-1553845785.119 : downloading piece:{"taskID":"00a0503ea12457638ebbef5d0bfae51f9e8e0a0a349312c211f26f53beb93cdc","superNode":"127.0.0.1","dstCid":"127.0.0.1-95953-1553845720.488","range":"67108864-71303167","result":503,"status":701,"pieceSize":4194304,"pieceNum":16} +``` + +that means that the image download is done by Dragonfly. + +If you need to ensure that if the image is transferred through other peer nodes, you can execute the following command: + +```bash +docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log | grep -v cdnnode +``` + +If the above command does not output the result, the mirror does not complete the transmission through other peer nodes. Otherwise, the transmission is completed through other peer nodes. From e74a5bae4897bd2d85efedc239234bb12694ed76 Mon Sep 17 00:00:00 2001 From: santong Date: Wed, 30 Jun 2021 15:15:36 +0800 Subject: [PATCH 4/8] docs:modify user guide Signed-off-by: santong --- docs/en/README.md | 5 +- docs/en/cli-reference/cdn.md | 2 +- docs/en/cli-reference/dfget.1 | 0 docs/en/config/cdn-nginx.conf | 2 +- .../en/{quick_start => quick-start}/README.md | 12 +- .../{download_files.md => download-files.md} | 0 .../{install_cdn.md => install-cdn.md} | 2 +- .../{install_client.md => install-client.md} | 2 +- ...{install_manager.md => install-manager.md} | 2 +- ...tall_scheduler.md => install-scheduler.md} | 2 +- ...oyment.md => multi-machines-deployment.md} | 0 docs/en/user_guide/download_files.md | 131 -------------- docs/en/user_guide/install_cdn.md | 167 ------------------ docs/en/user_guide/install_client.md | 136 -------------- docs/en/user_guide/install_manager.md | 167 ------------------ docs/en/user_guide/install_scheduler.md | 137 -------------- .../user_guide/multi_machines_deployment.md | 118 ------------- 17 files changed, 15 insertions(+), 870 deletions(-) delete mode 100644 docs/en/cli-reference/dfget.1 rename docs/en/{quick_start => quick-start}/README.md (91%) rename docs/en/user-guide/{download_files.md => download-files.md} (100%) rename docs/en/user-guide/{install_cdn.md => install-cdn.md} (98%) rename docs/en/user-guide/{install_client.md => install-client.md} (98%) rename docs/en/user-guide/{install_manager.md => install-manager.md} (98%) rename docs/en/user-guide/{install_scheduler.md => install-scheduler.md} (97%) rename docs/en/user-guide/{multi_machines_deployment.md => multi-machines-deployment.md} (100%) delete mode 100644 docs/en/user_guide/download_files.md delete mode 100644 docs/en/user_guide/install_cdn.md delete mode 100644 docs/en/user_guide/install_client.md delete mode 100644 docs/en/user_guide/install_manager.md delete mode 100644 docs/en/user_guide/install_scheduler.md delete mode 100644 docs/en/user_guide/multi_machines_deployment.md diff --git a/docs/en/README.md b/docs/en/README.md index 85b0d09f95f..d1f70ba923f 100644 --- a/docs/en/README.md +++ b/docs/en/README.md @@ -10,7 +10,7 @@ Organization of document is as following: * [WIP dfget](cli-reference/dfget.md) * [WIP cdnsystem](cli-reference/cdn.md) * [WIP scheduler](cli-reference/scheduler.md) - * [TODO manager](./cli-reference/manager.md) + * [TODO manager](cli-reference/manager.md) * [TODO API Reference](#API-Reference) * [Ecosystem](#Ecosystem) * [Kubernetes Integration](ecosystem/Kubernetes-with-Dragonfly.md) @@ -29,7 +29,8 @@ Find `WIP` or `TODO` in this page and follow [CONTRIBUTING](../../CONTRIBUTING.m ## [WIP] User Guide -[User Guide]() helps all kinds of guidance end users need to experience Dragonfly. Not only the very brief [Quick Start](./quick-start), but the detailed binary installation and configure illustration. In addition, any concept and function which help users understand Dragonfly better would be included as well. +[User Guide]() helps all kinds of guidance end users need to experience Dragonfly. Not only the very brief [Quick Start](./quick-start), but the detailed +binary installation and configure illustration. In addition, any concept and function which help users understand Dragonfly better would be included as well. ## [WIP] CLI Reference diff --git a/docs/en/cli-reference/cdn.md b/docs/en/cli-reference/cdn.md index 01f905a5086..98894c008b8 100644 --- a/docs/en/cli-reference/cdn.md +++ b/docs/en/cli-reference/cdn.md @@ -25,7 +25,7 @@ go run cmd/cdnsystem/main.go --profiler --gc-initial-delay duration gc initial delay is the delay time from the start to the first GC execution (default 6s) --gc-meta-interval duration gc meta interval is the interval time to execute the GC meta (default 2m0s) -h, --help help for cdn - --home-dir string homeDir is the working directory of cdnNode (default "/Users/su*__*nweipeng1/cdn-system") + --home-dir string homeDir is the working directory of cdnNode (default "/Users/${HOMEDIR}/ftp/") --max-bandwidth rate network rate that cdnNode can use (default 200MB) --port int listenPort is the port that cdn server listens on (default 8003) --profiler profiler sets whether cdnNode HTTP server setups profiler diff --git a/docs/en/cli-reference/dfget.1 b/docs/en/cli-reference/dfget.1 deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/en/config/cdn-nginx.conf b/docs/en/config/cdn-nginx.conf index 4465a219625..d97657ca0f6 100644 --- a/docs/en/config/cdn-nginx.conf +++ b/docs/en/config/cdn-nginx.conf @@ -45,7 +45,7 @@ http { server { listen 8001; location / { - root /home/admin/cdn-system/repo; + root /home/admin/ftp; } } } diff --git a/docs/en/quick_start/README.md b/docs/en/quick-start/README.md similarity index 91% rename from docs/en/quick_start/README.md rename to docs/en/quick-start/README.md index 1fc52874407..c1b7c3d881a 100644 --- a/docs/en/quick_start/README.md +++ b/docs/en/quick-start/README.md @@ -105,13 +105,13 @@ then Dragonfly works successfully. ## SEE ALSO -- [multi machines deployment](../user-guide/multi_machines_deployment.md) - experience Dragonfly on multiple machines -- [install manager](../user-guide/install_manager.md) - how to install the Dragonfly manager -- [install cdn](../user-guide/install_cdn.md) - how to install the Dragonfly cdn -- [install scheduler](../user-guide/install_scheduler.md) - how to install the Dragonfly scheduler -- [install client](../user-guide/install_client.md) - how to install the Dragonfly client +- [multi machines deployment](../user-guide/multi-machines-deployment.md) - experience Dragonfly on multiple machines +- [install manager](../user-guide/install-manager.md) - how to install the Dragonfly manager +- [install cdn](../user-guide/install-cdn.md) - how to install the Dragonfly cdn +- [install scheduler](../user-guide/install-scheduler.md) - how to install the Dragonfly scheduler +- [install client](../user-guide/install-client.md) - how to install the Dragonfly client - [proxy](../user_guide/proxy/docker.md) - make Dragonfly as an HTTP proxy for docker daemon -- [download files](../user-guide/download_files.md) - download files with Dragonfly +- [download files](../user-guide/download-files.md) - download files with Dragonfly - Container Runtimes - [cri-o mirror](../user_guide/registry/cri-o.md) - make Dragonfly as Registry Mirror for CRIO daemon - [cri-containerd_mirror](../user_guide/registry/cri-containerd.md) - make Dragonfly as Registry Mirror for containerd daemon diff --git a/docs/en/user-guide/download_files.md b/docs/en/user-guide/download-files.md similarity index 100% rename from docs/en/user-guide/download_files.md rename to docs/en/user-guide/download-files.md diff --git a/docs/en/user-guide/install_cdn.md b/docs/en/user-guide/install-cdn.md similarity index 98% rename from docs/en/user-guide/install_cdn.md rename to docs/en/user-guide/install-cdn.md index 4dc3ce54af0..4960e1da22d 100644 --- a/docs/en/user-guide/install_cdn.md +++ b/docs/en/user-guide/install-cdn.md @@ -160,7 +160,7 @@ Let's take nginx as an example. telnet 127.0.0.1 8003 ``` -- [Install the Dragonfly client](install_client.md) and test if the downloading works. +- [Install the Dragonfly client](install-client.md) and test if the downloading works. ```sh dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" diff --git a/docs/en/user-guide/install_client.md b/docs/en/user-guide/install-client.md similarity index 98% rename from docs/en/user-guide/install_client.md rename to docs/en/user-guide/install-client.md index 08827f7b17c..f3dd59934e0 100644 --- a/docs/en/user-guide/install_client.md +++ b/docs/en/user-guide/install-client.md @@ -133,4 +133,4 @@ Test if the downloading works. dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" ``` -And test dfdaemon by [pulling an image](download_files.md). +And test dfdaemon by [pulling an image](download-files.md). diff --git a/docs/en/user-guide/install_manager.md b/docs/en/user-guide/install-manager.md similarity index 98% rename from docs/en/user-guide/install_manager.md rename to docs/en/user-guide/install-manager.md index 4dc3ce54af0..4960e1da22d 100644 --- a/docs/en/user-guide/install_manager.md +++ b/docs/en/user-guide/install-manager.md @@ -160,7 +160,7 @@ Let's take nginx as an example. telnet 127.0.0.1 8003 ``` -- [Install the Dragonfly client](install_client.md) and test if the downloading works. +- [Install the Dragonfly client](install-client.md) and test if the downloading works. ```sh dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" diff --git a/docs/en/user-guide/install_scheduler.md b/docs/en/user-guide/install-scheduler.md similarity index 97% rename from docs/en/user-guide/install_scheduler.md rename to docs/en/user-guide/install-scheduler.md index 0216d9a9005..d9e5c7adb34 100644 --- a/docs/en/user-guide/install_scheduler.md +++ b/docs/en/user-guide/install-scheduler.md @@ -130,7 +130,7 @@ cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort telnet 127.0.0.1 8003 ``` -- [Install the Dragonfly client](install_client.md) and test if the downloading works. +- [Install the Dragonfly client](install-client.md) and test if the downloading works. ```sh dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" diff --git a/docs/en/user-guide/multi_machines_deployment.md b/docs/en/user-guide/multi-machines-deployment.md similarity index 100% rename from docs/en/user-guide/multi_machines_deployment.md rename to docs/en/user-guide/multi-machines-deployment.md diff --git a/docs/en/user_guide/download_files.md b/docs/en/user_guide/download_files.md deleted file mode 100644 index e0c4c8b41e4..00000000000 --- a/docs/en/user_guide/download_files.md +++ /dev/null @@ -1,131 +0,0 @@ -# Downloading Files with Dragonfly - -Things are done differently when you download container images and download general files with Dragonfly. - -## Prerequisites - -- You are using Linux operating system. -- The supernode service is started. - - **Tip:** For more information on the dfget command, see [dfget](../cli_reference/dfget.md). For more information on the installation of supernodes, see [Installing Server](./install_server.md). - -## Downloading container images - -1. Config the supernodes with the configuration file. - - ```shell - cat < /etc/dragonfly/dfget.yml - nodes: - - supernode01:port - - supernode02:port - - supernode03:port - EOD - ``` - -2. Start the dfget proxy (dfdaemon). - - ```sh - # Start dfdaemon and specify the image repo URL. The default port is `65001`. - dfdaemon --registry https://xxx.xx.x - # Review dfdaemon logs - tailf ~/.small-dragonfly/logs/dfdaemon.log - ``` - - **Tip:** To list all available parameters for dfdaemon, run `dfdaemon -h`. - -3. Configure the Docker daemon. - - a. Modify the configuration file `/etc/docker/daemon.json`. - - ```sh - vi /etc/docker/daemon.json - ``` - - **Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). - - b. Add or update the configuration item `registry-mirrors` in the configuration file. - - ```sh - "registry-mirrors": ["http://127.0.0.1:65001"] - ``` - - c. Restart Docker daemon. - - ```bash - systemctl restart docker - ``` - - d. Add authentication info for the private docker registry in `~/.docker/config.json` if the registry is configured with auth. - - ```json - { - "auths": { - "https://index.docker.io/v1/": { - "auth": "${auth_value}" - } - } - } - ``` - - The ${auth_value} is `base64("${username}:${password}")`. - - ```bash - echo "${username}:${password}" | base64 - ``` - -4. Download an image with Dragonfly. - - ```bash - docker pull {imageName} - ``` - - **Note:** Don't include the image repo URL in {imageName}, because the repo URL has been specified with the `registry` parameter when starting dfdaemon. - -## Downloading General Files - -1. Specify the supernodes in one of the following ways. - - - Specifying with the configuration file. - - ```sh - cat < /etc/dragonfly/dfget.yml - nodes: - - supernode01:port - - supernode02:port - - supernode03:port - EOD - ``` - - - Specifying with the parameter in the command line. - - ```sh - dfget -u "http://www.taobao.com" -o /tmp/test.html --node supernode01:port,supernode02:port,supernode03:port - ``` - - **Note:** When using this method, you must add the `node` parameter whenever you run the dfget command. And the parameter in the command line takes precedence over the configuration file. - -2. Download general files with Dragonfly in one of the following ways. - - - Download files with the default `/etc/dragonfly/dfget.yml` configuration. - - ```sh - dfget --url "http://xxx.xx.x" - ``` - - **Tip:** To list all available parameters for dfget, run `dfget -h`. - - - Download files with your specified supernodes. - - ```sh - dfget --url "http://xxx.xx.x" --node "127.0.0.1:8002" - ``` - - - Download files to your specified output file. - - ```sh - dfget --url "http://xxx.xx.x" -o a.txt - ``` - -## After this Task - -To review the downloading log, run `less ~/.small-dragonfly/logs/dfclient.log`. diff --git a/docs/en/user_guide/install_cdn.md b/docs/en/user_guide/install_cdn.md deleted file mode 100644 index db4cb1a0c5a..00000000000 --- a/docs/en/user_guide/install_cdn.md +++ /dev/null @@ -1,167 +0,0 @@ -# Installing Dragonfly CDN Server - -This topic explains how to install the Dragonfly cdn server. - -## Context - -Install CDN in one of the following ways: - -- Deploying with Docker. -- Deploying with physical machines: Recommended for production usage. - -## Prerequisites - -When deploying with Docker, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Docker|1.12.0+ - -When deploying with physical machines, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Golang|1.12.x -Nginx|0.8+ - -## Procedure - When Deploying with Docker - -### Get cdn image - -You can get it from [DockerHub](https://hub.docker.com/) directly. - -1. Obtain the latest Docker image of the cdn. - - ```sh - docker pull d7yio/cdn - ``` - -Or you can build your own cdn image. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly2.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly2 - ``` - -3. Build the Docker image. - - ```sh - TAG="1.0.0" - make docker-build-cdn D7Y_VERSION=$TAG - ``` - -4. Obtain the latest Docker image ID of the cdn. - - ```sh - docker image ls | grep 'cdn' | awk '{print $3}' | head -n1 - ``` - -### Start cdn - -**NOTE:** Replace ${cdnDockerImageId} with the ID obtained at the previous step. - -```sh -docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/cdn:/home/admin/cdn ${cdnDockerImageId} ---download-port=8001 -``` - -## Procedure - When Deploying with Physical Machines - -### Get cdn executable file - -1. Download a binary package of the cdn. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. - com/dragonflyoss/Dragonfly2/releases). - - ```sh - version=1.0.0 - wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz - ``` - -2. Unzip the package. - - ```bash - # Replace `xxx` with the installation directory. - tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx - ``` - -3. Move the `cdn` to your `PATH` environment variable to make sure you can directly use `cdn` command. - -Or you can build your own cdn executable file. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly2.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly2 - ``` - -3. Compile the source code. - - ```sh - make build-cdn && make install-cdn - ``` - -### Start cdn - -```sh -cdnHomeDir=/home/admin/cdn -cdnDownloadPort=8001 -cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort -``` - -### Start file server - -You can start a file server in any way. However, the following conditions must be met: - -- It must be rooted at `${cdnHomeDir}/repo` which is defined in the previous step. -- It must listen on the port `cdnDownloadPort` which is defined in the previous step. - -Let's take nginx as an example. - -1. Add the following configuration items to the Nginx configuration file. - - ```conf - server { - # Must be ${cdnDownloadPort} - listen 8001; - location / { - # Must be ${cdnHomeDir}/repo - root /home/admin/cdn/repo; - } - } - ``` - -2. Start Nginx. - - ```sh - sudo nginx - ``` - -## After this Task - -- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. - - ```sh - telnet 127.0.0.1 8001 - telnet 127.0.0.1 8003 - ``` - -- [Install the Dragonfly client](./install_client.md) and test if the downloading works. - - ```sh - dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" - ``` diff --git a/docs/en/user_guide/install_client.md b/docs/en/user_guide/install_client.md deleted file mode 100644 index 387434c7137..00000000000 --- a/docs/en/user_guide/install_client.md +++ /dev/null @@ -1,136 +0,0 @@ -# Installing Dragonfly Client - -This topic explains how to install the Dragonfly `dfclient`. - -## Context - -Install the `dfclient` in one of the following ways: - -- Deploying with Docker. -- Deploying with physical machines. - -## Prerequisites - -When deploying with Docker, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Docker|1.12.0+ - -When deploying with physical machines, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Golang|1.12.x - -## Procedure - When Deploying with Docker - -### Get dfclient image - -You can get it from [DockerHub](https://hub.docker.com/) directly. - -1. Obtain the latest Docker image ID of the SuperNode. - - ```sh - docker pull dragonflyoss/dfclient:1.0.0 - ``` - -Or you can build your own dfclient image. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly - ``` - -3. Build the Docker image. - - ```sh - TAG="1.0.0" - make docker-build-client DF_VERSION=$TAG - ``` - -4. Obtain the latest Docker image ID of the `dfclient`. - - ```sh - docker image ls | grep 'dfclient' | awk '{print $3}' | head -n1 - ``` - -### Start the dfdaemon - -**NOTE:** You should prepare the [config files](../config) which should locate under `/etc/dragonfly` by default. - -```sh -version=1.0.0 -# Replace ${supernode} with your own supernode node with format `ip:port=weight`. -SUPERNODE=$supernode -docker run -d --name dfclient --restart=always -p 65001:65001 -v $HOME/.small-dragonfly:/root/.small-dragonfly -v /etc/dragonfly:/etc/dragonfly dragonflyoss/dfclient:$version --node $SUPERNODE -``` - -## Procedure - When Deploying with Physical Machines - -### Get dfclient executable file - -1. Download a binary package of the SuperNode. You can download one of the latest builds for Dragonfly on the [github releases page](https://github.com/dragonflyoss/Dragonfly/releases). - - ```sh - version=1.0.0 - wget https://github.com/dragonflyoss/Dragonfly/releases/download/v$version/Dragonfly_$version_linux_amd64.tar.gz - ``` - -2. Unzip the package. - - ```bash - # Replace `xxx` with the installation directory. - tar -zxf Dragonfly_1.0.0_linux_amd64.tar.gz -C xxx - ``` - -3. Move the `dfget` and `dfdaemon` to your `PATH` environment variable to make sure you can directly use `dfget` and `dfdaemon` command. - -Or you can build your own dfclient executable files. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly - ``` - -3. Build `dfdaemon` and `dfget`. - - ```sh - make build-client && make install-client - ``` - -### Start the dfdaemon - -**NOTE:** You can ignore this step when using only dfget for file distribution . - -```sh -# Replace ${supernode} with your own supernode node with format `ip:port=weight`. -SUPERNODE=$supernode -dfdaemon --node $SUPERNODE -``` - -## After this Task - -Test if the downloading works. - -```sh -dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" -``` - -And test dfdaemon by [pulling an image](./download_files.md). diff --git a/docs/en/user_guide/install_manager.md b/docs/en/user_guide/install_manager.md deleted file mode 100644 index db4cb1a0c5a..00000000000 --- a/docs/en/user_guide/install_manager.md +++ /dev/null @@ -1,167 +0,0 @@ -# Installing Dragonfly CDN Server - -This topic explains how to install the Dragonfly cdn server. - -## Context - -Install CDN in one of the following ways: - -- Deploying with Docker. -- Deploying with physical machines: Recommended for production usage. - -## Prerequisites - -When deploying with Docker, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Docker|1.12.0+ - -When deploying with physical machines, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Golang|1.12.x -Nginx|0.8+ - -## Procedure - When Deploying with Docker - -### Get cdn image - -You can get it from [DockerHub](https://hub.docker.com/) directly. - -1. Obtain the latest Docker image of the cdn. - - ```sh - docker pull d7yio/cdn - ``` - -Or you can build your own cdn image. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly2.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly2 - ``` - -3. Build the Docker image. - - ```sh - TAG="1.0.0" - make docker-build-cdn D7Y_VERSION=$TAG - ``` - -4. Obtain the latest Docker image ID of the cdn. - - ```sh - docker image ls | grep 'cdn' | awk '{print $3}' | head -n1 - ``` - -### Start cdn - -**NOTE:** Replace ${cdnDockerImageId} with the ID obtained at the previous step. - -```sh -docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/cdn:/home/admin/cdn ${cdnDockerImageId} ---download-port=8001 -``` - -## Procedure - When Deploying with Physical Machines - -### Get cdn executable file - -1. Download a binary package of the cdn. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. - com/dragonflyoss/Dragonfly2/releases). - - ```sh - version=1.0.0 - wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz - ``` - -2. Unzip the package. - - ```bash - # Replace `xxx` with the installation directory. - tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx - ``` - -3. Move the `cdn` to your `PATH` environment variable to make sure you can directly use `cdn` command. - -Or you can build your own cdn executable file. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly2.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly2 - ``` - -3. Compile the source code. - - ```sh - make build-cdn && make install-cdn - ``` - -### Start cdn - -```sh -cdnHomeDir=/home/admin/cdn -cdnDownloadPort=8001 -cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort -``` - -### Start file server - -You can start a file server in any way. However, the following conditions must be met: - -- It must be rooted at `${cdnHomeDir}/repo` which is defined in the previous step. -- It must listen on the port `cdnDownloadPort` which is defined in the previous step. - -Let's take nginx as an example. - -1. Add the following configuration items to the Nginx configuration file. - - ```conf - server { - # Must be ${cdnDownloadPort} - listen 8001; - location / { - # Must be ${cdnHomeDir}/repo - root /home/admin/cdn/repo; - } - } - ``` - -2. Start Nginx. - - ```sh - sudo nginx - ``` - -## After this Task - -- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. - - ```sh - telnet 127.0.0.1 8001 - telnet 127.0.0.1 8003 - ``` - -- [Install the Dragonfly client](./install_client.md) and test if the downloading works. - - ```sh - dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" - ``` diff --git a/docs/en/user_guide/install_scheduler.md b/docs/en/user_guide/install_scheduler.md deleted file mode 100644 index cac3edacbd7..00000000000 --- a/docs/en/user_guide/install_scheduler.md +++ /dev/null @@ -1,137 +0,0 @@ -# Installing Dragonfly Scheduler Server - -This topic explains how to install the Dragonfly scheduler server. - -## Context - -Install scheduler in one of the following ways: - -- Deploying with Docker. -- Deploying with physical machines. - -## Prerequisites - -When deploying with Docker, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Docker|1.12.0+ - -When deploying with physical machines, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Golang|1.12.x -Nginx|0.8+ - -## Procedure - When Deploying with Docker - -### Get scheduler image - -You can get it from [DockerHub](https://hub.docker.com/) directly. - -1. Obtain the latest Docker image of the scheduler. - - ```sh - docker pull d7yio/scheduler - ``` - -Or you can build your own scheduler image. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly2.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly2 - ``` - -3. Build the Docker image. - - ```sh - TAG="1.0.0" - make docker-build-scheduler D7Y_VERSION=$TAG - ``` - -4. Obtain the latest Docker image ID of the scheduler. - - ```sh - docker image ls | grep 'scheduler' | awk '{print $3}' | head -n1 - ``` - -### Start scheduler - -**NOTE:** Replace ${schedulerDockerImageId} with the ID obtained at the previous step. - -```sh -docker run -d --name scheduler --restart=always -p 8002 -v /home/admin/scheduler:/home/admin/scheduler ${schedulerDockerImageId} ---download-port=8001 -``` - -## Procedure - When Deploying with Physical Machines - -### Get scheduler executable file - -1. Download a binary package of the scheduler. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. - com/dragonflyoss/Dragonfly2/releases). - - ```sh - version=1.0.0 - wget https://github.com/dragonflyoss/Dragonfly2/releases/download/v$version/Dragonfly2_$version_linux_amd64.tar.gz - ``` - -2. Unzip the package. - - ```bash - # Replace `xxx` with the installation directory. - tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx - ``` - -3. Move the `scheduler` to your `PATH` environment variable to make sure you can directly use `scheduler` command. - -Or you can build your own scheduler executable file. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly2.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly2 - ``` - -3. Compile the source code. - - ```sh - make build-scheduler && make install-scheduler - ``` - -### Start scheduler - -```sh -schedulerHomeDir=/home/admin/scheduler -cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort -``` -## After this Task - -- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. - - ```sh - telnet 127.0.0.1 8001 - telnet 127.0.0.1 8003 - ``` - -- [Install the Dragonfly client](./install_client.md) and test if the downloading works. - - ```sh - dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" - ``` diff --git a/docs/en/user_guide/multi_machines_deployment.md b/docs/en/user_guide/multi_machines_deployment.md deleted file mode 100644 index 6c24ad43d3f..00000000000 --- a/docs/en/user_guide/multi_machines_deployment.md +++ /dev/null @@ -1,118 +0,0 @@ -# Dragonfly multi-machines deployment - -The multi-machines deployment documentation is designed to help you fully experience Dragonfly on multiple machines. - -If you are using Dragonfly in your production environment to handle production image distribution, please refer to supernode and dfget's detailed production parameter configuration. - -## Prerequisites - -Assuming that experiment requires us to prepare three host machines, one to play a role of supernode, and the other two for dfclient. Then the topology of the three nodes cluster is like the following: - -![quick start cluster topology](../images/quick-start-topo.png) - -Then, we must provide: - -1. three host nodes in a LAN, and we assume that 3 machine IPs are replaced by the following names. - - - **dfsupernode**: Dragonfly server - - **dfclient0**: Dragonfly client one - - **dfclient1**: Dragonfly client two - -2. every node has deployed docker daemon - -## Step 1: Deploy Dragonfly Server (SuperNode) - -Deploy the Dragonfly server (Supernode) on the machine `dfsupernode`. - -```bash -docker run -d --name supernode \ - --restart=always \ - -p 8001:8001 \ - -p 8002:8002 \ - -v /home/admin/supernode:/home/admin/supernode \ - dragonflyoss/supernode:1.0.2 --download-port=8001 -``` - -## Step 2: Deploy Dragonfly Client (dfclient) - -The following operations should be performed both on the client machine `dfclient0`, `dfclient1`. - -### Prepare the configuration file - -Dragonfly's configuration file is located in the `/etc/dragonfly` directory by default. When using the container to deploy the client, you need to mount the configuration file to the container. - -Configure the Dragonfly Supernode address for the client: - -```bash -cat < /etc/dragonfly/dfget.yml -nodes: - - dfsupernode -EOD -``` - -### Start Dragonfly Client - -```bash -docker run -d --name dfclient \ - --restart=always \ - -p 65001:65001 \ - -v /etc/dragonfly:/etc/dragonfly \ - -v $HOME/.small-dragonfly:/root/.small-dragonfly \ - dragonflyoss/dfclient:1.0.2 --registry https://index.docker.io -``` - -**NOTE**: The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the others. - -## Step 3. Configure Docker Daemon - -We need to modify the Docker Daemon configuration to use the Dragonfly as a pull through registry both on the client machine `dfclient0`, `dfclient1`. - -1. Add or update the configuration item `registry-mirrors` in the configuration file`/etc/docker/daemon.json`. - -```json -{ - "registry-mirrors": ["http://127.0.0.1:65001"] -} -``` - -**Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). - -2. Restart Docker Daemon. - -```bash -systemctl restart docker -``` - -## Step 4: Pull images with Dragonfly - -Through the above steps, we can start to validate if Dragonfly works as expected. - -And you can pull the image as usual on either `dfclient0` or `dfclient1`, for example: - -```bash -docker pull nginx:latest -``` - -## Step 5: Validate Dragonfly - -You can execute the following command to check if the nginx image is distributed via Dragonfly. - -```bash -docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log -``` - -If the output of command above has content like - -``` -2019-03-29 15:49:53.913 INFO sign:96027-1553845785.119 : downloading piece:{"taskID":"00a0503ea12457638ebbef5d0bfae51f9e8e0a0a349312c211f26f53beb93cdc","superNode":"127.0.0.1","dstCid":"127.0.0.1-95953-1553845720.488","range":"67108864-71303167","result":503,"status":701,"pieceSize":4194304,"pieceNum":16} -``` - -that means that the image download is done by Dragonfly. - -If you need to ensure that if the image is transferred through other peer nodes, you can execute the following command: - -```bash -docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log | grep -v cdnnode -``` - -If the above command does not output the result, the mirror does not complete the transmission through other peer nodes. Otherwise, the transmission is completed through other peer nodes. From f47e255627c290bdf9b7dd092567813eb5e00c9b Mon Sep 17 00:00:00 2001 From: santong Date: Wed, 30 Jun 2021 15:29:29 +0800 Subject: [PATCH 5/8] docs:modify user guide Signed-off-by: santong --- docs/en/cli-reference/cdn.md | 23 +++----- docs/en/cli-reference/scheduler.md | 59 ++++++------------- docs/zh-CN/cli-reference/cdn.md | 53 ++++++++++++++++++ docs/zh-CN/cli-reference/scheduler.md | 51 +++++++++++++++++ docs/zh-CN/config/cdn.yaml | 81 +++++++++++++++++++++++++++ 5 files changed, 209 insertions(+), 58 deletions(-) create mode 100644 docs/zh-CN/cli-reference/cdn.md create mode 100644 docs/zh-CN/cli-reference/scheduler.md create mode 100644 docs/zh-CN/config/cdn.yaml diff --git a/docs/en/cli-reference/cdn.md b/docs/en/cli-reference/cdn.md index 98894c008b8..6e2a9d8b514 100644 --- a/docs/en/cli-reference/cdn.md +++ b/docs/en/cli-reference/cdn.md @@ -16,20 +16,11 @@ go run cmd/cdnsystem/main.go --profiler ### Options ``` - --advertise-ip string the cdn node ip is the ip we advertise to other peers in the p2p-network - --clean-ratio int CleanRatio is the ratio to clean the disk and it is based on 10. the value of CleanRatio should be [1-10] (default 1) - --config string the path of cdn configuration file (default "/etc/dragonfly/cdn.yaml") - --download-port int downloadPort is the port for download files from cdnNode (default 8001) - --fail-access-interval duration fail access interval is the interval time after failed to access the URL (default 3m0s) - --gc-disk-interval duration gc disk interval is the interval time to execute GC disk. (default 15s) - --gc-initial-delay duration gc initial delay is the delay time from the start to the first GC execution (default 6s) - --gc-meta-interval duration gc meta interval is the interval time to execute the GC meta (default 2m0s) - -h, --help help for cdn - --home-dir string homeDir is the working directory of cdnNode (default "/Users/${HOMEDIR}/ftp/") - --max-bandwidth rate network rate that cdnNode can use (default 200MB) - --port int listenPort is the port that cdn server listens on (default 8003) - --profiler profiler sets whether cdnNode HTTP server setups profiler - --system-bandwidth rate network rate reserved for system (default 20MB) - --task-expire-time duration task expire time is the time that a task is treated expired if the task is not accessed within the time (default 3m0s) - --young-gc-threshold file-size gc disk interval is the interval time to execute GC disk. (default 100GB) + --config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/cdn.yaml, it can also be + set by env var:CDN_CONFIG，The settings and uses of each configuration item can refer to cdn.yaml in config directory + --console whether logger output records to the stdout + -h, --help help for cdn + --jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces + --pprof-port int listen port for pprof, 0 represents random port (default -1) + --verbose whether logger use debug level ``` diff --git a/docs/en/cli-reference/scheduler.md b/docs/en/cli-reference/scheduler.md index 61ff35216d4..885cad11bb5 100644 --- a/docs/en/cli-reference/scheduler.md +++ b/docs/en/cli-reference/scheduler.md @@ -1,51 +1,26 @@ -## 调度器 +## Scheduler -调度器生成并维护下载过程中的P2P网络 - -### 说明 - -调度器是一个常驻后台运行的进程，用于接收和管理客户端的下载任务，通知CDN进行回源， 在下载过程中生成维护P2P网络，给客户端推送适合的下载节点 - -### 用法 +Scheduler is a long-running process which receives and manages download tasks from the client, notify the CDN to return to the source, +generate and maintain a P2P network during the download process, and push suitable download nodes to the client +## Try it ``` -scheduler [flags] +go run cmd/scheduler/main.go [Option] ``` +## Log configuration +set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal -### 可选参数 - +## Runtime metrics monitoring ``` - --config string the path of scheduler's configuration file (default "conf/scheduler.yaml") - -h, --help help for scheduler - --port int port is the port that scheduler server listens on (default 8002) - --sender-job-pool-size int sender-job-pool-size is used for scheduler and do not change it (default 10000) - --sender-num int sender-num is used for scheduler and do not change it (default 50) - --worker-job-pool-size int worker-job-pool-size is used for scheduler and do not change it (default 10000) - --worker-num int worker-num is used for scheduler and do not change it (default 12) +go run cmd/scheduler/main.go --profiler ``` - -### 使用示例 - -scheduler --config your-config-path/scheduler.yaml - -### 配置文件说明 +### Options ``` -server: - port: 8001 rpc 端口 - -scheduler: - -worker: - worker-num: 5 工作线程数 - worker-job-pool-size: 10000 工作队列长度 - sender-num: 10 发送消息线程数 - sender-job-pool-size: 10000 发送消息队列长度 - -cdn: - list: CDN列表 - - - - cdn-name : "cdn" CDN服务器的HostName - ip: "127.0.0.1" CDN服务器的IP地址 - rpcPort: 8003 CDN的RPC端口 - download-port: 8002 CDN的下载端口 + --config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/scheduler.yaml, it can + also be set by env var:SCHEDULER_CONFIG，The settings and uses of each configuration item can refer to scheduler.yaml in config directory + --console whether logger output records to the stdout + -h, --help help for cdn + --jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces + --pprof-port int listen port for pprof, 0 represents random port (default -1) + --verbose whether logger use debug level ``` diff --git a/docs/zh-CN/cli-reference/cdn.md b/docs/zh-CN/cli-reference/cdn.md new file mode 100644 index 00000000000..6eca49855a5 --- /dev/null +++ b/docs/zh-CN/cli-reference/cdn.md @@ -0,0 +1,53 @@ +## CDN + +CDN is a long-running process which caches downloaded data from source to avoid downloading the same files from source repeatedly + +## 用法 +``` +go run cmd/cdnsystem/main.go [Option] +``` +## Log configuration +set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal + +## Runtime metrics monitoring +``` +go run cmd/cdnsystem/main.go --profiler +``` +### 可选参数 + +``` + --config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/cdn.yaml, it can also be + set by env var:CDN_CONFIG，The settings and uses of each configuration item can refer to cdn.yaml in config directory + --console whether logger output records to the stdout + -h, --help help for cdn + --jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces + --pprof-port int listen port for pprof, 0 represents random port (default -1) + --verbose whether logger use debug level +``` + +### 二进制命令使用示例 + +scheduler --config your-config-path/scheduler.yaml + +### 配置文件说明 + +``` +server: + port: 8001 rpc 端口 + +scheduler: + +worker: + worker-num: 5 工作线程数 + worker-job-pool-size: 10000 工作队列长度 + sender-num: 10 发送消息线程数 + sender-job-pool-size: 10000 发送消息队列长度 + +cdn: + list: CDN列表 + - + - cdn-name : "cdn" CDN服务器的HostName + ip: "127.0.0.1" CDN服务器的IP地址 + rpcPort: 8003 CDN的RPC端口 + download-port: 8002 CDN的下载端口 +``` diff --git a/docs/zh-CN/cli-reference/scheduler.md b/docs/zh-CN/cli-reference/scheduler.md new file mode 100644 index 00000000000..9b647c47da8 --- /dev/null +++ b/docs/zh-CN/cli-reference/scheduler.md @@ -0,0 +1,51 @@ +## Scheduler + +Scheduler 生成并维护下载过程中的P2P网络 + +### 说明 + +Scheduler 是一个常驻后台运行的进程，用于接收和管理客户端的下载任务，通知CDN进行回源， 在下载过程中生成维护P2P网络，给客户端推送适合的下载节点 + +### 用法 +``` +scheduler [flags] +``` + +### 可选参数 + +``` + --config string the path of scheduler's configuration file (default "conf/scheduler.yaml") + -h, --help help for scheduler + --port int port is the port that scheduler server listens on (default 8002) + --sender-job-pool-size int sender-job-pool-size is used for scheduler and do not change it (default 10000) + --sender-num int sender-num is used for scheduler and do not change it (default 50) + --worker-job-pool-size int worker-job-pool-size is used for scheduler and do not change it (default 10000) + --worker-num int worker-num is used for scheduler and do not change it (default 12) +``` + +### 二进制命令使用示例 + +scheduler --config your-config-path/scheduler.yaml + +### 配置文件说明 + +``` +server: + port: 8001 rpc 端口 + +scheduler: + +worker: + worker-num: 5 工作线程数 + worker-job-pool-size: 10000 工作队列长度 + sender-num: 10 发送消息线程数 + sender-job-pool-size: 10000 发送消息队列长度 + +cdn: + list: CDN列表 + - + - cdn-name : "cdn" CDN服务器的HostName + ip: "127.0.0.1" CDN服务器的IP地址 + rpcPort: 8003 CDN的RPC端口 + download-port: 8002 CDN的下载端口 +``` diff --git a/docs/zh-CN/config/cdn.yaml b/docs/zh-CN/config/cdn.yaml new file mode 100644 index 00000000000..06ef9f517d1 --- /dev/null +++ b/docs/zh-CN/config/cdn.yaml @@ -0,0 +1,81 @@ +# 这个文件是 CDN 系统的配置模版 +# 你可以通过修改这里的配置项来自定义你的 CDN 系统 +--- +base: # 基础配置项 + # listenPort cdn 服务的监听端口. + # default: 8003 + listenPort: 8003 + + # DownloadPort CDN提供文件下载的端口. + # 你需要先启动一个文件服务器，并且该文件服务器监听该下载端口。. + # default: 8001 + downloadPort: 8001 + + # SystemReservedBandwidth is the network bandwidth reserved for system software. + # default: 20 MB, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte. + systemReservedBandwidth: 20M + + # MaxBandwidth is the network bandwidth that cdn can use. + # default: 1G, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte. + maxBandwidth: 1G + + # FailAccessInterval is the interval time after failed to access the URL. + # If a task failed to be downloaded from the source, it will not be retried in the time since the last failure. + # default: 3m + failAccessInterval: 3m + + # GCInitialDelay is the delay time from the start to the first GC execution. + # default: 6s + gcInitialDelay: 6s + + # GCMetaInterval is the interval time to execute GC meta. + # default: 2m0s + gcMetaInterval: 2m + + # gcStorageInterval is the interval time to execute GC storage. + # default: 15s + gcStorageInterval: 15s + + # TaskExpireTime when a task is not accessed within the taskExpireTime, + # and it will be treated to be expired. + # default: 3m0s + taskExpireTime: 3m + + # StoragePattern is the pattern of storage policy, [disk/hybrid] + storagePattern: disk + +plugins: # 插件配置 + storage: # 存储插件配置 + - name: disk + enable: true + config: + baseDir: /tmp/cdnsystem2 + gcConfig: + youngGCThreshold: 100G + fullGCThreshold: 5G + cleanRatio: 1 + intervalThreshold: 2h + - name: memory + enable: true + config: + baseDir: /tmp/memory/dragonfly + gcConfig: + youngGCThreshold: 100G + fullGCThreshold: 5G + cleanRatio: 3 + intervalThreshold: 2h + +# Console shows log on console +# default: false +console: false + +# Whether to enable debug level logger and enable pprof +# default: false +verbose: false + +# listen port for pprof, only valid when the verbose option is true +# default is random port +pprofPort: 0 + +# the service address that provides the configuration item +configServer: 127.0.0.1:8004 \ No newline at end of file From cfcb58f2a66a4d2986549675eadc9bb36ce3178a Mon Sep 17 00:00:00 2001 From: santong Date: Wed, 30 Jun 2021 16:53:31 +0800 Subject: [PATCH 6/8] docs: add readme Signed-off-by: santong --- FAQ.md | 1 + README.md | 27 ++-- ROADMAP.md | 1 + api/README.md | 4 +- docs/en/README.md | 25 ++-- docs/en/api-reference/api.md | 0 docs/en/cli-reference/manager.md | 26 ++++ docs/en/quick-start/README.md | 117 ----------------- docs/en/user-guide/README.md | 0 docs/en/user-guide/install/README.md | 4 + .../user-guide/{ => install}/install-cdn.md | 0 .../{ => install}/install-client.md | 4 +- .../{ => install}/install-manager.md | 0 .../{ => install}/install-scheduler.md | 0 docs/en/user-guide/quick-start.md | 121 +++++++++++++++++- 15 files changed, 190 insertions(+), 140 deletions(-) create mode 100644 FAQ.md create mode 100644 ROADMAP.md create mode 100644 docs/en/api-reference/api.md create mode 100644 docs/en/cli-reference/manager.md delete mode 100644 docs/en/quick-start/README.md create mode 100644 docs/en/user-guide/README.md create mode 100644 docs/en/user-guide/install/README.md rename docs/en/user-guide/{ => install}/install-cdn.md (100%) rename docs/en/user-guide/{ => install}/install-client.md (94%) rename docs/en/user-guide/{ => install}/install-manager.md (100%) rename docs/en/user-guide/{ => install}/install-scheduler.md (100%) diff --git a/FAQ.md b/FAQ.md new file mode 100644 index 00000000000..32914fe4ef7 --- /dev/null +++ b/FAQ.md @@ -0,0 +1 @@ +FAQ \ No newline at end of file diff --git a/README.md b/README.md index 7a09c3655e9..3738fcecf15 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,17 @@ Provide efficient, stable, secure, low-cost file and image distribution services to be the best practice and standard solution in the related Cloud-Native area. +## Introduction + +Dragonfly is an open source intelligent P2P based image and file distribution system. Its goal is to tackle all distribution problems in cloud native scenarios. Currently Dragonfly focuses on being: + +- Simple: well-defined user-facing API (HTTP), non-invasive to all container engines; +- Efficient: CDN support, P2P based file distribution to save enterprise bandwidth; +- Intelligent: host level speed limit, intelligent flow control due to host detection; +- Secure: block transmission encryption, HTTPS connection support. + +Dragonfly is now hosted by the Cloud Native Computing Foundation (CNCF) as an Incubating Level Project. Originally it was born to solve all kinds of distribution at very large scales, such as application distribution, cache distribution, log distribution, image distribution, and so on. + ## Features - Implement P2P files distribution with various storage types (HDFS, storage services from various cloud vendors, Maven, Yum, etc.) through a unified back-to-source adapter layer. @@ -45,9 +56,8 @@ Provide efficient, stable, secure, low-cost file and image distribution services ## Getting Started -- [Introduction][introduction] -- [Installation][installation] - [Quick start][quickstart] +- [Installation][installation] ## Documentation You can find the Dragonfly documentation [on the website][website]. @@ -64,12 +74,11 @@ You should check out our [CONTRIBUTING][contributing] and develop the project to ## Code of Conduct Please refer to our [Code of Conduct][codeconduct]. -[contributing]: CONTRIBUTING.md -[codeconduct]: CODE_OF_CONDUCT.md -[introduction]: https://github.com/dragonflyoss/Dragonfly2 -[installation]: https://github.com/dragonflyoss/Dragonfly2 -[quickstart]: https://github.com/dragonflyoss/Dragonfly2 +[arch]: docs/en/images/arch.png +[logo-linear]: docs/en/images/logo/dragonfly-linear.svg +[quickstart]: docs/en/user-guide/quick-start.md +[installation]: docs/en/user-guide/install/README.md [website]: https://d7y.io [discussion]: https://github.com/dragonflyoss/Dragonfly2/discussions -[logo-linear]: docs/en/images/logo/dragonfly-linear.svg -[arch]: docs/en/images/arch.png +[contributing]: CONTRIBUTING.md +[codeconduct]: CODE_OF_CONDUCT.md diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 00000000000..594457daf0b --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1 @@ +ROADMAP \ No newline at end of file diff --git a/api/README.md b/api/README.md index 2bbcdc260d8..b59dab13c19 100644 --- a/api/README.md +++ b/api/README.md @@ -3,6 +3,8 @@ We encourage users to experience Dragonfly in different ways. When doing this, there are three ways users could choose to interact with dragonfly mostly: * For end-users, command line tool `dfget`, `dfdaemon` is mostly used. -* For developers, Dragonfly manager's raw API is the original thing they would make use of. For more details about API docs, please refer to [api.md](../docs/api_reference/api.md). We should also keep it in mind that doc [apis.md](../docs/api_reference/api.md) is automatically generated by [swagger2markup](https://github.com/Swagger2Markup/swagger2markup). Please **DO NOT** edit [api.md](../docs/api_reference/api.md) directly. +* For developers, Dragonfly manager's raw API is the original thing they would make use of. For more details about API docs, please refer to + [api.md](/docs/en/api-reference/api.md). We should also keep it in mind that doc [apis.md](../docs/en/api-reference/api.md) is automatically generated by + [swagger2markup](https://github.com/Swagger2Markup/swagger2markup). Please **DO NOT** edit [api.md](../docs/en/api-reference/api.md) directly. Directory `/api` mainly describes the second part **Dragonfly Manager's Raw API**. diff --git a/docs/en/README.md b/docs/en/README.md index d1f70ba923f..e6828bfbb75 100644 --- a/docs/en/README.md +++ b/docs/en/README.md @@ -25,14 +25,17 @@ Find `WIP` or `TODO` in this page and follow [CONTRIBUTING](../../CONTRIBUTING.m ## Quick Start -[Quick Started](user-guide/quick-start.md) is exactly what you need if you would give Dragonfly a try. This document includes what are the prerequisites, how to install Dragonfly and how to experience Dragonfly's usage. +[Quick Started](user-guide/quick-start.md) is exactly what you need if you would give Dragonfly a try. This document includes what are the +prerequisites, +how to install Dragonfly and how to experience Dragonfly's usage. -## [WIP] User Guide +## User Guide -[User Guide]() helps all kinds of guidance end users need to experience Dragonfly. Not only the very brief [Quick Start](./quick-start), but the detailed -binary installation and configure illustration. In addition, any concept and function which help users understand Dragonfly better would be included as well. +[User Guide](user-guide/README.md) helps all kinds of guidance end users need to experience Dragonfly. Not only the very brief [Quick Start] +(user-guide/quick-start.md), but the detailed binary installation and configure illustration. In addition, any concept and function which help users +understand Dragonfly better would be included as well. -## [WIP] CLI Reference +## CLI Reference For almost all users, commandline is the first reference you may need. Document in directory [CLI Reference](cli-reference) is about command detailed usage of Dragonfly CLI including `dfget`, `cdnsystem`, `scheduler` and `manager`. You can get introductions, synopsis, examples, options about command. Last but not least, Dragonfly can guarantee commandline docs is strongly consistent with Dragonfly CLI's source code. What's more, all commandline docs are auto generated via source code. @@ -40,13 +43,15 @@ For almost all users, commandline is the first reference you may need. Document Commandline is the easiest way to experience Dragonfly's ability. API extension will bring more further experience of Dragonfly. Commandline is just one kind of combination usage of API, if you wish to hack or take more advantages of Dragonfly, please see [API Reference](./api-reference). Like command line document, all API docs are auto generated via source code. -## [WIP] Ecosystem +## Ecosystem -Ecosystem documents show connections between Dragonfly and popular tool or system in cloud native ecosystem. They guide end users how to experience cloud native systems with Dragonfly, such as other CNCF project Kubernetes and Harbor. +Ecosystem documents show connections between Dragonfly and popular tool or system in cloud native ecosystem. They guide end users how to experience cloud +native systems with Dragonfly, such as other CNCF project [Kubernetes](ecosystem/Kubernetes-with-Dragonfly.md) and [Harbor](ecosystem/Harbor-with-Dragonfly.md). -## [WIP] Developer Guide +## Developer Guide -Develop Guide helps (potential) developers/contributors to understand the theory inside Dragonfly rather than the interface it exposes. With better understanding of how Dragonfly is designed, developer could learn source code of Dragonfly much easier and know how to debug, test and hack. +[Develop Guide](development/local.md) helps (potential) developers/contributors to understand the theory inside Dragonfly rather than the interface it exposes. With +better understanding of how Dragonfly is designed, developer could learn source code of Dragonfly much easier and know how to debug, test and hack. ### [TODO] Design Doc @@ -67,7 +72,7 @@ For more details, please refer to [test](./test-guide). The folder `/docs` does not contain all the document about Dragonfly. There are still other really helpful documents in other path of this repo, like: -* [TODO FAQ.md](../FAQ.md) +* [TODO FAQ.md](../../FAQ.md) * [CHANGELOG.md](../../CHANGELOG.md) * [TODO ROADMAP.md](../../ROADMAP.md) * others. diff --git a/docs/en/api-reference/api.md b/docs/en/api-reference/api.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/en/cli-reference/manager.md b/docs/en/cli-reference/manager.md new file mode 100644 index 00000000000..b6edfda6dd2 --- /dev/null +++ b/docs/en/cli-reference/manager.md @@ -0,0 +1,26 @@ +## Manager + +Scheduler is a long-running process which receives and manages download tasks from the client, notify the CDN to return to the source, +generate and maintain a P2P network during the download process, and push suitable download nodes to the client +## Try it +``` +go run cmd/scheduler/main.go [Option] +``` +## Log configuration +set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal + +## Runtime metrics monitoring +``` +go run cmd/scheduler/main.go --profiler +``` +### Options + +``` + --config string the path of configuration file with yaml extension name, default is /Users/${USER_HOME}/.dragonfly/config/scheduler.yaml, it can + also be set by env var:SCHEDULER_CONFIG，The settings and uses of each configuration item can refer to scheduler.yaml in config directory + --console whether logger output records to the stdout + -h, --help help for cdn + --jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces + --pprof-port int listen port for pprof, 0 represents random port (default -1) + --verbose whether logger use debug level +``` diff --git a/docs/en/quick-start/README.md b/docs/en/quick-start/README.md deleted file mode 100644 index c1b7c3d881a..00000000000 --- a/docs/en/quick-start/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# Dragonfly Quick Start - -Dragonfly Quick Start document aims to help you to quick start Dragonfly journey. This experiment is quite easy and simplified. - -If you are using Dragonfly in your **production environment** to handle production image distribution, please refer to supernode and dfget's detailed production parameter configuration. - -## Prerequisites - -All steps in this document is doing on the same machine using the docker container, so make sure the docker container engine installed and started on your machine. You can also refer to the documentation: [multi-machine deployment](../user_guide/multi_machines_deployment.md) to experience Dragonfly. - -## Step 1: Deploy Dragonfly Manager Server - -```bash -docker run -d --name supernode \ - --restart=always \ - -p 8001:8001 \ - -p 8002:8002 \ - -v /home/admin/supernode:/home/admin/supernode \ - dragonflyoss/supernode:1.0.2 -``` - -## Step 2: Deploy Dragonfly CDN Server - -```bash -docker run -d --name supernode \ - --restart=always \ - -p 8001:8001 \ - -p 8002:8002 \ - -v /home/admin/supernode:/home/admin/supernode \ - dragonflyoss/supernode:1.0.2 -``` - -## Step 3: Deploy Dragonfly Scheduler Server - -```bash -docker run -d --name supernode \ - --restart=always \ - -p 8001:8001 \ - -p 8002:8002 \ - -v /home/admin/supernode:/home/admin/supernode \ - dragonflyoss/supernode:1.0.2 -``` - -## Step 2: Deploy Dragonfly Client - -```bash -SUPERNODE_IP=`docker inspect supernode -f '{{.NetworkSettings.Networks.bridge.IPAddress}}'` -docker run -d --name dfclient \ - --restart=always \ - -p 65001:65001 \ - -v $HOME/.small-dragonfly:/root/.small-dragonfly \ - dragonflyoss/dfclient:1.0.2 --registry https://index.docker.io --node $SUPERNODE_IP -``` - -**NOTE**: - -- The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the other **non-https image registries**. -- The `--node` parameter specifies the supernode's address in the format of **HOST:IP**. And the default value `8002` will be used if the port is not specified. Here we use `docker inspect` to get the ip of supernode container as the host value. Since the supernode container exposes its ports, you can specify this parameter to node ip address as well. - -## Step 3. Configure Docker Daemon - -We need to modify the Docker Daemon configuration to use the Dragonfly as a pull through registry. - -1. Add or update the configuration item `registry-mirrors` in the configuration file`/etc/docker/daemon.json`. - -```json -{ - "registry-mirrors": ["http://127.0.0.1:65001"] -} -``` - -**Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). - -2. Restart Docker Daemon. - -```bash -systemctl restart docker -``` - -## Step 4: Pull images with Dragonfly - -Through the above steps, we can start to validate if Dragonfly works as expected. - -And you can pull the image as usual, for example: - -```bash -docker pull nginx:latest -``` - -## Step 5: Validate Dragonfly - -You can execute the following command to check if the nginx image is distributed via Dragonfly. - -```bash -docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log -``` - -If the output of command above has content like - -``` -2019-03-29 15:49:53.913 INFO sign:96027-1553845785.119 : downloading piece:{"taskID":"00a0503ea12457638ebbef5d0bfae51f9e8e0a0a349312c211f26f53beb93cdc","superNode":"127.0.0.1","dstCid":"127.0.0.1-95953-1553845720.488","range":"67108864-71303167","result":503,"status":701,"pieceSize":4194304,"pieceNum":16} -``` - -then Dragonfly works successfully. - -## SEE ALSO - -- [multi machines deployment](../user-guide/multi-machines-deployment.md) - experience Dragonfly on multiple machines -- [install manager](../user-guide/install-manager.md) - how to install the Dragonfly manager -- [install cdn](../user-guide/install-cdn.md) - how to install the Dragonfly cdn -- [install scheduler](../user-guide/install-scheduler.md) - how to install the Dragonfly scheduler -- [install client](../user-guide/install-client.md) - how to install the Dragonfly client -- [proxy](../user_guide/proxy/docker.md) - make Dragonfly as an HTTP proxy for docker daemon -- [download files](../user-guide/download-files.md) - download files with Dragonfly -- Container Runtimes - - [cri-o mirror](../user_guide/registry/cri-o.md) - make Dragonfly as Registry Mirror for CRIO daemon - - [cri-containerd_mirror](../user_guide/registry/cri-containerd.md) - make Dragonfly as Registry Mirror for containerd daemon diff --git a/docs/en/user-guide/README.md b/docs/en/user-guide/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/en/user-guide/install/README.md b/docs/en/user-guide/install/README.md new file mode 100644 index 00000000000..48ca323e049 --- /dev/null +++ b/docs/en/user-guide/install/README.md @@ -0,0 +1,4 @@ +[install-client]() is the installation instructions of client +[install-cdn]() is the installation instructions of CDN +[install-cdn]() is the installation instructions of CDN + diff --git a/docs/en/user-guide/install-cdn.md b/docs/en/user-guide/install/install-cdn.md similarity index 100% rename from docs/en/user-guide/install-cdn.md rename to docs/en/user-guide/install/install-cdn.md diff --git a/docs/en/user-guide/install-client.md b/docs/en/user-guide/install/install-client.md similarity index 94% rename from docs/en/user-guide/install-client.md rename to docs/en/user-guide/install/install-client.md index f3dd59934e0..93582eb48fc 100644 --- a/docs/en/user-guide/install-client.md +++ b/docs/en/user-guide/install/install-client.md @@ -66,7 +66,7 @@ Or you can build your own dfclient image. ### Start the dfdaemon -**NOTE:** You should prepare the [config files](../config) which should locate under `/etc/dragonfly` by default. +**NOTE:** You should prepare the [config files](../../config) which should locate under `/etc/dragonfly` by default. ```sh version=1.0.0 @@ -133,4 +133,4 @@ Test if the downloading works. dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" ``` -And test dfdaemon by [pulling an image](download-files.md). +And test dfdaemon by [pulling an image](../download-files.md). diff --git a/docs/en/user-guide/install-manager.md b/docs/en/user-guide/install/install-manager.md similarity index 100% rename from docs/en/user-guide/install-manager.md rename to docs/en/user-guide/install/install-manager.md diff --git a/docs/en/user-guide/install-scheduler.md b/docs/en/user-guide/install/install-scheduler.md similarity index 100% rename from docs/en/user-guide/install-scheduler.md rename to docs/en/user-guide/install/install-scheduler.md diff --git a/docs/en/user-guide/quick-start.md b/docs/en/user-guide/quick-start.md index b073b7d834a..284953ed329 100644 --- a/docs/en/user-guide/quick-start.md +++ b/docs/en/user-guide/quick-start.md @@ -15,4 +15,123 @@ This table describes some container runtimes version and documents. | CRI-O | All | [Link](./registry-mirror/cri-o.md) | Yes | crictl pull docker.io/library/alpine:latest | When using Dragonfly in Kubernetes, we recommend to use `Containerd with CRI` and `CRI-O`, deploying document can be -found in [Kubernetes-with-Dragonfly](../ecosystem/Kubernetes-with-Dragonfly.md). \ No newline at end of file +found in [Kubernetes-with-Dragonfly](../ecosystem/Kubernetes-with-Dragonfly.md). + + +# Dragonfly Quick Start + +Dragonfly Quick Start document aims to help you to quick start Dragonfly journey. This experiment is quite easy and simplified. + +If you are using Dragonfly in your **production environment** to handle production image distribution, please refer to supernode and dfget's detailed production parameter configuration. + +## Prerequisites + +All steps in this document is doing on the same machine using the docker container, so make sure the docker container engine installed and started on your machine. You can also refer to the documentation: [multi-machine deployment](../user_guide/multi_machines_deployment.md) to experience Dragonfly. + +## Step 1: Deploy Dragonfly Manager Server + +```bash +docker run -d --name supernode \ + --restart=always \ + -p 8001:8001 \ + -p 8002:8002 \ + -v /home/admin/supernode:/home/admin/supernode \ + dragonflyoss/supernode:1.0.2 +``` + +## Step 2: Deploy Dragonfly CDN Server + +```bash +docker run -d --name supernode \ + --restart=always \ + -p 8001:8001 \ + -p 8002:8002 \ + -v /home/admin/supernode:/home/admin/supernode \ + dragonflyoss/supernode:1.0.2 +``` + +## Step 3: Deploy Dragonfly Scheduler Server + +```bash +docker run -d --name supernode \ + --restart=always \ + -p 8001:8001 \ + -p 8002:8002 \ + -v /home/admin/supernode:/home/admin/supernode \ + dragonflyoss/supernode:1.0.2 +``` + +## Step 2: Deploy Dragonfly Client + +```bash +SUPERNODE_IP=`docker inspect supernode -f '{{.NetworkSettings.Networks.bridge.IPAddress}}'` +docker run -d --name dfclient \ + --restart=always \ + -p 65001:65001 \ + -v $HOME/.small-dragonfly:/root/.small-dragonfly \ + dragonflyoss/dfclient:1.0.2 --registry https://index.docker.io --node $SUPERNODE_IP +``` + +**NOTE**: + +- The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the other **non-https image registries**. +- The `--node` parameter specifies the supernode's address in the format of **HOST:IP**. And the default value `8002` will be used if the port is not specified. Here we use `docker inspect` to get the ip of supernode container as the host value. Since the supernode container exposes its ports, you can specify this parameter to node ip address as well. + +## Step 3. Configure Docker Daemon + +We need to modify the Docker Daemon configuration to use the Dragonfly as a pull through registry. + +1. Add or update the configuration item `registry-mirrors` in the configuration file`/etc/docker/daemon.json`. + +```json +{ + "registry-mirrors": ["http://127.0.0.1:65001"] +} +``` + +**Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). + +2. Restart Docker Daemon. + +```bash +systemctl restart docker +``` + +## Step 4: Pull images with Dragonfly + +Through the above steps, we can start to validate if Dragonfly works as expected. + +And you can pull the image as usual, for example: + +```bash +docker pull nginx:latest +``` + +## Step 5: Validate Dragonfly + +You can execute the following command to check if the nginx image is distributed via Dragonfly. + +```bash +docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log +``` + +If the output of command above has content like + +``` +2019-03-29 15:49:53.913 INFO sign:96027-1553845785.119 : downloading piece:{"taskID":"00a0503ea12457638ebbef5d0bfae51f9e8e0a0a349312c211f26f53beb93cdc","superNode":"127.0.0.1","dstCid":"127.0.0.1-95953-1553845720.488","range":"67108864-71303167","result":503,"status":701,"pieceSize":4194304,"pieceNum":16} +``` + +then Dragonfly works successfully. + +## SEE ALSO + +- [multi machines deployment](../user-guide/multi-machines-deployment.md) - experience Dragonfly on multiple machines +- [install manager](../user-guide/install/install-manager.md) - how to install the Dragonfly manager +- [install cdn](../user-guide/install/install-cdn.md) - how to install the Dragonfly cdn +- [install scheduler](../user-guide/install/install-scheduler.md) - how to install the Dragonfly scheduler +- [install client](../user-guide/install/install-client.md) - how to install the Dragonfly client +- [proxy](../user_guide/proxy/docker.md) - make Dragonfly as an HTTP proxy for docker daemon +- [download files](../user-guide/download-files.md) - download files with Dragonfly +- Container Runtimes + - [cri-o mirror](../user_guide/registry/cri-o.md) - make Dragonfly as Registry Mirror for CRIO daemon + - [cri-containerd_mirror](../user_guide/registry/cri-containerd.md) - make Dragonfly as Registry Mirror for containerd daemon \ No newline at end of file From 60ecd837d980f71b65b6284ecf3f6102ca1b2ce0 Mon Sep 17 00:00:00 2001 From: santong Date: Wed, 30 Jun 2021 17:54:06 +0800 Subject: [PATCH 7/8] docs: add user guide Signed-off-by: santong --- docs/en/cli-reference/cdn.md | 2 +- docs/en/cli-reference/dfget.md | 4 +- docs/en/cli-reference/manager.md | 10 +- docs/en/cli-reference/scheduler.md | 2 +- docs/en/user-guide/install/README.md | 7 +- docs/en/user-guide/install/install-cdn.md | 12 +- docs/en/user-guide/install/install-client.md | 136 ------------------ docs/en/user-guide/install/install-manager.md | 84 +++-------- .../user-guide/install/install-scheduler.md | 19 +-- docs/en/user-guide/quick-start.md | 115 +-------------- docs/zh-CN/cli-reference/cdn.md | 35 +---- docs/zh-CN/cli-reference/manager.md | 20 +++ docs/zh-CN/cli-reference/scheduler.md | 37 +---- 13 files changed, 76 insertions(+), 407 deletions(-) delete mode 100644 docs/en/user-guide/install/install-client.md create mode 100644 docs/zh-CN/cli-reference/manager.md diff --git a/docs/en/cli-reference/cdn.md b/docs/en/cli-reference/cdn.md index 6e2a9d8b514..a7f006cd4c7 100644 --- a/docs/en/cli-reference/cdn.md +++ b/docs/en/cli-reference/cdn.md @@ -7,7 +7,7 @@ CDN is a long-running process which caches downloaded data from source to avoid go run cmd/cdnsystem/main.go [Option] ``` ## Log configuration -set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal +set environment variable console=true if you want to print logs to Terminal ## Runtime metrics monitoring ``` diff --git a/docs/en/cli-reference/dfget.md b/docs/en/cli-reference/dfget.md index a42e30798ea..c66b04f2b69 100644 --- a/docs/en/cli-reference/dfget.md +++ b/docs/en/cli-reference/dfget.md @@ -21,7 +21,7 @@ dfget --schedulers 127.0.0.1:8002 -o /path/to/output -u "http://example.com/obje ## Log configuration -set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal +set environment variable console=true if you want to print logs to Terminal ### Options @@ -64,7 +64,7 @@ set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Te ## Log configuration -set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal +set environment variable console=true if you want to print logs to Terminal ### Options diff --git a/docs/en/cli-reference/manager.md b/docs/en/cli-reference/manager.md index b6edfda6dd2..5163eff078c 100644 --- a/docs/en/cli-reference/manager.md +++ b/docs/en/cli-reference/manager.md @@ -1,13 +1,15 @@ ## Manager -Scheduler is a long-running process which receives and manages download tasks from the client, notify the CDN to return to the source, -generate and maintain a P2P network during the download process, and push suitable download nodes to the client +Manager is a process that runs in the background and plays the role of the brain of each subsystem cluster in Dragonfly. It is used to manage the dynamic +configuration of each system module and provide functions such as heartbeat keeping alive, monitoring the market, and product functions. + ## Try it ``` -go run cmd/scheduler/main.go [Option] +go run cmd/manager/main.go [Option] ``` + ## Log configuration -set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal +set environment variable console=local if you want to print logs to Terminal ## Runtime metrics monitoring ``` diff --git a/docs/en/cli-reference/scheduler.md b/docs/en/cli-reference/scheduler.md index 885cad11bb5..7ee366cd17b 100644 --- a/docs/en/cli-reference/scheduler.md +++ b/docs/en/cli-reference/scheduler.md @@ -7,7 +7,7 @@ generate and maintain a P2P network during the download process, and push suitab go run cmd/scheduler/main.go [Option] ``` ## Log configuration -set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal +set environment variable console=true if you want to print logs to Terminal ## Runtime metrics monitoring ``` diff --git a/docs/en/user-guide/install/README.md b/docs/en/user-guide/install/README.md index 48ca323e049..77ed3e9f891 100644 --- a/docs/en/user-guide/install/README.md +++ b/docs/en/user-guide/install/README.md @@ -1,4 +1,5 @@ -[install-client]() is the installation instructions of client -[install-cdn]() is the installation instructions of CDN -[install-cdn]() is the installation instructions of CDN +[install-client](./install-client.md) is the installation instructions of client +[install-manager](./install-manager.md) is the installation instructions of manager +[install-scheduler](./install-scheduler.md) is the installation instructions of scheduler +[install-cdn](./install-cdn.md) is the installation instructions of CDN diff --git a/docs/en/user-guide/install/install-cdn.md b/docs/en/user-guide/install/install-cdn.md index 4960e1da22d..4ff61df83c4 100644 --- a/docs/en/user-guide/install/install-cdn.md +++ b/docs/en/user-guide/install/install-cdn.md @@ -1,6 +1,6 @@ # Installing Dragonfly CDN Server -This topic explains how to install the Dragonfly cdn server. +This topic explains how to install the Dragonfly CDN server. ## Context @@ -70,7 +70,7 @@ Or you can build your own cdn image. **NOTE:** Replace ${cdnDockerImageId} with the ID obtained at the previous step. ```sh -docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/cdn:/home/admin/cdn ${cdnDockerImageId} +docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/ftp:/home/admin/ftp ${cdnDockerImageId} --download-port=8001 ``` @@ -127,7 +127,7 @@ cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort You can start a file server in any way. However, the following conditions must be met: -- It must be rooted at `${cdnHomeDir}/repo` which is defined in the previous step. +- It must be rooted at `${cdnHomeDir}/ftp` which is defined in the previous step. - It must listen on the port `cdnDownloadPort` which is defined in the previous step. Let's take nginx as an example. @@ -159,9 +159,3 @@ Let's take nginx as an example. telnet 127.0.0.1 8001 telnet 127.0.0.1 8003 ``` - -- [Install the Dragonfly client](install-client.md) and test if the downloading works. - - ```sh - dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" - ``` diff --git a/docs/en/user-guide/install/install-client.md b/docs/en/user-guide/install/install-client.md deleted file mode 100644 index 93582eb48fc..00000000000 --- a/docs/en/user-guide/install/install-client.md +++ /dev/null @@ -1,136 +0,0 @@ -# Installing Dragonfly Client - -This topic explains how to install the Dragonfly `dfclient`. - -## Context - -Install the `dfclient` in one of the following ways: - -- Deploying with Docker. -- Deploying with physical machines. - -## Prerequisites - -When deploying with Docker, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Docker|1.12.0+ - -When deploying with physical machines, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Golang|1.12.x - -## Procedure - When Deploying with Docker - -### Get dfclient image - -You can get it from [DockerHub](https://hub.docker.com/) directly. - -1. Obtain the latest Docker image ID of the SuperNode. - - ```sh - docker pull dragonflyoss/dfclient:1.0.0 - ``` - -Or you can build your own dfclient image. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly - ``` - -3. Build the Docker image. - - ```sh - TAG="1.0.0" - make docker-build-client DF_VERSION=$TAG - ``` - -4. Obtain the latest Docker image ID of the `dfclient`. - - ```sh - docker image ls | grep 'dfclient' | awk '{print $3}' | head -n1 - ``` - -### Start the dfdaemon - -**NOTE:** You should prepare the [config files](../../config) which should locate under `/etc/dragonfly` by default. - -```sh -version=1.0.0 -# Replace ${supernode} with your own supernode node with format `ip:port=weight`. -SUPERNODE=$supernode -docker run -d --name dfclient --restart=always -p 65001:65001 -v $HOME/.small-dragonfly:/root/.small-dragonfly -v /etc/dragonfly:/etc/dragonfly dragonflyoss/dfclient:$version --node $SUPERNODE -``` - -## Procedure - When Deploying with Physical Machines - -### Get dfclient executable file - -1. Download a binary package of the SuperNode. You can download one of the latest builds for Dragonfly on the [github releases page](https://github.com/dragonflyoss/Dragonfly/releases). - - ```sh - version=1.0.0 - wget https://github.com/dragonflyoss/Dragonfly/releases/download/v$version/Dragonfly_$version_linux_amd64.tar.gz - ``` - -2. Unzip the package. - - ```bash - # Replace `xxx` with the installation directory. - tar -zxf Dragonfly_1.0.0_linux_amd64.tar.gz -C xxx - ``` - -3. Move the `dfget` and `dfdaemon` to your `PATH` environment variable to make sure you can directly use `dfget` and `dfdaemon` command. - -Or you can build your own dfclient executable files. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly - ``` - -3. Build `dfdaemon` and `dfget`. - - ```sh - make build-client && make install-client - ``` - -### Start the dfdaemon - -**NOTE:** You can ignore this step when using only dfget for file distribution . - -```sh -# Replace ${supernode} with your own supernode node with format `ip:port=weight`. -SUPERNODE=$supernode -dfdaemon --node $SUPERNODE -``` - -## After this Task - -Test if the downloading works. - -```sh -dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" -``` - -And test dfdaemon by [pulling an image](../download-files.md). diff --git a/docs/en/user-guide/install/install-manager.md b/docs/en/user-guide/install/install-manager.md index 4960e1da22d..620d5784ef5 100644 --- a/docs/en/user-guide/install/install-manager.md +++ b/docs/en/user-guide/install/install-manager.md @@ -1,13 +1,13 @@ -# Installing Dragonfly CDN Server +# Installing Dragonfly Manager Server -This topic explains how to install the Dragonfly cdn server. +This topic explains how to install the Dragonfly manager server. ## Context -Install CDN in one of the following ways: +Install manager in one of the following ways: -- Deploying with Docker. -- Deploying with physical machines: Recommended for production usage. +- Deploying with Docker: Recommended for production usage. +- Deploying with physical machines. ## Prerequisites @@ -28,17 +28,17 @@ Nginx|0.8+ ## Procedure - When Deploying with Docker -### Get cdn image +### Get manager image You can get it from [DockerHub](https://hub.docker.com/) directly. -1. Obtain the latest Docker image of the cdn. +1. Obtain the latest Docker image of the manager. ```sh - docker pull d7yio/cdn + docker pull d7yio/manager ``` -Or you can build your own cdn image. +Or you can build your own manager image. 1. Obtain the source code of Dragonfly. @@ -56,29 +56,28 @@ Or you can build your own cdn image. ```sh TAG="1.0.0" - make docker-build-cdn D7Y_VERSION=$TAG + make docker-build-manager D7Y_VERSION=$TAG ``` -4. Obtain the latest Docker image ID of the cdn. +4. Obtain the latest Docker image ID of the manager. ```sh - docker image ls | grep 'cdn' | awk '{print $3}' | head -n1 + docker image ls | grep 'manager' | awk '{print $3}' | head -n1 ``` -### Start cdn +### Start manager -**NOTE:** Replace ${cdnDockerImageId} with the ID obtained at the previous step. +**NOTE:** Replace ${managerDockerImageId} with the ID obtained at the previous step. ```sh -docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/admin/cdn:/home/admin/cdn ${cdnDockerImageId} ---download-port=8001 +docker run -d --name manager --restart=always -p 8004:8004 ${managerDockerImageId} ``` ## Procedure - When Deploying with Physical Machines ### Get cdn executable file -1. Download a binary package of the cdn. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. +1. Download a binary package of the manager. You can download one of the latest builds for Dragonfly on the [github releases page](https://github. com/dragonflyoss/Dragonfly2/releases). ```sh @@ -93,9 +92,9 @@ docker run -d --name cdn --restart=always -p 8001:8001 -p 8003:8003 -v /home/adm tar -zxf Dragonfly2_1.0.0_linux_amd64.tar.gz -C xxx ``` -3. Move the `cdn` to your `PATH` environment variable to make sure you can directly use `cdn` command. +3. Move the `manager` to your `PATH` environment variable to make sure you can directly use `manager` command. -Or you can build your own cdn executable file. +Or you can build your own manager executable file. 1. Obtain the source code of Dragonfly. @@ -112,56 +111,19 @@ Or you can build your own cdn executable file. 3. Compile the source code. ```sh - make build-cdn && make install-cdn + make build-manaager && make install-manager ``` -### Start cdn +### Start manager ```sh -cdnHomeDir=/home/admin/cdn -cdnDownloadPort=8001 -cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort +manager --options ``` -### Start file server - -You can start a file server in any way. However, the following conditions must be met: - -- It must be rooted at `${cdnHomeDir}/repo` which is defined in the previous step. -- It must listen on the port `cdnDownloadPort` which is defined in the previous step. - -Let's take nginx as an example. - -1. Add the following configuration items to the Nginx configuration file. - - ```conf - server { - # Must be ${cdnDownloadPort} - listen 8001; - location / { - # Must be ${cdnHomeDir}/repo - root /home/admin/cdn/repo; - } - } - ``` - -2. Start Nginx. - - ```sh - sudo nginx - ``` - ## After this Task -- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. - - ```sh - telnet 127.0.0.1 8001 - telnet 127.0.0.1 8003 - ``` - -- [Install the Dragonfly client](install-client.md) and test if the downloading works. +- After manager is installed, run the following commands to verify if **manager** is started, and if Port `8004` and `8003` is available. ```sh - dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" + telnet 127.0.0.1 8004 ``` diff --git a/docs/en/user-guide/install/install-scheduler.md b/docs/en/user-guide/install/install-scheduler.md index d9e5c7adb34..c005b3da394 100644 --- a/docs/en/user-guide/install/install-scheduler.md +++ b/docs/en/user-guide/install/install-scheduler.md @@ -6,7 +6,7 @@ This topic explains how to install the Dragonfly scheduler server. Install scheduler in one of the following ways: -- Deploying with Docker. +- Deploying with Docker: Recommended for production usage. - Deploying with physical machines. ## Prerequisites @@ -70,8 +70,7 @@ Or you can build your own scheduler image. **NOTE:** Replace ${schedulerDockerImageId} with the ID obtained at the previous step. ```sh -docker run -d --name scheduler --restart=always -p 8002 -v /home/admin/scheduler:/home/admin/scheduler ${schedulerDockerImageId} ---download-port=8001 +docker run -d --name scheduler --restart=always -p 8002 ${schedulerDockerImageId} ``` ## Procedure - When Deploying with Physical Machines @@ -118,20 +117,12 @@ Or you can build your own scheduler executable file. ### Start scheduler ```sh -schedulerHomeDir=/home/admin/scheduler -cdn --home-dir=$cdnHomeDir --port=8003 --download-port=$cdnDownloadPort +scheduler --options ``` ## After this Task -- After cdn is installed, run the following commands to verify if Nginx and **cdn** are started, and if Port `8001` and `8003` are available. +- After scheduler is installed, run the following commands to verify if **scheduler** is started, and if Port `8002` is available. ```sh - telnet 127.0.0.1 8001 - telnet 127.0.0.1 8003 - ``` - -- [Install the Dragonfly client](install-client.md) and test if the downloading works. - - ```sh - dfget --url "http://${resourceUrl}" --output ./resource.png --supernode "127.0.0.1:8002=1" + telnet 127.0.0.1 8002 ``` diff --git a/docs/en/user-guide/quick-start.md b/docs/en/user-guide/quick-start.md index 284953ed329..c7d0da439d8 100644 --- a/docs/en/user-guide/quick-start.md +++ b/docs/en/user-guide/quick-start.md @@ -9,120 +9,13 @@ This table describes some container runtimes version and documents. | Runtime | Version | Document | CRI Support | Pull Command | | --- | --- | --- | --- | --- | -| Docker | All | [Link](./proxy/docker.md) | No | docker pull docker.io/library/alpine | | Containerd without CRI | All | [Link](./proxy/containerd.md) | No | ctr image pull docker.io/library/alpine | -| Containerd with CRI | v1.1.0+ | [Link](registry-mirror/cri-containerd.md) | Yes | crictl pull docker.io/library/alpine:latest | +| Containerd with CRI | v1.1.0+ | [Link](./registry-mirror/cri-containerd.md) | Yes | crictl pull docker.io/library/alpine:latest | | CRI-O | All | [Link](./registry-mirror/cri-o.md) | Yes | crictl pull docker.io/library/alpine:latest | When using Dragonfly in Kubernetes, we recommend to use `Containerd with CRI` and `CRI-O`, deploying document can be found in [Kubernetes-with-Dragonfly](../ecosystem/Kubernetes-with-Dragonfly.md). - -# Dragonfly Quick Start - -Dragonfly Quick Start document aims to help you to quick start Dragonfly journey. This experiment is quite easy and simplified. - -If you are using Dragonfly in your **production environment** to handle production image distribution, please refer to supernode and dfget's detailed production parameter configuration. - -## Prerequisites - -All steps in this document is doing on the same machine using the docker container, so make sure the docker container engine installed and started on your machine. You can also refer to the documentation: [multi-machine deployment](../user_guide/multi_machines_deployment.md) to experience Dragonfly. - -## Step 1: Deploy Dragonfly Manager Server - -```bash -docker run -d --name supernode \ - --restart=always \ - -p 8001:8001 \ - -p 8002:8002 \ - -v /home/admin/supernode:/home/admin/supernode \ - dragonflyoss/supernode:1.0.2 -``` - -## Step 2: Deploy Dragonfly CDN Server - -```bash -docker run -d --name supernode \ - --restart=always \ - -p 8001:8001 \ - -p 8002:8002 \ - -v /home/admin/supernode:/home/admin/supernode \ - dragonflyoss/supernode:1.0.2 -``` - -## Step 3: Deploy Dragonfly Scheduler Server - -```bash -docker run -d --name supernode \ - --restart=always \ - -p 8001:8001 \ - -p 8002:8002 \ - -v /home/admin/supernode:/home/admin/supernode \ - dragonflyoss/supernode:1.0.2 -``` - -## Step 2: Deploy Dragonfly Client - -```bash -SUPERNODE_IP=`docker inspect supernode -f '{{.NetworkSettings.Networks.bridge.IPAddress}}'` -docker run -d --name dfclient \ - --restart=always \ - -p 65001:65001 \ - -v $HOME/.small-dragonfly:/root/.small-dragonfly \ - dragonflyoss/dfclient:1.0.2 --registry https://index.docker.io --node $SUPERNODE_IP -``` - -**NOTE**: - -- The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the other **non-https image registries**. -- The `--node` parameter specifies the supernode's address in the format of **HOST:IP**. And the default value `8002` will be used if the port is not specified. Here we use `docker inspect` to get the ip of supernode container as the host value. Since the supernode container exposes its ports, you can specify this parameter to node ip address as well. - -## Step 3. Configure Docker Daemon - -We need to modify the Docker Daemon configuration to use the Dragonfly as a pull through registry. - -1. Add or update the configuration item `registry-mirrors` in the configuration file`/etc/docker/daemon.json`. - -```json -{ - "registry-mirrors": ["http://127.0.0.1:65001"] -} -``` - -**Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). - -2. Restart Docker Daemon. - -```bash -systemctl restart docker -``` - -## Step 4: Pull images with Dragonfly - -Through the above steps, we can start to validate if Dragonfly works as expected. - -And you can pull the image as usual, for example: - -```bash -docker pull nginx:latest -``` - -## Step 5: Validate Dragonfly - -You can execute the following command to check if the nginx image is distributed via Dragonfly. - -```bash -docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log -``` - -If the output of command above has content like - -``` -2019-03-29 15:49:53.913 INFO sign:96027-1553845785.119 : downloading piece:{"taskID":"00a0503ea12457638ebbef5d0bfae51f9e8e0a0a349312c211f26f53beb93cdc","superNode":"127.0.0.1","dstCid":"127.0.0.1-95953-1553845720.488","range":"67108864-71303167","result":503,"status":701,"pieceSize":4194304,"pieceNum":16} -``` - -then Dragonfly works successfully. - ## SEE ALSO - [multi machines deployment](../user-guide/multi-machines-deployment.md) - experience Dragonfly on multiple machines @@ -130,8 +23,8 @@ then Dragonfly works successfully. - [install cdn](../user-guide/install/install-cdn.md) - how to install the Dragonfly cdn - [install scheduler](../user-guide/install/install-scheduler.md) - how to install the Dragonfly scheduler - [install client](../user-guide/install/install-client.md) - how to install the Dragonfly client -- [proxy](../user_guide/proxy/docker.md) - make Dragonfly as an HTTP proxy for docker daemon +- [proxy](../user-guide/proxy/containerd.md) - make Dragonfly as an HTTP proxy for docker daemon - [download files](../user-guide/download-files.md) - download files with Dragonfly - Container Runtimes - - [cri-o mirror](../user_guide/registry/cri-o.md) - make Dragonfly as Registry Mirror for CRIO daemon - - [cri-containerd_mirror](../user_guide/registry/cri-containerd.md) - make Dragonfly as Registry Mirror for containerd daemon \ No newline at end of file + - [cri-o mirror](../user-guide/registry-mirror/cri-o.md) - make Dragonfly as Registry Mirror for CRIO daemon + - [cri-containerd_mirror](../user-guide/registry-mirror/cri-containerd.md) - make Dragonfly as Registry Mirror for containerd daemon \ No newline at end of file diff --git a/docs/zh-CN/cli-reference/cdn.md b/docs/zh-CN/cli-reference/cdn.md index 6eca49855a5..d59e1fc7c59 100644 --- a/docs/zh-CN/cli-reference/cdn.md +++ b/docs/zh-CN/cli-reference/cdn.md @@ -1,13 +1,13 @@ ## CDN -CDN is a long-running process which caches downloaded data from source to avoid downloading the same files from source repeatedly +CDN 是一个长时间运行的服务进程，它缓存从源下载的数据，以避免重复从源下载相同的文件 ## 用法 ``` go run cmd/cdnsystem/main.go [Option] ``` -## Log configuration -set environment variable DF_ACTIVE_PROFILE=local if you want to print logs to Terminal +## 输出日志配置 +如果你期望把系统执行日志输出到命令终端，设置环境变量 console=true ## Runtime metrics monitoring ``` @@ -23,31 +23,4 @@ go run cmd/cdnsystem/main.go --profiler --jaeger string jaeger endpoint url, like: http://localhost:14250/api/traces --pprof-port int listen port for pprof, 0 represents random port (default -1) --verbose whether logger use debug level -``` - -### 二进制命令使用示例 - -scheduler --config your-config-path/scheduler.yaml - -### 配置文件说明 - -``` -server: - port: 8001 rpc 端口 - -scheduler: - -worker: - worker-num: 5 工作线程数 - worker-job-pool-size: 10000 工作队列长度 - sender-num: 10 发送消息线程数 - sender-job-pool-size: 10000 发送消息队列长度 - -cdn: - list: CDN列表 - - - - cdn-name : "cdn" CDN服务器的HostName - ip: "127.0.0.1" CDN服务器的IP地址 - rpcPort: 8003 CDN的RPC端口 - download-port: 8002 CDN的下载端口 -``` +``` \ No newline at end of file diff --git a/docs/zh-CN/cli-reference/manager.md b/docs/zh-CN/cli-reference/manager.md new file mode 100644 index 00000000000..35357d6b574 --- /dev/null +++ b/docs/zh-CN/cli-reference/manager.md @@ -0,0 +1,20 @@ +## Scheduler + +Manager 是一个常驻后台运行的进程，它在蜻蜓中扮演各个子系统集群大脑的角色， 用于管理各个系统模块依赖的动态配置，以及提供心跳保活、监控大盘和产品化的功能。 + +### 用法 +``` +go run cmd/manager/main.go [Option] +``` + +### 可选参数 + +``` + --config string the path of scheduler's configuration file (default "conf/scheduler.yaml") + -h, --help help for scheduler + --port int port is the port that scheduler server listens on (default 8002) + --sender-job-pool-size int sender-job-pool-size is used for scheduler and do not change it (default 10000) + --sender-num int sender-num is used for scheduler and do not change it (default 50) + --worker-job-pool-size int worker-job-pool-size is used for scheduler and do not change it (default 10000) + --worker-num int worker-num is used for scheduler and do not change it (default 12) +``` \ No newline at end of file diff --git a/docs/zh-CN/cli-reference/scheduler.md b/docs/zh-CN/cli-reference/scheduler.md index 9b647c47da8..2d1f188f43b 100644 --- a/docs/zh-CN/cli-reference/scheduler.md +++ b/docs/zh-CN/cli-reference/scheduler.md @@ -1,14 +1,10 @@ -## Scheduler - -Scheduler 生成并维护下载过程中的P2P网络 - -### 说明 +## Scheduler Scheduler 是一个常驻后台运行的进程，用于接收和管理客户端的下载任务，通知CDN进行回源， 在下载过程中生成维护P2P网络，给客户端推送适合的下载节点 ### 用法 ``` -scheduler [flags] +go run cmd/scheduler/main.go [Option] ``` ### 可选参数 @@ -21,31 +17,4 @@ scheduler [flags] --sender-num int sender-num is used for scheduler and do not change it (default 50) --worker-job-pool-size int worker-job-pool-size is used for scheduler and do not change it (default 10000) --worker-num int worker-num is used for scheduler and do not change it (default 12) -``` - -### 二进制命令使用示例 - -scheduler --config your-config-path/scheduler.yaml - -### 配置文件说明 - -``` -server: - port: 8001 rpc 端口 - -scheduler: - -worker: - worker-num: 5 工作线程数 - worker-job-pool-size: 10000 工作队列长度 - sender-num: 10 发送消息线程数 - sender-job-pool-size: 10000 发送消息队列长度 - -cdn: - list: CDN列表 - - - - cdn-name : "cdn" CDN服务器的HostName - ip: "127.0.0.1" CDN服务器的IP地址 - rpcPort: 8003 CDN的RPC端口 - download-port: 8002 CDN的下载端口 -``` +``` \ No newline at end of file From 26b51a8bb6dea84e5b4d7bb9c81f388baed2f04b Mon Sep 17 00:00:00 2001 From: santong Date: Wed, 30 Jun 2021 18:05:54 +0800 Subject: [PATCH 8/8] docs: add user guide Signed-off-by: santong --- docs/en/config/scheduler.yaml | 2 +- docs/en/user-guide/download-files.md | 131 ------------------ .../user-guide/multi-machines-deployment.md | 118 ---------------- docs/en/user-guide/quick-start.md | 3 - 4 files changed, 1 insertion(+), 253 deletions(-) delete mode 100644 docs/en/user-guide/download-files.md delete mode 100644 docs/en/user-guide/multi-machines-deployment.md diff --git a/docs/en/config/scheduler.yaml b/docs/en/config/scheduler.yaml index c4d3ac9d0a0..6551734d054 100644 --- a/docs/en/config/scheduler.yaml +++ b/docs/en/config/scheduler.yaml @@ -4,7 +4,7 @@ debug: false server: - # ListenPort is the ip and port supernode server listens on. + # ListenPort is the ip and port scheduler server listens on. # default: 8002 port: 8002 diff --git a/docs/en/user-guide/download-files.md b/docs/en/user-guide/download-files.md deleted file mode 100644 index e0c4c8b41e4..00000000000 --- a/docs/en/user-guide/download-files.md +++ /dev/null @@ -1,131 +0,0 @@ -# Downloading Files with Dragonfly - -Things are done differently when you download container images and download general files with Dragonfly. - -## Prerequisites - -- You are using Linux operating system. -- The supernode service is started. - - **Tip:** For more information on the dfget command, see [dfget](../cli_reference/dfget.md). For more information on the installation of supernodes, see [Installing Server](./install_server.md). - -## Downloading container images - -1. Config the supernodes with the configuration file. - - ```shell - cat < /etc/dragonfly/dfget.yml - nodes: - - supernode01:port - - supernode02:port - - supernode03:port - EOD - ``` - -2. Start the dfget proxy (dfdaemon). - - ```sh - # Start dfdaemon and specify the image repo URL. The default port is `65001`. - dfdaemon --registry https://xxx.xx.x - # Review dfdaemon logs - tailf ~/.small-dragonfly/logs/dfdaemon.log - ``` - - **Tip:** To list all available parameters for dfdaemon, run `dfdaemon -h`. - -3. Configure the Docker daemon. - - a. Modify the configuration file `/etc/docker/daemon.json`. - - ```sh - vi /etc/docker/daemon.json - ``` - - **Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). - - b. Add or update the configuration item `registry-mirrors` in the configuration file. - - ```sh - "registry-mirrors": ["http://127.0.0.1:65001"] - ``` - - c. Restart Docker daemon. - - ```bash - systemctl restart docker - ``` - - d. Add authentication info for the private docker registry in `~/.docker/config.json` if the registry is configured with auth. - - ```json - { - "auths": { - "https://index.docker.io/v1/": { - "auth": "${auth_value}" - } - } - } - ``` - - The ${auth_value} is `base64("${username}:${password}")`. - - ```bash - echo "${username}:${password}" | base64 - ``` - -4. Download an image with Dragonfly. - - ```bash - docker pull {imageName} - ``` - - **Note:** Don't include the image repo URL in {imageName}, because the repo URL has been specified with the `registry` parameter when starting dfdaemon. - -## Downloading General Files - -1. Specify the supernodes in one of the following ways. - - - Specifying with the configuration file. - - ```sh - cat < /etc/dragonfly/dfget.yml - nodes: - - supernode01:port - - supernode02:port - - supernode03:port - EOD - ``` - - - Specifying with the parameter in the command line. - - ```sh - dfget -u "http://www.taobao.com" -o /tmp/test.html --node supernode01:port,supernode02:port,supernode03:port - ``` - - **Note:** When using this method, you must add the `node` parameter whenever you run the dfget command. And the parameter in the command line takes precedence over the configuration file. - -2. Download general files with Dragonfly in one of the following ways. - - - Download files with the default `/etc/dragonfly/dfget.yml` configuration. - - ```sh - dfget --url "http://xxx.xx.x" - ``` - - **Tip:** To list all available parameters for dfget, run `dfget -h`. - - - Download files with your specified supernodes. - - ```sh - dfget --url "http://xxx.xx.x" --node "127.0.0.1:8002" - ``` - - - Download files to your specified output file. - - ```sh - dfget --url "http://xxx.xx.x" -o a.txt - ``` - -## After this Task - -To review the downloading log, run `less ~/.small-dragonfly/logs/dfclient.log`. diff --git a/docs/en/user-guide/multi-machines-deployment.md b/docs/en/user-guide/multi-machines-deployment.md deleted file mode 100644 index 6c24ad43d3f..00000000000 --- a/docs/en/user-guide/multi-machines-deployment.md +++ /dev/null @@ -1,118 +0,0 @@ -# Dragonfly multi-machines deployment - -The multi-machines deployment documentation is designed to help you fully experience Dragonfly on multiple machines. - -If you are using Dragonfly in your production environment to handle production image distribution, please refer to supernode and dfget's detailed production parameter configuration. - -## Prerequisites - -Assuming that experiment requires us to prepare three host machines, one to play a role of supernode, and the other two for dfclient. Then the topology of the three nodes cluster is like the following: - -![quick start cluster topology](../images/quick-start-topo.png) - -Then, we must provide: - -1. three host nodes in a LAN, and we assume that 3 machine IPs are replaced by the following names. - - - **dfsupernode**: Dragonfly server - - **dfclient0**: Dragonfly client one - - **dfclient1**: Dragonfly client two - -2. every node has deployed docker daemon - -## Step 1: Deploy Dragonfly Server (SuperNode) - -Deploy the Dragonfly server (Supernode) on the machine `dfsupernode`. - -```bash -docker run -d --name supernode \ - --restart=always \ - -p 8001:8001 \ - -p 8002:8002 \ - -v /home/admin/supernode:/home/admin/supernode \ - dragonflyoss/supernode:1.0.2 --download-port=8001 -``` - -## Step 2: Deploy Dragonfly Client (dfclient) - -The following operations should be performed both on the client machine `dfclient0`, `dfclient1`. - -### Prepare the configuration file - -Dragonfly's configuration file is located in the `/etc/dragonfly` directory by default. When using the container to deploy the client, you need to mount the configuration file to the container. - -Configure the Dragonfly Supernode address for the client: - -```bash -cat < /etc/dragonfly/dfget.yml -nodes: - - dfsupernode -EOD -``` - -### Start Dragonfly Client - -```bash -docker run -d --name dfclient \ - --restart=always \ - -p 65001:65001 \ - -v /etc/dragonfly:/etc/dragonfly \ - -v $HOME/.small-dragonfly:/root/.small-dragonfly \ - dragonflyoss/dfclient:1.0.2 --registry https://index.docker.io -``` - -**NOTE**: The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the others. - -## Step 3. Configure Docker Daemon - -We need to modify the Docker Daemon configuration to use the Dragonfly as a pull through registry both on the client machine `dfclient0`, `dfclient1`. - -1. Add or update the configuration item `registry-mirrors` in the configuration file`/etc/docker/daemon.json`. - -```json -{ - "registry-mirrors": ["http://127.0.0.1:65001"] -} -``` - -**Tip:** For more information on `/etc/docker/daemon.json`, see [Docker documentation](https://docs.docker.com/registry/recipes/mirror/#configure-the-cache). - -2. Restart Docker Daemon. - -```bash -systemctl restart docker -``` - -## Step 4: Pull images with Dragonfly - -Through the above steps, we can start to validate if Dragonfly works as expected. - -And you can pull the image as usual on either `dfclient0` or `dfclient1`, for example: - -```bash -docker pull nginx:latest -``` - -## Step 5: Validate Dragonfly - -You can execute the following command to check if the nginx image is distributed via Dragonfly. - -```bash -docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log -``` - -If the output of command above has content like - -``` -2019-03-29 15:49:53.913 INFO sign:96027-1553845785.119 : downloading piece:{"taskID":"00a0503ea12457638ebbef5d0bfae51f9e8e0a0a349312c211f26f53beb93cdc","superNode":"127.0.0.1","dstCid":"127.0.0.1-95953-1553845720.488","range":"67108864-71303167","result":503,"status":701,"pieceSize":4194304,"pieceNum":16} -``` - -that means that the image download is done by Dragonfly. - -If you need to ensure that if the image is transferred through other peer nodes, you can execute the following command: - -```bash -docker exec dfclient grep 'downloading piece' /root/.small-dragonfly/logs/dfclient.log | grep -v cdnnode -``` - -If the above command does not output the result, the mirror does not complete the transmission through other peer nodes. Otherwise, the transmission is completed through other peer nodes. diff --git a/docs/en/user-guide/quick-start.md b/docs/en/user-guide/quick-start.md index c7d0da439d8..6253719b3cd 100644 --- a/docs/en/user-guide/quick-start.md +++ b/docs/en/user-guide/quick-start.md @@ -18,13 +18,10 @@ found in [Kubernetes-with-Dragonfly](../ecosystem/Kubernetes-with-Dragonfly.md). ## SEE ALSO -- [multi machines deployment](../user-guide/multi-machines-deployment.md) - experience Dragonfly on multiple machines - [install manager](../user-guide/install/install-manager.md) - how to install the Dragonfly manager - [install cdn](../user-guide/install/install-cdn.md) - how to install the Dragonfly cdn - [install scheduler](../user-guide/install/install-scheduler.md) - how to install the Dragonfly scheduler -- [install client](../user-guide/install/install-client.md) - how to install the Dragonfly client - [proxy](../user-guide/proxy/containerd.md) - make Dragonfly as an HTTP proxy for docker daemon -- [download files](../user-guide/download-files.md) - download files with Dragonfly - Container Runtimes - [cri-o mirror](../user-guide/registry-mirror/cri-o.md) - make Dragonfly as Registry Mirror for CRIO daemon - [cri-containerd_mirror](../user-guide/registry-mirror/cri-containerd.md) - make Dragonfly as Registry Mirror for containerd daemon \ No newline at end of file