Skip to content

Commit

Permalink
log search: return zip instead of tar or gzip (#724)
Browse files Browse the repository at this point in the history
  • Loading branch information
baurine authored Sep 8, 2020
1 parent f57b245 commit 1dc0480
Show file tree
Hide file tree
Showing 4 changed files with 103 additions and 184 deletions.
103 changes: 25 additions & 78 deletions pkg/apiserver/logsearch/pack.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,8 @@
package logsearch

import (
"archive/tar"
"fmt"
"io"
"net/http"
"os"
"path"

"github.com/gin-gonic/gin"
"github.com/pingcap/log"
Expand All @@ -28,86 +24,37 @@ import (
"github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/utils"
)

func packLogsAsTarball(tasks []*TaskModel, w io.Writer) {
tw := tar.NewWriter(w)
defer tw.Close()

for _, task := range tasks {
if task.LogStorePath == nil && task.SlowLogStorePath == nil {
continue
}
if task.LogStorePath != nil {
if err := dumpLog(*task.LogStorePath, tw); err != nil {
log.Warn("Failed to pack log",
zap.Any("task", task),
zap.Error(err))
continue
}
}
if task.SlowLogStorePath != nil {
if err := dumpLog(*task.SlowLogStorePath, tw); err != nil {
log.Warn("Failed to pack slow log",
zap.Any("task", task),
zap.Error(err))
continue
}
}
}
}

func dumpLog(savedPath string, tw *tar.Writer) error {
f, err := os.Open(savedPath)
if err != nil {
return err
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return err
}
err = tw.WriteHeader(&tar.Header{
Name: path.Base(savedPath),
Mode: int64(fi.Mode()),
ModTime: fi.ModTime(),
Size: fi.Size(),
})
if err != nil {
return err
}

_, err = io.Copy(tw, f)
if err != nil {
return err
}
return nil
}

func serveTaskForDownload(task *TaskModel, c *gin.Context) {
if task.LogStorePath == nil && task.SlowLogStorePath == nil {
logPath := task.LogStorePath
if logPath == nil {
logPath = task.SlowLogStorePath
}
if logPath == nil {
utils.MakeInvalidRequestErrorWithMessage(c, "Log is not ready")
return
}
reader, writer := io.Pipe()
go func() {
defer writer.Close()
packLogsAsTarball([]*TaskModel{task}, writer)
}()
contentType := "application/tar"
extraHeaders := map[string]string{
"Content-Disposition": fmt.Sprintf(`attachment; filename="logs-%s.tar"`, task.Target.FileName()),
}
c.DataFromReader(http.StatusOK, -1, contentType, reader, extraHeaders)
c.FileAttachment(*logPath, fmt.Sprintf("logs-%s.zip", task.Target.FileName()))
}

func serveMultipleTaskForDownload(tasks []*TaskModel, c *gin.Context) {
reader, writer := io.Pipe()
go func() {
defer writer.Close()
packLogsAsTarball(tasks, writer)
}()
contentType := "application/tar"
extraHeaders := map[string]string{
"Content-Disposition": `attachment; filename="logs.tar"`,
filePaths := make([]string, 0, len(tasks))
for _, task := range tasks {
logPath := task.LogStorePath
if logPath == nil {
logPath = task.SlowLogStorePath
}
if logPath == nil {
c.Status(http.StatusInternalServerError)
_ = c.Error(utils.ErrInvalidRequest.New("Some logs are not available"))
return
}
filePaths = append(filePaths, *logPath)
}

c.Writer.Header().Set("Content-type", "application/octet-stream")
c.Writer.Header().Set("Content-Disposition", "attachment; filename=\"logs.zip\"")
err := utils.StreamZipPack(c.Writer, filePaths, false)
if err != nil {
log.Error("Stream zip pack failed", zap.Error(err))
}
c.DataFromReader(http.StatusOK, -1, contentType, reader, extraHeaders)
}
38 changes: 12 additions & 26 deletions pkg/apiserver/profiling/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"time"

"github.com/gin-gonic/gin"
"github.com/pingcap/log"
"go.uber.org/zap"

"github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/user"
"github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/utils"
Expand Down Expand Up @@ -228,21 +230,13 @@ func (s *Service) downloadGroup(c *gin.Context) {
filePathes[i] = task.FilePath
}

temp, err := ioutil.TempFile("", fmt.Sprintf("taskgroup_%d", taskGroupID))
fileName := fmt.Sprintf("profiling_pack_%d.zip", taskGroupID)
c.Writer.Header().Set("Content-type", "application/octet-stream")
c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName))
err = utils.StreamZipPack(c.Writer, filePathes, true)
if err != nil {
_ = c.Error(err)
return
}

err = createTarball(temp, filePathes)
defer temp.Close()
if err != nil {
_ = c.Error(err)
return
log.Error("Stream zip pack failed", zap.Error(err))
}

fileName := fmt.Sprintf("profiling_pack_%d.tar.gz", taskGroupID)
c.FileAttachment(temp.Name(), fileName)
}

// @ID downloadProfilingSingle
Expand Down Expand Up @@ -275,21 +269,13 @@ func (s *Service) downloadSingle(c *gin.Context) {
return
}

temp, err := ioutil.TempFile("", fmt.Sprintf("task_%d", taskID))
fileName := fmt.Sprintf("profiling_%d.zip", taskID)
c.Writer.Header().Set("Content-type", "application/octet-stream")
c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName))
err = utils.StreamZipPack(c.Writer, []string{task.FilePath}, true)
if err != nil {
_ = c.Error(err)
return
}

err = createTarball(temp, []string{task.FilePath})
defer temp.Close()
if err != nil {
_ = c.Error(err)
return
log.Error("Stream zip pack failed", zap.Error(err))
}

fileName := fmt.Sprintf("profiling_%d.tar.gz", taskID)
c.FileAttachment(temp.Name(), fileName)
}

// @ID viewProfilingSingle
Expand Down
80 changes: 0 additions & 80 deletions pkg/apiserver/profiling/util.go

This file was deleted.

66 changes: 66 additions & 0 deletions pkg/apiserver/utils/zip.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
"archive/zip"
"io"
"os"
)

func StreamZipPack(w io.Writer, files []string, needCompress bool) error {
pack := zip.NewWriter(w)
defer pack.Close()

for _, file := range files {
err := streamZipFile(pack, file, needCompress)
if err != nil {
return err
}
}

return nil
}

func streamZipFile(zipPack *zip.Writer, file string, needCompress bool) error {
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()

fileInfo, err := f.Stat()
if err != nil {
return err
}

zipMethod := zip.Store // no compress
if needCompress {
zipMethod = zip.Deflate // compress
}
zipFile, err := zipPack.CreateHeader(&zip.FileHeader{
Name: fileInfo.Name(),
Method: zipMethod,
})
if err != nil {
return err
}

_, err = io.Copy(zipFile, f)
if err != nil {
return err
}

return nil
}

0 comments on commit 1dc0480

Please sign in to comment.