Skip to content

Commit

Permalink
修改write文件的goroutine模式，尝试提高大批量单个重复文件的处理速度
Browse files Browse the repository at this point in the history
  • Loading branch information
@tanganyu1114
tanganyu1114 committed Dec 29, 2020
1 parent ee67ed5 commit 7d0e7ec
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 212 deletions.
184 changes: 0 additions & 184 deletions model/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,187 +148,3 @@ func NewLink(stat bool, src, dist string) *link {
dist: dist,
}
}

/*type fileMap map[string]struct {
files []string
size int64
}
type FileMaps interface {
Select(hash string) bool
Insert(hash, file string, size int)
}
func (f fileMap) Select(hash string) bool {
_, ok := f[hash]
return ok
}
func (f fileMap) Insert(record Record) {
if record.Select().Stats {
f[record.Select().Hash] = struct {
files []string
size int64
}{
files: append(f[record.Select().Hash].files, record.Select().File),
size: int64(record.Select().Size)}
}
}
type PathResult interface {
AddTotalNum()
AddTotalSize(size int64)
AddErrInfo(path string)
}
type pathResult struct {
totalNum int
totalSize int64
errNum int
errPath []string
}
func NewPathResult() *pathResult {
return &pathResult{
totalNum: 0,
totalSize: 0,
errNum: 0,
errPath: nil,
}
}
func (p pathResult) AddTotalSize(size int64) {
p.totalSize += size
}
func (p pathResult) AddTotalNum() {
p.totalNum += 1
}
func (p pathResult) AddErrInfo(path string) {
p.errNum += 1
p.errPath = append(p.errPath, path)
}
type Record interface {
Select() *record
}
type record struct {
Stats bool
Hash string
File string
Size int
}
func NewRecord(stats bool, hash, file string, size int) *record {
return &record{
Stats: stats,
Hash: hash,
File: file,
Size: size,
}
}
func (r record) Select() *record {
return &r
}
type Result interface {
Insert(record Record)
Select() (totalSize int64, totalNum int, errNum int, errFile []string)
}
type path struct {
totalNum int
errNum int
errPath []string
}
type read struct {
totalSize int
totalNum int
errNum int
errFile []string
}
func NewRead() *read {
return &read{
totalSize: 0,
totalNum: 0,
errNum: 0,
errFile: make([]string, 0),
}
}
type write struct {
totalSize int
totalNum int
errNum int
errFile []string
}
func NewWrite() *write {
return &write{
totalSize: 0,
totalNum: 0,
errNum: 0,
errFile: make([]string, 0),
}
}
type link struct {
totalSize int
totalNum int
errNum int
errFile []string
}
func NewLink() *link {
return &link{
totalSize: 0,
totalNum: 0,
errNum: 0,
errFile: make([]string, 0),
}
}
func (r read) Select() (totalSize int, totalNum int, errNum int, errFile []string) {
return r.totalSize, r.totalNum, r.errNum, r.errFile
}
func (r write) Select() (totalSize int, totalNum int, errNum int, errFile []string) {
return r.totalSize, r.totalNum, r.errNum, r.errFile
}
func (r link) Select() (totalSize int, totalNum int, errNum int, errFile []string) {
return r.totalSize, r.totalNum, r.errNum, r.errFile
}
func (r read) Insert(record Record) {
r.totalNum += 1
r.totalSize += record.Select().Size
if !record.Select().Stats {
r.errNum += 1
r.errFile = append(r.errFile, record.Select().File)
}
}
func (r write) Insert(record Record) {
r.totalNum += 1
r.totalSize += record.Select().Size
if !record.Select().Stats {
r.errNum += 1
r.errFile = append(r.errFile, record.Select().File)
}
}
func (r link) Insert(record Record) {
r.totalNum += 1
r.totalSize += record.Select().Size
if !record.Select().Stats {
r.errNum += 1
r.errFile = append(r.errFile, record.Select().File)
}
}
*/
57 changes: 29 additions & 28 deletions write/write.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,33 +40,33 @@ func Write(dm string) {
func dryRunFile() {
wg := sync.WaitGroup{}
for hash, files := range model.FileMap {
wg.Add(1)
model.ControlCH <- 1
go func(hash string, files []string) {
defer wg.Done()
defer func() { <-model.ControlCH }()
if len(files) > 1 {
for _, file := range files[1:] {
if len(files) > 1 {
for _, file := range files[1:] {
wg.Add(1)
model.ControlCH <- 1
go func(hash string, file string) {
defer wg.Done()
defer func() { <-model.ControlCH }()
fmt.Printf("[INFO]: Duplicate File: %s\n", file)
rd := model.NewWrite(true, hash, file, model.FileSize[hash])
model.RecordCH <- rd
}
}(hash, file)
}
}(hash, files)
}
}
wg.Wait()
}

func removeFile() {
wg := sync.WaitGroup{}
for hash, files := range model.FileMap {
wg.Add(1)
model.ControlCH <- 1
go func(hash string, files []string) {
defer wg.Done()
defer func() { <-model.ControlCH }()
if len(files) > 1 {
for _, file := range files[1:] {
if len(files) > 1 {
for _, file := range files[1:] {
wg.Add(1)
model.ControlCH <- 1
go func(hash string, file string) {
defer wg.Done()
defer func() { <-model.ControlCH }()
var stat = true
err := os.Remove(file)
if err != nil {
Expand All @@ -77,24 +77,24 @@ func removeFile() {
}
rd := model.NewWrite(stat, hash, file, model.FileSize[hash])
model.RecordCH <- rd
}
}(hash, file)
}

}(hash, files)
}
}
wg.Wait()
}

func linkFile() {
wg := sync.WaitGroup{}
for hash, files := range model.FileMap {
wg.Add(1)
model.ControlCH <- 1
go func(hash string, files []string) {
defer wg.Done()
defer func() { <-model.ControlCH }()
if len(files) > 1 {
for _, file := range files[1:] {
if len(files) > 1 {
for _, file := range files[1:] {
wg.Add(1)
model.ControlCH <- 1
go func(hash string, file string) {
defer wg.Done()
defer func() { <-model.ControlCH }()

var stat = true
err := os.Link(files[0], file)
if err != nil {
Expand All @@ -105,9 +105,10 @@ func linkFile() {
}
rd := model.NewLink(stat, files[0], file)
model.RecordCH <- rd
}

}(hash, file)
}
}(hash, files)
}
}
wg.Wait()
}

0 comments on commit 7d0e7ec

Please sign in to comment.