-
Notifications
You must be signed in to change notification settings - Fork 2
/
parser_csv.go
121 lines (99 loc) · 2.44 KB
/
parser_csv.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
/*
* Copyright (c) 2016 Yanko Bolanos
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*
*/
package main
import (
"bytes"
"encoding/csv"
"log"
"strings"
"time"
)
type CSVParser struct {
App string
AppVer string
Filename string
Hostname string
FieldsOrder []string
Table []Attribute
Delimiter rune
}
func NewCSVParser(app, appVer, filename, hostname string, fieldsOrder []string, defaultTable []Attribute, options []string) *CSVParser {
delimiter := rune(',')
parsedOptions := ParseOptions(options)
for k, v := range parsedOptions {
if k == "delimiter" {
runes := []rune(v)
if len(runes) > 0 {
delimiter = runes[0]
}
}
}
return &CSVParser{
App: app,
AppVer: appVer,
Filename: filename,
Hostname: hostname,
FieldsOrder: fieldsOrder,
Table: defaultTable,
Delimiter: delimiter,
}
}
func (p *CSVParser) Init(defaults, fieldMappings map[string]string, FieldsOrder []string, defaultTable []Attribute) {
}
func (p *CSVParser) GetTable() []Attribute {
return p.Table
}
func (p *CSVParser) Defaults() map[string]string {
d := make(map[string]string)
for _, k := range p.Table {
d[k.Key] = "\\N"
}
d["app"] = p.App
d["app_ver"] = p.AppVer
d["filename"] = p.Filename
d["hostname"] = p.Hostname
d["ingest_datetime"] = time.Now().UTC().Format(ISO_8601)
d["event_datetime"] = d["ingest_datetime"]
return d
}
func (p *CSVParser) Parse(line string) (map[string]string, error) {
result := p.Defaults()
r := csv.NewReader(strings.NewReader(line))
r.Comma = p.Delimiter
var cleanLogLine bytes.Buffer
record, err := r.Read()
if err != nil {
log.Printf(err.Error())
return result, err
}
if len(record) != len(p.FieldsOrder) {
return result, ErrCSVFieldsOrderDoNotMatch
}
for i, field := range p.FieldsOrder {
value := record[i]
skipField := false
if field == "" {
skipField = true
}
_, ok := result[field]
if skipField || !ok {
cleanLogLine.WriteString(value)
cleanLogLine.WriteString(" ")
}
if isNull(value) {
result[field] = "\\N"
} else {
result[field] = value
}
}
srcByte := cleanupPairs.ReplaceAll(cleanLogLine.Bytes(), []byte{})
srcByte = cleanupSpaces.ReplaceAll(srcByte, []byte{})
result["log_line"] = strings.TrimSpace(string(srcByte))
return result, err
}