Close #115 migrate tests to new format

Makefile

@@ -40,17 +40,17 @@ asset:
 	cd io/fs; \
 	$(FLAGS_SHARED) go run -tags=dev assets_gen.go
 
-build: peg asset
+build: peg
 	cd gnparser; \
 	$(GOCLEAN); \
 	$(FLAGS_SHARED) $(NO_C) $(GOBUILD)
 
-install: peg asset
+install: peg
 	cd gnparser; \
 	$(GOCLEAN); \
 	$(FLAGS_SHARED) $(NO_C) $(GOINSTALL)
 
-release: peg asset dockerhub
+release: peg dockerhub
 	cd gnparser; \
 	$(GOCLEAN); \
 	$(FLAGS_LINUX) $(NO_C) $(GOBUILD); \

config/config.go

@@ -19,6 +19,9 @@ type Config struct {
 	// a stream of name-strings.
 	JobsNum int
 
+	// BatchSize sets the maximum number of elements in names-strings slice.
+	BatchSize int
+
 	// KeepHTMLTags can be set to true when it is desirable to clean up names from
 	// a few HTML tags often present in names-strings that were planned to be
 	// presented via an HTML page.
@@ -43,6 +46,7 @@ func NewConfig(opts ...Option) Config {
 	cfg := Config{
 		Format:       format.CSV,
 		JobsNum:      runtime.NumCPU(),
+		BatchSize:    50_000,
 		KeepHTMLTags: false,
 		Port:         8080,
 	}
@@ -92,9 +96,27 @@ func OptWithDetails(b bool) Option {
 	}
 }
 
+// OptBatchSize sets the max number of names in a batch.
+func OptBatchSize(i int) Option {
+	return func(cfg *Config) {
+		if i <= 0 {
+			log.Println("Batch size should be a positive number")
+			return
+		}
+		cfg.BatchSize = i
+	}
+}
+
 // OptPort sets a port for web-service.
 func OptPort(i int) Option {
 	return func(cfg *Config) {
 		cfg.Port = i
 	}
 }
+
+// OptIsTest sets a test flag.
+func OptIsTest(b bool) Option {
+	return func(cfg *Config) {
+		cfg.IsTest = b
+	}
+}

config/config_test.go

@@ -14,6 +14,7 @@ func TestNew(t *testing.T) {
 	deflt := config.Config{
 		Format:       format.CSV,
 		JobsNum:      runtime.NumCPU(),
+		BatchSize:    50_000,
 		KeepHTMLTags: false,
 		WithDetails:  false,
 		Port:         8080,
@@ -28,6 +29,7 @@ func TestNewOpts(t *testing.T) {
 	updt := config.Config{
 		Format:       format.CompactJSON,
 		JobsNum:      161,
+		BatchSize:    1,
 		KeepHTMLTags: true,
 		WithDetails:  true,
 		Port:         8989,
@@ -39,6 +41,7 @@ func opts() []config.Option {
 	return []config.Option{
 		config.OptFormat("compact"),
 		config.OptJobsNum(161),
+		config.OptBatchSize(1),
 		config.OptKeepHTMLTags(true),
 		config.OptWithDetails(true),
 		config.OptPort(8989),

entity/output/annotation.go

@@ -11,24 +11,24 @@ const (
 	NoAnnot Annotation = iota
 	SurrogateAnnot
 	ComparisonAnnot
+	ApproximationAnnot
 	BOLDAnnot
 	HybridAnnot
-	ApproximationAnnot
 	NamedHybridAnnot
 	HybridFormulaAnnot
 	NothoHybridAnnot
 )
 
 var annotMap = map[Annotation]string{
 	NoAnnot:            "",
-	SurrogateAnnot:     "a surrogate name",
-	ComparisonAnnot:    "comparison",
-	BOLDAnnot:          "BOLD surrogate",
-	HybridAnnot:        "a hybrid name",
-	ApproximationAnnot: "approximation",
-	NamedHybridAnnot:   "named hybrid",
-	HybridFormulaAnnot: "hybrid formula",
-	NothoHybridAnnot:   "notho- hybrid",
+	SurrogateAnnot:     "SURROGATE",
+	ComparisonAnnot:    "COMPARISON",
+	ApproximationAnnot: "APPROXIMATION",
+	BOLDAnnot:          "BOLD_SURROGATE",
+	HybridAnnot:        "HYBRID",
+	NamedHybridAnnot:   "NAMED_HYBRID",
+	HybridFormulaAnnot: "HYBRID_FORMULA",
+	NothoHybridAnnot:   "NOTHO_HYBRID",
 }
 
 var annotStrMap = func() map[string]Annotation {

entity/output/annotation_test.go

@@ -14,9 +14,9 @@ func TestStringAnnot(t *testing.T) {
 		res   string
 	}{
 		{out.NoAnnot, ""},
-		{out.ComparisonAnnot, "Comparison"},
-		{out.ApproxSurrogateAnnot, "Approx. surrogate"},
-		{out.SurrogateAnnot, "Surrogate"},
+		{out.ComparisonAnnot, "COMPARISON"},
+		{out.ApproximationAnnot, "APPROXIMATION"},
+		{out.SurrogateAnnot, "SURROGATE"},
 	}
 
 	for i := range data {
@@ -37,7 +37,7 @@ func TestJSONAnnot(t *testing.T) {
 		{dataOb{"None", out.NoAnnot, []int{}},
 			`{"f1":"None","annot":"","f2":[]}`},
 		{dataOb{"Comparison", out.ComparisonAnnot, []int{2, 3, 4}},
-			`{"f1":"Comparison","annot":"Comparison","f2":[2,3,4]}`},
+			`{"f1":"Comparison","annot":"COMPARISON","f2":[2,3,4]}`},
 	}
 	enc := encode.GNjson{}
 	var dob dataOb

entity/output/details.go

@@ -1,7 +1,7 @@
 package output
 
 type Uninomial struct {
-	Uninomial  string      `json:"uninomial"`
+	Value      string      `json:"uninomial"`
 	Rank       string      `json:"rank,omitempty"`
 	Parent     string      `json:"parent,omitempty"`
 	Authorship *Authorship `json:"authorship,omitempty"`
@@ -27,15 +27,15 @@ type InfraSpeciesElem struct {
 
 type Comparison struct {
 	Genus             string      `json:"genus"`
-	Species           string      `json:"species"`
-	SpeciesAuthorship *Authorship `json:"speciesAuthorship,omitempty"`
+	Species           string      `json:"species,omitempty"`
+	SpeciesAuthorship *Authorship `json:"authorship,omitempty"`
 	CompMarker        string      `json:"comparisonMarker"`
 }
 
 type Approximation struct {
 	Genus             string      `json:"genus"`
-	Species           string      `json:"species"`
-	SpeciesAuthorship *Authorship `json:"speciesAuthorship"`
+	Species           string      `json:"species,omitempty"`
+	SpeciesAuthorship *Authorship `json:"authorship,omitempty"`
 	ApproxMarker      string      `json:"approximationMarker,omitempty"`
 	Ignored           string      `json:"ignored,omitempty"`
 }

entity/output/output.go

@@ -45,7 +45,7 @@ func (p Parsed) csvOutput() string {
 		simple,
 		full,
 		authorship,
-		strconv.Itoa(p.OverallQuality),
+		strconv.Itoa(p.ParseQuality),
 	}
 	return gncsv.ToCSV(res)
 }

entity/output/parsed.go

@@ -6,15 +6,15 @@ import (
 
 type Parsed struct {
 	Parsed          bool             `json:"parsed"`
-	OverallQuality  int              `json:"parseQuality"`
+	ParseQuality    int              `json:"parseQuality"`
 	QualityWarnings []QualityWarning `json:"qualityWarnings,omitempty"`
 	Verbatim        string           `json:"verbatim"`
 	Normalized      string           `json:"normalized,omitempty"`
 	Canonical       *Canonical       `json:"canonical,omitempty"`
 	Cardinality     int              `json:"cardinality"`
 	Authorship      *Authorship      `json:"authorship,omitempty"`
-	Bacteria        *tb.Tribool      `json:"isBacteria,omitempty"`
-	Virus           bool             `json:"isVirus,omitempty"`
+	Bacteria        *tb.Tribool      `json:"bacteria,omitempty"`
+	Virus           bool             `json:"virus,omitempty"`
 	Hybrid          *Annotation      `json:"hybrid,omitempty"`
 	Surrogate       *Annotation      `json:"surrogate,omitempty"`
 	Tail            string           `json:"tail,omitempty"`
@@ -53,7 +53,7 @@ type Authors struct {
 
 type Year struct {
 	Value         string `json:"year"`
-	IsApproximate bool   `json:"isApproximate"`
+	IsApproximate bool   `json:"isApproximate,omitempty"`
 }
 
 type Position struct {

entity/output/pos_words.go

@@ -28,8 +28,8 @@ const (
 
 var wordTypeMap = map[WordType]string{
 	UnknownType:          "word",
-	ComparisonType:       "comparison",
-	ApproxType:           "approximation",
+	ComparisonType:       "comparisonMarker",
+	ApproxType:           "approximationMarker",
 	AuthorWordType:       "authorWord",
 	AuthorWordFiliusType: "authorWordFilius",
 	GenusType:            "genus",

entity/parser/ast.go

@@ -26,14 +26,13 @@ type scientificNameNode struct {
 	annotation    *o.Annotation
 	tail          string
 	parserVersion string
-	warnings      []o.Warning
+	warnings      map[o.Warning]struct{}
 }
 
 func (p *Engine) newScientificNameNode() {
 	n := p.root.up
 	var name nameData
 	var tail string
-	var annot o.Annotation
 
 	for n != nil {
 		switch n.token32.pegRule {
@@ -44,17 +43,6 @@ func (p *Engine) newScientificNameNode() {
 		}
 		n = n.next
 	}
-	warns := make([]o.Warning, len(p.warnings))
-	i := 0
-	for k := range p.warnings {
-		warns[i] = k
-		i++
-	}
-	if str.IsBoldSurrogate(tail) {
-		p.cardinality = 0
-		annot = o.BOLDAnnot
-		p.surrogate = &annot
-	}
 	if p.tail != "" && tail == "" {
 		tail = p.tail
 	}
@@ -65,7 +53,6 @@ func (p *Engine) newScientificNameNode() {
 		surrogate:   p.surrogate,
 		bacteria:    p.bacteria,
 		tail:        tail,
-		warnings:    warns,
 	}
 	p.sn = &sn
 }
@@ -85,7 +72,6 @@ func (p *Engine) tailValue(n *node32) string {
 	if t.begin == t.end {
 		return ""
 	}
-	p.addWarn(o.TailWarn)
 	return string(p.buffer[t.begin:t.end])
 }
 
@@ -198,7 +184,6 @@ type namedGenusHybridNode struct {
 func (p *Engine) newNamedGenusHybridNode(n *node32) *namedGenusHybridNode {
 	var nhn *namedGenusHybridNode
 	var name nameData
-	var annot o.Annotation
 	n = n.up
 	if n.token32.pegRule != ruleHybridChar {
 		return nhn
@@ -219,9 +204,6 @@ func (p *Engine) newNamedGenusHybridNode(n *node32) *namedGenusHybridNode {
 	case ruleNameSpecies:
 		name = p.newSpeciesNode(n)
 	case ruleNameApprox:
-		annot = o.ApproximationAnnot
-		p.surrogate = &annot
-		p.addWarn(o.NameApproxWarn)
 		name = p.newApproxNode(n)
 	}
 	nhn = &namedGenusHybridNode{
@@ -332,9 +314,6 @@ func (p *Engine) newSingleName(n *node32) nameData {
 	case ruleNameSpecies:
 		name = p.newSpeciesNode(n)
 	case ruleNameApprox:
-		p.addWarn(o.NameApproxWarn)
-		annot = o.ApproximationAnnot
-		p.surrogate = &annot
 		name = p.newApproxNode(n)
 	case ruleNameComp:
 		p.addWarn(o.NameComparisonWarn)
@@ -362,12 +341,15 @@ type approxNode struct {
 
 func (p *Engine) newApproxNode(n *node32) *approxNode {
 	var an *approxNode
+	annot := o.ApproximationAnnot
+	p.surrogate = &annot
+	p.addWarn(o.NameApproxWarn)
 	if n.token32.pegRule != ruleNameApprox {
 		return an
 	}
 	var gen *wordNode
 	var spEp *spEpithetNode
-	var annot *wordNode
+	var appr *wordNode
 	var ign string
 	n = n.up
 	for n != nil {
@@ -377,7 +359,7 @@ func (p *Engine) newApproxNode(n *node32) *approxNode {
 		case ruleSpeciesEpithet:
 			spEp = p.newSpeciesEpithetNode(n)
 		case ruleApproximation:
-			annot = p.newWordNode(n, o.ApproxType)
+			appr = p.newWordNode(n, o.ApproxType)
 		case ruleApproxNameIgnored:
 			ign = p.nodeValue(n)
 		}
@@ -386,7 +368,7 @@ func (p *Engine) newApproxNode(n *node32) *approxNode {
 	an = &approxNode{
 		Genus:     gen,
 		SpEpithet: spEp,
-		Approx:    annot,
+		Approx:    appr,
 		Ignored:   ign,
 	}
 	p.cardinality = 0

entity/parser/name.go

@@ -525,7 +525,7 @@ func (u *uninomialNode) lastAuthorship() *authorshipNode {
 }
 
 func (u *uninomialNode) details() o.Details {
-	ud := o.Uninomial{Uninomial: u.Word.NormValue}
+	ud := o.Uninomial{Value: u.Word.NormValue}
 	if u.Authorship != nil {
 		ud.Authorship = u.Authorship.details()
 	}
@@ -568,9 +568,9 @@ func (u *uninomialComboNode) lastAuthorship() *authorshipNode {
 
 func (u *uninomialComboNode) details() o.Details {
 	ud := o.Uninomial{
-		Uninomial: u.Uninomial2.Word.NormValue,
-		Rank:      u.Rank.Word.NormValue,
-		Parent:    u.Uninomial1.Word.NormValue,
+		Value:  u.Uninomial2.Word.NormValue,
+		Rank:   u.Rank.Word.NormValue,
+		Parent: u.Uninomial1.Word.NormValue,
 	}
 	if u.Uninomial2.Authorship != nil {
 		ud.Authorship = u.Uninomial2.Authorship.details()

entity/parser/output.go

@@ -20,11 +20,12 @@ func (sn *scientificNameNode) ToOutput(withDetails bool) o.Parsed {
 	}
 
 	res.Parsed = true
-	res.OverallQuality, res.QualityWarnings = processWarnings(sn.warnings)
+	res.ParseQuality, res.QualityWarnings = qualityWarnings(sn.warnings)
 	res.Normalized = sn.Normalized()
 	res.Cardinality = sn.cardinality
 	res.Authorship = sn.LastAuthorship(withDetails)
 	res.Hybrid = sn.hybrid
+	res.Surrogate = sn.surrogate
 	res.Bacteria = sn.bacteria
 	res.Tail = sn.tail
 	if withDetails {
@@ -34,7 +35,7 @@ func (sn *scientificNameNode) ToOutput(withDetails bool) o.Parsed {
 	return res
 }
 
-func processWarnings(ws []o.Warning) (int, []o.QualityWarning) {
+func qualityWarnings(ws map[o.Warning]struct{}) (int, []o.QualityWarning) {
 	warns := prepareWarnings(ws)
 	quality := 1
 	if len(warns) > 0 {
@@ -43,10 +44,12 @@ func processWarnings(ws []o.Warning) (int, []o.QualityWarning) {
 	return quality, warns
 }
 
-func prepareWarnings(ws []o.Warning) []o.QualityWarning {
+func prepareWarnings(ws map[o.Warning]struct{}) []o.QualityWarning {
 	res := make([]o.QualityWarning, len(ws))
-	for i := range ws {
-		res[i] = ws[i].NewQualityWarning()
+	var i int
+	for k := range ws {
+		res[i] = k.NewQualityWarning()
+		i++
 	}
 
 	sort.Slice(res, func(i, j int) bool {