Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement elastic search issue indexer interface #6150

Closed
wants to merge 11 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
6 changes: 6 additions & 0 deletions .drone.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ services:
pull: default
image: gitea/test-openldap:latest

- name: elasticsearch
pull: default
environment:
- "discovery.type=single-node"
image: elastic/elasticsearch:5.6.16

steps:
- name: fetch-tags
pull: default
Expand Down
5 changes: 2 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ require (
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
github.com/denisenkom/go-mssqldb v0.0.0-20190121005146-b04fd42d9952
github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect
github.com/emirpasic/gods v1.12.0
Expand Down Expand Up @@ -123,6 +123,7 @@ require (
gopkg.in/ini.v1 v1.42.0
gopkg.in/ldap.v3 v3.0.2
gopkg.in/macaron.v1 v1.3.2
gopkg.in/olivere/elastic.v5 v5.0.79
gopkg.in/redis.v2 v2.3.2 // indirect
gopkg.in/src-d/go-billy.v4 v4.3.0
gopkg.in/src-d/go-git.v4 v4.12.0
Expand All @@ -134,5 +135,3 @@ require (
xorm.io/builder v0.3.5
xorm.io/core v0.6.3
)

replace github.com/denisenkom/go-mssqldb => github.com/denisenkom/go-mssqldb v0.0.0-20180314172330-6a30f4e59a44
15 changes: 13 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,9 @@ github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.0.0-20180314172330-6a30f4e59a44 h1:x0uHqLQTSEL9LKic8sWDt3ASkq07ve5ojIIUl5uF64M=
github.com/denisenkom/go-mssqldb v0.0.0-20180314172330-6a30f4e59a44/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc=
github.com/denisenkom/go-mssqldb v0.0.0-20190121005146-b04fd42d9952/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc=
github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0 h1:epsH3lb7KVbXHYk7LYGN5EiE0MxcevHU85CKITJ0wUY=
github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
Expand Down Expand Up @@ -97,6 +98,8 @@ github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQD
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw+Q=
github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gliderlabs/ssh v0.1.3/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
Expand Down Expand Up @@ -144,6 +147,8 @@ github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561/go.mod h1:YgYOrVn3N
github.com/gogits/cron v0.0.0-20160810035002-7f3990acf183 h1:EBTlva3AOSb80G3JSwY6ZMdILEZJ1JKuewrbqrNjWuE=
github.com/gogits/cron v0.0.0-20160810035002-7f3990acf183/go.mod h1:pX+V62FFmklia2fhP3P4YSY6iJdPO5jIDKFQ5fEd5QE=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
Expand Down Expand Up @@ -232,6 +237,8 @@ github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de h1:nyxwRdWHAVxpFcDThedEg
github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de/go.mod h1:3q8WtuPQsoRbatJuy3nvq/hRSvuBJrHHr+ybPPiNvHQ=
github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af h1:UaWHNBdukWrSG3DRvHFR/hyfg681fceqQDYVTBncKfQ=
github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af/go.mod h1:Cqz6pqow14VObJ7peltM+2n3PWOz7yTrfUuGbVFkzN0=
github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5 h1:0x4qcEHDpruK6ML/m/YSlFUUu0UpRD3I2PHsNCuGnyA=
github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/markbates/going v1.0.0/go.mod h1:I6mnB4BPnEeqo85ynXIx1ZFLLbtiLHNXVgWeFO9OGOA=
github.com/markbates/goth v1.49.0 h1:qQ4Ti4WaqAxNAggOC+4s5M85sMVfMJwQn/Xkp73wfgI=
github.com/markbates/goth v1.49.0/go.mod h1:zZmAw0Es0Dpm7TT/4AdN14QrkiWLMrrU9Xei1o+/mdA=
Expand Down Expand Up @@ -317,6 +324,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY=
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff h1:86HlEv0yBCry9syNuylzqznKXDK11p6D0DT596yNMys=
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs=
Expand Down Expand Up @@ -357,6 +365,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190122013713-64072686203f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand Down Expand Up @@ -431,6 +440,8 @@ gopkg.in/ldap.v3 v3.0.2 h1:R6RBtabK6e1GO0eQKtkyOFbAHO73QesLzI2w2DZ6b9w=
gopkg.in/ldap.v3 v3.0.2/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw=
gopkg.in/macaron.v1 v1.3.2 h1:AvWIaPmwBUA87/OWzePkoxeaw6YJWDfBt1pDFPBnLf8=
gopkg.in/macaron.v1 v1.3.2/go.mod h1:PrsiawTWAGZs6wFbT5hlr7SQ2Ns9h7cUVtcUu4lQOVo=
gopkg.in/olivere/elastic.v5 v5.0.79 h1:q+FQfSQxl+xIHoEwq8RGBsb5pRB9f8rfaLh4D9jx18A=
gopkg.in/olivere/elastic.v5 v5.0.79/go.mod h1:uhHoB4o3bvX5sorxBU29rPcmBQdV2Qfg0FBrx5D6pV0=
gopkg.in/redis.v2 v2.3.2 h1:GPVIIB/JnL1wvfULefy3qXmPu1nfNu2d0yA09FHgwfs=
gopkg.in/redis.v2 v2.3.2/go.mod h1:4wl9PJ/CqzeHk3LVq1hNLHH8krm3+AXEgut4jVc++LU=
gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek=
Expand Down
9 changes: 9 additions & 0 deletions integrations/issue_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,15 @@ func testNewIssue(t *testing.T, session *TestSession, user, repo, title, content
val = htmlDoc.doc.Find(".comment-list .comments .comment .render-content p").First().Text()
assert.Equal(t, content, val)

if setting.Indexer.IssueType == "elasticsearch" {
req := NewRequest(t, "GET", path.Join(user, repo, "issues?type=all&state=open&labels=0&milestone=0&assignee=0&q=template"))
resp := session.MakeRequest(t, req, http.StatusOK)

htmlDoc := NewHTMLParser(t, resp.Body)
text := htmlDoc.doc.Find("issue list").Find("title has-emoji").Text()
assert.EqualValues(t, text, "The template has changed")
}

return issueURL
}

Expand Down
3 changes: 3 additions & 0 deletions integrations/mysql.ini.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ PASSWD = {{TEST_MYSQL_PASSWORD}}
SSL_MODE = disable

[indexer]
ISSUE_INDEXER_TYPE = elasticsearch
ISSUE_INDEXER_CONN_STR = http://elastic:changeme@elasticsearch:9200
ISSUE_INDEXER_QUEUE_TYPE = noqueue
ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve
REPO_INDEXER_ENABLED = true
REPO_INDEXER_PATH = integrations/indexers-mysql/repos.bleve
Expand Down
253 changes: 253 additions & 0 deletions modules/indexer/issues/elastic_search.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,253 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package issues

import (
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"time"

"code.gitea.io/gitea/modules/log"

"gopkg.in/olivere/elastic.v5"
)

var (
_ Indexer = &ElasticSearchIndexer{}
)

// ElasticSearchIndexer implements Indexer interface
type ElasticSearchIndexer struct {
client *elastic.Client
indexerName string
typeName string
customMappingFilePath string
}

type elasticLogger struct {
*log.Logger
}

func (l elasticLogger) Printf(format string, args ...interface{}) {
l.Logger.Log(2, l.Logger.GetLevel(), format, args...)
zeripath marked this conversation as resolved.
Show resolved Hide resolved
}

// NewElasticSearchIndexer creates a new elasticsearch indexer
func NewElasticSearchIndexer(url, indexerName, customMappingFilePath string) (*ElasticSearchIndexer, error) {
opts := []elastic.ClientOptionFunc{
elastic.SetURL(url),
elastic.SetSniff(false),
elastic.SetHealthcheckInterval(10 * time.Second),
elastic.SetGzip(false),
}

logger := elasticLogger{log.GetLogger(log.DEFAULT)}

if logger.GetLevel() == log.TRACE || logger.GetLevel() == log.DEBUG {
opts = append(opts, elastic.SetTraceLog(logger))
} else if logger.GetLevel() == log.ERROR || logger.GetLevel() == log.CRITICAL || logger.GetLevel() == log.FATAL {
opts = append(opts, elastic.SetErrorLog(logger))
} else if logger.GetLevel() == log.INFO || logger.GetLevel() == log.WARN {
opts = append(opts, elastic.SetInfoLog(logger))
}

client, err := elastic.NewClient(opts...)
if err != nil {
return nil, err
}

return &ElasticSearchIndexer{
client: client,
indexerName: indexerName,
typeName: "indexer_data",
customMappingFilePath: customMappingFilePath,
}, nil
}

const (
defaultMapping = `{
"settings":{
"number_of_shards":1,
"number_of_replicas":0
},
"mappings":{
"indexer_data":{
"properties":{
"id":{
"type":"integer",
"index": false
},
"repo_id":{
"type":"integer",
"index": true
},
"title":{
"type":"text",
"index": true
},
"content":{
"type":"text",
"index": true
},
"comments":{
"type" : "text",
"index": true
}
}
}
}
}`
)

// Init will initial the indexer
func (b *ElasticSearchIndexer) Init() (bool, error) {
ctx := context.Background()
exists, err := b.client.IndexExists(b.indexerName).Do(ctx)
if err != nil {
return false, err
}

if !exists {
var mapping = defaultMapping
if len(b.customMappingFilePath) > 0 {
bs, err := ioutil.ReadFile(b.customMappingFilePath)
if err != nil {
return false, err
}
mapping = string(bs)
}

createIndex, err := b.client.CreateIndex(b.indexerName).BodyString(mapping).Do(ctx)
if err != nil {
return false, err
}
if !createIndex.Acknowledged {
return false, errors.New("init failed")
}

return false, nil
}
return true, nil
}

// Index will save the index data
func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
if len(issues) == 0 {
return nil
} else if len(issues) == 1 {
issue := issues[0]
_, err := b.client.Index().
Index(b.indexerName).
Type(b.typeName).
Id(fmt.Sprintf("%d", issue.ID)).
BodyJson(map[string]interface{}{
"id": issue.ID,
"repo_id": issue.RepoID,
"title": issue.Title,
"content": issue.Content,
"comments": issue.Comments,
}).
Do(context.Background())
return err
}

reqs := make([]elastic.BulkableRequest, 0)
for _, issue := range issues {
reqs = append(reqs,
elastic.NewBulkIndexRequest().
Index(b.indexerName).
Type(b.typeName).
Id(fmt.Sprintf("%d", issue.ID)).
Doc(map[string]interface{}{
"id": issue.ID,
"repo_id": issue.RepoID,
"title": issue.Title,
"content": issue.Content,
"comments": issue.Comments,
}),
)
}

_, err := b.client.Bulk().
Index(b.indexerName).
Type(b.typeName).
Add(reqs...).
Do(context.Background())
return err
}

// Delete deletes indexes by ids
func (b *ElasticSearchIndexer) Delete(ids ...int64) error {
if len(ids) == 0 {
return nil
} else if len(ids) == 1 {
_, err := b.client.Delete().
Index(b.indexerName).
Type(b.typeName).
Id(fmt.Sprintf("%d", ids[0])).
Do(context.Background())
return err
}

reqs := make([]elastic.BulkableRequest, 0)
for _, id := range ids {
reqs = append(reqs,
elastic.NewBulkDeleteRequest().
Index(b.indexerName).
Type(b.typeName).
Id(fmt.Sprintf("%d", id)),
)
}

_, err := b.client.Bulk().
Index(b.indexerName).
Type(b.typeName).
Add(reqs...).
Do(context.Background())
return err
}

// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *ElasticSearchIndexer) Search(keyword string, repoID int64, limit, start int) (*SearchResult, error) {
kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
query := elastic.NewBoolQuery()
query = query.Filter(kwQuery)
if repoID > 0 {
repoQuery := elastic.NewTermQuery("repo_id", repoID)
query = query.Must(repoQuery)
}
searchResult, err := b.client.Search().
Index(b.indexerName).
Type(b.typeName).
Query(query).
Sort("id", true).
From(start).Size(limit).
Do(context.Background())
if err != nil {
return nil, err
}

hits := make([]Match, 0, limit)
for _, hit := range searchResult.Hits.Hits {
items := make(map[string]interface{})
err := json.Unmarshal(*hit.Source, &items)
if err != nil {
return nil, err
}
hits = append(hits, Match{
ID: int64(items["id"].(float64)),
RepoID: int64(items["repo_id"].(float64)),
})
}

return &SearchResult{
Total: searchResult.TotalHits(),
Hits: hits,
}, nil
}
Loading