mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-09-07 15:33:26 +00:00
Automatically pause queue if index service is unavailable (#15066)
* Handle keyword search error when issue indexer service is not available * Implement automatic disabling and resume of code indexer queue
This commit is contained in:
parent
2649eddcf0
commit
8038610a42
28 changed files with 522 additions and 151 deletions
|
@ -187,6 +187,27 @@ make lint-frontend
|
||||||
|
|
||||||
Note: When working on frontend code, set `USE_SERVICE_WORKER` to `false` in `app.ini` to prevent undesirable caching of frontend assets.
|
Note: When working on frontend code, set `USE_SERVICE_WORKER` to `false` in `app.ini` to prevent undesirable caching of frontend assets.
|
||||||
|
|
||||||
|
### Configuring local ElasticSearch instance
|
||||||
|
|
||||||
|
Start local ElasticSearch instance using docker:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
mkdir -p $(pwd)/data/elasticsearch
|
||||||
|
sudo chown -R 1000:1000 $(pwd)/data/elasticsearch
|
||||||
|
docker run --rm -p 127.0.0.1:9200:9200 -p 127.0.0.1:9300:9300 -e "discovery.type=single-node" -v "$(pwd)/data/elasticsearch:/usr/share/elasticsearch/data" docker.elastic.co/elasticsearch/elasticsearch:7.16.3
|
||||||
|
```
|
||||||
|
|
||||||
|
Configure `app.ini`:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
[indexer]
|
||||||
|
ISSUE_INDEXER_TYPE = elasticsearch
|
||||||
|
ISSUE_INDEXER_CONN_STR = http://elastic:changeme@localhost:9200
|
||||||
|
REPO_INDEXER_ENABLED = true
|
||||||
|
REPO_INDEXER_TYPE = elasticsearch
|
||||||
|
REPO_INDEXER_CONN_STR = http://elastic:changeme@localhost:9200
|
||||||
|
```
|
||||||
|
|
||||||
### Building and adding SVGs
|
### Building and adding SVGs
|
||||||
|
|
||||||
SVG icons are built using the `make svg` target which compiles the icon sources defined in `build/generate-svg.js` into the output directory `public/img/svg`. Custom icons can be added in the `web_src/svg` directory.
|
SVG icons are built using the `make svg` target which compiles the icon sources defined in `build/generate-svg.js` into the output directory `public/img/svg`. Custom icons can be added in the `web_src/svg` directory.
|
||||||
|
|
|
@ -17,10 +17,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func resultFilenames(t testing.TB, doc *HTMLDoc) []string {
|
func resultFilenames(t testing.TB, doc *HTMLDoc) []string {
|
||||||
resultsSelection := doc.doc.Find(".repository.search")
|
filenameSelections := doc.doc.Find(".repository.search").Find(".repo-search-result").Find(".header").Find("span.file")
|
||||||
assert.EqualValues(t, 1, resultsSelection.Length(),
|
|
||||||
"Invalid template (repo search template has changed?)")
|
|
||||||
filenameSelections := resultsSelection.Find(".repo-search-result").Find(".header").Find("span.file")
|
|
||||||
result := make([]string, filenameSelections.Length())
|
result := make([]string, filenameSelections.Length())
|
||||||
filenameSelections.Each(func(i int, selection *goquery.Selection) {
|
filenameSelections.Each(func(i int, selection *goquery.Selection) {
|
||||||
result[i] = selection.Text()
|
result[i] = selection.Text()
|
||||||
|
|
|
@ -65,6 +65,7 @@ type Engine interface {
|
||||||
Query(...interface{}) ([]map[string][]byte, error)
|
Query(...interface{}) ([]map[string][]byte, error)
|
||||||
Cols(...string) *xorm.Session
|
Cols(...string) *xorm.Session
|
||||||
Context(ctx context.Context) *xorm.Session
|
Context(ctx context.Context) *xorm.Session
|
||||||
|
Ping() error
|
||||||
}
|
}
|
||||||
|
|
||||||
// TableInfo returns table's information via an object
|
// TableInfo returns table's information via an object
|
||||||
|
|
|
@ -1859,7 +1859,7 @@ func GetRepoIssueStats(repoID, uid int64, filterMode int, isPull bool) (numOpen,
|
||||||
}
|
}
|
||||||
|
|
||||||
// SearchIssueIDsByKeyword search issues on database
|
// SearchIssueIDsByKeyword search issues on database
|
||||||
func SearchIssueIDsByKeyword(kw string, repoIDs []int64, limit, start int) (int64, []int64, error) {
|
func SearchIssueIDsByKeyword(ctx context.Context, kw string, repoIDs []int64, limit, start int) (int64, []int64, error) {
|
||||||
repoCond := builder.In("repo_id", repoIDs)
|
repoCond := builder.In("repo_id", repoIDs)
|
||||||
subQuery := builder.Select("id").From("issue").Where(repoCond)
|
subQuery := builder.Select("id").From("issue").Where(repoCond)
|
||||||
kw = strings.ToUpper(kw)
|
kw = strings.ToUpper(kw)
|
||||||
|
@ -1884,7 +1884,7 @@ func SearchIssueIDsByKeyword(kw string, repoIDs []int64, limit, start int) (int6
|
||||||
ID int64
|
ID int64
|
||||||
UpdatedUnix int64
|
UpdatedUnix int64
|
||||||
}, 0, limit)
|
}, 0, limit)
|
||||||
err := db.GetEngine(db.DefaultContext).Distinct("id", "updated_unix").Table("issue").Where(cond).
|
err := db.GetEngine(ctx).Distinct("id", "updated_unix").Table("issue").Where(cond).
|
||||||
OrderBy("`updated_unix` DESC").Limit(limit, start).
|
OrderBy("`updated_unix` DESC").Limit(limit, start).
|
||||||
Find(&res)
|
Find(&res)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -1894,7 +1894,7 @@ func SearchIssueIDsByKeyword(kw string, repoIDs []int64, limit, start int) (int6
|
||||||
ids = append(ids, r.ID)
|
ids = append(ids, r.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
total, err := db.GetEngine(db.DefaultContext).Distinct("id").Table("issue").Where(cond).Count()
|
total, err := db.GetEngine(ctx).Distinct("id").Table("issue").Where(cond).Count()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, err
|
return 0, nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
package models
|
package models
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"sync"
|
"sync"
|
||||||
|
@ -303,23 +304,23 @@ func TestIssue_loadTotalTimes(t *testing.T) {
|
||||||
|
|
||||||
func TestIssue_SearchIssueIDsByKeyword(t *testing.T) {
|
func TestIssue_SearchIssueIDsByKeyword(t *testing.T) {
|
||||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||||
total, ids, err := SearchIssueIDsByKeyword("issue2", []int64{1}, 10, 0)
|
total, ids, err := SearchIssueIDsByKeyword(context.TODO(), "issue2", []int64{1}, 10, 0)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, 1, total)
|
assert.EqualValues(t, 1, total)
|
||||||
assert.EqualValues(t, []int64{2}, ids)
|
assert.EqualValues(t, []int64{2}, ids)
|
||||||
|
|
||||||
total, ids, err = SearchIssueIDsByKeyword("first", []int64{1}, 10, 0)
|
total, ids, err = SearchIssueIDsByKeyword(context.TODO(), "first", []int64{1}, 10, 0)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, 1, total)
|
assert.EqualValues(t, 1, total)
|
||||||
assert.EqualValues(t, []int64{1}, ids)
|
assert.EqualValues(t, []int64{1}, ids)
|
||||||
|
|
||||||
total, ids, err = SearchIssueIDsByKeyword("for", []int64{1}, 10, 0)
|
total, ids, err = SearchIssueIDsByKeyword(context.TODO(), "for", []int64{1}, 10, 0)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, 5, total)
|
assert.EqualValues(t, 5, total)
|
||||||
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
|
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
|
||||||
|
|
||||||
// issue1's comment id 2
|
// issue1's comment id 2
|
||||||
total, ids, err = SearchIssueIDsByKeyword("good", []int64{1}, 10, 0)
|
total, ids, err = SearchIssueIDsByKeyword(context.TODO(), "good", []int64{1}, 10, 0)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, 1, total)
|
assert.EqualValues(t, 1, total)
|
||||||
assert.EqualValues(t, []int64{1}, ids)
|
assert.EqualValues(t, []int64{1}, ids)
|
||||||
|
@ -464,7 +465,7 @@ func TestCorrectIssueStats(t *testing.T) {
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
// Now we will get all issueID's that match the "Bugs are nasty" query.
|
// Now we will get all issueID's that match the "Bugs are nasty" query.
|
||||||
total, ids, err := SearchIssueIDsByKeyword("Bugs are nasty", []int64{1}, issueAmount, 0)
|
total, ids, err := SearchIssueIDsByKeyword(context.TODO(), "Bugs are nasty", []int64{1}, issueAmount, 0)
|
||||||
|
|
||||||
// Just to be sure.
|
// Just to be sure.
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
|
@ -21,6 +21,7 @@ import (
|
||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/cache"
|
"code.gitea.io/gitea/modules/cache"
|
||||||
"code.gitea.io/gitea/modules/git"
|
"code.gitea.io/gitea/modules/git"
|
||||||
|
code_indexer "code.gitea.io/gitea/modules/indexer/code"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/markup/markdown"
|
"code.gitea.io/gitea/modules/markup/markdown"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
@ -522,6 +523,9 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
|
||||||
ctx.Data["ExposeAnonSSH"] = setting.SSH.ExposeAnonymous
|
ctx.Data["ExposeAnonSSH"] = setting.SSH.ExposeAnonymous
|
||||||
ctx.Data["DisableHTTP"] = setting.Repository.DisableHTTPGit
|
ctx.Data["DisableHTTP"] = setting.Repository.DisableHTTPGit
|
||||||
ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
|
ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
|
||||||
|
if setting.Indexer.RepoIndexerEnabled {
|
||||||
|
ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
|
||||||
|
}
|
||||||
ctx.Data["CloneLink"] = repo.CloneLink()
|
ctx.Data["CloneLink"] = repo.CloneLink()
|
||||||
ctx.Data["WikiCloneLink"] = repo.WikiCloneLink()
|
ctx.Data["WikiCloneLink"] = repo.WikiCloneLink()
|
||||||
|
|
||||||
|
|
|
@ -271,6 +271,15 @@ func (b *BleveIndexer) Close() {
|
||||||
log.Info("PID: %d Repository Indexer closed", os.Getpid())
|
log.Info("PID: %d Repository Indexer closed", os.Getpid())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetAvailabilityChangeCallback does nothing
|
||||||
|
func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ping does nothing
|
||||||
|
func (b *BleveIndexer) Ping() bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// Index indexes the data
|
// Index indexes the data
|
||||||
func (b *BleveIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
|
func (b *BleveIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
|
||||||
batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
|
batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
|
||||||
|
@ -319,7 +328,7 @@ func (b *BleveIndexer) Delete(repoID int64) error {
|
||||||
|
|
||||||
// Search searches for files in the specified repo.
|
// Search searches for files in the specified repo.
|
||||||
// Returns the matching file-paths
|
// Returns the matching file-paths
|
||||||
func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
|
func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
|
||||||
var (
|
var (
|
||||||
indexerQuery query.Query
|
indexerQuery query.Query
|
||||||
keywordQuery query.Query
|
keywordQuery query.Query
|
||||||
|
@ -372,7 +381,7 @@ func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, p
|
||||||
searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
|
searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := b.indexer.Search(searchRequest)
|
result, err := b.indexer.SearchInContext(ctx, searchRequest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, nil, err
|
return 0, nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,16 +7,20 @@ package code
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
repo_model "code.gitea.io/gitea/models/repo"
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
"code.gitea.io/gitea/modules/analyze"
|
"code.gitea.io/gitea/modules/analyze"
|
||||||
"code.gitea.io/gitea/modules/charset"
|
"code.gitea.io/gitea/modules/charset"
|
||||||
"code.gitea.io/gitea/modules/git"
|
"code.gitea.io/gitea/modules/git"
|
||||||
|
"code.gitea.io/gitea/modules/graceful"
|
||||||
"code.gitea.io/gitea/modules/json"
|
"code.gitea.io/gitea/modules/json"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
@ -39,8 +43,12 @@ var _ Indexer = &ElasticSearchIndexer{}
|
||||||
|
|
||||||
// ElasticSearchIndexer implements Indexer interface
|
// ElasticSearchIndexer implements Indexer interface
|
||||||
type ElasticSearchIndexer struct {
|
type ElasticSearchIndexer struct {
|
||||||
client *elastic.Client
|
client *elastic.Client
|
||||||
indexerAliasName string
|
indexerAliasName string
|
||||||
|
available bool
|
||||||
|
availabilityCallback func(bool)
|
||||||
|
stopTimer chan struct{}
|
||||||
|
lock sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
type elasticLogger struct {
|
type elasticLogger struct {
|
||||||
|
@ -78,7 +86,23 @@ func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, bo
|
||||||
indexer := &ElasticSearchIndexer{
|
indexer := &ElasticSearchIndexer{
|
||||||
client: client,
|
client: client,
|
||||||
indexerAliasName: indexerName,
|
indexerAliasName: indexerName,
|
||||||
|
available: true,
|
||||||
|
stopTimer: make(chan struct{}),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ticker := time.NewTicker(10 * time.Second)
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
indexer.checkAvailability()
|
||||||
|
case <-indexer.stopTimer:
|
||||||
|
ticker.Stop()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
exists, err := indexer.init()
|
exists, err := indexer.init()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
indexer.Close()
|
indexer.Close()
|
||||||
|
@ -123,17 +147,17 @@ func (b *ElasticSearchIndexer) realIndexerName() string {
|
||||||
|
|
||||||
// Init will initialize the indexer
|
// Init will initialize the indexer
|
||||||
func (b *ElasticSearchIndexer) init() (bool, error) {
|
func (b *ElasticSearchIndexer) init() (bool, error) {
|
||||||
ctx := context.Background()
|
ctx := graceful.GetManager().HammerContext()
|
||||||
exists, err := b.client.IndexExists(b.realIndexerName()).Do(ctx)
|
exists, err := b.client.IndexExists(b.realIndexerName()).Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, b.checkError(err)
|
||||||
}
|
}
|
||||||
if !exists {
|
if !exists {
|
||||||
mapping := defaultMapping
|
mapping := defaultMapping
|
||||||
|
|
||||||
createIndex, err := b.client.CreateIndex(b.realIndexerName()).BodyString(mapping).Do(ctx)
|
createIndex, err := b.client.CreateIndex(b.realIndexerName()).BodyString(mapping).Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, b.checkError(err)
|
||||||
}
|
}
|
||||||
if !createIndex.Acknowledged {
|
if !createIndex.Acknowledged {
|
||||||
return false, fmt.Errorf("create index %s with %s failed", b.realIndexerName(), mapping)
|
return false, fmt.Errorf("create index %s with %s failed", b.realIndexerName(), mapping)
|
||||||
|
@ -143,7 +167,7 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
|
||||||
// check version
|
// check version
|
||||||
r, err := b.client.Aliases().Do(ctx)
|
r, err := b.client.Aliases().Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
realIndexerNames := r.IndicesByAlias(b.indexerAliasName)
|
realIndexerNames := r.IndicesByAlias(b.indexerAliasName)
|
||||||
|
@ -152,10 +176,10 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
|
||||||
Add(b.realIndexerName(), b.indexerAliasName).
|
Add(b.realIndexerName(), b.indexerAliasName).
|
||||||
Do(ctx)
|
Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, b.checkError(err)
|
||||||
}
|
}
|
||||||
if !res.Acknowledged {
|
if !res.Acknowledged {
|
||||||
return false, fmt.Errorf("")
|
return false, fmt.Errorf("create alias %s to index %s failed", b.indexerAliasName, b.realIndexerName())
|
||||||
}
|
}
|
||||||
} else if len(realIndexerNames) >= 1 && realIndexerNames[0] < b.realIndexerName() {
|
} else if len(realIndexerNames) >= 1 && realIndexerNames[0] < b.realIndexerName() {
|
||||||
log.Warn("Found older gitea indexer named %s, but we will create a new one %s and keep the old NOT DELETED. You can delete the old version after the upgrade succeed.",
|
log.Warn("Found older gitea indexer named %s, but we will create a new one %s and keep the old NOT DELETED. You can delete the old version after the upgrade succeed.",
|
||||||
|
@ -165,16 +189,30 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
|
||||||
Add(b.realIndexerName(), b.indexerAliasName).
|
Add(b.realIndexerName(), b.indexerAliasName).
|
||||||
Do(ctx)
|
Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, b.checkError(err)
|
||||||
}
|
}
|
||||||
if !res.Acknowledged {
|
if !res.Acknowledged {
|
||||||
return false, fmt.Errorf("")
|
return false, fmt.Errorf("change alias %s to index %s failed", b.indexerAliasName, b.realIndexerName())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return exists, nil
|
return exists, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
|
||||||
|
func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||||
|
b.lock.Lock()
|
||||||
|
defer b.lock.Unlock()
|
||||||
|
b.availabilityCallback = callback
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ping checks if elastic is available
|
||||||
|
func (b *ElasticSearchIndexer) Ping() bool {
|
||||||
|
b.lock.RLock()
|
||||||
|
defer b.lock.RUnlock()
|
||||||
|
return b.available
|
||||||
|
}
|
||||||
|
|
||||||
func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
|
func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
|
||||||
// Ignore vendored files in code search
|
// Ignore vendored files in code search
|
||||||
if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
|
if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
|
||||||
|
@ -190,7 +228,7 @@ func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.Wr
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if size, err = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64); err != nil {
|
if size, err = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64); err != nil {
|
||||||
return nil, fmt.Errorf("Misformatted git cat-file output: %v", err)
|
return nil, fmt.Errorf("misformatted git cat-file output: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -274,8 +312,8 @@ func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repos
|
||||||
_, err := b.client.Bulk().
|
_, err := b.client.Bulk().
|
||||||
Index(b.indexerAliasName).
|
Index(b.indexerAliasName).
|
||||||
Add(reqs...).
|
Add(reqs...).
|
||||||
Do(context.Background())
|
Do(ctx)
|
||||||
return err
|
return b.checkError(err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -284,8 +322,8 @@ func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repos
|
||||||
func (b *ElasticSearchIndexer) Delete(repoID int64) error {
|
func (b *ElasticSearchIndexer) Delete(repoID int64) error {
|
||||||
_, err := b.client.DeleteByQuery(b.indexerAliasName).
|
_, err := b.client.DeleteByQuery(b.indexerAliasName).
|
||||||
Query(elastic.NewTermsQuery("repo_id", repoID)).
|
Query(elastic.NewTermsQuery("repo_id", repoID)).
|
||||||
Do(context.Background())
|
Do(graceful.GetManager().HammerContext())
|
||||||
return err
|
return b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// indexPos find words positions for start and the following end on content. It will
|
// indexPos find words positions for start and the following end on content. It will
|
||||||
|
@ -366,7 +404,7 @@ func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search searches for codes and language stats by given conditions.
|
// Search searches for codes and language stats by given conditions.
|
||||||
func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
|
func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
|
||||||
searchType := esMultiMatchTypeBestFields
|
searchType := esMultiMatchTypeBestFields
|
||||||
if isMatch {
|
if isMatch {
|
||||||
searchType = esMultiMatchTypePhrasePrefix
|
searchType = esMultiMatchTypePhrasePrefix
|
||||||
|
@ -407,9 +445,9 @@ func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string,
|
||||||
).
|
).
|
||||||
Sort("repo_id", true).
|
Sort("repo_id", true).
|
||||||
From(start).Size(pageSize).
|
From(start).Size(pageSize).
|
||||||
Do(context.Background())
|
Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, nil, err
|
return 0, nil, nil, b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return convertResult(searchResult, kw, pageSize)
|
return convertResult(searchResult, kw, pageSize)
|
||||||
|
@ -421,9 +459,9 @@ func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string,
|
||||||
Aggregation("language", aggregation).
|
Aggregation("language", aggregation).
|
||||||
Query(query).
|
Query(query).
|
||||||
Size(0). // We only needs stats information
|
Size(0). // We only needs stats information
|
||||||
Do(context.Background())
|
Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, nil, err
|
return 0, nil, nil, b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
query = query.Must(langQuery)
|
query = query.Must(langQuery)
|
||||||
|
@ -438,9 +476,9 @@ func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string,
|
||||||
).
|
).
|
||||||
Sort("repo_id", true).
|
Sort("repo_id", true).
|
||||||
From(start).Size(pageSize).
|
From(start).Size(pageSize).
|
||||||
Do(context.Background())
|
Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, nil, err
|
return 0, nil, nil, b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
total, hits, _, err := convertResult(searchResult, kw, pageSize)
|
total, hits, _, err := convertResult(searchResult, kw, pageSize)
|
||||||
|
@ -449,4 +487,51 @@ func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close implements indexer
|
// Close implements indexer
|
||||||
func (b *ElasticSearchIndexer) Close() {}
|
func (b *ElasticSearchIndexer) Close() {
|
||||||
|
select {
|
||||||
|
case <-b.stopTimer:
|
||||||
|
default:
|
||||||
|
close(b.stopTimer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ElasticSearchIndexer) checkError(err error) error {
|
||||||
|
var opErr *net.OpError
|
||||||
|
if !(elastic.IsConnErr(err) || (errors.As(err, &opErr) && (opErr.Op == "dial" || opErr.Op == "read"))) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
b.setAvailability(false)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ElasticSearchIndexer) checkAvailability() {
|
||||||
|
if b.Ping() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request cluster state to check if elastic is available again
|
||||||
|
_, err := b.client.ClusterState().Do(graceful.GetManager().ShutdownContext())
|
||||||
|
if err != nil {
|
||||||
|
b.setAvailability(false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
b.setAvailability(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ElasticSearchIndexer) setAvailability(available bool) {
|
||||||
|
b.lock.Lock()
|
||||||
|
defer b.lock.Unlock()
|
||||||
|
|
||||||
|
if b.available == available {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
b.available = available
|
||||||
|
if b.availabilityCallback != nil {
|
||||||
|
// Call the callback from within the lock to ensure that the ordering remains correct
|
||||||
|
b.availabilityCallback(b.available)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -42,9 +42,11 @@ type SearchResultLanguages struct {
|
||||||
|
|
||||||
// Indexer defines an interface to index and search code contents
|
// Indexer defines an interface to index and search code contents
|
||||||
type Indexer interface {
|
type Indexer interface {
|
||||||
|
Ping() bool
|
||||||
|
SetAvailabilityChangeCallback(callback func(bool))
|
||||||
Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error
|
Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error
|
||||||
Delete(repoID int64) error
|
Delete(repoID int64) error
|
||||||
Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
|
Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
|
||||||
Close()
|
Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -140,6 +142,7 @@ func Init() {
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
unhandled := make([]queue.Data, 0, len(data))
|
||||||
for _, datum := range data {
|
for _, datum := range data {
|
||||||
indexerData, ok := datum.(*IndexerData)
|
indexerData, ok := datum.(*IndexerData)
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -150,10 +153,14 @@ func Init() {
|
||||||
|
|
||||||
if err := index(ctx, indexer, indexerData.RepoID); err != nil {
|
if err := index(ctx, indexer, indexerData.RepoID); err != nil {
|
||||||
log.Error("index: %v", err)
|
log.Error("index: %v", err)
|
||||||
continue
|
if indexer.Ping() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Add back to queue
|
||||||
|
unhandled = append(unhandled, datum)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return unhandled
|
||||||
}
|
}
|
||||||
|
|
||||||
indexerQueue = queue.CreateUniqueQueue("code_indexer", handler, &IndexerData{})
|
indexerQueue = queue.CreateUniqueQueue("code_indexer", handler, &IndexerData{})
|
||||||
|
@ -212,6 +219,18 @@ func Init() {
|
||||||
|
|
||||||
indexer.set(rIndexer)
|
indexer.set(rIndexer)
|
||||||
|
|
||||||
|
if queue, ok := indexerQueue.(queue.Pausable); ok {
|
||||||
|
rIndexer.SetAvailabilityChangeCallback(func(available bool) {
|
||||||
|
if !available {
|
||||||
|
log.Info("Code index queue paused")
|
||||||
|
queue.Pause()
|
||||||
|
} else {
|
||||||
|
log.Info("Code index queue resumed")
|
||||||
|
queue.Resume()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// Start processing the queue
|
// Start processing the queue
|
||||||
go graceful.GetManager().RunWithShutdownFns(indexerQueue.Run)
|
go graceful.GetManager().RunWithShutdownFns(indexerQueue.Run)
|
||||||
|
|
||||||
|
@ -262,6 +281,17 @@ func UpdateRepoIndexer(repo *repo_model.Repository) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsAvailable checks if issue indexer is available
|
||||||
|
func IsAvailable() bool {
|
||||||
|
idx, err := indexer.get()
|
||||||
|
if err != nil {
|
||||||
|
log.Error("IsAvailable(): unable to get indexer: %v", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return idx.Ping()
|
||||||
|
}
|
||||||
|
|
||||||
// populateRepoIndexer populate the repo indexer with pre-existing data. This
|
// populateRepoIndexer populate the repo indexer with pre-existing data. This
|
||||||
// should only be run when the indexer is created for the first time.
|
// should only be run when the indexer is created for the first time.
|
||||||
func populateRepoIndexer(ctx context.Context) {
|
func populateRepoIndexer(ctx context.Context) {
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
package code
|
package code
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -65,7 +66,7 @@ func testIndexer(name string, t *testing.T, indexer Indexer) {
|
||||||
|
|
||||||
for _, kw := range keywords {
|
for _, kw := range keywords {
|
||||||
t.Run(kw.Keyword, func(t *testing.T) {
|
t.Run(kw.Keyword, func(t *testing.T) {
|
||||||
total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10, false)
|
total, res, langs, err := indexer.Search(context.TODO(), kw.RepoIDs, "", kw.Keyword, 1, 10, false)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, len(kw.IDs), total)
|
assert.EqualValues(t, len(kw.IDs), total)
|
||||||
assert.Len(t, langs, kw.Langs)
|
assert.Len(t, langs, kw.Langs)
|
||||||
|
|
|
@ -6,6 +6,7 @@ package code
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/highlight"
|
"code.gitea.io/gitea/modules/highlight"
|
||||||
|
@ -106,12 +107,12 @@ func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, erro
|
||||||
}
|
}
|
||||||
|
|
||||||
// PerformSearch perform a search on a repository
|
// PerformSearch perform a search on a repository
|
||||||
func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) {
|
func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) {
|
||||||
if len(keyword) == 0 {
|
if len(keyword) == 0 {
|
||||||
return 0, nil, nil, nil
|
return 0, nil, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
|
total, results, resultLanguages, err := indexer.Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, nil, err
|
return 0, nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
repo_model "code.gitea.io/gitea/models/repo"
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
var indexer = newWrappedIndexer()
|
var indexer = newWrappedIndexer()
|
||||||
|
@ -56,6 +57,26 @@ func (w *wrappedIndexer) get() (Indexer, error) {
|
||||||
return w.internal, nil
|
return w.internal, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
|
||||||
|
func (w *wrappedIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||||
|
indexer, err := w.get()
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Failed to get indexer: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
indexer.SetAvailabilityChangeCallback(callback)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ping checks if elastic is available
|
||||||
|
func (w *wrappedIndexer) Ping() bool {
|
||||||
|
indexer, err := w.get()
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("Failed to get indexer: %v", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return indexer.Ping()
|
||||||
|
}
|
||||||
|
|
||||||
func (w *wrappedIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
|
func (w *wrappedIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
|
||||||
indexer, err := w.get()
|
indexer, err := w.get()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -72,12 +93,12 @@ func (w *wrappedIndexer) Delete(repoID int64) error {
|
||||||
return indexer.Delete(repoID)
|
return indexer.Delete(repoID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
|
func (w *wrappedIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
|
||||||
indexer, err := w.get()
|
indexer, err := w.get()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, nil, err
|
return 0, nil, nil, err
|
||||||
}
|
}
|
||||||
return indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
|
return indexer.Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *wrappedIndexer) Close() {
|
func (w *wrappedIndexer) Close() {
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
package issues
|
package issues
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
@ -186,6 +187,15 @@ func (b *BleveIndexer) Init() (bool, error) {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetAvailabilityChangeCallback does nothing
|
||||||
|
func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ping does nothing
|
||||||
|
func (b *BleveIndexer) Ping() bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// Close will close the bleve indexer
|
// Close will close the bleve indexer
|
||||||
func (b *BleveIndexer) Close() {
|
func (b *BleveIndexer) Close() {
|
||||||
if b.indexer != nil {
|
if b.indexer != nil {
|
||||||
|
@ -229,7 +239,7 @@ func (b *BleveIndexer) Delete(ids ...int64) error {
|
||||||
|
|
||||||
// Search searches for issues by given conditions.
|
// Search searches for issues by given conditions.
|
||||||
// Returns the matching issue IDs
|
// Returns the matching issue IDs
|
||||||
func (b *BleveIndexer) Search(keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
|
func (b *BleveIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
|
||||||
var repoQueriesP []*query.NumericRangeQuery
|
var repoQueriesP []*query.NumericRangeQuery
|
||||||
for _, repoID := range repoIDs {
|
for _, repoID := range repoIDs {
|
||||||
repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "RepoID"))
|
repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "RepoID"))
|
||||||
|
@ -249,7 +259,7 @@ func (b *BleveIndexer) Search(keyword string, repoIDs []int64, limit, start int)
|
||||||
search := bleve.NewSearchRequestOptions(indexerQuery, limit, start, false)
|
search := bleve.NewSearchRequestOptions(indexerQuery, limit, start, false)
|
||||||
search.SortBy([]string{"-_score"})
|
search.SortBy([]string{"-_score"})
|
||||||
|
|
||||||
result, err := b.indexer.Search(search)
|
result, err := b.indexer.SearchInContext(ctx, search)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
package issues
|
package issues
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -84,7 +85,7 @@ func TestBleveIndexAndSearch(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, kw := range keywords {
|
for _, kw := range keywords {
|
||||||
res, err := indexer.Search(kw.Keyword, []int64{2}, 10, 0)
|
res, err := indexer.Search(context.TODO(), kw.Keyword, []int64{2}, 10, 0)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
ids := make([]int64, 0, len(res.Hits))
|
ids := make([]int64, 0, len(res.Hits))
|
||||||
|
|
|
@ -4,33 +4,47 @@
|
||||||
|
|
||||||
package issues
|
package issues
|
||||||
|
|
||||||
import "code.gitea.io/gitea/models"
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
)
|
||||||
|
|
||||||
// DBIndexer implements Indexer interface to use database's like search
|
// DBIndexer implements Indexer interface to use database's like search
|
||||||
type DBIndexer struct{}
|
type DBIndexer struct{}
|
||||||
|
|
||||||
// Init dummy function
|
// Init dummy function
|
||||||
func (db *DBIndexer) Init() (bool, error) {
|
func (i *DBIndexer) Init() (bool, error) {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetAvailabilityChangeCallback dummy function
|
||||||
|
func (i *DBIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ping checks if database is available
|
||||||
|
func (i *DBIndexer) Ping() bool {
|
||||||
|
return db.GetEngine(db.DefaultContext).Ping() != nil
|
||||||
|
}
|
||||||
|
|
||||||
// Index dummy function
|
// Index dummy function
|
||||||
func (db *DBIndexer) Index(issue []*IndexerData) error {
|
func (i *DBIndexer) Index(issue []*IndexerData) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete dummy function
|
// Delete dummy function
|
||||||
func (db *DBIndexer) Delete(ids ...int64) error {
|
func (i *DBIndexer) Delete(ids ...int64) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close dummy function
|
// Close dummy function
|
||||||
func (db *DBIndexer) Close() {
|
func (i *DBIndexer) Close() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search dummy function
|
// Search dummy function
|
||||||
func (db *DBIndexer) Search(kw string, repoIDs []int64, limit, start int) (*SearchResult, error) {
|
func (i *DBIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) {
|
||||||
total, ids, err := models.SearchIssueIDsByKeyword(kw, repoIDs, limit, start)
|
total, ids, err := models.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,9 +8,12 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/graceful"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
|
||||||
"github.com/olivere/elastic/v7"
|
"github.com/olivere/elastic/v7"
|
||||||
|
@ -20,8 +23,12 @@ var _ Indexer = &ElasticSearchIndexer{}
|
||||||
|
|
||||||
// ElasticSearchIndexer implements Indexer interface
|
// ElasticSearchIndexer implements Indexer interface
|
||||||
type ElasticSearchIndexer struct {
|
type ElasticSearchIndexer struct {
|
||||||
client *elastic.Client
|
client *elastic.Client
|
||||||
indexerName string
|
indexerName string
|
||||||
|
available bool
|
||||||
|
availabilityCallback func(bool)
|
||||||
|
stopTimer chan struct{}
|
||||||
|
lock sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
type elasticLogger struct {
|
type elasticLogger struct {
|
||||||
|
@ -56,10 +63,27 @@ func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, er
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return &ElasticSearchIndexer{
|
indexer := &ElasticSearchIndexer{
|
||||||
client: client,
|
client: client,
|
||||||
indexerName: indexerName,
|
indexerName: indexerName,
|
||||||
}, nil
|
available: true,
|
||||||
|
stopTimer: make(chan struct{}),
|
||||||
|
}
|
||||||
|
|
||||||
|
ticker := time.NewTicker(10 * time.Second)
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
indexer.checkAvailability()
|
||||||
|
case <-indexer.stopTimer:
|
||||||
|
ticker.Stop()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return indexer, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -93,10 +117,10 @@ const (
|
||||||
|
|
||||||
// Init will initialize the indexer
|
// Init will initialize the indexer
|
||||||
func (b *ElasticSearchIndexer) Init() (bool, error) {
|
func (b *ElasticSearchIndexer) Init() (bool, error) {
|
||||||
ctx := context.Background()
|
ctx := graceful.GetManager().HammerContext()
|
||||||
exists, err := b.client.IndexExists(b.indexerName).Do(ctx)
|
exists, err := b.client.IndexExists(b.indexerName).Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
|
@ -104,7 +128,7 @@ func (b *ElasticSearchIndexer) Init() (bool, error) {
|
||||||
|
|
||||||
createIndex, err := b.client.CreateIndex(b.indexerName).BodyString(mapping).Do(ctx)
|
createIndex, err := b.client.CreateIndex(b.indexerName).BodyString(mapping).Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, b.checkError(err)
|
||||||
}
|
}
|
||||||
if !createIndex.Acknowledged {
|
if !createIndex.Acknowledged {
|
||||||
return false, errors.New("init failed")
|
return false, errors.New("init failed")
|
||||||
|
@ -115,6 +139,20 @@ func (b *ElasticSearchIndexer) Init() (bool, error) {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
|
||||||
|
func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||||
|
b.lock.Lock()
|
||||||
|
defer b.lock.Unlock()
|
||||||
|
b.availabilityCallback = callback
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ping checks if elastic is available
|
||||||
|
func (b *ElasticSearchIndexer) Ping() bool {
|
||||||
|
b.lock.RLock()
|
||||||
|
defer b.lock.RUnlock()
|
||||||
|
return b.available
|
||||||
|
}
|
||||||
|
|
||||||
// Index will save the index data
|
// Index will save the index data
|
||||||
func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
|
func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
|
||||||
if len(issues) == 0 {
|
if len(issues) == 0 {
|
||||||
|
@ -131,8 +169,8 @@ func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
|
||||||
"content": issue.Content,
|
"content": issue.Content,
|
||||||
"comments": issue.Comments,
|
"comments": issue.Comments,
|
||||||
}).
|
}).
|
||||||
Do(context.Background())
|
Do(graceful.GetManager().HammerContext())
|
||||||
return err
|
return b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
reqs := make([]elastic.BulkableRequest, 0)
|
reqs := make([]elastic.BulkableRequest, 0)
|
||||||
|
@ -154,8 +192,8 @@ func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
|
||||||
_, err := b.client.Bulk().
|
_, err := b.client.Bulk().
|
||||||
Index(b.indexerName).
|
Index(b.indexerName).
|
||||||
Add(reqs...).
|
Add(reqs...).
|
||||||
Do(context.Background())
|
Do(graceful.GetManager().HammerContext())
|
||||||
return err
|
return b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete deletes indexes by ids
|
// Delete deletes indexes by ids
|
||||||
|
@ -166,8 +204,8 @@ func (b *ElasticSearchIndexer) Delete(ids ...int64) error {
|
||||||
_, err := b.client.Delete().
|
_, err := b.client.Delete().
|
||||||
Index(b.indexerName).
|
Index(b.indexerName).
|
||||||
Id(fmt.Sprintf("%d", ids[0])).
|
Id(fmt.Sprintf("%d", ids[0])).
|
||||||
Do(context.Background())
|
Do(graceful.GetManager().HammerContext())
|
||||||
return err
|
return b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
reqs := make([]elastic.BulkableRequest, 0)
|
reqs := make([]elastic.BulkableRequest, 0)
|
||||||
|
@ -182,13 +220,13 @@ func (b *ElasticSearchIndexer) Delete(ids ...int64) error {
|
||||||
_, err := b.client.Bulk().
|
_, err := b.client.Bulk().
|
||||||
Index(b.indexerName).
|
Index(b.indexerName).
|
||||||
Add(reqs...).
|
Add(reqs...).
|
||||||
Do(context.Background())
|
Do(graceful.GetManager().HammerContext())
|
||||||
return err
|
return b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search searches for issues by given conditions.
|
// Search searches for issues by given conditions.
|
||||||
// Returns the matching issue IDs
|
// Returns the matching issue IDs
|
||||||
func (b *ElasticSearchIndexer) Search(keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
|
func (b *ElasticSearchIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
|
||||||
kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
|
kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
|
||||||
query := elastic.NewBoolQuery()
|
query := elastic.NewBoolQuery()
|
||||||
query = query.Must(kwQuery)
|
query = query.Must(kwQuery)
|
||||||
|
@ -205,9 +243,9 @@ func (b *ElasticSearchIndexer) Search(keyword string, repoIDs []int64, limit, st
|
||||||
Query(query).
|
Query(query).
|
||||||
Sort("_score", false).
|
Sort("_score", false).
|
||||||
From(start).Size(limit).
|
From(start).Size(limit).
|
||||||
Do(context.Background())
|
Do(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, b.checkError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
hits := make([]Match, 0, limit)
|
hits := make([]Match, 0, limit)
|
||||||
|
@ -225,4 +263,51 @@ func (b *ElasticSearchIndexer) Search(keyword string, repoIDs []int64, limit, st
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close implements indexer
|
// Close implements indexer
|
||||||
func (b *ElasticSearchIndexer) Close() {}
|
func (b *ElasticSearchIndexer) Close() {
|
||||||
|
select {
|
||||||
|
case <-b.stopTimer:
|
||||||
|
default:
|
||||||
|
close(b.stopTimer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ElasticSearchIndexer) checkError(err error) error {
|
||||||
|
var opErr *net.OpError
|
||||||
|
if !(elastic.IsConnErr(err) || (errors.As(err, &opErr) && (opErr.Op == "dial" || opErr.Op == "read"))) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
b.setAvailability(false)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ElasticSearchIndexer) checkAvailability() {
|
||||||
|
if b.Ping() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request cluster state to check if elastic is available again
|
||||||
|
_, err := b.client.ClusterState().Do(graceful.GetManager().ShutdownContext())
|
||||||
|
if err != nil {
|
||||||
|
b.setAvailability(false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
b.setAvailability(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ElasticSearchIndexer) setAvailability(available bool) {
|
||||||
|
b.lock.Lock()
|
||||||
|
defer b.lock.Unlock()
|
||||||
|
|
||||||
|
if b.available == available {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
b.available = available
|
||||||
|
if b.availabilityCallback != nil {
|
||||||
|
// Call the callback from within the lock to ensure that the ordering remains correct
|
||||||
|
b.availabilityCallback(b.available)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -47,9 +47,11 @@ type SearchResult struct {
|
||||||
// Indexer defines an interface to indexer issues contents
|
// Indexer defines an interface to indexer issues contents
|
||||||
type Indexer interface {
|
type Indexer interface {
|
||||||
Init() (bool, error)
|
Init() (bool, error)
|
||||||
|
Ping() bool
|
||||||
|
SetAvailabilityChangeCallback(callback func(bool))
|
||||||
Index(issue []*IndexerData) error
|
Index(issue []*IndexerData) error
|
||||||
Delete(ids ...int64) error
|
Delete(ids ...int64) error
|
||||||
Search(kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
|
Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
|
||||||
Close()
|
Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -111,6 +113,7 @@ func InitIssueIndexer(syncReindex bool) {
|
||||||
}
|
}
|
||||||
|
|
||||||
iData := make([]*IndexerData, 0, len(data))
|
iData := make([]*IndexerData, 0, len(data))
|
||||||
|
unhandled := make([]queue.Data, 0, len(data))
|
||||||
for _, datum := range data {
|
for _, datum := range data {
|
||||||
indexerData, ok := datum.(*IndexerData)
|
indexerData, ok := datum.(*IndexerData)
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -119,13 +122,34 @@ func InitIssueIndexer(syncReindex bool) {
|
||||||
}
|
}
|
||||||
log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete)
|
log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete)
|
||||||
if indexerData.IsDelete {
|
if indexerData.IsDelete {
|
||||||
_ = indexer.Delete(indexerData.IDs...)
|
if err := indexer.Delete(indexerData.IDs...); err != nil {
|
||||||
|
log.Error("Error whilst deleting from index: %v Error: %v", indexerData.IDs, err)
|
||||||
|
if indexer.Ping() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Add back to queue
|
||||||
|
unhandled = append(unhandled, datum)
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
iData = append(iData, indexerData)
|
iData = append(iData, indexerData)
|
||||||
}
|
}
|
||||||
|
if len(unhandled) > 0 {
|
||||||
|
for _, indexerData := range iData {
|
||||||
|
unhandled = append(unhandled, indexerData)
|
||||||
|
}
|
||||||
|
return unhandled
|
||||||
|
}
|
||||||
if err := indexer.Index(iData); err != nil {
|
if err := indexer.Index(iData); err != nil {
|
||||||
log.Error("Error whilst indexing: %v Error: %v", iData, err)
|
log.Error("Error whilst indexing: %v Error: %v", iData, err)
|
||||||
|
if indexer.Ping() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Add back to queue
|
||||||
|
for _, indexerData := range iData {
|
||||||
|
unhandled = append(unhandled, indexerData)
|
||||||
|
}
|
||||||
|
return unhandled
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -193,6 +217,18 @@ func InitIssueIndexer(syncReindex bool) {
|
||||||
log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
|
log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if queue, ok := issueIndexerQueue.(queue.Pausable); ok {
|
||||||
|
holder.get().SetAvailabilityChangeCallback(func(available bool) {
|
||||||
|
if !available {
|
||||||
|
log.Info("Issue index queue paused")
|
||||||
|
queue.Pause()
|
||||||
|
} else {
|
||||||
|
log.Info("Issue index queue resumed")
|
||||||
|
queue.Resume()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// Start processing the queue
|
// Start processing the queue
|
||||||
go graceful.GetManager().RunWithShutdownFns(issueIndexerQueue.Run)
|
go graceful.GetManager().RunWithShutdownFns(issueIndexerQueue.Run)
|
||||||
|
|
||||||
|
@ -334,7 +370,7 @@ func DeleteRepoIssueIndexer(repo *repo_model.Repository) {
|
||||||
|
|
||||||
// SearchIssuesByKeyword search issue ids by keywords and repo id
|
// SearchIssuesByKeyword search issue ids by keywords and repo id
|
||||||
// WARNNING: You have to ensure user have permission to visit repoIDs' issues
|
// WARNNING: You have to ensure user have permission to visit repoIDs' issues
|
||||||
func SearchIssuesByKeyword(repoIDs []int64, keyword string) ([]int64, error) {
|
func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) {
|
||||||
var issueIDs []int64
|
var issueIDs []int64
|
||||||
indexer := holder.get()
|
indexer := holder.get()
|
||||||
|
|
||||||
|
@ -342,7 +378,7 @@ func SearchIssuesByKeyword(repoIDs []int64, keyword string) ([]int64, error) {
|
||||||
log.Error("SearchIssuesByKeyword(): unable to get indexer!")
|
log.Error("SearchIssuesByKeyword(): unable to get indexer!")
|
||||||
return nil, fmt.Errorf("unable to get issue indexer")
|
return nil, fmt.Errorf("unable to get issue indexer")
|
||||||
}
|
}
|
||||||
res, err := indexer.Search(keyword, repoIDs, 50, 0)
|
res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -351,3 +387,14 @@ func SearchIssuesByKeyword(repoIDs []int64, keyword string) ([]int64, error) {
|
||||||
}
|
}
|
||||||
return issueIDs, nil
|
return issueIDs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsAvailable checks if issue indexer is available
|
||||||
|
func IsAvailable() bool {
|
||||||
|
indexer := holder.get()
|
||||||
|
if indexer == nil {
|
||||||
|
log.Error("IsAvailable(): unable to get indexer!")
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return indexer.Ping()
|
||||||
|
}
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
package issues
|
package issues
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
@ -56,19 +57,19 @@ func TestBleveSearchIssues(t *testing.T) {
|
||||||
|
|
||||||
time.Sleep(5 * time.Second)
|
time.Sleep(5 * time.Second)
|
||||||
|
|
||||||
ids, err := SearchIssuesByKeyword([]int64{1}, "issue2")
|
ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, []int64{2}, ids)
|
assert.EqualValues(t, []int64{2}, ids)
|
||||||
|
|
||||||
ids, err = SearchIssuesByKeyword([]int64{1}, "first")
|
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, []int64{1}, ids)
|
assert.EqualValues(t, []int64{1}, ids)
|
||||||
|
|
||||||
ids, err = SearchIssuesByKeyword([]int64{1}, "for")
|
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
|
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
|
||||||
|
|
||||||
ids, err = SearchIssuesByKeyword([]int64{1}, "good")
|
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, []int64{1}, ids)
|
assert.EqualValues(t, []int64{1}, ids)
|
||||||
}
|
}
|
||||||
|
@ -79,19 +80,19 @@ func TestDBSearchIssues(t *testing.T) {
|
||||||
setting.Indexer.IssueType = "db"
|
setting.Indexer.IssueType = "db"
|
||||||
InitIssueIndexer(true)
|
InitIssueIndexer(true)
|
||||||
|
|
||||||
ids, err := SearchIssuesByKeyword([]int64{1}, "issue2")
|
ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, []int64{2}, ids)
|
assert.EqualValues(t, []int64{2}, ids)
|
||||||
|
|
||||||
ids, err = SearchIssuesByKeyword([]int64{1}, "first")
|
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, []int64{1}, ids)
|
assert.EqualValues(t, []int64{1}, ids)
|
||||||
|
|
||||||
ids, err = SearchIssuesByKeyword([]int64{1}, "for")
|
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
|
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)
|
||||||
|
|
||||||
ids, err = SearchIssuesByKeyword([]int64{1}, "good")
|
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, []int64{1}, ids)
|
assert.EqualValues(t, []int64{1}, ids)
|
||||||
}
|
}
|
||||||
|
|
|
@ -268,6 +268,7 @@ search = Search
|
||||||
code = Code
|
code = Code
|
||||||
search.fuzzy = Fuzzy
|
search.fuzzy = Fuzzy
|
||||||
search.match = Match
|
search.match = Match
|
||||||
|
code_search_unavailable = Currently code search is not available. Please contact your site administrator.
|
||||||
repo_no_results = No matching repositories found.
|
repo_no_results = No matching repositories found.
|
||||||
user_no_results = No matching users found.
|
user_no_results = No matching users found.
|
||||||
org_no_results = No matching organizations found.
|
org_no_results = No matching organizations found.
|
||||||
|
@ -1262,6 +1263,7 @@ issues.filter_sort.moststars = Most stars
|
||||||
issues.filter_sort.feweststars = Fewest stars
|
issues.filter_sort.feweststars = Fewest stars
|
||||||
issues.filter_sort.mostforks = Most forks
|
issues.filter_sort.mostforks = Most forks
|
||||||
issues.filter_sort.fewestforks = Fewest forks
|
issues.filter_sort.fewestforks = Fewest forks
|
||||||
|
issues.keyword_search_unavailable = Currently searhing by keyword is not available. Please contact your site administrator.
|
||||||
issues.action_open = Open
|
issues.action_open = Open
|
||||||
issues.action_close = Close
|
issues.action_close = Close
|
||||||
issues.action_label = Label
|
issues.action_label = Label
|
||||||
|
@ -1707,6 +1709,8 @@ search.search_repo = Search repository
|
||||||
search.fuzzy = Fuzzy
|
search.fuzzy = Fuzzy
|
||||||
search.match = Match
|
search.match = Match
|
||||||
search.results = Search results for "%s" in <a href="%s">%s</a>
|
search.results = Search results for "%s" in <a href="%s">%s</a>
|
||||||
|
search.code_no_results = No source code matching your search term found.
|
||||||
|
search.code_search_unavailable = Currently code search is not available. Please contact your site administrator.
|
||||||
|
|
||||||
settings = Settings
|
settings = Settings
|
||||||
settings.desc = Settings is where you can manage the settings for the repository
|
settings.desc = Settings is where you can manage the settings for the repository
|
||||||
|
|
|
@ -188,7 +188,7 @@ func SearchIssues(ctx *context.APIContext) {
|
||||||
}
|
}
|
||||||
var issueIDs []int64
|
var issueIDs []int64
|
||||||
if len(keyword) > 0 && len(repoIDs) > 0 {
|
if len(keyword) > 0 && len(repoIDs) > 0 {
|
||||||
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(repoIDs, keyword); err != nil {
|
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil {
|
||||||
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
|
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -379,7 +379,7 @@ func ListIssues(ctx *context.APIContext) {
|
||||||
var issueIDs []int64
|
var issueIDs []int64
|
||||||
var labelIDs []int64
|
var labelIDs []int64
|
||||||
if len(keyword) > 0 {
|
if len(keyword) > 0 {
|
||||||
issueIDs, err = issue_indexer.SearchIssuesByKeyword([]int64{ctx.Repo.Repository.ID}, keyword)
|
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
|
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -87,17 +87,27 @@ func Code(ctx *context.Context) {
|
||||||
|
|
||||||
ctx.Data["RepoMaps"] = rightRepoMap
|
ctx.Data["RepoMaps"] = rightRepoMap
|
||||||
|
|
||||||
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
|
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ctx.ServerError("SearchResults", err)
|
if code_indexer.IsAvailable() {
|
||||||
return
|
ctx.ServerError("SearchResults", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.Data["CodeIndexerUnavailable"] = true
|
||||||
|
} else {
|
||||||
|
ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
|
||||||
}
|
}
|
||||||
// if non-login user or isAdmin, no need to check UnitTypeCode
|
// if non-login user or isAdmin, no need to check UnitTypeCode
|
||||||
} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
|
} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
|
||||||
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
|
total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ctx.ServerError("SearchResults", err)
|
if code_indexer.IsAvailable() {
|
||||||
return
|
ctx.ServerError("SearchResults", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.Data["CodeIndexerUnavailable"] = true
|
||||||
|
} else {
|
||||||
|
ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
|
||||||
}
|
}
|
||||||
|
|
||||||
loadRepoIDs := make([]int64, 0, len(searchResults))
|
loadRepoIDs := make([]int64, 0, len(searchResults))
|
||||||
|
|
|
@ -161,10 +161,13 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
|
||||||
|
|
||||||
var issueIDs []int64
|
var issueIDs []int64
|
||||||
if len(keyword) > 0 {
|
if len(keyword) > 0 {
|
||||||
issueIDs, err = issue_indexer.SearchIssuesByKeyword([]int64{repo.ID}, keyword)
|
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ctx.ServerError("issueIndexer.Search", err)
|
if issue_indexer.IsAvailable() {
|
||||||
return
|
ctx.ServerError("issueIndexer.Search", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.Data["IssueIndexerUnavailable"] = true
|
||||||
}
|
}
|
||||||
if len(issueIDs) == 0 {
|
if len(issueIDs) == 0 {
|
||||||
forceEmpty = true
|
forceEmpty = true
|
||||||
|
|
|
@ -30,11 +30,16 @@ func Search(ctx *context.Context) {
|
||||||
queryType := ctx.FormTrim("t")
|
queryType := ctx.FormTrim("t")
|
||||||
isMatch := queryType == "match"
|
isMatch := queryType == "match"
|
||||||
|
|
||||||
total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch([]int64{ctx.Repo.Repository.ID},
|
total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch(ctx, []int64{ctx.Repo.Repository.ID},
|
||||||
language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
|
language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ctx.ServerError("SearchResults", err)
|
if code_indexer.IsAvailable() {
|
||||||
return
|
ctx.ServerError("SearchResults", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.Data["CodeIndexerUnavailable"] = true
|
||||||
|
} else {
|
||||||
|
ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
|
||||||
}
|
}
|
||||||
ctx.Data["Keyword"] = keyword
|
ctx.Data["Keyword"] = keyword
|
||||||
ctx.Data["Language"] = language
|
ctx.Data["Language"] = language
|
||||||
|
|
|
@ -438,7 +438,7 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
|
||||||
|
|
||||||
// Execute keyword search for issues.
|
// Execute keyword search for issues.
|
||||||
// USING NON-FINAL STATE OF opts FOR A QUERY.
|
// USING NON-FINAL STATE OF opts FOR A QUERY.
|
||||||
issueIDsFromSearch, err := issueIDsFromSearch(ctxUser, keyword, opts)
|
issueIDsFromSearch, err := issueIDsFromSearch(ctx, ctxUser, keyword, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ctx.ServerError("issueIDsFromSearch", err)
|
ctx.ServerError("issueIDsFromSearch", err)
|
||||||
return
|
return
|
||||||
|
@ -673,7 +673,7 @@ func getRepoIDs(reposQuery string) []int64 {
|
||||||
return repoIDs
|
return repoIDs
|
||||||
}
|
}
|
||||||
|
|
||||||
func issueIDsFromSearch(ctxUser *user_model.User, keyword string, opts *models.IssuesOptions) ([]int64, error) {
|
func issueIDsFromSearch(ctx *context.Context, ctxUser *user_model.User, keyword string, opts *models.IssuesOptions) ([]int64, error) {
|
||||||
if len(keyword) == 0 {
|
if len(keyword) == 0 {
|
||||||
return []int64{}, nil
|
return []int64{}, nil
|
||||||
}
|
}
|
||||||
|
@ -682,7 +682,7 @@ func issueIDsFromSearch(ctxUser *user_model.User, keyword string, opts *models.I
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %v", err)
|
return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %v", err)
|
||||||
}
|
}
|
||||||
issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(searchRepoIDs, keyword)
|
issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("SearchIssuesByKeyword: %v", err)
|
return nil, fmt.Errorf("SearchIssuesByKeyword: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,21 +5,25 @@
|
||||||
<form class="ui form ignore-dirty" style="max-width: 100%">
|
<form class="ui form ignore-dirty" style="max-width: 100%">
|
||||||
<input type="hidden" name="tab" value="{{$.TabName}}">
|
<input type="hidden" name="tab" value="{{$.TabName}}">
|
||||||
<div class="ui fluid action input">
|
<div class="ui fluid action input">
|
||||||
<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
|
<input name="q" value="{{.Keyword}}"{{if .CodeIndexerUnavailable }} disabled{{end}} placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
|
||||||
<div class="ui dropdown selection">
|
<div class="ui dropdown selection{{if .CodeIndexerUnavailable }} disabled{{end}}">
|
||||||
<input name="t" type="hidden" value="{{.queryType}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
|
<input name="t" type="hidden" value="{{.queryType}}"{{if .CodeIndexerUnavailable }} disabled{{end}}>{{svg "octicon-triangle-down" 14 "dropdown icon"}}
|
||||||
<div class="text">{{.i18n.Tr (printf "explore.search.%s" (or .queryType "fuzzy"))}}</div>
|
<div class="text">{{.i18n.Tr (printf "explore.search.%s" (or .queryType "fuzzy"))}}</div>
|
||||||
<div class="menu transition hidden" tabindex="-1" style="display: block !important;">
|
<div class="menu transition hidden" tabindex="-1" style="display: block !important;">
|
||||||
<div class="item" data-value="">{{.i18n.Tr "explore.search.fuzzy"}}</div>
|
<div class="item" data-value="">{{.i18n.Tr "explore.search.fuzzy"}}</div>
|
||||||
<div class="item" data-value="match">{{.i18n.Tr "explore.search.match"}}</div>
|
<div class="item" data-value="match">{{.i18n.Tr "explore.search.match"}}</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<button class="ui blue button">{{.i18n.Tr "explore.search"}}</button>
|
<button class="ui blue button"{{if .CodeIndexerUnavailable }} disabled{{end}}>{{.i18n.Tr "explore.search"}}</button>
|
||||||
</div>
|
</div>
|
||||||
</form>
|
</form>
|
||||||
<div class="ui divider"></div>
|
<div class="ui divider"></div>
|
||||||
<div class="ui user list">
|
<div class="ui user list">
|
||||||
{{if .SearchResults}}
|
{{if .CodeIndexerUnavailable }}
|
||||||
|
<div class="ui error message">
|
||||||
|
<p>{{$.i18n.Tr "explore.code_search_unavailable"}}</p>
|
||||||
|
</div>
|
||||||
|
{{else if .SearchResults}}
|
||||||
<h3>
|
<h3>
|
||||||
{{.i18n.Tr "explore.code_search_results" (.Keyword|Escape) | Str2html }}
|
{{.i18n.Tr "explore.code_search_results" (.Keyword|Escape) | Str2html }}
|
||||||
</h3>
|
</h3>
|
||||||
|
|
|
@ -13,9 +13,12 @@
|
||||||
<div class="ui repo-search">
|
<div class="ui repo-search">
|
||||||
<form class="ui form ignore-dirty" action="{{.RepoLink}}/search" method="get">
|
<form class="ui form ignore-dirty" action="{{.RepoLink}}/search" method="get">
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<div class="ui action input">
|
<div class="ui action input{{if .CodeIndexerUnavailable }} disabled left icon tooltip{{end}}"{{if .CodeIndexerUnavailable }} data-content="{{.i18n.Tr "repo.search.code_search_unavailable"}}"{{end}}>
|
||||||
<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
|
<input name="q" value="{{.Keyword}}"{{if .CodeIndexerUnavailable }} disabled{{end}} placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
|
||||||
<button class="ui icon button" type="submit">
|
{{if .CodeIndexerUnavailable }}
|
||||||
|
<i class="icon df ac jc">{{svg "octicon-alert"}}</i>
|
||||||
|
{{end}}
|
||||||
|
<button class="ui icon button"{{if .CodeIndexerUnavailable }} disabled{{end}} type="submit">
|
||||||
{{svg "octicon-search"}}
|
{{svg "octicon-search"}}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -5,60 +5,68 @@
|
||||||
<div class="ui repo-search">
|
<div class="ui repo-search">
|
||||||
<form class="ui form ignore-dirty" method="get">
|
<form class="ui form ignore-dirty" method="get">
|
||||||
<div class="ui fluid action input">
|
<div class="ui fluid action input">
|
||||||
<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
|
<input name="q" value="{{.Keyword}}"{{if .CodeIndexerUnavailable }} disabled{{end}} placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
|
||||||
<div class="ui dropdown selection">
|
<div class="ui dropdown selection{{if .CodeIndexerUnavailable }} disabled{{end}}">
|
||||||
<input name="t" type="hidden" value="{{.queryType}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
|
<input name="t" type="hidden"{{if .CodeIndexerUnavailable }} disabled{{end}} value="{{.queryType}}">{{svg "octicon-triangle-down" 14 "dropdown icon"}}
|
||||||
<div class="text">{{.i18n.Tr (printf "repo.search.%s" (or .queryType "fuzzy"))}}</div>
|
<div class="text">{{.i18n.Tr (printf "repo.search.%s" (or .queryType "fuzzy"))}}</div>
|
||||||
<div class="menu transition hidden" tabindex="-1" style="display: block !important;">
|
<div class="menu transition hidden" tabindex="-1" style="display: block !important;">
|
||||||
<div class="item" data-value="">{{.i18n.Tr "repo.search.fuzzy"}}</div>
|
<div class="item" data-value="">{{.i18n.Tr "repo.search.fuzzy"}}</div>
|
||||||
<div class="item" data-value="match">{{.i18n.Tr "repo.search.match"}}</div>
|
<div class="item" data-value="match">{{.i18n.Tr "repo.search.match"}}</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<button class="ui icon button" type="submit">{{svg "octicon-search" 16}}</button>
|
<button class="ui icon button"{{if .CodeIndexerUnavailable }} disabled{{end}} type="submit">{{svg "octicon-search" 16}}</button>
|
||||||
</div>
|
</div>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
{{if .Keyword}}
|
{{if .CodeIndexerUnavailable }}
|
||||||
|
<div class="ui error message">
|
||||||
|
<p>{{$.i18n.Tr "repo.search.code_search_unavailable"}}</p>
|
||||||
|
</div>
|
||||||
|
{{else if .Keyword}}
|
||||||
<h3>
|
<h3>
|
||||||
{{.i18n.Tr "repo.search.results" (.Keyword|Escape) (.RepoLink|Escape) (.RepoName|Escape) | Str2html }}
|
{{.i18n.Tr "repo.search.results" (.Keyword|Escape) (.RepoLink|Escape) (.RepoName|Escape) | Str2html }}
|
||||||
</h3>
|
</h3>
|
||||||
<div class="df ac fw">
|
{{if .SearchResults}}
|
||||||
{{range $term := .SearchResultLanguages}}
|
<div class="df ac fw">
|
||||||
<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{$.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
|
{{range $term := .SearchResultLanguages}}
|
||||||
<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
|
<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{$.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
|
||||||
{{$term.Language}}
|
<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
|
||||||
<div class="detail">{{$term.Count}}</div>
|
{{$term.Language}}
|
||||||
</a>
|
<div class="detail">{{$term.Count}}</div>
|
||||||
{{end}}
|
</a>
|
||||||
</div>
|
{{end}}
|
||||||
<div class="repository search">
|
</div>
|
||||||
{{range $result := .SearchResults}}
|
<div class="repository search">
|
||||||
<div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
|
{{range $result := .SearchResults}}
|
||||||
<h4 class="ui top attached normal header">
|
<div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
|
||||||
<span class="file">{{.Filename}}</span>
|
<h4 class="ui top attached normal header">
|
||||||
<a class="ui basic tiny button" rel="nofollow" href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments .Filename}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
|
<span class="file">{{.Filename}}</span>
|
||||||
</h4>
|
<a class="ui basic tiny button" rel="nofollow" href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments .Filename}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
|
||||||
<div class="ui attached table segment">
|
</h4>
|
||||||
<div class="file-body file-code code-view">
|
<div class="ui attached table segment">
|
||||||
<table>
|
<div class="file-body file-code code-view">
|
||||||
<tbody>
|
<table>
|
||||||
<tr>
|
<tbody>
|
||||||
<td class="lines-num">
|
<tr>
|
||||||
{{range .LineNumbers}}
|
<td class="lines-num">
|
||||||
<a href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
|
{{range .LineNumbers}}
|
||||||
{{end}}
|
<a href="{{$.SourcePath}}/src/commit/{{PathEscape $result.CommitID}}/{{PathEscapeSegments $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
|
||||||
</td>
|
{{end}}
|
||||||
<td class="lines-code chroma"><code class="code-inner">{{.FormattedLines | Safe}}</code></td>
|
</td>
|
||||||
</tr>
|
<td class="lines-code chroma"><code class="code-inner">{{.FormattedLines | Safe}}</code></td>
|
||||||
</tbody>
|
</tr>
|
||||||
</table>
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{{template "shared/searchbottom" dict "root" $ "result" .}}
|
||||||
</div>
|
</div>
|
||||||
{{template "shared/searchbottom" dict "root" $ "result" .}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{template "base/paginate" .}}
|
||||||
</div>
|
{{else}}
|
||||||
{{template "base/paginate" .}}
|
<div>{{$.i18n.Tr "repo.search.code_no_results"}}</div>
|
||||||
|
{{end}}
|
||||||
{{end}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -139,5 +139,10 @@
|
||||||
</div>
|
</div>
|
||||||
</li>
|
</li>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
{{if .IssueIndexerUnavailable}}
|
||||||
|
<div class="ui error message">
|
||||||
|
<p>{{$.i18n.Tr "repo.issues.keyword_search_unavailable"}}</p>
|
||||||
|
</div>
|
||||||
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
{{template "base/paginate" .}}
|
{{template "base/paginate" .}}
|
||||||
|
|
Loading…
Reference in a new issue