3c45cf8494
Move langauge detection to separate module to be more reusable Add option to disable vendored file exclusion from file search Allways show all language stats for search
50 lines
1.3 KiB
Go
Vendored
50 lines
1.3 KiB
Go
Vendored
// Copyright (c) 2014 Couchbase, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package single
|
|
|
|
import (
|
|
"github.com/blevesearch/bleve/analysis"
|
|
"github.com/blevesearch/bleve/registry"
|
|
)
|
|
|
|
const Name = "single"
|
|
|
|
type SingleTokenTokenizer struct {
|
|
}
|
|
|
|
func NewSingleTokenTokenizer() *SingleTokenTokenizer {
|
|
return &SingleTokenTokenizer{}
|
|
}
|
|
|
|
func (t *SingleTokenTokenizer) Tokenize(input []byte) analysis.TokenStream {
|
|
return analysis.TokenStream{
|
|
&analysis.Token{
|
|
Term: input,
|
|
Position: 1,
|
|
Start: 0,
|
|
End: len(input),
|
|
Type: analysis.AlphaNumeric,
|
|
},
|
|
}
|
|
}
|
|
|
|
func SingleTokenTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
|
|
return NewSingleTokenTokenizer(), nil
|
|
}
|
|
|
|
func init() {
|
|
registry.RegisterTokenizer(Name, SingleTokenTokenizerConstructor)
|
|
}
|