feat(core): Add a new string index: shingles

Author: harshil-goel

dgraph/cmd/alpha/dashboard.go

@@ -52,6 +52,7 @@ func keywordHandler(w http.ResponseWriter, r *http.Request) {
 		"@normalize",
 		"after",
 		"allofterms",
+		"shingles",
 		"alloftext",
 		"and",
 		"anyofterms",

dql/parser.go

@@ -1700,7 +1700,7 @@ func validFuncName(name string) bool {
 	}
 
 	switch name {
-	case "regexp", "anyofterms", "allofterms", "alloftext", "anyoftext",
+	case "regexp", "anyofterms", "allofterms", "alloftext", "anyoftext", "shingles",
 		"has", "uid", "uid_in", "anyof", "allof", "type", "match", "similar_to":
 		return true
 	}

query/query.go

@@ -1007,7 +1007,7 @@ func calculatePaginationParams(sg *SubGraph) (int32, int32) {
 	shouldExclude := false
 	if sg.SrcFunc != nil {
 		switch sg.SrcFunc.Name {
-		case "regexp", "alloftext", "allofterms", "match":
+		case "regexp", "alloftext", "allofterms", "match", "shingles":
 			shouldExclude = true
 		default:
 			shouldExclude = false
@@ -2746,7 +2746,7 @@ func isValidArg(a string) bool {
 // isValidFuncName checks if fn passed is valid keyword.
 func isValidFuncName(f string) bool {
 	switch f {
-	case "anyofterms", "allofterms", "val", "regexp", "anyoftext", "alloftext",
+	case "anyofterms", "allofterms", "val", "regexp", "anyoftext", "alloftext", "shingles",
 		"has", "uid", "uid_in", "anyof", "allof", "type", "match", "similar_to":
 		return true
 	}

tok/tok.go

@@ -48,6 +48,7 @@ const (
 	IdentSha       = 0xC
 	IdentBigFloat  = 0xD
 	IdentVFloat    = 0xE
+	IdentShingles  = 0xF
 	IdentCustom    = 0x80
 	IdentDelimiter = 0x1f // ASCII 31 - Unit separator
 )
@@ -98,6 +99,7 @@ func init() {
 	registerTokenizer(HashTokenizer{})
 	registerTokenizer(TermTokenizer{})
 	registerTokenizer(FullTextTokenizer{})
+	registerTokenizer(ShinglesTokenizer{})
 	registerTokenizer(Sha256Tokenizer{})
 	setupBleve()
 }
@@ -424,6 +426,57 @@ func (t ExactTokenizer) Prefix() []byte {
 	return prefix
 }
 
+type ShinglesTokenizer struct {
+	lang string
+}
+
+func (t ShinglesTokenizer) Name() string { return "shingles" }
+func (t ShinglesTokenizer) Type() string { return "string" }
+func (t ShinglesTokenizer) Tokens(v interface{}) ([]string, error) {
+	str, ok := v.(string)
+	if !ok || str == "" {
+		return []string{}, nil
+	}
+	lang := LangBase(t.lang)
+
+	// Step 1: Lowercase, normalize, basic tokenization
+	tokens := fulltextAnalyzer.Analyze([]byte(str))
+
+	// Step 2: Remove stopwords
+	tokens = filterStopwords(lang, tokens)
+
+	// Step 3: Apply stemming
+	tokens = filterStemmers(lang, tokens)
+
+	// Step 4: Generate shingles (bigrams and trigrams)
+	shingled := make([]string, 0, len(tokens))
+	n := len(tokens)
+
+	for i := 0; i < n; i++ {
+		// unigram
+		shingled = append(shingled, string(tokens[i].Term))
+
+		// bigram
+		if i+1 < n {
+			shingled = append(shingled, string(tokens[i].Term)+" "+string(tokens[i+1].Term))
+		}
+		// trigram
+		if i+2 < n {
+			shingled = append(shingled, string(tokens[i].Term)+" "+string(tokens[i+1].Term)+" "+string(tokens[i+2].Term))
+		}
+
+		if i+3 < n {
+			shingled = append(shingled, string(tokens[i].Term)+" "+string(tokens[i+1].Term)+" "+string(tokens[i+2].Term)+" "+string(tokens[i+3].Term))
+		}
+	}
+
+	// Step 5: Deduplicate
+	return x.RemoveDuplicates(shingled), nil
+}
+func (t ShinglesTokenizer) Identifier() byte { return IdentShingles }
+func (t ShinglesTokenizer) IsSortable() bool { return false }
+func (t ShinglesTokenizer) IsLossy() bool    { return true }
+
 // FullTextTokenizer generates full-text tokens from string data.
 type FullTextTokenizer struct{ lang string }
 

tok/tokens.go

@@ -59,6 +59,10 @@ func GetTermTokens(funcArgs []string) ([]string, error) {
 	return GetTokens(IdentTerm, funcArgs...)
 }
 
+func GetShinglesTokens(funcArgs []string, lang string) ([]string, error) {
+	return BuildTokens(funcArgs[0], ShinglesTokenizer{lang: lang})
+}
+
 // GetFullTextTokens returns the full-text tokens for the given value.
 func GetFullTextTokens(funcArgs []string, lang string) ([]string, error) {
 	if l := len(funcArgs); l != 1 {

worker/task.go

@@ -210,6 +210,7 @@ const (
 	geoFn
 	passwordFn
 	regexFn
+	shinglesFn
 	fullTextSearchFn
 	hasFn
 	uidInFn
@@ -248,6 +249,8 @@ func parseFuncTypeHelper(name string) (FuncType, string) {
 		return passwordFn, f
 	case "regexp":
 		return regexFn, f
+	case "shingles":
+		return shinglesFn, f
 	case "alloftext", "anyoftext":
 		return fullTextSearchFn, f
 	case "has":
@@ -311,7 +314,7 @@ func (srcFn *functionContext) needsValuePostings(typ types.TypeID) (bool, error)
 			return false, nil
 		}
 		return true, nil
-	case geoFn, regexFn, fullTextSearchFn, standardFn, hasFn, customIndexFn, matchFn:
+	case geoFn, regexFn, fullTextSearchFn, standardFn, hasFn, customIndexFn, matchFn, shinglesFn:
 		// All of these require an index, hence would require fetching uid postings.
 		return false, nil
 	case uidInFn, compareScalarFn:
@@ -819,7 +822,7 @@ func (qs *queryState) handleUidPostings(
 				} else {
 					key = x.DataKey(q.Attr, q.UidList.Uids[i])
 				}
-			case geoFn, regexFn, fullTextSearchFn, standardFn, customIndexFn, matchFn,
+			case geoFn, regexFn, fullTextSearchFn, standardFn, customIndexFn, matchFn, shinglesFn,
 				compareAttrFn:
 				key = x.IndexKey(q.Attr, srcFn.tokens[i])
 			default:
@@ -1189,7 +1192,7 @@ func needsStringFiltering(srcFn *functionContext, langs []string, attr string) b
 	return langForFunc(langs) != "." &&
 		(srcFn.fnType == standardFn || srcFn.fnType == hasFn ||
 			srcFn.fnType == fullTextSearchFn || srcFn.fnType == compareAttrFn ||
-			srcFn.fnType == customIndexFn)
+			srcFn.fnType == customIndexFn || srcFn.fnType == shinglesFn)
 }
 
 func (qs *queryState) handleCompareScalarFunction(ctx context.Context, arg funcArgs) error {
@@ -1704,6 +1707,11 @@ func (qs *queryState) filterStringFunction(arg funcArgs) error {
 	case hasFn:
 		// Dont do anything, as filtering based on lang is already
 		// done above.
+	case shinglesFn:
+		filter.tokens = arg.srcFn.tokens
+		filter.match = defaultMatch
+		filter.tokName = "shingles"
+		filtered = matchStrings(filtered, values, &filter)
 	case fullTextSearchFn:
 		filter.tokens = arg.srcFn.tokens
 		filter.match = defaultMatch
@@ -2024,7 +2032,7 @@ func parseSrcFn(ctx context.Context, q *pb.Query) (*functionContext, error) {
 			return nil, err
 		}
 		fc.n = len(q.UidList.Uids)
-	case standardFn, fullTextSearchFn:
+	case standardFn, fullTextSearchFn, shinglesFn:
 		// srcfunc 0th val is func name and [2:] are args.
 		// we tokenize the arguments of the query.
 		if err = ensureArgsCount(q.SrcFunc, 1); err != nil {

worker/tokens.go

@@ -21,6 +21,8 @@ import (
 func verifyStringIndex(ctx context.Context, attr string, funcType FuncType) (string, bool) {
 	var requiredTokenizer tok.Tokenizer
 	switch funcType {
+	case shinglesFn:
+		requiredTokenizer = tok.ShinglesTokenizer{}
 	case fullTextSearchFn:
 		requiredTokenizer = tok.FullTextTokenizer{}
 	case matchFn:
@@ -63,6 +65,9 @@ func getStringTokens(funcArgs []string, lang string, funcType FuncType) ([]strin
 	if funcType == fullTextSearchFn {
 		return tok.GetFullTextTokens(funcArgs, lang)
 	}
+	if funcType == shinglesFn {
+		return tok.GetShinglesTokens(funcArgs, lang)
+	}
 	return tok.GetTermTokens(funcArgs)
 }