Skip to content

Commit 5fccaa1

Browse files
liuxy0551liuxy0551
authored
feat: #410 optimize processCandidates tokenIndexOffset (#411)
* test: test suggestion wordRanges with range when processCandidates without tokenIndexOffset * feat: #410 optimize processCandidates tokenIndexOffset
1 parent 26219b8 commit 5fccaa1

23 files changed

+334
-75
lines changed

‎src/parser/common/basicSQL.ts

+3-11
Original file line numberDiff line numberDiff line change
@@ -73,15 +73,13 @@ export abstract class BasicSQL<
7373
/**
7474
* Convert candidates to suggestions
7575
* @param candidates candidate list
76-
* @param allTokens all tokens from input
76+
* @param allTokens slice all tokens from input by tokenIndexOffset
7777
* @param caretTokenIndex tokenIndex of caretPosition
78-
* @param tokenIndexOffset offset of the tokenIndex in the candidates compared to the tokenIndex in allTokens
7978
*/
8079
protected abstract processCandidates(
8180
candidates: CandidatesCollection,
8281
allTokens: Token[],
83-
caretTokenIndex: number,
84-
tokenIndexOffset: number
82+
caretTokenIndex: number
8583
): Suggestions<Token>;
8684

8785
/**
@@ -539,13 +537,7 @@ export abstract class BasicSQL<
539537
core.preferredRules = this.preferredRules;
540538

541539
const candidates = core.collectCandidates(caretTokenIndex, parseTree);
542-
const originalSuggestions = this.processCandidates(
543-
candidates,
544-
allTokens,
545-
caretTokenIndex,
546-
0
547-
// tokenIndexOffset
548-
);
540+
const originalSuggestions = this.processCandidates(candidates, allTokens, caretTokenIndex);
549541

550542
const syntaxSuggestions: SyntaxSuggestion<WordRange>[] = originalSuggestions.syntax.map(
551543
(syntaxCtx) => {

‎src/parser/flink/index.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -68,19 +68,14 @@ export class FlinkSQL extends BasicSQL<FlinkSqlLexer, ProgramContext, FlinkSqlPa
6868
protected processCandidates(
6969
candidates: CandidatesCollection,
7070
allTokens: Token[],
71-
caretTokenIndex: number,
72-
tokenIndexOffset: number
71+
caretTokenIndex: number
7372
): Suggestions<Token> {
7473
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
7574
const keywords: string[] = [];
7675

7776
for (let candidate of candidates.rules) {
7877
const [ruleType, candidateRule] = candidate;
79-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
80-
const tokenRanges = allTokens.slice(
81-
startTokenIndex,
82-
caretTokenIndex + tokenIndexOffset + 1
83-
);
78+
const tokenRanges = allTokens.slice(candidateRule.startTokenIndex, caretTokenIndex + 1);
8479

8580
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
8681
switch (ruleType) {

‎src/parser/hive/index.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -69,18 +69,13 @@ export class HiveSQL extends BasicSQL<HiveSqlLexer, ProgramContext, HiveSqlParse
6969
protected processCandidates(
7070
candidates: CandidatesCollection,
7171
allTokens: Token[],
72-
caretTokenIndex: number,
73-
tokenIndexOffset: number
72+
caretTokenIndex: number
7473
): Suggestions<Token> {
7574
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
7675
const keywords: string[] = [];
7776
for (let candidate of candidates.rules) {
7877
const [ruleType, candidateRule] = candidate;
79-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
80-
const tokenRanges = allTokens.slice(
81-
startTokenIndex,
82-
caretTokenIndex + tokenIndexOffset + 1
83-
);
78+
const tokenRanges = allTokens.slice(candidateRule.startTokenIndex, caretTokenIndex + 1);
8479

8580
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
8681
switch (ruleType) {

‎src/parser/impala/index.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -67,18 +67,13 @@ export class ImpalaSQL extends BasicSQL<ImpalaSqlLexer, ProgramContext, ImpalaSq
6767
protected processCandidates(
6868
candidates: CandidatesCollection,
6969
allTokens: Token[],
70-
caretTokenIndex: number,
71-
tokenIndexOffset: number
70+
caretTokenIndex: number
7271
): Suggestions<Token> {
7372
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
7473
const keywords: string[] = [];
7574
for (let candidate of candidates.rules) {
7675
const [ruleType, candidateRule] = candidate;
77-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
78-
const tokenRanges = allTokens.slice(
79-
startTokenIndex,
80-
caretTokenIndex + tokenIndexOffset + 1
81-
);
76+
const tokenRanges = allTokens.slice(candidateRule.startTokenIndex, caretTokenIndex + 1);
8277

8378
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
8479
switch (ruleType) {

‎src/parser/mysql/index.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -66,19 +66,14 @@ export class MySQL extends BasicSQL<MySqlLexer, ProgramContext, MySqlParser> {
6666
protected processCandidates(
6767
candidates: CandidatesCollection,
6868
allTokens: Token[],
69-
caretTokenIndex: number,
70-
tokenIndexOffset: number
69+
caretTokenIndex: number
7170
): Suggestions<Token> {
7271
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
7372
const keywords: string[] = [];
7473

7574
for (const candidate of candidates.rules) {
7675
const [ruleType, candidateRule] = candidate;
77-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
78-
const tokenRanges = allTokens.slice(
79-
startTokenIndex,
80-
caretTokenIndex + tokenIndexOffset + 1
81-
);
76+
const tokenRanges = allTokens.slice(candidateRule.startTokenIndex, caretTokenIndex + 1);
8277

8378
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
8479
switch (ruleType) {

‎src/parser/postgresql/index.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -72,18 +72,13 @@ export class PostgreSQL extends BasicSQL<PostgreSqlLexer, ProgramContext, Postgr
7272
protected processCandidates(
7373
candidates: CandidatesCollection,
7474
allTokens: Token[],
75-
caretTokenIndex: number,
76-
tokenIndexOffset: number
75+
caretTokenIndex: number
7776
): Suggestions<Token> {
7877
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
7978
const keywords: string[] = [];
8079
for (let candidate of candidates.rules) {
8180
const [ruleType, candidateRule] = candidate;
82-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
83-
const tokenRanges = allTokens.slice(
84-
startTokenIndex,
85-
caretTokenIndex + tokenIndexOffset + 1
86-
);
81+
const tokenRanges = allTokens.slice(candidateRule.startTokenIndex, caretTokenIndex + 1);
8782

8883
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
8984
switch (ruleType) {

‎src/parser/spark/index.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -67,19 +67,14 @@ export class SparkSQL extends BasicSQL<SparkSqlLexer, ProgramContext, SparkSqlPa
6767
protected processCandidates(
6868
candidates: CandidatesCollection,
6969
allTokens: Token[],
70-
caretTokenIndex: number,
71-
tokenIndexOffset: number
70+
caretTokenIndex: number
7271
): Suggestions<Token> {
7372
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
7473
const keywords: string[] = [];
7574

7675
for (const candidate of candidates.rules) {
7776
const [ruleType, candidateRule] = candidate;
78-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
79-
const tokenRanges = allTokens.slice(
80-
startTokenIndex,
81-
caretTokenIndex + tokenIndexOffset + 1
82-
);
77+
const tokenRanges = allTokens.slice(candidateRule.startTokenIndex, caretTokenIndex + 1);
8378

8479
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
8580
switch (ruleType) {

‎src/parser/trino/index.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -69,19 +69,14 @@ export class TrinoSQL extends BasicSQL<TrinoSqlLexer, ProgramContext, TrinoSqlPa
6969
protected processCandidates(
7070
candidates: CandidatesCollection,
7171
allTokens: Token[],
72-
caretTokenIndex: number,
73-
tokenIndexOffset: number
72+
caretTokenIndex: number
7473
): Suggestions<Token> {
7574
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
7675
const keywords: string[] = [];
7776

7877
for (let candidate of candidates.rules) {
7978
const [ruleType, candidateRule] = candidate;
80-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
81-
const tokenRanges = allTokens.slice(
82-
startTokenIndex,
83-
caretTokenIndex + tokenIndexOffset + 1
84-
);
79+
const tokenRanges = allTokens.slice(candidateRule.startTokenIndex, caretTokenIndex + 1);
8580

8681
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
8782
switch (ruleType) {

‎test/parser/flink/suggestion/multipleStatement.test.ts

+19-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,25 @@ describe('FlinkSQL Multiple Statements Syntax Suggestion', () => {
3636
);
3737

3838
expect(suggestion).not.toBeUndefined();
39-
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
39+
expect(suggestion?.wordRanges?.length).toBe(2);
40+
expect(suggestion?.wordRanges).toEqual([
41+
{
42+
text: 'db',
43+
line: 16,
44+
startIndex: 306,
45+
endIndex: 307,
46+
startColumn: 14,
47+
endColumn: 16,
48+
},
49+
{
50+
text: '.',
51+
line: 16,
52+
startIndex: 308,
53+
endIndex: 308,
54+
startColumn: 16,
55+
endColumn: 17,
56+
},
57+
]);
4058
});
4159

4260
test('Insert into table ', () => {

‎test/parser/flink/suggestion/syntaxSuggestion.test.ts

+11-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,17 @@ describe('Flink SQL Syntax Suggestion', () => {
3232
);
3333

3434
expect(suggestion).not.toBeUndefined();
35-
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['cat']);
35+
expect(suggestion?.wordRanges?.length).toBe(1);
36+
expect(suggestion?.wordRanges).toEqual([
37+
{
38+
text: 'cat',
39+
line: 1,
40+
startIndex: 13,
41+
endIndex: 15,
42+
startColumn: 14,
43+
endColumn: 17,
44+
},
45+
]);
3646
});
3747

3848
test('Select table', () => {

‎test/parser/hive/suggestion/multipleStatement.test.ts

+19-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,25 @@ describe('HiveSQL Multiple Statements Syntax Suggestion', () => {
3636
);
3737

3838
expect(suggestion).not.toBeUndefined();
39-
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
39+
expect(suggestion?.wordRanges?.length).toBe(2);
40+
expect(suggestion?.wordRanges).toEqual([
41+
{
42+
text: 'db',
43+
line: 9,
44+
startIndex: 272,
45+
endIndex: 273,
46+
startColumn: 14,
47+
endColumn: 16,
48+
},
49+
{
50+
text: '.',
51+
line: 9,
52+
startIndex: 274,
53+
endIndex: 274,
54+
startColumn: 16,
55+
endColumn: 17,
56+
},
57+
]);
4058
});
4159

4260
test('Insert into table ', () => {

‎test/parser/hive/suggestion/syntaxSuggestion.test.ts

+27-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,33 @@ describe('Hive SQL Syntax Suggestion', () => {
3232
);
3333

3434
expect(suggestion).not.toBeUndefined();
35-
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.', 'tb']);
35+
expect(suggestion?.wordRanges?.length).toBe(3);
36+
expect(suggestion?.wordRanges).toEqual([
37+
{
38+
text: 'db',
39+
line: 1,
40+
startIndex: 12,
41+
endIndex: 13,
42+
startColumn: 13,
43+
endColumn: 15,
44+
},
45+
{
46+
text: '.',
47+
line: 1,
48+
startIndex: 14,
49+
endIndex: 14,
50+
startColumn: 15,
51+
endColumn: 16,
52+
},
53+
{
54+
text: 'tb',
55+
line: 1,
56+
startIndex: 15,
57+
endIndex: 16,
58+
startColumn: 16,
59+
endColumn: 18,
60+
},
61+
]);
3662
});
3763

3864
test('Select table ', () => {

‎test/parser/impala/suggestion/multipleStatement.test.ts

+19-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,25 @@ describe('ImpalaSQL Multiple Statements Syntax Suggestion', () => {
3636
);
3737

3838
expect(suggestion).not.toBeUndefined();
39-
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
39+
expect(suggestion?.wordRanges?.length).toBe(2);
40+
expect(suggestion?.wordRanges).toEqual([
41+
{
42+
text: 'db',
43+
line: 9,
44+
startIndex: 203,
45+
endIndex: 204,
46+
startColumn: 14,
47+
endColumn: 16,
48+
},
49+
{
50+
text: '.',
51+
line: 9,
52+
startIndex: 205,
53+
endIndex: 205,
54+
startColumn: 16,
55+
endColumn: 17,
56+
},
57+
]);
4058
});
4159

4260
test('Insert into table ', () => {

‎test/parser/impala/suggestion/syntaxSuggestion.test.ts

+27-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,33 @@ describe('Impala SQL Syntax Suggestion', () => {
2626
);
2727

2828
expect(suggestion).not.toBeUndefined();
29-
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['cat', '.', 'a']);
29+
expect(suggestion?.wordRanges?.length).toBe(3);
30+
expect(suggestion?.wordRanges).toEqual([
31+
{
32+
text: 'cat',
33+
line: 1,
34+
startIndex: 14,
35+
endIndex: 16,
36+
startColumn: 15,
37+
endColumn: 18,
38+
},
39+
{
40+
text: '.',
41+
line: 1,
42+
startIndex: 17,
43+
endIndex: 17,
44+
startColumn: 18,
45+
endColumn: 19,
46+
},
47+
{
48+
text: 'a',
49+
line: 1,
50+
startIndex: 18,
51+
endIndex: 18,
52+
startColumn: 19,
53+
endColumn: 20,
54+
},
55+
]);
3056
});
3157

3258
test('Function call', () => {

‎test/parser/mysql/suggestion/multipleStatement.test.ts

+19-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,25 @@ describe('MySQL Multiple Statements Syntax Suggestion', () => {
3636
);
3737

3838
expect(suggestion).not.toBeUndefined();
39-
expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
39+
expect(suggestion?.wordRanges?.length).toBe(2);
40+
expect(suggestion?.wordRanges).toEqual([
41+
{
42+
text: 'db',
43+
line: 9,
44+
startIndex: 306,
45+
endIndex: 307,
46+
startColumn: 14,
47+
endColumn: 16,
48+
},
49+
{
50+
text: '.',
51+
line: 9,
52+
startIndex: 308,
53+
endIndex: 308,
54+
startColumn: 16,
55+
endColumn: 17,
56+
},
57+
]);
4058
});
4159

4260
test('Insert into table ', () => {

0 commit comments

Comments
 (0)