Skip to content

Commit

Permalink
cleaner non-recursive system to go through rerunWithout rule list and…
Browse files Browse the repository at this point in the history 
… a general nonspecific combine function

Signed-off-by: Paul Sebastian <[email protected]>
  • Loading branch information
@paulstn
paulstn committed Jan 9, 2025
1 parent c4023ad commit e2e3843
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 114 deletions.
2 changes: 1 addition & 1 deletion src/plugins/data/public/antlr/opensearch_sql/code_completion.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ describe('sql code_completion', () => {
});
});

it.skip('should suggest aggregate functions when appropriate', async () => {
it('should suggest aggregate functions when appropriate', async () => {
const result = await getSimpleSuggestions('SELECT * FROM test-index WHERE ');

SQL_SYMBOLS.AGREGATE_FUNCTIONS.forEach((func) => {
Expand Down
5 changes: 3 additions & 2 deletions src/plugins/data/public/antlr/opensearch_sql/opensearch_sql_autocomplete.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,9 +178,10 @@ export function processVisitedRules(
break;
}
case OpenSearchSQLParser.RULE_predicate: {
rerunWithoutRules.push(ruleId);
rerunWithoutRules.push(ruleId); // rerun to fetch aggs by blocking pred

/**
* creates a list of the tokens from the start of the pedicate to the end
* create a list of the tokens from the start of the pedicate to the end
* intentionally omit all tokens with type SPACE
* now we know we only have "significant tokens"
*/
Expand Down
1 change: 0 additions & 1 deletion src/plugins/data/public/antlr/shared/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,5 +156,4 @@ export interface ParsingSubject<A extends AutocompleteResultBase, L, P> {
query: string;
cursor: CursorPosition;
context?: ParserRuleContext;
previousResult?: A;
}
198 changes: 88 additions & 110 deletions src/plugins/data/public/antlr/shared/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ export const formatFieldsToSuggestions = (
return fieldSuggestions;
};

export const parseQuery = <
const singleParseQuery = <
A extends AutocompleteResultBase,
L extends LexerType,
P extends ParserType
Expand All @@ -198,8 +198,7 @@ export const parseQuery = <
query,
cursor,
context,
previousResult,
}: ParsingSubject<A, L, P>): AutocompleteResultBase => {
}: ParsingSubject<A, L, P>): A => {
const parser = createParser(Lexer, Parser, query);
const { tokenStream } = parser;
const errorListener = new GeneralErrorListener(tokenDictionary.SPACE);
Expand Down Expand Up @@ -239,122 +238,101 @@ export const parseQuery = <
suggestKeywords,
};

// console.clear();
// console.log('cursorTokenIndex', cursorTokenIndex);
// console.log('Formatted Token Stream:');
// let index = 0;
// try {
// while (true) {
// const token = tokenStream.get(index);
// if (!token) break;

// const isCurrentToken = index === cursorTokenIndex;
// const tokenInfo = `Token ${index}: ${token.text} (Type: ${token.type})`;

// if (isCurrentToken) {
// console.log(`%c${tokenInfo}`, 'background-color: red; font-weight: bold;');
// } else {
// console.log(tokenInfo);
// }

// index++;
// }
// } catch (error) {
// console.error('Error while iterating through token stream:', error);
// }

const currentResult = enrichAutocompleteResult(
result,
rules,
tokenStream,
cursorTokenIndex,
cursor,
query
);

// combine previous and current result
const combinedResult: A = {};

// look at every field in both results then combine inside of combinedResult

if (previousResult) {
Object.keys({ ...currentResult, ...previousResult }).forEach((key) => {
const currentField = currentResult[key as keyof A];
const previousField = previousResult[key as keyof A];

if (currentField && previousField) {
combinedResult[key as keyof A] = {
...currentField,
suggestKeywords: [
...(previousField.suggestKeywords ?? []),
...(currentField.suggestKeywords ?? []),
],
};
} else if (currentField) {
combinedResult[key as keyof A] = currentField;
} else if (previousField) {
combinedResult[key as keyof A] = previousField;
}
});
}

// // combine previous and current result
// if (previousResultKeywords) {
// // only need to modify initial results if there are context keywords
// if (!currentResult?.suggestKeywords) {
// // set initial keywords to be context keywords
// currentResult.suggestKeywords = previousResultKeywords;
// } else {
// // merge initial and context keywords
// const combined = [...currentResult.suggestKeywords, ...previousResultKeywords];

// // ES6 magic to filter out duplicate objects based on id field
// currentResult.suggestKeywords = combined.filter(
// (item, index, self) => index === self.findIndex((other) => other.id === item.id)
// );
// }
// }

// return when we don't have any more rerun and combine rules
if (!currentResult?.rerunWithoutRules || currentResult.rerunWithoutRules.length === 0) {
return currentResult;
}

// pop the top/last rule in the rerun and combine list
const nextRuleToAvoid = currentResult.rerunWithoutRules.pop();
if (!nextRuleToAvoid) {
// rerunWithoutRules being undefined or empty should lead to early return up above
throw new Error('nextRuleToAvoid is undefined');
}

const nextRulesToVisit = new Set(
Array.from(rulesToVisit).filter((rule) => rule !== nextRuleToAvoid)
);

/**
* TODO:
*
* need to handle the issue where the previous results findings from things like
* if we should suggest operators, persist. rn only the keywords are persisting
* across runs
*
* might have to fix the issue in a way where we grab all characteristics across runs,
* this thing might break for ppl if we need characteristics from later runs, where we
* unlease descendants
*/
return enrichAutocompleteResult(result, rules, tokenStream, cursorTokenIndex, cursor, query);
};

// call parseQuery again without specified rule and with this run's result
return parseQuery({
export const parseQuery = <
A extends AutocompleteResultBase,
L extends LexerType,
P extends ParserType
>({
Lexer,
Parser,
tokenDictionary,
ignoredTokens,
rulesToVisit,
getParseTree,
enrichAutocompleteResult,
query,
cursor,
context,
}: ParsingSubject<A, L, P>): AutocompleteResultBase => {
const result = singleParseQuery({
Lexer,
Parser,
tokenDictionary,
ignoredTokens,
rulesToVisit: nextRulesToVisit,
rulesToVisit,
getParseTree,
enrichAutocompleteResult,
query,
cursor,
context,
previousResultKeywords: currentResult,
});

if (!result.rerunWithoutRules) {
return result;
}

// go through each rule in list and run a singleParseQuery without it, combining results
result.rerunWithoutRules.forEach((rule) => {
const modifiedRulesToVisit = new Set(rulesToVisit);
modifiedRulesToVisit.delete(rule);

const nextResult = singleParseQuery({
Lexer,
Parser,
tokenDictionary,
ignoredTokens,
rulesToVisit: modifiedRulesToVisit,
getParseTree,
enrichAutocompleteResult,
query,
cursor,
context,
});

// combine result and nextResult
for (const [field, value] of Object.entries(result)) {
if (field === 'suggestColumns') {
// combine tables inside of suggestColumns
result.suggestColumns = {
tables: [
...(result.suggestColumns?.tables ?? []),
...(nextResult.suggestColumns?.tables ?? []),
],
};
continue;
}

switch (typeof value) {
case 'boolean':
// if a boolean is true, keep overall result true
result[field as keyof A] ||= nextResult[field as keyof A];
break;
case 'undefined':
// use the latter result
result[field as keyof A] = nextResult[field as keyof A];
break;
case 'object':
if (Array.isArray(value)) {
// combine arrays
const combined = [
...((result[field as keyof A] as any[]) ?? []),
...((nextResult[field as keyof A] as any[]) ?? []),
];
// ES6 magic to filter out duplicate objects based on id field
(result[field as keyof A] as any[]) = combined.filter(
(item, index, self) => index === self.findIndex((other) => other.id === item.id)
);
break;
}
default:
// use the initial result
break;
}
}
});

return result;
};

0 comments on commit e2e3843

Please sign in to comment.