diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a9c213d1..ad9542b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,9 +21,6 @@ jobs: java-version: '17' distribution: 'temurin' - - name: Generate Grammar Source - run: ./gradlew generateGrammarSource - - name: Run Tests run: ./gradlew check diff --git a/build.gradle.kts b/build.gradle.kts index 716b41b2..67e5f409 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -47,7 +47,6 @@ changelog { } dependencies { - implementation(project(":codegpt-core")) implementation(project(":codegpt-telemetry")) implementation(project(":codegpt-treesitter")) diff --git a/codegpt-core/build.gradle.kts b/codegpt-core/build.gradle.kts deleted file mode 100644 index ccf90305..00000000 --- a/codegpt-core/build.gradle.kts +++ /dev/null @@ -1,19 +0,0 @@ -plugins { - id("codegpt.java-conventions") - id("antlr") -} - -dependencies { - antlr("org.antlr:antlr4:4.13.1") - api("com.github.jelmerk:hnswlib-core:1.1.2") - api("com.github.jelmerk:hnswlib-utils:1.1.2") - implementation("org.antlr:antlr4-runtime:4.13.1") - implementation("org.json:json:20240303") -} - -tasks { - generateGrammarSource { - outputDirectory = file("src/main/java/grammar") - arguments = arguments + listOf("-package", "grammar") - } -} \ No newline at end of file diff --git a/codegpt-core/src/main/antlr/JSON.g4 b/codegpt-core/src/main/antlr/JSON.g4 deleted file mode 100644 index 3a67ccbc..00000000 --- a/codegpt-core/src/main/antlr/JSON.g4 +++ /dev/null @@ -1,82 +0,0 @@ - -/** Taken from "The Definitive ANTLR 4 Reference" by Terence Parr */ - -// Derived from https://json.org -grammar JSON; - -json - : value EOF - ; - -obj - : '{' pair (',' pair)* '}' - | '{' '}' - ; - -pair - : STRING ':' value - ; - -arr - : '[' value (',' value)* ']' - | '[' ']' - ; - -value - : STRING - | NUMBER - | obj - | arr - | 'true' - | 'false' - | 'null' - ; - - -STRING - : '"' (ESC | SAFECODEPOINT)* '"' - ; - - -fragment ESC - : '\\' (["\\/bfnrt] | UNICODE) - ; - - -fragment UNICODE - : 'u' HEX HEX HEX HEX - ; - - -fragment HEX - : [0-9a-fA-F] - ; - - -fragment SAFECODEPOINT - : ~ ["\\\u0000-\u001F] - ; - - -NUMBER - : '-'? INT ('.' [0-9] +)? EXP? - ; - - -fragment INT - // integer part forbis leading 0s (e.g. `01`) - : '0' | [1-9] [0-9]* - ; - -// no leading zeros - -fragment EXP - // exponent number permits leading 0s (e.g. `1e01`) - : [Ee] [+\-]? [0-9]+ - ; - -// \- since - means "range" inside [...] - -WS - : [ \t\n\r] + -> skip - ; diff --git a/codegpt-core/src/main/antlr/JavaLexer.g4 b/codegpt-core/src/main/antlr/JavaLexer.g4 deleted file mode 100644 index a340c6ab..00000000 --- a/codegpt-core/src/main/antlr/JavaLexer.g4 +++ /dev/null @@ -1,241 +0,0 @@ -/* - [The "BSD licence"] - Copyright (c) 2013 Terence Parr, Sam Harwell - Copyright (c) 2017 Ivan Kochurkin (upgrade to Java 8) - Copyright (c) 2021 Michał Lorek (upgrade to Java 11) - Copyright (c) 2022 Michał Lorek (upgrade to Java 17) - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -lexer grammar JavaLexer; - -// Keywords - -ABSTRACT: 'abstract'; -ASSERT: 'assert'; -BOOLEAN: 'boolean'; -BREAK: 'break'; -BYTE: 'byte'; -CASE: 'case'; -CATCH: 'catch'; -CHAR: 'char'; -CLASS: 'class'; -CONST: 'const'; -CONTINUE: 'continue'; -DEFAULT: 'default'; -DO: 'do'; -DOUBLE: 'double'; -ELSE: 'else'; -ENUM: 'enum'; -EXTENDS: 'extends'; -FINAL: 'final'; -FINALLY: 'finally'; -FLOAT: 'float'; -FOR: 'for'; -IF: 'if'; -GOTO: 'goto'; -IMPLEMENTS: 'implements'; -IMPORT: 'import'; -INSTANCEOF: 'instanceof'; -INT: 'int'; -INTERFACE: 'interface'; -LONG: 'long'; -NATIVE: 'native'; -NEW: 'new'; -PACKAGE: 'package'; -PRIVATE: 'private'; -PROTECTED: 'protected'; -PUBLIC: 'public'; -RETURN: 'return'; -SHORT: 'short'; -STATIC: 'static'; -STRICTFP: 'strictfp'; -SUPER: 'super'; -SWITCH: 'switch'; -SYNCHRONIZED: 'synchronized'; -THIS: 'this'; -THROW: 'throw'; -THROWS: 'throws'; -TRANSIENT: 'transient'; -TRY: 'try'; -VOID: 'void'; -VOLATILE: 'volatile'; -WHILE: 'while'; - -// Module related keywords -MODULE: 'module'; -OPEN: 'open'; -REQUIRES: 'requires'; -EXPORTS: 'exports'; -OPENS: 'opens'; -TO: 'to'; -USES: 'uses'; -PROVIDES: 'provides'; -WITH: 'with'; -TRANSITIVE: 'transitive'; - -// Local Variable Type Inference -VAR: 'var'; // reserved type name - -// Switch Expressions -YIELD: 'yield'; // reserved type name from Java 14 - -// Records -RECORD: 'record'; - -// Sealed Classes -SEALED: 'sealed'; -PERMITS: 'permits'; -NON_SEALED: 'non-sealed'; - -// Literals - -DECIMAL_LITERAL: ('0' | [1-9] (Digits? | '_'+ Digits)) [lL]?; -HEX_LITERAL: '0' [xX] [0-9a-fA-F] ([0-9a-fA-F_]* [0-9a-fA-F])? [lL]?; -OCT_LITERAL: '0' '_'* [0-7] ([0-7_]* [0-7])? [lL]?; -BINARY_LITERAL: '0' [bB] [01] ([01_]* [01])? [lL]?; - -FLOAT_LITERAL: (Digits '.' Digits? | '.' Digits) ExponentPart? [fFdD]? - | Digits (ExponentPart [fFdD]? | [fFdD]) - ; - -HEX_FLOAT_LITERAL: '0' [xX] (HexDigits '.'? | HexDigits? '.' HexDigits) [pP] [+-]? Digits [fFdD]?; - -BOOL_LITERAL: 'true' - | 'false' - ; - -CHAR_LITERAL: '\'' (~['\\\r\n] | EscapeSequence) '\''; - -STRING_LITERAL: '"' (~["\\\r\n] | EscapeSequence)* '"'; - -TEXT_BLOCK: '"""' [ \t]* [\r\n] (. | EscapeSequence)*? '"""'; - -NULL_LITERAL: 'null'; - -// Separators - -LPAREN: '('; -RPAREN: ')'; -LBRACE: '{'; -RBRACE: '}'; -LBRACK: '['; -RBRACK: ']'; -SEMI: ';'; -COMMA: ','; -DOT: '.'; - -// Operators - -ASSIGN: '='; -GT: '>'; -LT: '<'; -BANG: '!'; -TILDE: '~'; -QUESTION: '?'; -COLON: ':'; -EQUAL: '=='; -LE: '<='; -GE: '>='; -NOTEQUAL: '!='; -AND: '&&'; -OR: '||'; -INC: '++'; -DEC: '--'; -ADD: '+'; -SUB: '-'; -MUL: '*'; -DIV: '/'; -BITAND: '&'; -BITOR: '|'; -CARET: '^'; -MOD: '%'; - -ADD_ASSIGN: '+='; -SUB_ASSIGN: '-='; -MUL_ASSIGN: '*='; -DIV_ASSIGN: '/='; -AND_ASSIGN: '&='; -OR_ASSIGN: '|='; -XOR_ASSIGN: '^='; -MOD_ASSIGN: '%='; -LSHIFT_ASSIGN: '<<='; -RSHIFT_ASSIGN: '>>='; -URSHIFT_ASSIGN: '>>>='; - -// Java 8 tokens - -ARROW: '->'; -COLONCOLON: '::'; - -// Additional symbols not defined in the lexical specification - -AT: '@'; -ELLIPSIS: '...'; - -// Whitespace and comments - -WS: [ \t\r\n\u000C]+ -> channel(HIDDEN); -COMMENT: '/*' .*? '*/' -> channel(HIDDEN); -LINE_COMMENT: '//' ~[\r\n]* -> channel(HIDDEN); - -// Identifiers - -IDENTIFIER: Letter LetterOrDigit*; - -// Fragment rules - -fragment ExponentPart - : [eE] [+-]? Digits - ; - -fragment EscapeSequence - : '\\' 'u005c'? [btnfr"'\\] - | '\\' 'u005c'? ([0-3]? [0-7])? [0-7] - | '\\' 'u'+ HexDigit HexDigit HexDigit HexDigit - ; - -fragment HexDigits - : HexDigit ((HexDigit | '_')* HexDigit)? - ; - -fragment HexDigit - : [0-9a-fA-F] - ; - -fragment Digits - : [0-9] ([0-9_]* [0-9])? - ; - -fragment LetterOrDigit - : Letter - | [0-9] - ; - -fragment Letter - : [a-zA-Z$_] // these are the "java letters" below 0x7F - | ~[\u0000-\u007F\uD800-\uDBFF] // covers all characters above 0x7F which are not a surrogate - | [\uD800-\uDBFF] [\uDC00-\uDFFF] // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF - ; diff --git a/codegpt-core/src/main/antlr/JavaParser.g4 b/codegpt-core/src/main/antlr/JavaParser.g4 deleted file mode 100644 index bc249c89..00000000 --- a/codegpt-core/src/main/antlr/JavaParser.g4 +++ /dev/null @@ -1,780 +0,0 @@ -/* - [The "BSD licence"] - Copyright (c) 2013 Terence Parr, Sam Harwell - Copyright (c) 2017 Ivan Kochurkin (upgrade to Java 8) - Copyright (c) 2021 Michał Lorek (upgrade to Java 11) - Copyright (c) 2022 Michał Lorek (upgrade to Java 17) - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -parser grammar JavaParser; - -options { tokenVocab=JavaLexer; } - -compilationUnit - : packageDeclaration? (importDeclaration | ';')* (typeDeclaration | ';')* - | moduleDeclaration EOF - ; - -packageDeclaration - : annotation* PACKAGE qualifiedName ';' - ; - -importDeclaration - : IMPORT STATIC? qualifiedName ('.' '*')? ';' - ; - -typeDeclaration - : classOrInterfaceModifier* - (classDeclaration | enumDeclaration | interfaceDeclaration | annotationTypeDeclaration | recordDeclaration) - ; - -modifier - : classOrInterfaceModifier - | NATIVE - | SYNCHRONIZED - | TRANSIENT - | VOLATILE - ; - -classOrInterfaceModifier - : annotation - | PUBLIC - | PROTECTED - | PRIVATE - | STATIC - | ABSTRACT - | FINAL // FINAL for class only -- does not apply to interfaces - | STRICTFP - | SEALED // Java17 - | NON_SEALED // Java17 - ; - -variableModifier - : FINAL - | annotation - ; - -classDeclaration - : CLASS identifier typeParameters? - (EXTENDS typeType)? - (IMPLEMENTS typeList)? - (PERMITS typeList)? // Java17 - classBody - ; - -typeParameters - : '<' typeParameter (',' typeParameter)* '>' - ; - -typeParameter - : annotation* identifier (EXTENDS annotation* typeBound)? - ; - -typeBound - : typeType ('&' typeType)* - ; - -enumDeclaration - : ENUM identifier (IMPLEMENTS typeList)? '{' enumConstants? ','? enumBodyDeclarations? '}' - ; - -enumConstants - : enumConstant (',' enumConstant)* - ; - -enumConstant - : annotation* identifier arguments? classBody? - ; - -enumBodyDeclarations - : ';' classBodyDeclaration* - ; - -interfaceDeclaration - : INTERFACE identifier typeParameters? (EXTENDS typeList)? (PERMITS typeList)? interfaceBody - ; - -classBody - : '{' classBodyDeclaration* '}' - ; - -interfaceBody - : '{' interfaceBodyDeclaration* '}' - ; - -classBodyDeclaration - : ';' - | STATIC? block - | modifier* memberDeclaration - ; - -memberDeclaration - : recordDeclaration //Java17 - | methodDeclaration - | genericMethodDeclaration - | fieldDeclaration - | constructorDeclaration - | genericConstructorDeclaration - | interfaceDeclaration - | annotationTypeDeclaration - | classDeclaration - | enumDeclaration - ; - -/* We use rule this even for void methods which cannot have [] after parameters. - This simplifies grammar and we can consider void to be a type, which - renders the [] matching as a context-sensitive issue or a semantic check - for invalid return type after parsing. - */ -methodDeclaration - : typeTypeOrVoid identifier formalParameters ('[' ']')* - (THROWS qualifiedNameList)? - methodBody - ; - -methodBody - : block - | ';' - ; - -typeTypeOrVoid - : typeType - | VOID - ; - -genericMethodDeclaration - : typeParameters methodDeclaration - ; - -genericConstructorDeclaration - : typeParameters constructorDeclaration - ; - -constructorDeclaration - : identifier formalParameters (THROWS qualifiedNameList)? constructorBody=block - ; - -compactConstructorDeclaration - : modifier* identifier constructorBody=block - ; - -fieldDeclaration - : typeType variableDeclarators ';' - ; - -interfaceBodyDeclaration - : modifier* interfaceMemberDeclaration - | ';' - ; - -interfaceMemberDeclaration - : recordDeclaration // Java17 - | constDeclaration - | interfaceMethodDeclaration - | genericInterfaceMethodDeclaration - | interfaceDeclaration - | annotationTypeDeclaration - | classDeclaration - | enumDeclaration - ; - -constDeclaration - : typeType constantDeclarator (',' constantDeclarator)* ';' - ; - -constantDeclarator - : identifier ('[' ']')* '=' variableInitializer - ; - -// Early versions of Java allows brackets after the method name, eg. -// public int[] return2DArray() [] { ... } -// is the same as -// public int[][] return2DArray() { ... } -interfaceMethodDeclaration - : interfaceMethodModifier* interfaceCommonBodyDeclaration - ; - -// Java8 -interfaceMethodModifier - : annotation - | PUBLIC - | ABSTRACT - | DEFAULT - | STATIC - | STRICTFP - ; - -genericInterfaceMethodDeclaration - : interfaceMethodModifier* typeParameters interfaceCommonBodyDeclaration - ; - -interfaceCommonBodyDeclaration - : annotation* typeTypeOrVoid identifier formalParameters ('[' ']')* (THROWS qualifiedNameList)? methodBody - ; - -variableDeclarators - : variableDeclarator (',' variableDeclarator)* - ; - -variableDeclarator - : variableDeclaratorId ('=' variableInitializer)? - ; - -variableDeclaratorId - : identifier ('[' ']')* - ; - -variableInitializer - : arrayInitializer - | expression - ; - -arrayInitializer - : '{' (variableInitializer (',' variableInitializer)* ','? )? '}' - ; - -classOrInterfaceType - : (identifier typeArguments? '.')* typeIdentifier typeArguments? - ; - -typeArgument - : typeType - | annotation* '?' ((EXTENDS | SUPER) typeType)? - ; - -qualifiedNameList - : qualifiedName (',' qualifiedName)* - ; - -formalParameters - : '(' ( receiverParameter? - | receiverParameter (',' formalParameterList)? - | formalParameterList? - ) ')' - ; - -receiverParameter - : typeType (identifier '.')* THIS - ; - -formalParameterList - : formalParameter (',' formalParameter)* (',' lastFormalParameter)? - | lastFormalParameter - ; - -formalParameter - : variableModifier* typeType variableDeclaratorId - ; - -lastFormalParameter - : variableModifier* typeType annotation* '...' variableDeclaratorId - ; - -// local variable type inference -lambdaLVTIList - : lambdaLVTIParameter (',' lambdaLVTIParameter)* - ; - -lambdaLVTIParameter - : variableModifier* VAR identifier - ; - -qualifiedName - : identifier ('.' identifier)* - ; - -literal - : integerLiteral - | floatLiteral - | CHAR_LITERAL - | STRING_LITERAL - | BOOL_LITERAL - | NULL_LITERAL - | TEXT_BLOCK // Java17 - ; - -integerLiteral - : DECIMAL_LITERAL - | HEX_LITERAL - | OCT_LITERAL - | BINARY_LITERAL - ; - -floatLiteral - : FLOAT_LITERAL - | HEX_FLOAT_LITERAL - ; - -// ANNOTATIONS -altAnnotationQualifiedName - : (identifier DOT)* '@' identifier - ; - -annotation - : ('@' qualifiedName | altAnnotationQualifiedName) ('(' ( elementValuePairs | elementValue )? ')')? - ; - -elementValuePairs - : elementValuePair (',' elementValuePair)* - ; - -elementValuePair - : identifier '=' elementValue - ; - -elementValue - : expression - | annotation - | elementValueArrayInitializer - ; - -elementValueArrayInitializer - : '{' (elementValue (',' elementValue)*)? ','? '}' - ; - -annotationTypeDeclaration - : '@' INTERFACE identifier annotationTypeBody - ; - -annotationTypeBody - : '{' annotationTypeElementDeclaration* '}' - ; - -annotationTypeElementDeclaration - : modifier* annotationTypeElementRest - | ';' // this is not allowed by the grammar, but apparently allowed by the actual compiler - ; - -annotationTypeElementRest - : typeType annotationMethodOrConstantRest ';' - | classDeclaration ';'? - | interfaceDeclaration ';'? - | enumDeclaration ';'? - | annotationTypeDeclaration ';'? - | recordDeclaration ';'? // Java17 - ; - -annotationMethodOrConstantRest - : annotationMethodRest - | annotationConstantRest - ; - -annotationMethodRest - : identifier '(' ')' defaultValue? - ; - -annotationConstantRest - : variableDeclarators - ; - -defaultValue - : DEFAULT elementValue - ; - -// MODULES - Java9 - -moduleDeclaration - : OPEN? MODULE qualifiedName moduleBody - ; - -moduleBody - : '{' moduleDirective* '}' - ; - -moduleDirective - : REQUIRES requiresModifier* qualifiedName ';' - | EXPORTS qualifiedName (TO qualifiedName)? ';' - | OPENS qualifiedName (TO qualifiedName)? ';' - | USES qualifiedName ';' - | PROVIDES qualifiedName WITH qualifiedName ';' - ; - -requiresModifier - : TRANSITIVE - | STATIC - ; - -// RECORDS - Java 17 - -recordDeclaration - : RECORD identifier typeParameters? recordHeader - (IMPLEMENTS typeList)? - recordBody - ; - -recordHeader - : '(' recordComponentList? ')' - ; - -recordComponentList - : recordComponent (',' recordComponent)* - ; - -recordComponent - : typeType identifier - ; - -recordBody - : '{' (classBodyDeclaration | compactConstructorDeclaration)* '}' - ; - -// STATEMENTS / BLOCKS - -block - : '{' blockStatement* '}' - ; - -blockStatement - : localVariableDeclaration ';' - | localTypeDeclaration - | statement - ; - -localVariableDeclaration - : variableModifier* (VAR identifier '=' expression | typeType variableDeclarators) - ; - -identifier - : IDENTIFIER - | MODULE - | OPEN - | REQUIRES - | EXPORTS - | OPENS - | TO - | USES - | PROVIDES - | WITH - | TRANSITIVE - | YIELD - | SEALED - | PERMITS - | RECORD - | VAR - ; - -typeIdentifier // Identifiers that are not restricted for type declarations - : IDENTIFIER - | MODULE - | OPEN - | REQUIRES - | EXPORTS - | OPENS - | TO - | USES - | PROVIDES - | WITH - | TRANSITIVE - | SEALED - | PERMITS - | RECORD - ; - -localTypeDeclaration - : classOrInterfaceModifier* - (classDeclaration | interfaceDeclaration | recordDeclaration) - ; - -statement - : blockLabel=block - | ASSERT expression (':' expression)? ';' - | IF parExpression statement (ELSE statement)? - | FOR '(' forControl ')' statement - | WHILE parExpression statement - | DO statement WHILE parExpression ';' - | TRY block (catchClause+ finallyBlock? | finallyBlock) - | TRY resourceSpecification block catchClause* finallyBlock? - | SWITCH parExpression '{' switchBlockStatementGroup* switchLabel* '}' - | SYNCHRONIZED parExpression block - | RETURN expression? ';' - | THROW expression ';' - | BREAK identifier? ';' - | CONTINUE identifier? ';' - | YIELD expression ';' // Java17 - | SEMI - | statementExpression=expression ';' - | switchExpression ';'? // Java17 - | identifierLabel=identifier ':' statement - ; - -catchClause - : CATCH '(' variableModifier* catchType identifier ')' block - ; - -catchType - : qualifiedName ('|' qualifiedName)* - ; - -finallyBlock - : FINALLY block - ; - -resourceSpecification - : '(' resources ';'? ')' - ; - -resources - : resource (';' resource)* - ; - -resource - : variableModifier* ( classOrInterfaceType variableDeclaratorId | VAR identifier ) '=' expression - | qualifiedName - ; - -/** Matches cases then statements, both of which are mandatory. - * To handle empty cases at the end, we add switchLabel* to statement. - */ -switchBlockStatementGroup - : switchLabel+ blockStatement+ - ; - -switchLabel - : CASE (constantExpression=expression | enumConstantName=IDENTIFIER | typeType varName=identifier) ':' - | DEFAULT ':' - ; - -forControl - : enhancedForControl - | forInit? ';' expression? ';' forUpdate=expressionList? - ; - -forInit - : localVariableDeclaration - | expressionList - ; - -enhancedForControl - : variableModifier* (typeType | VAR) variableDeclaratorId ':' expression - ; - -// EXPRESSIONS - -parExpression - : '(' expression ')' - ; - -expressionList - : expression (',' expression)* - ; - -methodCall - : (identifier | THIS | SUPER) arguments - ; - -expression - // Expression order in accordance with https://introcs.cs.princeton.edu/java/11precedence/ - // Level 16, Primary, array and member access - : primary - | expression '[' expression ']' - | expression bop='.' - ( - identifier - | methodCall - | THIS - | NEW nonWildcardTypeArguments? innerCreator - | SUPER superSuffix - | explicitGenericInvocation - ) - // Method calls and method references are part of primary, and hence level 16 precedence - | methodCall - | expression '::' typeArguments? identifier - | typeType '::' (typeArguments? identifier | NEW) - | classType '::' typeArguments? NEW - - | switchExpression // Java17 - - // Level 15 Post-increment/decrement operators - | expression postfix=('++' | '--') - - // Level 14, Unary operators - | prefix=('+'|'-'|'++'|'--'|'~'|'!') expression - - // Level 13 Cast and object creation - | '(' annotation* typeType ('&' typeType)* ')' expression - | NEW creator - - // Level 12 to 1, Remaining operators - | expression bop=('*'|'/'|'%') expression // Level 12, Multiplicative operators - | expression bop=('+'|'-') expression // Level 11, Additive operators - | expression ('<' '<' | '>' '>' '>' | '>' '>') expression // Level 10, Shift operators - | expression bop=('<=' | '>=' | '>' | '<') expression // Level 9, Relational operators - | expression bop=INSTANCEOF (typeType | pattern) - | expression bop=('==' | '!=') expression // Level 8, Equality Operators - | expression bop='&' expression // Level 7, Bitwise AND - | expression bop='^' expression // Level 6, Bitwise XOR - | expression bop='|' expression // Level 5, Bitwise OR - | expression bop='&&' expression // Level 4, Logic AND - | expression bop='||' expression // Level 3, Logic OR - | expression bop='?' expression ':' expression // Level 2, Ternary - // Level 1, Assignment - | expression - bop=('=' | '+=' | '-=' | '*=' | '/=' | '&=' | '|=' | '^=' | '>>=' | '>>>=' | '<<=' | '%=') - expression - - // Level 0, Lambda Expression - | lambdaExpression // Java8 - ; - -// Java17 -pattern - : variableModifier* typeType annotation* identifier - ; - -// Java8 -lambdaExpression - : lambdaParameters '->' lambdaBody - ; - -// Java8 -lambdaParameters - : identifier - | '(' formalParameterList? ')' - | '(' identifier (',' identifier)* ')' - | '(' lambdaLVTIList? ')' - ; - -// Java8 -lambdaBody - : expression - | block - ; - -primary - : '(' expression ')' - | THIS - | SUPER - | literal - | identifier - | typeTypeOrVoid '.' CLASS - | nonWildcardTypeArguments (explicitGenericInvocationSuffix | THIS arguments) - ; - -// Java17 -switchExpression - : SWITCH parExpression '{' switchLabeledRule* '}' - ; - -// Java17 -switchLabeledRule - : CASE (expressionList | NULL_LITERAL | guardedPattern) (ARROW | COLON) switchRuleOutcome - | DEFAULT (ARROW | COLON) switchRuleOutcome - ; - -// Java17 -guardedPattern - : '(' guardedPattern ')' - | variableModifier* typeType annotation* identifier ('&&' expression)* - | guardedPattern '&&' expression - ; - -// Java17 -switchRuleOutcome - : block - | blockStatement* - ; - -classType - : (classOrInterfaceType '.')? annotation* identifier typeArguments? - ; - -creator - : nonWildcardTypeArguments? createdName classCreatorRest - | createdName arrayCreatorRest - ; - -createdName - : identifier typeArgumentsOrDiamond? ('.' identifier typeArgumentsOrDiamond?)* - | primitiveType - ; - -innerCreator - : identifier nonWildcardTypeArgumentsOrDiamond? classCreatorRest - ; - -arrayCreatorRest - : ('[' ']')+ arrayInitializer - | ('[' expression ']')+ ('[' ']')* - ; - -classCreatorRest - : arguments classBody? - ; - -explicitGenericInvocation - : nonWildcardTypeArguments explicitGenericInvocationSuffix - ; - -typeArgumentsOrDiamond - : '<' '>' - | typeArguments - ; - -nonWildcardTypeArgumentsOrDiamond - : '<' '>' - | nonWildcardTypeArguments - ; - -nonWildcardTypeArguments - : '<' typeList '>' - ; - -typeList - : typeType (',' typeType)* - ; - -typeType - : annotation* (classOrInterfaceType | primitiveType) (annotation* '[' ']')* - ; - -primitiveType - : BOOLEAN - | CHAR - | BYTE - | SHORT - | INT - | LONG - | FLOAT - | DOUBLE - ; - -typeArguments - : '<' typeArgument (',' typeArgument)* '>' - ; - -superSuffix - : arguments - | '.' typeArguments? identifier arguments? - ; - -explicitGenericInvocationSuffix - : SUPER superSuffix - | identifier arguments - ; - -arguments - : '(' expressionList? ')' - ; diff --git a/codegpt-core/src/main/antlr/KotlinLexer.g4 b/codegpt-core/src/main/antlr/KotlinLexer.g4 deleted file mode 100644 index 98663a79..00000000 --- a/codegpt-core/src/main/antlr/KotlinLexer.g4 +++ /dev/null @@ -1,587 +0,0 @@ -/** - * Kotlin Grammar for ANTLR v4 - * - * Based on: - * http://jetbrains.github.io/kotlin-spec/#_grammars_and_parsing - * and - * http://kotlinlang.org/docs/reference/grammar.html - * - * Tested on - * https://github.com/JetBrains/kotlin/tree/master/compiler/testData/psi - */ - -lexer grammar KotlinLexer; - -import UnicodeClasses; - -ShebangLine - : '#!' ~[\u000A\u000D]* - -> channel(HIDDEN) - ; - -DelimitedComment - : '/*' ( DelimitedComment | . )*? '*/' - -> channel(HIDDEN) - ; - -LineComment - : '//' ~[\u000A\u000D]* - -> channel(HIDDEN) - ; - -WS - : [\u0020\u0009\u000C] - -> skip - ; - -NL: '\u000A' | '\u000D' '\u000A' ; - -//SEPARATORS & OPERATIONS - -RESERVED: '...' ; -DOT: '.' ; -COMMA: ',' ; -LPAREN: '(' -> pushMode(Inside) ; -RPAREN: ')' ; -LSQUARE: '[' -> pushMode(Inside) ; -RSQUARE: ']' ; -LCURL: '{' ; -RCURL: '}' ; -MULT: '*' ; -MOD: '%' ; -DIV: '/' ; -ADD: '+' ; -SUB: '-' ; -INCR: '++' ; -DECR: '--' ; -CONJ: '&&' ; -DISJ: '||' ; -EXCL: '!' ; -COLON: ':' ; -SEMICOLON: ';' ; -ASSIGNMENT: '=' ; -ADD_ASSIGNMENT: '+=' ; -SUB_ASSIGNMENT: '-=' ; -MULT_ASSIGNMENT: '*=' ; -DIV_ASSIGNMENT: '/=' ; -MOD_ASSIGNMENT: '%=' ; -ARROW: '->' ; -DOUBLE_ARROW: '=>' ; -RANGE: '..' ; -COLONCOLON: '::' ; -Q_COLONCOLON: '?::' ; -DOUBLE_SEMICOLON: ';;' ; -HASH: '#' ; -AT: '@' ; -QUEST: '?' ; -ELVIS: '?:' ; -LANGLE: '<' ; -RANGLE: '>' ; -LE: '<=' ; -GE: '>=' ; -EXCL_EQ: '!=' ; -EXCL_EQEQ: '!==' ; -AS_SAFE: 'as?' ; -EQEQ: '==' ; -EQEQEQ: '===' ; -SINGLE_QUOTE: '\'' ; - -//KEYWORDS - -RETURN_AT: 'return@' Identifier ; -CONTINUE_AT: 'continue@' Identifier ; -BREAK_AT: 'break@' Identifier ; - -FILE: '@file' ; -PACKAGE: 'package' ; -IMPORT: 'import' ; -CLASS: 'class' ; -INTERFACE: 'interface' ; -FUN: 'fun' ; -OBJECT: 'object' ; -VAL: 'val' ; -VAR: 'var' ; -TYPE_ALIAS: 'typealias' ; -CONSTRUCTOR: 'constructor' ; -BY: 'by' ; -COMPANION: 'companion' ; -INIT: 'init' ; -THIS: 'this' ; -SUPER: 'super' ; -TYPEOF: 'typeof' ; -WHERE: 'where' ; -IF: 'if' ; -ELSE: 'else' ; -WHEN: 'when' ; -TRY: 'try' ; -CATCH: 'catch' ; -FINALLY: 'finally' ; -FOR: 'for' ; -DO: 'do' ; -WHILE: 'while' ; -THROW: 'throw' ; -RETURN: 'return' ; -CONTINUE: 'continue' ; -BREAK: 'break' ; -AS: 'as' ; -IS: 'is' ; -IN: 'in' ; -NOT_IS: '!is' (WS | NL)+ ; -NOT_IN: '!in' (WS | NL)+ ; -OUT: 'out' ; -FIELD: '@field' ; -PROPERTY: '@property' ; -GET: '@get' ; -SET: '@set' ; -GETTER: 'get' ; -SETTER: 'set' ; -RECEIVER: '@receiver' ; -PARAM: '@param' ; -SETPARAM: '@setparam' ; -DELEGATE: '@delegate' ; -DYNAMIC: 'dynamic' ; - -//MODIFIERS - -PUBLIC: 'public' ; -PRIVATE: 'private' ; -PROTECTED: 'protected' ; -INTERNAL: 'internal' ; -ENUM: 'enum' ; -SEALED: 'sealed' ; -ANNOTATION: 'annotation' ; -DATA: 'data' ; -INNER: 'inner' ; -TAILREC: 'tailrec' ; -OPERATOR: 'operator' ; -INLINE: 'inline' ; -INFIX: 'infix' ; -EXTERNAL: 'external' ; -SUSPEND: 'suspend' ; -OVERRIDE: 'override' ; -ABSTRACT: 'abstract' ; -FINAL: 'final' ; -OPEN: 'open' ; -CONST: 'const' ; -LATEINIT: 'lateinit' ; -VARARG: 'vararg' ; -NOINLINE: 'noinline' ; -CROSSINLINE: 'crossinline' ; -REIFIED: 'reified' ; - -// - -QUOTE_OPEN: '"' -> pushMode(LineString) ; -TRIPLE_QUOTE_OPEN: '"""' -> pushMode(MultiLineString) ; - -RealLiteral - : FloatLiteral - | DoubleLiteral - ; - -FloatLiteral - : (DoubleLiteral | IntegerLiteral) [fF] - ; - -DoubleLiteral - : ( (DecDigitNoZero DecDigit* | '0')? '.' - | (DecDigitNoZero (DecDigit | '_')* DecDigit)? '.') - ( DecDigit+ - | DecDigit (DecDigit | '_')+ DecDigit - | DecDigit+ [eE] ('+' | '-')? DecDigit+ - | DecDigit+ [eE] ('+' | '-')? DecDigit (DecDigit | '_')+ DecDigit - | DecDigit (DecDigit | '_')+ DecDigit [eE] ('+' | '-')? DecDigit+ - | DecDigit (DecDigit | '_')+ DecDigit [eE] ('+' | '-')? DecDigit (DecDigit | '_')+ DecDigit - ) - ; - -LongLiteral - : (IntegerLiteral | HexLiteral | BinLiteral) 'L' - ; - -IntegerLiteral - : ('0' - | DecDigitNoZero DecDigit* - | DecDigitNoZero (DecDigit | '_')+ DecDigit - | DecDigitNoZero DecDigit* [eE] ('+' | '-')? DecDigit+ - | DecDigitNoZero DecDigit* [eE] ('+' | '-')? DecDigit (DecDigit | '_')+ DecDigit - | DecDigitNoZero (DecDigit | '_')+ DecDigit [eE] ('+' | '-')? DecDigit+ - | DecDigitNoZero (DecDigit | '_')+ DecDigit [eE] ('+' | '-')? DecDigit (DecDigit | '_')+ DecDigit - ) - ; - -fragment DecDigit - : UNICODE_CLASS_ND - ; - -fragment DecDigitNoZero - : UNICODE_CLASS_ND_NoZeros - ; - -fragment UNICODE_CLASS_ND_NoZeros - : '\u0031'..'\u0039' - | '\u0661'..'\u0669' - | '\u06f1'..'\u06f9' - | '\u07c1'..'\u07c9' - | '\u0967'..'\u096f' - | '\u09e7'..'\u09ef' - | '\u0a67'..'\u0a6f' - | '\u0ae7'..'\u0aef' - | '\u0b67'..'\u0b6f' - | '\u0be7'..'\u0bef' - | '\u0c67'..'\u0c6f' - | '\u0ce7'..'\u0cef' - | '\u0d67'..'\u0d6f' - | '\u0de7'..'\u0def' - | '\u0e51'..'\u0e59' - | '\u0ed1'..'\u0ed9' - | '\u0f21'..'\u0f29' - | '\u1041'..'\u1049' - | '\u1091'..'\u1099' - | '\u17e1'..'\u17e9' - | '\u1811'..'\u1819' - | '\u1947'..'\u194f' - | '\u19d1'..'\u19d9' - | '\u1a81'..'\u1a89' - | '\u1a91'..'\u1a99' - | '\u1b51'..'\u1b59' - | '\u1bb1'..'\u1bb9' - | '\u1c41'..'\u1c49' - | '\u1c51'..'\u1c59' - | '\ua621'..'\ua629' - | '\ua8d1'..'\ua8d9' - | '\ua901'..'\ua909' - | '\ua9d1'..'\ua9d9' - | '\ua9f1'..'\ua9f9' - | '\uaa51'..'\uaa59' - | '\uabf1'..'\uabf9' - | '\uff11'..'\uff19' - ; - -HexLiteral - : '0' [xX] HexDigit (HexDigit | '_')* - ; - -fragment HexDigit - : [0-9a-fA-F] - ; - -BinLiteral - : '0' [bB] BinDigit (BinDigit | '_')* - ; - -fragment BinDigit - : [01] - ; - -BooleanLiteral - : 'true' - | 'false' - ; - -NullLiteral - : 'null' - ; - -Identifier - : (Letter | '_') (Letter | '_' | DecDigit)* - | '`' ~('`')+ '`' - ; - -LabelReference - : '@' Identifier - ; - -LabelDefinition - : Identifier '@' - ; - -FieldIdentifier - : '$' Identifier - ; - -CharacterLiteral - : '\'' (EscapeSeq | .) '\'' - ; - -fragment EscapeSeq - : UniCharacterLiteral - | EscapedIdentifier - ; - -fragment UniCharacterLiteral - : '\\' 'u' HexDigit HexDigit HexDigit HexDigit - ; - -fragment EscapedIdentifier - : '\\' ('t' | 'b' | 'r' | 'n' | '\'' | '"' | '\\' | '$') - ; - -fragment Letter - : UNICODE_CLASS_LL - | UNICODE_CLASS_LM - | UNICODE_CLASS_LO - | UNICODE_CLASS_LT - | UNICODE_CLASS_LU - | UNICODE_CLASS_NL - ; - - -mode Inside ; - -Inside_RPAREN: ')' -> popMode, type(RPAREN) ; -Inside_RSQUARE: ']' -> popMode, type(RSQUARE); - -Inside_LPAREN: LPAREN -> pushMode(Inside), type(LPAREN) ; -Inside_LSQUARE: LSQUARE -> pushMode(Inside), type(LSQUARE) ; - -Inside_LCURL: LCURL -> type(LCURL) ; -Inside_RCURL: RCURL -> type(RCURL) ; -Inside_DOT: DOT -> type(DOT) ; -Inside_COMMA: COMMA -> type(COMMA) ; -Inside_MULT: MULT -> type(MULT) ; -Inside_MOD: MOD -> type(MOD) ; -Inside_DIV: DIV -> type(DIV) ; -Inside_ADD: ADD -> type(ADD) ; -Inside_SUB: SUB -> type(SUB) ; -Inside_INCR: INCR -> type(INCR) ; -Inside_DECR: DECR -> type(DECR) ; -Inside_CONJ: CONJ -> type(CONJ) ; -Inside_DISJ: DISJ -> type(DISJ) ; -Inside_EXCL: EXCL -> type(EXCL) ; -Inside_COLON: COLON -> type(COLON) ; -Inside_SEMICOLON: SEMICOLON -> type(SEMICOLON) ; -Inside_ASSIGNMENT: ASSIGNMENT -> type(ASSIGNMENT) ; -Inside_ADD_ASSIGNMENT: ADD_ASSIGNMENT -> type(ADD_ASSIGNMENT) ; -Inside_SUB_ASSIGNMENT: SUB_ASSIGNMENT -> type(SUB_ASSIGNMENT) ; -Inside_MULT_ASSIGNMENT: MULT_ASSIGNMENT -> type(MULT_ASSIGNMENT) ; -Inside_DIV_ASSIGNMENT: DIV_ASSIGNMENT -> type(DIV_ASSIGNMENT) ; -Inside_MOD_ASSIGNMENT: MOD_ASSIGNMENT -> type(MOD_ASSIGNMENT) ; -Inside_ARROW: ARROW -> type(ARROW) ; -Inside_DOUBLE_ARROW: DOUBLE_ARROW -> type(DOUBLE_ARROW) ; -Inside_RANGE: RANGE -> type(RANGE) ; -Inside_RESERVED: RESERVED -> type(RESERVED) ; -Inside_COLONCOLON: COLONCOLON -> type(COLONCOLON) ; -Inside_Q_COLONCOLON: Q_COLONCOLON -> type(Q_COLONCOLON) ; -Inside_DOUBLE_SEMICOLON: DOUBLE_SEMICOLON -> type(DOUBLE_SEMICOLON) ; -Inside_HASH: HASH -> type(HASH) ; -Inside_AT: AT -> type(AT) ; -Inside_QUEST: QUEST -> type(QUEST) ; -Inside_ELVIS: ELVIS -> type(ELVIS) ; -Inside_LANGLE: LANGLE -> type(LANGLE) ; -Inside_RANGLE: RANGLE -> type(RANGLE) ; -Inside_LE: LE -> type(LE) ; -Inside_GE: GE -> type(GE) ; -Inside_EXCL_EQ: EXCL_EQ -> type(EXCL_EQ) ; -Inside_EXCL_EQEQ: EXCL_EQEQ -> type(EXCL_EQEQ) ; -Inside_NOT_IS: NOT_IS -> type(NOT_IS) ; -Inside_NOT_IN: NOT_IN -> type(NOT_IN) ; -Inside_AS_SAFE: AS_SAFE -> type(AS_SAFE) ; -Inside_EQEQ: EQEQ -> type(EQEQ) ; -Inside_EQEQEQ: EQEQEQ -> type(EQEQEQ) ; -Inside_SINGLE_QUOTE: SINGLE_QUOTE -> type(SINGLE_QUOTE) ; -Inside_QUOTE_OPEN: QUOTE_OPEN -> pushMode(LineString), type(QUOTE_OPEN) ; -Inside_TRIPLE_QUOTE_OPEN: TRIPLE_QUOTE_OPEN -> pushMode(MultiLineString), type(TRIPLE_QUOTE_OPEN) ; - -Inside_VAL: VAL -> type(VAL) ; -Inside_VAR: VAR -> type(VAR) ; -Inside_OBJECT: OBJECT -> type(OBJECT) ; -Inside_SUPER: SUPER -> type(SUPER) ; -Inside_IN: IN -> type(IN) ; -Inside_OUT: OUT -> type(OUT) ; -Inside_FIELD: FIELD -> type(FIELD) ; -Inside_FILE: FILE -> type(FILE) ; -Inside_PROPERTY: PROPERTY -> type(PROPERTY) ; -Inside_GET: GET -> type(GET) ; -Inside_SET: SET -> type(SET) ; -Inside_RECEIVER: RECEIVER -> type(RECEIVER) ; -Inside_PARAM: PARAM -> type(PARAM) ; -Inside_SETPARAM: SETPARAM -> type(SETPARAM) ; -Inside_DELEGATE: DELEGATE -> type(DELEGATE) ; -Inside_THROW: THROW -> type(THROW) ; -Inside_RETURN: RETURN -> type(RETURN) ; -Inside_CONTINUE: CONTINUE -> type(CONTINUE) ; -Inside_BREAK: BREAK -> type(BREAK) ; -Inside_RETURN_AT: RETURN_AT -> type(RETURN_AT) ; -Inside_CONTINUE_AT: CONTINUE_AT -> type(CONTINUE_AT) ; -Inside_BREAK_AT: BREAK_AT -> type(BREAK_AT) ; -Inside_IF: IF -> type(IF) ; -Inside_ELSE: ELSE -> type(ELSE) ; -Inside_WHEN: WHEN -> type(WHEN) ; -Inside_TRY: TRY -> type(TRY) ; -Inside_CATCH: CATCH -> type(CATCH) ; -Inside_FINALLY: FINALLY -> type(FINALLY) ; -Inside_FOR: FOR -> type(FOR) ; -Inside_DO: DO -> type(DO) ; -Inside_WHILE: WHILE -> type(WHILE) ; - -Inside_PUBLIC: PUBLIC -> type(PUBLIC) ; -Inside_PRIVATE: PRIVATE -> type(PRIVATE) ; -Inside_PROTECTED: PROTECTED -> type(PROTECTED) ; -Inside_INTERNAL: INTERNAL -> type(INTERNAL) ; -Inside_ENUM: ENUM -> type(ENUM) ; -Inside_SEALED: SEALED -> type(SEALED) ; -Inside_ANNOTATION: ANNOTATION -> type(ANNOTATION) ; -Inside_DATA: DATA -> type(DATA) ; -Inside_INNER: INNER -> type(INNER) ; -Inside_TAILREC: TAILREC -> type(TAILREC) ; -Inside_OPERATOR: OPERATOR -> type(OPERATOR) ; -Inside_INLINE: INLINE -> type(INLINE) ; -Inside_INFIX: INFIX -> type(INFIX) ; -Inside_EXTERNAL: EXTERNAL -> type(EXTERNAL) ; -Inside_SUSPEND: SUSPEND -> type(SUSPEND) ; -Inside_OVERRIDE: OVERRIDE -> type(OVERRIDE) ; -Inside_ABSTRACT: ABSTRACT -> type(ABSTRACT) ; -Inside_FINAL: FINAL -> type(FINAL) ; -Inside_OPEN: OPEN -> type(OPEN) ; -Inside_CONST: CONST -> type(CONST) ; -Inside_LATEINIT: LATEINIT -> type(LATEINIT) ; -Inside_VARARG: VARARG -> type(VARARG) ; -Inside_NOINLINE: NOINLINE -> type(NOINLINE) ; -Inside_CROSSINLINE: CROSSINLINE -> type(CROSSINLINE) ; -Inside_REIFIED: REIFIED -> type(REIFIED) ; - -Inside_BooleanLiteral: BooleanLiteral -> type(BooleanLiteral) ; -Inside_IntegerLiteral: IntegerLiteral -> type(IntegerLiteral) ; -Inside_HexLiteral: HexLiteral -> type(HexLiteral) ; -Inside_BinLiteral: BinLiteral -> type(BinLiteral) ; -Inside_CharacterLiteral: CharacterLiteral -> type(CharacterLiteral) ; -Inside_RealLiteral: RealLiteral -> type(RealLiteral) ; -Inside_NullLiteral: NullLiteral -> type(NullLiteral) ; - -Inside_LongLiteral: LongLiteral -> type(LongLiteral) ; - -Inside_Identifier: Identifier -> type(Identifier) ; -Inside_LabelReference: LabelReference -> type(LabelReference) ; -Inside_LabelDefinition: LabelDefinition -> type(LabelDefinition) ; -Inside_Comment: (LineComment | DelimitedComment) -> channel(HIDDEN) ; -Inside_WS: WS -> skip ; -Inside_NL: NL -> skip ; - - -mode LineString ; - -QUOTE_CLOSE - : '"' -> popMode - ; - -LineStrRef - : FieldIdentifier - ; - -LineStrText - : ~('\\' | '"' | '$')+ | '$' - ; - -LineStrEscapedChar - : '\\' . - | UniCharacterLiteral - ; - -LineStrExprStart - : '${' -> pushMode(StringExpression) - ; - - -mode MultiLineString ; - -TRIPLE_QUOTE_CLOSE - : MultiLineStringQuote? '"""' -> popMode - ; - -MultiLineStringQuote - : '"'+ - ; - -MultiLineStrRef - : FieldIdentifier - ; - -MultiLineStrText - : ~('\\' | '"' | '$')+ | '$' - ; - -MultiLineStrEscapedChar - : '\\' . - ; - -MultiLineStrExprStart - : '${' -> pushMode(StringExpression) - ; - -MultiLineNL: NL -> skip ; - - -mode StringExpression ; - -StrExpr_RCURL: RCURL -> popMode, type(RCURL) ; - -StrExpr_LPAREN: LPAREN -> pushMode(Inside), type(LPAREN) ; -StrExpr_LSQUARE: LSQUARE -> pushMode(Inside), type(LSQUARE) ; - -StrExpr_RPAREN: ')' -> type(RPAREN) ; -StrExpr_RSQUARE: ']' -> type(RSQUARE); -StrExpr_LCURL: LCURL -> pushMode(StringExpression), type(LCURL) ; -StrExpr_DOT: DOT -> type(DOT) ; -StrExpr_COMMA: COMMA -> type(COMMA) ; -StrExpr_MULT: MULT -> type(MULT) ; -StrExpr_MOD: MOD -> type(MOD) ; -StrExpr_DIV: DIV -> type(DIV) ; -StrExpr_ADD: ADD -> type(ADD) ; -StrExpr_SUB: SUB -> type(SUB) ; -StrExpr_INCR: INCR -> type(INCR) ; -StrExpr_DECR: DECR -> type(DECR) ; -StrExpr_CONJ: CONJ -> type(CONJ) ; -StrExpr_DISJ: DISJ -> type(DISJ) ; -StrExpr_EXCL: EXCL -> type(EXCL) ; -StrExpr_COLON: COLON -> type(COLON) ; -StrExpr_SEMICOLON: SEMICOLON -> type(SEMICOLON) ; -StrExpr_ASSIGNMENT: ASSIGNMENT -> type(ASSIGNMENT) ; -StrExpr_ADD_ASSIGNMENT: ADD_ASSIGNMENT -> type(ADD_ASSIGNMENT) ; -StrExpr_SUB_ASSIGNMENT: SUB_ASSIGNMENT -> type(SUB_ASSIGNMENT) ; -StrExpr_MULT_ASSIGNMENT: MULT_ASSIGNMENT -> type(MULT_ASSIGNMENT) ; -StrExpr_DIV_ASSIGNMENT: DIV_ASSIGNMENT -> type(DIV_ASSIGNMENT) ; -StrExpr_MOD_ASSIGNMENT: MOD_ASSIGNMENT -> type(MOD_ASSIGNMENT) ; -StrExpr_ARROW: ARROW -> type(ARROW) ; -StrExpr_DOUBLE_ARROW: DOUBLE_ARROW -> type(DOUBLE_ARROW) ; -StrExpr_RANGE: RANGE -> type(RANGE) ; -StrExpr_COLONCOLON: COLONCOLON -> type(COLONCOLON) ; -StrExpr_Q_COLONCOLON: Q_COLONCOLON -> type(Q_COLONCOLON) ; -StrExpr_DOUBLE_SEMICOLON: DOUBLE_SEMICOLON -> type(DOUBLE_SEMICOLON) ; -StrExpr_HASH: HASH -> type(HASH) ; -StrExpr_AT: AT -> type(AT) ; -StrExpr_QUEST: QUEST -> type(QUEST) ; -StrExpr_ELVIS: ELVIS -> type(ELVIS) ; -StrExpr_LANGLE: LANGLE -> type(LANGLE) ; -StrExpr_RANGLE: RANGLE -> type(RANGLE) ; -StrExpr_LE: LE -> type(LE) ; -StrExpr_GE: GE -> type(GE) ; -StrExpr_EXCL_EQ: EXCL_EQ -> type(EXCL_EQ) ; -StrExpr_EXCL_EQEQ: EXCL_EQEQ -> type(EXCL_EQEQ) ; -StrExpr_AS: AS -> type(IS) ; -StrExpr_IS: IS -> type(IN) ; -StrExpr_IN: IN ; -StrExpr_NOT_IS: NOT_IS -> type(NOT_IS) ; -StrExpr_NOT_IN: NOT_IN -> type(NOT_IN) ; -StrExpr_AS_SAFE: AS_SAFE -> type(AS_SAFE) ; -StrExpr_EQEQ: EQEQ -> type(EQEQ) ; -StrExpr_EQEQEQ: EQEQEQ -> type(EQEQEQ) ; -StrExpr_SINGLE_QUOTE: SINGLE_QUOTE -> type(SINGLE_QUOTE) ; -StrExpr_QUOTE_OPEN: QUOTE_OPEN -> pushMode(LineString), type(QUOTE_OPEN) ; -StrExpr_TRIPLE_QUOTE_OPEN: TRIPLE_QUOTE_OPEN -> pushMode(MultiLineString), type(TRIPLE_QUOTE_OPEN) ; - -StrExpr_BooleanLiteral: BooleanLiteral -> type(BooleanLiteral) ; -StrExpr_IntegerLiteral: IntegerLiteral -> type(IntegerLiteral) ; -StrExpr_HexLiteral: HexLiteral -> type(HexLiteral) ; -StrExpr_BinLiteral: BinLiteral -> type(BinLiteral) ; -StrExpr_CharacterLiteral: CharacterLiteral -> type(CharacterLiteral) ; -StrExpr_RealLiteral: RealLiteral -> type(RealLiteral) ; -StrExpr_NullLiteral: NullLiteral -> type(NullLiteral) ; -StrExpr_LongLiteral: LongLiteral -> type(LongLiteral) ; - -StrExpr_Identifier: Identifier -> type(Identifier) ; -StrExpr_LabelReference: LabelReference -> type(LabelReference) ; -StrExpr_LabelDefinition: LabelDefinition -> type(LabelDefinition) ; -StrExpr_Comment: (LineComment | DelimitedComment) -> channel(HIDDEN) ; -StrExpr_WS: WS -> skip ; -StrExpr_NL: NL -> skip ; diff --git a/codegpt-core/src/main/antlr/KotlinParser.g4 b/codegpt-core/src/main/antlr/KotlinParser.g4 deleted file mode 100644 index 147f0a3b..00000000 --- a/codegpt-core/src/main/antlr/KotlinParser.g4 +++ /dev/null @@ -1,796 +0,0 @@ -/** - * Kotlin Grammar for ANTLR v4 - * - * Based on: - * http://jetbrains.github.io/kotlin-spec/#_grammars_and_parsing - * and - * http://kotlinlang.org/docs/reference/grammar.html - * - * Tested on - * https://github.com/JetBrains/kotlin/tree/master/compiler/testData/psi - */ - -parser grammar KotlinParser; - -options { tokenVocab = KotlinLexer; } - -kotlinFile - : NL* preamble anysemi* (topLevelObject (anysemi+ topLevelObject?)*)? EOF - ; - -script - : NL* preamble anysemi* (expression (anysemi+ expression?)*)? EOF - ; - -preamble - : fileAnnotations? packageHeader importList - ; - -fileAnnotations - : fileAnnotation+ - ; - -fileAnnotation - : (FILE COLON (LSQUARE unescapedAnnotation+ RSQUARE | unescapedAnnotation) semi?)+ - ; - -packageHeader - : (modifierList? PACKAGE identifier semi?)? - ; - -importList - : importHeader* - ; - -importHeader - : IMPORT identifier (DOT MULT | importAlias)? semi? - ; - -importAlias - : AS simpleIdentifier - ; - -topLevelObject - : classDeclaration - | objectDeclaration - | functionDeclaration - | propertyDeclaration - | typeAlias - ; - -classDeclaration - : modifierList? (CLASS | INTERFACE) NL* simpleIdentifier - (NL* typeParameters)? (NL* primaryConstructor)? - (NL* COLON NL* delegationSpecifiers)? - (NL* typeConstraints)? - (NL* classBody | NL* enumClassBody)? - ; - -primaryConstructor - : modifierList? (CONSTRUCTOR NL*)? classParameters - ; - -classParameters - : LPAREN (classParameter (COMMA classParameter)* COMMA?)? RPAREN - ; - -classParameter - : modifierList? (VAL | VAR)? simpleIdentifier COLON type (ASSIGNMENT expression)? - ; - -delegationSpecifiers - : annotations* delegationSpecifier (NL* COMMA NL* annotations* delegationSpecifier)* - ; - -delegationSpecifier - : constructorInvocation - | userType - | explicitDelegation - ; - -constructorInvocation - : userType callSuffix - ; - -explicitDelegation - : userType NL* BY NL* expression - ; - -classBody - : LCURL NL* classMemberDeclaration* NL* RCURL - ; - -classMemberDeclaration - : (classDeclaration - | functionDeclaration - | objectDeclaration - | companionObject - | propertyDeclaration - | anonymousInitializer - | secondaryConstructor - | typeAlias) anysemi+ - ; - -anonymousInitializer - : INIT NL* block - ; - -secondaryConstructor - : modifierList? CONSTRUCTOR NL* functionValueParameters (NL* COLON NL* constructorDelegationCall)? NL* block? - ; - -constructorDelegationCall - : THIS NL* valueArguments - | SUPER NL* valueArguments - ; - -enumClassBody - : LCURL NL* enumEntries? (NL* SEMICOLON NL* classMemberDeclaration*)? NL* RCURL - ; - -enumEntries - : (enumEntry NL*)+ SEMICOLON? - ; - -enumEntry - : annotations* simpleIdentifier (NL* valueArguments)? (NL* classBody)? (NL* COMMA)? - ; - -functionDeclaration - : modifierList? FUN - (NL* type NL* DOT)? - (NL* typeParameters)? - (NL* receiverType NL* DOT)? - (NL* identifier)? - NL* functionValueParameters - (NL* COLON NL* type)? - (NL* typeConstraints)? - (NL* functionBody)? - ; - -functionValueParameters - : LPAREN (functionValueParameter (COMMA functionValueParameter)* COMMA?)? RPAREN - ; - -functionValueParameter - : modifierList? parameter (ASSIGNMENT expression)? - ; - -parameter - : simpleIdentifier COLON type - ; - -receiverType - : typeModifierList? (parenthesizedType | nullableType | typeReference) - ; - -functionBody - : block - | ASSIGNMENT NL* expression - ; - -objectDeclaration - : modifierList? OBJECT - NL* simpleIdentifier - (NL* primaryConstructor)? - (NL* COLON NL* delegationSpecifiers)? - (NL* classBody)? - ; - -companionObject - : modifierList? COMPANION NL* modifierList? OBJECT - (NL* simpleIdentifier)? - (NL* COLON NL* delegationSpecifiers)? - (NL* classBody)? - ; - -propertyDeclaration - : modifierList? (VAL | VAR) - (NL* typeParameters)? - (NL* type NL* DOT)? - (NL* (multiVariableDeclaration | variableDeclaration)) - (NL* typeConstraints)? - (NL* (BY | ASSIGNMENT) NL* expression)? - (NL* getter (semi setter)? | NL* setter (semi getter)?)? - ; - -multiVariableDeclaration - : LPAREN variableDeclaration (COMMA variableDeclaration)* RPAREN - ; - -variableDeclaration - : simpleIdentifier (COLON type)? - ; - -getter - : modifierList? GETTER - | modifierList? GETTER NL* LPAREN RPAREN (NL* COLON NL* type)? NL* (block | ASSIGNMENT NL* expression) - ; - -setter - : modifierList? SETTER - | modifierList? SETTER NL* LPAREN (annotations | parameterModifier)* (simpleIdentifier | parameter) RPAREN NL* functionBody - ; - -typeAlias - : modifierList? TYPE_ALIAS NL* simpleIdentifier (NL* typeParameters)? NL* ASSIGNMENT NL* type - ; - -typeParameters - : LANGLE NL* typeParameter (NL* COMMA NL* typeParameter)* (NL* COMMA)? NL* RANGLE - ; - -typeParameter - : modifierList? NL* (simpleIdentifier | MULT) (NL* COLON NL* type)? - ; - -type - : typeModifierList? - ( functionType - | parenthesizedType - | nullableType - | typeReference) - ; - -typeModifierList - : (annotations | SUSPEND NL*)+ - ; - -parenthesizedType - : LPAREN type RPAREN - ; - -nullableType - : (typeReference | parenthesizedType) NL* QUEST+ - ; - -typeReference - : LPAREN typeReference RPAREN - | userType - | DYNAMIC - ; - -functionType - : (functionTypeReceiver NL* DOT NL*)? functionTypeParameters NL* ARROW (NL* type) - ; - -functionTypeReceiver - : parenthesizedType - | nullableType - | typeReference - ; - -userType - : simpleUserType (NL* DOT NL* simpleUserType)* - ; - -simpleUserType - : simpleIdentifier (NL* typeArguments)? - ; - -//parameters for functionType -functionTypeParameters - : LPAREN NL* (parameter | type)? (NL* COMMA NL* (parameter | type))* (NL* COMMA)? NL* RPAREN - ; - -typeConstraints - : WHERE NL* typeConstraint (NL* COMMA NL* typeConstraint)* - ; - -typeConstraint - : annotations* simpleIdentifier NL* COLON NL* type - ; - -block - : LCURL statements RCURL - ; - -statements - : anysemi* (statement (anysemi+ statement?)*)? - ; - -statement - : declaration - | blockLevelExpression - ; - -blockLevelExpression - : annotations* NL* expression - ; - -declaration - : labelDefinition* - ( classDeclaration - | functionDeclaration - | propertyDeclaration - | typeAlias) - ; - -expression - : disjunction (assignmentOperator disjunction)* - ; - -disjunction - : conjunction (NL* DISJ NL* conjunction)* - ; - -conjunction - : equalityComparison (NL* CONJ NL* equalityComparison)* - ; - -equalityComparison - : comparison (equalityOperation NL* comparison)* - ; - -comparison - : namedInfix (comparisonOperator NL* namedInfix)? - ; - -namedInfix - : elvisExpression ((inOperator NL* elvisExpression)+ | (isOperator NL* type))? - ; - -elvisExpression - : infixFunctionCall (NL* ELVIS NL* infixFunctionCall)* - ; - -infixFunctionCall - : rangeExpression (simpleIdentifier NL* rangeExpression)* - ; - -rangeExpression - : additiveExpression (RANGE NL* additiveExpression)* - ; - -additiveExpression - : multiplicativeExpression (additiveOperator NL* multiplicativeExpression)* - ; - -multiplicativeExpression - : typeRHS (multiplicativeOperation NL* typeRHS)* - ; - -typeRHS - : prefixUnaryExpression (NL* typeOperation prefixUnaryExpression)* - ; - -prefixUnaryExpression - : prefixUnaryOperation* postfixUnaryExpression - ; - -postfixUnaryExpression - : (atomicExpression | callableReference) postfixUnaryOperation* - ; - -atomicExpression - : parenthesizedExpression - | literalConstant - | functionLiteral - | thisExpression // THIS labelReference? - | superExpression // SUPER (LANGLE type RANGLE)? labelReference? - | conditionalExpression // ifExpression, whenExpression - | tryExpression - | objectLiteral - | jumpExpression - | loopExpression - | collectionLiteral - | simpleIdentifier - | VAL identifier - ; - -parenthesizedExpression - : LPAREN expression RPAREN - ; - -callSuffix - : typeArguments valueArguments? annotatedLambda* - | valueArguments annotatedLambda* - | annotatedLambda+ - ; - -annotatedLambda - : unescapedAnnotation* LabelDefinition? NL* functionLiteral - ; - -arrayAccess - : LSQUARE (expression (COMMA expression)*)? RSQUARE - ; - -valueArguments - : LPAREN (valueArgument (COMMA valueArgument)* (NL* COMMA)?)? RPAREN - ; - -typeArguments - : LANGLE NL* typeProjection (NL* COMMA typeProjection)* (NL* COMMA)? NL* RANGLE QUEST? - ; - -typeProjection - : typeProjectionModifierList? type | MULT - ; - -typeProjectionModifierList - : varianceAnnotation+ - ; - -valueArgument - : (simpleIdentifier NL* ASSIGNMENT NL*)? MULT? NL* expression - ; - -literalConstant - : BooleanLiteral - | IntegerLiteral - | stringLiteral - | HexLiteral - | BinLiteral - | CharacterLiteral - | RealLiteral - | NullLiteral - | LongLiteral - ; - -stringLiteral - : lineStringLiteral - | multiLineStringLiteral - ; - -lineStringLiteral - : QUOTE_OPEN (lineStringContent | lineStringExpression)* QUOTE_CLOSE - ; - -multiLineStringLiteral - : TRIPLE_QUOTE_OPEN (multiLineStringContent | multiLineStringExpression | lineStringLiteral | MultiLineStringQuote)* TRIPLE_QUOTE_CLOSE - ; - -lineStringContent - : LineStrText - | LineStrEscapedChar - | LineStrRef - ; - -lineStringExpression - : LineStrExprStart expression RCURL - ; - -multiLineStringContent - : MultiLineStrText - | MultiLineStrEscapedChar - | MultiLineStrRef - ; - -multiLineStringExpression - : MultiLineStrExprStart expression RCURL - ; - -functionLiteral - : annotations* - ( LCURL NL* statements NL* RCURL - | LCURL NL* lambdaParameters NL* ARROW NL* statements NL* RCURL ) - ; - -lambdaParameters - : lambdaParameter? (NL* COMMA NL* lambdaParameter)* - ; - -lambdaParameter - : variableDeclaration - | multiVariableDeclaration (NL* COLON NL* type)? - ; - -// https://kotlinlang.org/docs/reference/grammar.html#objectLiteral -objectLiteral - : OBJECT (NL* COLON NL* delegationSpecifiers)? NL* classBody? - ; - -collectionLiteral - : LSQUARE expression? (COMMA expression)* RSQUARE - ; - -thisExpression - : THIS LabelReference? - ; - -superExpression - : SUPER (LANGLE NL* type NL* RANGLE)? LabelReference? - ; - -conditionalExpression - : ifExpression - | whenExpression - ; - -ifExpression - : IF NL* LPAREN expression RPAREN NL* controlStructureBody? SEMICOLON? - (NL* ELSE NL* controlStructureBody?)? - ; - -controlStructureBody - : block - | expression - ; - -whenExpression - : WHEN NL* (LPAREN expression RPAREN)? NL* LCURL NL* (whenEntry NL*)* NL* RCURL - ; - -whenEntry - : whenCondition (NL* COMMA NL* whenCondition)* NL* ARROW NL* controlStructureBody semi? - | ELSE NL* ARROW NL* controlStructureBody - ; - -whenCondition - : expression - | rangeTest - | typeTest - ; - -rangeTest - : inOperator NL* expression - ; - -typeTest - : isOperator NL* type - ; - -tryExpression - : TRY NL* block (NL* catchBlock)* (NL* finallyBlock)? - ; - -catchBlock - : CATCH NL* LPAREN annotations* simpleIdentifier COLON userType RPAREN NL* block - ; - -finallyBlock - : FINALLY NL* block - ; - -loopExpression - : forExpression - | whileExpression - | doWhileExpression - ; - -forExpression - : FOR NL* LPAREN annotations* (variableDeclaration | multiVariableDeclaration) IN expression RPAREN NL* controlStructureBody? - ; - -whileExpression - : WHILE NL* LPAREN expression RPAREN NL* controlStructureBody? - ; - -doWhileExpression - : DO NL* controlStructureBody? NL* WHILE NL* LPAREN expression RPAREN - ; - -jumpExpression - : THROW NL* expression - | (RETURN | RETURN_AT) expression? - | CONTINUE | CONTINUE_AT - | BREAK | BREAK_AT - ; - -callableReference - : (userType (QUEST NL*)*)? NL* (COLONCOLON | Q_COLONCOLON) NL* (identifier | CLASS) - | THIS NL* COLONCOLON NL* CLASS - ; - -assignmentOperator - : ASSIGNMENT - | ADD_ASSIGNMENT - | SUB_ASSIGNMENT - | MULT_ASSIGNMENT - | DIV_ASSIGNMENT - | MOD_ASSIGNMENT - ; - -equalityOperation - : EXCL_EQ - | EXCL_EQEQ - | EQEQ - | EQEQEQ - ; - -comparisonOperator - : LANGLE - | RANGLE - | LE - | GE - ; - -inOperator - : IN | NOT_IN - ; - -isOperator - : IS | NOT_IS - ; - -additiveOperator - : ADD | SUB - ; - -multiplicativeOperation - : MULT - | DIV - | MOD - ; - -typeOperation - : AS - | AS_SAFE - | COLON - ; - -prefixUnaryOperation - : INCR - | DECR - | ADD - | SUB - | EXCL - | annotations - | labelDefinition - ; - -postfixUnaryOperation - : INCR | DECR | EXCL EXCL - | callSuffix - | arrayAccess - | NL* memberAccessOperator postfixUnaryExpression - ; - -memberAccessOperator - : DOT | QUEST DOT - ; - -modifierList - : (annotations | modifier)+ - ; - -modifier - : (classModifier - | memberModifier - | visibilityModifier - | varianceAnnotation - | functionModifier - | propertyModifier - | inheritanceModifier - | parameterModifier - | typeParameterModifier) NL* - ; - -classModifier - : ENUM - | SEALED - | ANNOTATION - | DATA - | INNER - ; - -memberModifier - : OVERRIDE - | LATEINIT - ; - -visibilityModifier - : PUBLIC - | PRIVATE - | INTERNAL - | PROTECTED - ; - -varianceAnnotation - : IN | OUT - ; - -functionModifier - : TAILREC - | OPERATOR - | INFIX - | INLINE - | EXTERNAL - | SUSPEND - ; - -propertyModifier - : CONST - ; - -inheritanceModifier - : ABSTRACT - | FINAL - | OPEN - ; - -parameterModifier - : VARARG - | NOINLINE - | CROSSINLINE - ; - -typeParameterModifier - : REIFIED - ; - -labelDefinition - : LabelDefinition NL* - ; - -annotations - : (annotation | annotationList) NL* - ; - -annotation - : annotationUseSiteTarget NL* COLON NL* unescapedAnnotation - | LabelReference (NL* DOT NL* simpleIdentifier)* (NL* typeArguments)? (NL* valueArguments)? - ; - -annotationList - : annotationUseSiteTarget COLON LSQUARE unescapedAnnotation+ RSQUARE - | AT LSQUARE unescapedAnnotation+ RSQUARE - ; - -annotationUseSiteTarget - : FIELD - | FILE - | PROPERTY - | GET - | SET - | RECEIVER - | PARAM - | SETPARAM - | DELEGATE - ; - -unescapedAnnotation - : identifier typeArguments? valueArguments? - ; - -identifier - : simpleIdentifier (NL* DOT simpleIdentifier)* - ; - -simpleIdentifier - : Identifier - //soft keywords: - | ABSTRACT - | ANNOTATION - | BY - | CATCH - | COMPANION - | CONSTRUCTOR - | CROSSINLINE - | DATA - | DYNAMIC - | ENUM - | EXTERNAL - | FINAL - | FINALLY - | GETTER - | IMPORT - | INFIX - | INIT - | INLINE - | INNER - | INTERNAL - | LATEINIT - | NOINLINE - | OPEN - | OPERATOR - | OUT - | OVERRIDE - | PRIVATE - | PROTECTED - | PUBLIC - | REIFIED - | SEALED - | TAILREC - | SETTER - | VARARG - | WHERE - //strong keywords - | CONST - | SUSPEND - ; - -semi: NL+ | NL* SEMICOLON NL*; - -anysemi: NL | SEMICOLON; diff --git a/codegpt-core/src/main/antlr/PythonLexer.g4 b/codegpt-core/src/main/antlr/PythonLexer.g4 deleted file mode 100644 index afc5d5fc..00000000 --- a/codegpt-core/src/main/antlr/PythonLexer.g4 +++ /dev/null @@ -1,759 +0,0 @@ -/* -Python grammar. -The MIT License (MIT). -Copyright (c) 2014, Bart Kiers, bart@big-o.nl -Copyright (c) 2019, Dmitriy Litovchenko, Dmitry.Litovchenko1@yandex.ru, Positive Technologies -Copyright (c) 2019, Nikita Subbotin, sub.nik.and@gmail.com, Positive Technologies -Copyright (c) 2019, Ivan Kochurkin, kvanttt@gmail.com, Positive Technologies - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -lexer grammar PythonLexer; - -options { superClass=PythonLexerBase; } - -// Insert here @header for C++ lexer. - -// Artificial tokens only for parser purposes - -tokens { INDENT, DEDENT, LINE_BREAK } - -// Keywords - -DEF : 'def'; -RETURN : 'return'; -RAISE : 'raise'; -FROM : 'from'; -IMPORT : 'import'; -NONLOCAL : 'nonlocal'; -AS : 'as'; -GLOBAL : 'global'; -ASSERT : 'assert'; -IF : 'if'; -ELIF : 'elif'; -ELSE : 'else'; -WHILE : 'while'; -FOR : 'for'; -IN : 'in'; -TRY : 'try'; -NONE : 'None'; -FINALLY : 'finally'; -WITH : 'with'; -EXCEPT : 'except'; -LAMBDA : 'lambda'; -OR : 'or'; -AND : 'and'; -NOT : 'not'; -IS : 'is'; -CLASS : 'class'; -YIELD : 'yield'; -DEL : 'del'; -PASS : 'pass'; -CONTINUE : 'continue'; -BREAK : 'break'; -ASYNC : 'async'; -AWAIT : 'await'; -PRINT : 'print'; -EXEC : 'exec'; -TRUE : 'True'; -FALSE : 'False'; - -// Operators - -DOT : '.'; -ELLIPSIS : '...'; -REVERSE_QUOTE : '`'; -STAR : '*'; -COMMA : ','; -COLON : ':'; -SEMI_COLON : ';'; -POWER : '**'; -ASSIGN : '='; -OR_OP : '|'; -XOR : '^'; -AND_OP : '&'; -LEFT_SHIFT : '<<'; -RIGHT_SHIFT : '>>'; -ADD : '+'; -MINUS : '-'; -DIV : '/'; -MOD : '%'; -IDIV : '//'; -NOT_OP : '~'; -LESS_THAN : '<'; -GREATER_THAN : '>'; -EQUALS : '=='; -GT_EQ : '>='; -LT_EQ : '<='; -NOT_EQ_1 : '<>'; -NOT_EQ_2 : '!='; -AT : '@'; -ARROW : '->'; -ADD_ASSIGN : '+='; -SUB_ASSIGN : '-='; -MULT_ASSIGN : '*='; -AT_ASSIGN : '@='; -DIV_ASSIGN : '/='; -MOD_ASSIGN : '%='; -AND_ASSIGN : '&='; -OR_ASSIGN : '|='; -XOR_ASSIGN : '^='; -LEFT_SHIFT_ASSIGN : '<<='; -RIGHT_SHIFT_ASSIGN : '>>='; -POWER_ASSIGN : '**='; -IDIV_ASSIGN : '//='; - -STRING : ([uU] | [fF] [rR]? | [rR] [fF]?)? (SHORT_STRING | LONG_STRING) - | ([bB] [rR]? | [rR] [bB]) (SHORT_BYTES | LONG_BYTES) - ; - -DECIMAL_INTEGER : [1-9] [0-9]* - | '0'+ - ; -OCT_INTEGER : '0' [oO] [0-7]+; -HEX_INTEGER : '0' [xX] [0-9a-fA-F]+; -BIN_INTEGER : '0' [bB] [01]+; - -IMAG_NUMBER : (EXPONENT_OR_POINT_FLOAT | [0-9]+) [jJ]; -FLOAT_NUMBER : EXPONENT_OR_POINT_FLOAT; - -OPEN_PAREN : '(' {this.IncIndentLevel();}; -CLOSE_PAREN : ')' {this.DecIndentLevel();}; -OPEN_BRACE : '{' {this.IncIndentLevel();}; -CLOSE_BRACE : '}' {this.DecIndentLevel();}; -OPEN_BRACKET : '[' {this.IncIndentLevel();}; -CLOSE_BRACKET : ']' {this.DecIndentLevel();}; - -NAME : ID_START ID_CONTINUE*; - -LINE_JOIN : '\\' [ \t]* RN -> channel(HIDDEN); -NEWLINE : RN {this.HandleNewLine();} -> channel(HIDDEN); -WS : [ \t]+ {this.HandleSpaces();} -> channel(HIDDEN); -COMMENT : '#' ~[\r\n\f]* -> channel(HIDDEN); - -// Fragments - -fragment SHORT_STRING - : '\'' ('\\' (RN | .) | ~[\\\r\n'])* '\'' - | '"' ('\\' (RN | .) | ~[\\\r\n"])* '"' - ; - -fragment LONG_STRING - : '\'\'\'' LONG_STRING_ITEM*? '\'\'\'' - | '"""' LONG_STRING_ITEM*? '"""' - ; - -fragment LONG_STRING_ITEM - : ~'\\' - | '\\' (RN | .) - ; - -fragment RN - : '\r'? '\n' - ; - -fragment EXPONENT_OR_POINT_FLOAT - : ([0-9]+ | POINT_FLOAT) [eE] [+-]? [0-9]+ - | POINT_FLOAT - ; - -fragment POINT_FLOAT - : [0-9]* '.' [0-9]+ - | [0-9]+ '.' - ; - -fragment SHORT_BYTES - : '\'' (SHORT_BYTES_CHAR_NO_SINGLE_QUOTE | BYTES_ESCAPE_SEQ)* '\'' - | '"' (SHORT_BYTES_CHAR_NO_DOUBLE_QUOTE | BYTES_ESCAPE_SEQ)* '"' - ; - -fragment LONG_BYTES - : '\'\'\'' LONG_BYTES_ITEM*? '\'\'\'' - | '"""' LONG_BYTES_ITEM*? '"""' - ; - -fragment LONG_BYTES_ITEM - : LONG_BYTES_CHAR - | BYTES_ESCAPE_SEQ - ; - -fragment SHORT_BYTES_CHAR_NO_SINGLE_QUOTE - : [\u0000-\u0009] - | [\u000B-\u000C] - | [\u000E-\u0026] - | [\u0028-\u005B] - | [\u005D-\u007F] - ; - -fragment SHORT_BYTES_CHAR_NO_DOUBLE_QUOTE - : [\u0000-\u0009] - | [\u000B-\u000C] - | [\u000E-\u0021] - | [\u0023-\u005B] - | [\u005D-\u007F] - ; - -/// Any ASCII character except "\" -fragment LONG_BYTES_CHAR - : [\u0000-\u005B] - | [\u005D-\u007F] - ; - -/// "\" -fragment BYTES_ESCAPE_SEQ - : '\\' [\u0000-\u007F] - ; - -/// All characters in id_start, plus characters in the categories Mn, Mc, Nd, Pc and others with the Other_ID_Continue property -fragment ID_CONTINUE - : ID_START - | [0-9] - | [\u0300-\u036F] - | [\u0483-\u0486] - | [\u0591-\u05B9] - | [\u05BB-\u05BD] - | '\u05BF' - | [\u05C1-\u05C2] - | [\u05C4-\u05C5] - | '\u05C7' - | [\u0610-\u0615] - | [\u064B-\u065E] - | [\u0660-\u0669] - | '\u0670' - | [\u06D6-\u06DC] - | [\u06DF-\u06E4] - | [\u06E7-\u06E8] - | [\u06EA-\u06ED] - | [\u06F0-\u06F9] - | '\u0711' - | [\u0730-\u074A] - | [\u07A6-\u07B0] - | [\u0901-\u0902] - | '\u0903' - | '\u093C' - | [\u093E-\u0940] - | [\u0941-\u0948] - | [\u0949-\u094C] - | '\u094D' - | [\u0951-\u0954] - | [\u0962-\u0963] - | [\u0966-\u096F] - | '\u0981' - | [\u0982-\u0983] - | '\u09BC' - | [\u09BE-\u09C0] - | [\u09C1-\u09C4] - | [\u09C7-\u09C8] - | [\u09CB-\u09CC] - | '\u09CD' - | '\u09D7' - | [\u09E2-\u09E3] - | [\u09E6-\u09EF] - | [\u0A01-\u0A02] - | '\u0A03' - | '\u0A3C' - | [\u0A3E-\u0A40] - | [\u0A41-\u0A42] - | [\u0A47-\u0A48] - | [\u0A4B-\u0A4D] - | [\u0A66-\u0A6F] - | [\u0A70-\u0A71] - | [\u0A81-\u0A82] - | '\u0A83' - | '\u0ABC' - | [\u0ABE-\u0AC0] - | [\u0AC1-\u0AC5] - | [\u0AC7-\u0AC8] - | '\u0AC9' - | [\u0ACB-\u0ACC] - | '\u0ACD' - | [\u0AE2-\u0AE3] - | [\u0AE6-\u0AEF] - | '\u0B01' - | [\u0B02-\u0B03] - | '\u0B3C' - | '\u0B3E' - | '\u0B3F' - | '\u0B40' - | [\u0B41-\u0B43] - | [\u0B47-\u0B48] - | [\u0B4B-\u0B4C] - | '\u0B4D' - | '\u0B56' - | '\u0B57' - | [\u0B66-\u0B6F] - | '\u0B82' - | [\u0BBE-\u0BBF] - | '\u0BC0' - | [\u0BC1-\u0BC2] - | [\u0BC6-\u0BC8] - | [\u0BCA-\u0BCC] - | '\u0BCD' - | '\u0BD7' - | [\u0BE6-\u0BEF] - | [\u0C01-\u0C03] - | [\u0C3E-\u0C40] - | [\u0C41-\u0C44] - | [\u0C46-\u0C48] - | [\u0C4A-\u0C4D] - | [\u0C55-\u0C56] - | [\u0C66-\u0C6F] - | [\u0C82-\u0C83] - | '\u0CBC' - | '\u0CBE' - | '\u0CBF' - | [\u0CC0-\u0CC4] - | '\u0CC6' - | [\u0CC7-\u0CC8] - | [\u0CCA-\u0CCB] - | [\u0CCC-\u0CCD] - | [\u0CD5-\u0CD6] - | [\u0CE6-\u0CEF] - | [\u0D02-\u0D03] - | [\u0D3E-\u0D40] - | [\u0D41-\u0D43] - | [\u0D46-\u0D48] - | [\u0D4A-\u0D4C] - | '\u0D4D' - | '\u0D57' - | [\u0D66-\u0D6F] - | [\u0D82-\u0D83] - | '\u0DCA' - | [\u0DCF-\u0DD1] - | [\u0DD2-\u0DD4] - | '\u0DD6' - | [\u0DD8-\u0DDF] - | [\u0DF2-\u0DF3] - | '\u0E31' - | [\u0E34-\u0E3A] - | [\u0E47-\u0E4E] - | [\u0E50-\u0E59] - | '\u0EB1' - | [\u0EB4-\u0EB9] - | [\u0EBB-\u0EBC] - | [\u0EC8-\u0ECD] - | [\u0ED0-\u0ED9] - | [\u0F18-\u0F19] - | [\u0F20-\u0F29] - | '\u0F35' - | '\u0F37' - | '\u0F39' - | [\u0F3E-\u0F3F] - | [\u0F71-\u0F7E] - | '\u0F7F' - | [\u0F80-\u0F84] - | [\u0F86-\u0F87] - | [\u0F90-\u0F97] - | [\u0F99-\u0FBC] - | '\u0FC6' - | '\u102C' - | [\u102D-\u1030] - | '\u1031' - | '\u1032' - | [\u1036-\u1037] - | '\u1038' - | '\u1039' - | [\u1040-\u1049] - | [\u1056-\u1057] - | [\u1058-\u1059] - | '\u135F' - | [\u1369-\u1371] - | [\u1712-\u1714] - | [\u1732-\u1734] - | [\u1752-\u1753] - | [\u1772-\u1773] - | '\u17B6' - | [\u17B7-\u17BD] - | [\u17BE-\u17C5] - | '\u17C6' - | [\u17C7-\u17C8] - | [\u17C9-\u17D3] - | '\u17DD' - | [\u17E0-\u17E9] - | [\u180B-\u180D] - | [\u1810-\u1819] - | '\u18A9' - | [\u1920-\u1922] - | [\u1923-\u1926] - | [\u1927-\u1928] - | [\u1929-\u192B] - | [\u1930-\u1931] - | '\u1932' - | [\u1933-\u1938] - | [\u1939-\u193B] - | [\u1946-\u194F] - | [\u19B0-\u19C0] - | [\u19C8-\u19C9] - | [\u19D0-\u19D9] - | [\u1A17-\u1A18] - | [\u1A19-\u1A1B] - | [\u1DC0-\u1DC3] - | [\u203F-\u2040] - | '\u2054' - | [\u20D0-\u20DC] - | '\u20E1' - | [\u20E5-\u20EB] - | [\u302A-\u302F] - | [\u3099-\u309A] - | '\uA802' - | '\uA806' - | '\uA80B' - | [\uA823-\uA824] - | [\uA825-\uA826] - | '\uA827' - | '\uFB1E' - | [\uFE00-\uFE0F] - | [\uFE20-\uFE23] - | [\uFE33-\uFE34] - | [\uFE4D-\uFE4F] - | [\uFF10-\uFF19] - | '\uFF3F' - ; - -/// All characters in general categories Lu, Ll, Lt, Lm, Lo, Nl, the underscore, and characters with the Other_ID_Start property -fragment ID_START - : '_' - | [A-Z] - | [a-z] - | '\u00AA' - | '\u00B5' - | '\u00BA' - | [\u00C0-\u00D6] - | [\u00D8-\u00F6] - | [\u00F8-\u01BA] - | '\u01BB' - | [\u01BC-\u01BF] - | [\u01C0-\u01C3] - | [\u01C4-\u0241] - | [\u0250-\u02AF] - | [\u02B0-\u02C1] - | [\u02C6-\u02D1] - | [\u02E0-\u02E4] - | '\u02EE' - | '\u037A' - | '\u0386' - | [\u0388-\u038A] - | '\u038C' - | [\u038E-\u03A1] - | [\u03A3-\u03CE] - | [\u03D0-\u03F5] - | [\u03F7-\u0481] - | [\u048A-\u04CE] - | [\u04D0-\u04F9] - | [\u0500-\u050F] - | [\u0531-\u0556] - | '\u0559' - | [\u0561-\u0587] - | [\u05D0-\u05EA] - | [\u05F0-\u05F2] - | [\u0621-\u063A] - | '\u0640' - | [\u0641-\u064A] - | [\u066E-\u066F] - | [\u0671-\u06D3] - | '\u06D5' - | [\u06E5-\u06E6] - | [\u06EE-\u06EF] - | [\u06FA-\u06FC] - | '\u06FF' - | '\u0710' - | [\u0712-\u072F] - | [\u074D-\u076D] - | [\u0780-\u07A5] - | '\u07B1' - | [\u0904-\u0939] - | '\u093D' - | '\u0950' - | [\u0958-\u0961] - | '\u097D' - | [\u0985-\u098C] - | [\u098F-\u0990] - | [\u0993-\u09A8] - | [\u09AA-\u09B0] - | '\u09B2' - | [\u09B6-\u09B9] - | '\u09BD' - | '\u09CE' - | [\u09DC-\u09DD] - | [\u09DF-\u09E1] - | [\u09F0-\u09F1] - | [\u0A05-\u0A0A] - | [\u0A0F-\u0A10] - | [\u0A13-\u0A28] - | [\u0A2A-\u0A30] - | [\u0A32-\u0A33] - | [\u0A35-\u0A36] - | [\u0A38-\u0A39] - | [\u0A59-\u0A5C] - | '\u0A5E' - | [\u0A72-\u0A74] - | [\u0A85-\u0A8D] - | [\u0A8F-\u0A91] - | [\u0A93-\u0AA8] - | [\u0AAA-\u0AB0] - | [\u0AB2-\u0AB3] - | [\u0AB5-\u0AB9] - | '\u0ABD' - | '\u0AD0' - | [\u0AE0-\u0AE1] - | [\u0B05-\u0B0C] - | [\u0B0F-\u0B10] - | [\u0B13-\u0B28] - | [\u0B2A-\u0B30] - | [\u0B32-\u0B33] - | [\u0B35-\u0B39] - | '\u0B3D' - | [\u0B5C-\u0B5D] - | [\u0B5F-\u0B61] - | '\u0B71' - | '\u0B83' - | [\u0B85-\u0B8A] - | [\u0B8E-\u0B90] - | [\u0B92-\u0B95] - | [\u0B99-\u0B9A] - | '\u0B9C' - | [\u0B9E-\u0B9F] - | [\u0BA3-\u0BA4] - | [\u0BA8-\u0BAA] - | [\u0BAE-\u0BB9] - | [\u0C05-\u0C0C] - | [\u0C0E-\u0C10] - | [\u0C12-\u0C28] - | [\u0C2A-\u0C33] - | [\u0C35-\u0C39] - | [\u0C60-\u0C61] - | [\u0C85-\u0C8C] - | [\u0C8E-\u0C90] - | [\u0C92-\u0CA8] - | [\u0CAA-\u0CB3] - | [\u0CB5-\u0CB9] - | '\u0CBD' - | '\u0CDE' - | [\u0CE0-\u0CE1] - | [\u0D05-\u0D0C] - | [\u0D0E-\u0D10] - | [\u0D12-\u0D28] - | [\u0D2A-\u0D39] - | [\u0D60-\u0D61] - | [\u0D85-\u0D96] - | [\u0D9A-\u0DB1] - | [\u0DB3-\u0DBB] - | '\u0DBD' - | [\u0DC0-\u0DC6] - | [\u0E01-\u0E30] - | [\u0E32-\u0E33] - | [\u0E40-\u0E45] - | '\u0E46' - | [\u0E81-\u0E82] - | '\u0E84' - | [\u0E87-\u0E88] - | '\u0E8A' - | '\u0E8D' - | [\u0E94-\u0E97] - | [\u0E99-\u0E9F] - | [\u0EA1-\u0EA3] - | '\u0EA5' - | '\u0EA7' - | [\u0EAA-\u0EAB] - | [\u0EAD-\u0EB0] - | [\u0EB2-\u0EB3] - | '\u0EBD' - | [\u0EC0-\u0EC4] - | '\u0EC6' - | [\u0EDC-\u0EDD] - | '\u0F00' - | [\u0F40-\u0F47] - | [\u0F49-\u0F6A] - | [\u0F88-\u0F8B] - | [\u1000-\u1021] - | [\u1023-\u1027] - | [\u1029-\u102A] - | [\u1050-\u1055] - | [\u10A0-\u10C5] - | [\u10D0-\u10FA] - | '\u10FC' - | [\u1100-\u1159] - | [\u115F-\u11A2] - | [\u11A8-\u11F9] - | [\u1200-\u1248] - | [\u124A-\u124D] - | [\u1250-\u1256] - | '\u1258' - | [\u125A-\u125D] - | [\u1260-\u1288] - | [\u128A-\u128D] - | [\u1290-\u12B0] - | [\u12B2-\u12B5] - | [\u12B8-\u12BE] - | '\u12C0' - | [\u12C2-\u12C5] - | [\u12C8-\u12D6] - | [\u12D8-\u1310] - | [\u1312-\u1315] - | [\u1318-\u135A] - | [\u1380-\u138F] - | [\u13A0-\u13F4] - | [\u1401-\u166C] - | [\u166F-\u1676] - | [\u1681-\u169A] - | [\u16A0-\u16EA] - | [\u16EE-\u16F0] - | [\u1700-\u170C] - | [\u170E-\u1711] - | [\u1720-\u1731] - | [\u1740-\u1751] - | [\u1760-\u176C] - | [\u176E-\u1770] - | [\u1780-\u17B3] - | '\u17D7' - | '\u17DC' - | [\u1820-\u1842] - | '\u1843' - | [\u1844-\u1877] - | [\u1880-\u18A8] - | [\u1900-\u191C] - | [\u1950-\u196D] - | [\u1970-\u1974] - | [\u1980-\u19A9] - | [\u19C1-\u19C7] - | [\u1A00-\u1A16] - | [\u1D00-\u1D2B] - | [\u1D2C-\u1D61] - | [\u1D62-\u1D77] - | '\u1D78' - | [\u1D79-\u1D9A] - | [\u1D9B-\u1DBF] - | [\u1E00-\u1E9B] - | [\u1EA0-\u1EF9] - | [\u1F00-\u1F15] - | [\u1F18-\u1F1D] - | [\u1F20-\u1F45] - | [\u1F48-\u1F4D] - | [\u1F50-\u1F57] - | '\u1F59' - | '\u1F5B' - | '\u1F5D' - | [\u1F5F-\u1F7D] - | [\u1F80-\u1FB4] - | [\u1FB6-\u1FBC] - | '\u1FBE' - | [\u1FC2-\u1FC4] - | [\u1FC6-\u1FCC] - | [\u1FD0-\u1FD3] - | [\u1FD6-\u1FDB] - | [\u1FE0-\u1FEC] - | [\u1FF2-\u1FF4] - | [\u1FF6-\u1FFC] - | '\u2071' - | '\u207F' - | [\u2090-\u2094] - | '\u2102' - | '\u2107' - | [\u210A-\u2113] - | '\u2115' - | '\u2118' - | [\u2119-\u211D] - | '\u2124' - | '\u2126' - | '\u2128' - | [\u212A-\u212D] - | '\u212E' - | [\u212F-\u2131] - | [\u2133-\u2134] - | [\u2135-\u2138] - | '\u2139' - | [\u213C-\u213F] - | [\u2145-\u2149] - | [\u2160-\u2183] - | [\u2C00-\u2C2E] - | [\u2C30-\u2C5E] - | [\u2C80-\u2CE4] - | [\u2D00-\u2D25] - | [\u2D30-\u2D65] - | '\u2D6F' - | [\u2D80-\u2D96] - | [\u2DA0-\u2DA6] - | [\u2DA8-\u2DAE] - | [\u2DB0-\u2DB6] - | [\u2DB8-\u2DBE] - | [\u2DC0-\u2DC6] - | [\u2DC8-\u2DCE] - | [\u2DD0-\u2DD6] - | [\u2DD8-\u2DDE] - | '\u3005' - | '\u3006' - | '\u3007' - | [\u3021-\u3029] - | [\u3031-\u3035] - | [\u3038-\u303A] - | '\u303B' - | '\u303C' - | [\u3041-\u3096] - | [\u309B-\u309C] - | [\u309D-\u309E] - | '\u309F' - | [\u30A1-\u30FA] - | [\u30FC-\u30FE] - | '\u30FF' - | [\u3105-\u312C] - | [\u3131-\u318E] - | [\u31A0-\u31B7] - | [\u31F0-\u31FF] - | [\u3400-\u4DB5] - | [\u4E00-\u9FBB] - | [\uA000-\uA014] - | '\uA015' - | [\uA016-\uA48C] - | [\uA800-\uA801] - | [\uA803-\uA805] - | [\uA807-\uA80A] - | [\uA80C-\uA822] - | [\uAC00-\uD7A3] - | [\uF900-\uFA2D] - | [\uFA30-\uFA6A] - | [\uFA70-\uFAD9] - | [\uFB00-\uFB06] - | [\uFB13-\uFB17] - | '\uFB1D' - | [\uFB1F-\uFB28] - | [\uFB2A-\uFB36] - | [\uFB38-\uFB3C] - | '\uFB3E' - | [\uFB40-\uFB41] - | [\uFB43-\uFB44] - | [\uFB46-\uFBB1] - | [\uFBD3-\uFD3D] - | [\uFD50-\uFD8F] - | [\uFD92-\uFDC7] - | [\uFDF0-\uFDFB] - | [\uFE70-\uFE74] - | [\uFE76-\uFEFC] - | [\uFF21-\uFF3A] - | [\uFF41-\uFF5A] - | [\uFF66-\uFF6F] - | '\uFF70' - | [\uFF71-\uFF9D] - | [\uFF9E-\uFF9F] - | [\uFFA0-\uFFBE] - | [\uFFC2-\uFFC7] - | [\uFFCA-\uFFCF] - | [\uFFD2-\uFFD7] - | [\uFFDA-\uFFDC] - ; diff --git a/codegpt-core/src/main/antlr/PythonParser.g4 b/codegpt-core/src/main/antlr/PythonParser.g4 deleted file mode 100644 index d192bce3..00000000 --- a/codegpt-core/src/main/antlr/PythonParser.g4 +++ /dev/null @@ -1,386 +0,0 @@ -/* -Python grammar. -The MIT License (MIT). -Copyright (c) 2014, Bart Kiers, bart@big-o.nl -Copyright (c) 2019, Dmitriy Litovchenko, Dmitry.Litovchenko1@yandex.ru, Positive Technologies -Copyright (c) 2019, Nikita Subbotin, sub.nik.and@gmail.com, Positive Technologies -Copyright (c) 2019, Ivan Kochurkin, kvanttt@gmail.com, Positive Technologies - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -parser grammar PythonParser; - -// Insert here @header for C++ parser. - -options { tokenVocab=PythonLexer; superClass=PythonParserBase; } - -root - : (single_input - | file_input - | eval_input)? EOF - ; - -// A single interactive statement; -single_input - : LINE_BREAK - | simple_stmt - | compound_stmt LINE_BREAK - ; - -// A module or sequence of commands read from an input file -file_input - : (LINE_BREAK | stmt)+ - ; - -// An input for the eval() and input() functions -eval_input - : testlist LINE_BREAK* - ; - -stmt - : simple_stmt - | compound_stmt - ; - -compound_stmt - : IF cond=test COLON suite elif_clause* else_clause? #if_stmt - | WHILE test COLON suite else_clause? #while_stmt - | ASYNC? FOR exprlist IN testlist COLON suite else_clause? #for_stmt - | TRY COLON suite (except_clause+ else_clause? finally_clause? | finally_clause) #try_stmt - | ASYNC? WITH with_item (COMMA with_item)* COLON suite #with_stmt - | decorator* (classdef | funcdef) #class_or_func_def_stmt - ; - -suite - : simple_stmt - | LINE_BREAK INDENT stmt+ DEDENT - ; - -decorator - : AT dotted_name (OPEN_PAREN arglist? CLOSE_PAREN)? LINE_BREAK - ; - -elif_clause - : ELIF test COLON suite - ; - -else_clause - : ELSE COLON suite - ; - -finally_clause - : FINALLY COLON suite - ; - -with_item - // NB compile.c makes sure that the default except clause is last - : test (AS expr)? - ; - -// Python 2 : EXCEPT test COMMA name -// Python 3 : EXCEPT test AS name -except_clause - : EXCEPT (test ({this.CheckVersion(2)}? COMMA name {this.SetVersion(2);} | {this.CheckVersion(3)}? AS name {this.SetVersion(3);})?)? COLON suite - ; - -classdef - : CLASS name (OPEN_PAREN arglist? CLOSE_PAREN)? COLON suite - ; - -funcdef - : ASYNC? DEF name OPEN_PAREN typedargslist? CLOSE_PAREN (ARROW test)? COLON suite - ; - -// python 3 paramters -// parameters list may have a trailing comma -typedargslist - : (def_parameters COMMA)? (args (COMMA def_parameters)? (COMMA kwargs)? | kwargs) COMMA? - | def_parameters COMMA? - ; - -args - : STAR named_parameter - ; - -kwargs - : POWER named_parameter - ; - -def_parameters - : def_parameter (COMMA def_parameter)* - ; - -// TODO: bare STAR parameter must follow named ones -def_parameter - : named_parameter (ASSIGN test)? - | STAR - ; - -named_parameter - : name (COLON test)? - ; - -simple_stmt - : small_stmt (SEMI_COLON small_stmt)* SEMI_COLON? (LINE_BREAK | EOF) - ; - -// TODO 1: left part augmented assignment should be `test` only, no stars or lists -// TODO 2: semantically annotated declaration is not an assignment -small_stmt - : testlist_star_expr assign_part? #expr_stmt - | {this.CheckVersion(2)}? PRINT ((test (COMMA test)* COMMA?) - | RIGHT_SHIFT test ((COMMA test)+ COMMA?)) {this.SetVersion(2);} #print_stmt // Python 2 - | DEL exprlist #del_stmt - | PASS #pass_stmt - | BREAK #break_stmt - | CONTINUE #continue_stmt - | RETURN testlist? #return_stmt - | RAISE (test (COMMA test (COMMA test)?)?)? (FROM test)? #raise_stmt - | yield_expr #yield_stmt - | IMPORT dotted_as_names #import_stmt - | FROM ((DOT | ELLIPSIS)* dotted_name | (DOT | ELLIPSIS)+) - IMPORT (STAR | OPEN_PAREN import_as_names CLOSE_PAREN | import_as_names) #from_stmt - | GLOBAL name (COMMA name)* #global_stmt - | {this.CheckVersion(2)}? EXEC expr (IN test (COMMA test)?)? {this.SetVersion(2);} #exec_stmt // Python 2 - | ASSERT test (COMMA test)? #assert_stmt - | {this.CheckVersion(3)}? NONLOCAL name (COMMA name)* {this.SetVersion(3);} #nonlocal_stmt // Python 3 - ; - -testlist_star_expr - : ((test | star_expr) COMMA)+ (test | star_expr)? - | testlist - ; - -star_expr - : STAR expr - ; - -assign_part - // if left expression in assign is bool literal, it's mean that is Python 2 here - : ASSIGN ( testlist_star_expr (ASSIGN testlist_star_expr)* (ASSIGN yield_expr)? - | yield_expr) - | {this.CheckVersion(3)}? COLON test (ASSIGN testlist)? {this.SetVersion(3);} // annassign Python3 rule - | op=( ADD_ASSIGN - | SUB_ASSIGN - | MULT_ASSIGN - | AT_ASSIGN - | DIV_ASSIGN - | MOD_ASSIGN - | AND_ASSIGN - | OR_ASSIGN - | XOR_ASSIGN - | LEFT_SHIFT_ASSIGN - | RIGHT_SHIFT_ASSIGN - | POWER_ASSIGN - | IDIV_ASSIGN - ) - (yield_expr | testlist) - ; - -exprlist - : expr (COMMA expr)* COMMA? - ; - -import_as_names - : import_as_name (COMMA import_as_name)* COMMA? - ; - -// TODO: that means we can use keyword True as the name here: `from foo import bar as True` -- no -import_as_name - : name (AS name)? - ; - -dotted_as_names - : dotted_as_name (COMMA dotted_as_name)* - ; - -dotted_as_name - : dotted_name (AS name)? - ; - -/* - * Warning! - * According to https://docs.python.org/3/reference/expressions.html#lambda LAMBDA should be followed by - * `parameter_list` (in our case it is `typedargslist`) - * But that's not true! `typedargslist` may have parameters with type hinting, but that's not permitted in lambda - * definition - */ -// https://docs.python.org/3/reference/expressions.html#operator-precedence -test - : logical_test (IF logical_test ELSE test)? - | LAMBDA varargslist? COLON test - ; - -// the same as `typedargslist`, but with no types -varargslist - : (vardef_parameters COMMA)? (varargs (COMMA vardef_parameters)? (COMMA varkwargs)? | varkwargs) COMMA? - | vardef_parameters COMMA? - ; - -vardef_parameters - : vardef_parameter (COMMA vardef_parameter)* - ; - -// TODO: bare STAR parameter must follow named ones -vardef_parameter - : name (ASSIGN test)? - | STAR - ; - -varargs - : STAR name - ; - -varkwargs - : POWER name - ; - -logical_test - : comparison - | NOT logical_test - | logical_test op=AND logical_test - | logical_test op=OR logical_test - ; - -comparison - : comparison (LESS_THAN | GREATER_THAN | EQUALS | GT_EQ | LT_EQ | NOT_EQ_1 | NOT_EQ_2 | optional=NOT? IN | IS optional=NOT?) comparison - | expr - ; - -expr - : AWAIT? atom trailer* - | expr op=POWER expr - | op=(ADD | MINUS | NOT_OP) expr - | expr op=(STAR | DIV | MOD | IDIV | AT) expr - | expr op=(ADD | MINUS) expr - | expr op=(LEFT_SHIFT | RIGHT_SHIFT) expr - | expr op=AND_OP expr - | expr op=XOR expr - | expr op=OR_OP expr - ; - -atom - : OPEN_PAREN (yield_expr | testlist_comp)? CLOSE_PAREN - | OPEN_BRACKET testlist_comp? CLOSE_BRACKET - | OPEN_BRACE dictorsetmaker? CLOSE_BRACE - | REVERSE_QUOTE testlist COMMA? REVERSE_QUOTE - | ELLIPSIS - | name - | PRINT - | EXEC - | MINUS? number - | NONE - | STRING+ - ; - -dictorsetmaker - : (test COLON test | POWER expr) (COMMA (test COLON test | POWER expr))* COMMA? // key_datum_list - | test COLON test comp_for // dict_comprehension - | testlist_comp - ; - -testlist_comp - : (test | star_expr) (comp_for | (COMMA (test | star_expr))* COMMA?) - ; - -testlist - : test (COMMA test)* COMMA? - ; - -dotted_name - : dotted_name DOT name - | name - ; - -name - : NAME - | TRUE - | FALSE - ; - -number - : integer - | IMAG_NUMBER - | FLOAT_NUMBER - ; - -integer - : DECIMAL_INTEGER - | OCT_INTEGER - | HEX_INTEGER - | BIN_INTEGER - ; - -yield_expr - : YIELD yield_arg? - ; - -yield_arg - : FROM test - | testlist - ; - -// TODO: this way we can pass: `f(x for x in i, a)`, but it's invalid. -// See: https://docs.python.org/3/reference/expressions.html#calls -trailer - : DOT name arguments? - | arguments - ; - -arguments - : OPEN_PAREN arglist? CLOSE_PAREN - | OPEN_BRACKET subscriptlist CLOSE_BRACKET - ; - -arglist - // The reason that keywords are test nodes instead of name is that using name - // results in an ambiguity. ast.c makes sure it's a name. - : argument (COMMA argument)* COMMA? - ; - -argument - : test (comp_for | ASSIGN test)? - | (POWER | STAR) test - ; - -// TODO: maybe inline? -subscriptlist - : subscript (COMMA subscript)* COMMA? - ; - -subscript - : ELLIPSIS - | test (COLON test? sliceop?)? - | COLON test? sliceop? - ; - -// TODO: maybe inline? -sliceop - : COLON test? - ; - -comp_for - : FOR exprlist IN logical_test comp_iter? - ; - -comp_iter - : comp_for - | IF test comp_iter? - ; diff --git a/codegpt-core/src/main/antlr/TypeScriptLexer.g4 b/codegpt-core/src/main/antlr/TypeScriptLexer.g4 deleted file mode 100644 index 3d71fe67..00000000 --- a/codegpt-core/src/main/antlr/TypeScriptLexer.g4 +++ /dev/null @@ -1,308 +0,0 @@ -lexer grammar TypeScriptLexer; - -channels { ERROR } - -options { - superClass=TypeScriptLexerBase; -} - - -MultiLineComment: '/*' .*? '*/' -> channel(HIDDEN); -SingleLineComment: '//' ~[\r\n\u2028\u2029]* -> channel(HIDDEN); -RegularExpressionLiteral: '/' RegularExpressionFirstChar RegularExpressionChar* {this.IsRegexPossible()}? '/' IdentifierPart*; - -OpenBracket: '['; -CloseBracket: ']'; -OpenParen: '('; -CloseParen: ')'; -OpenBrace: '{' {this.ProcessOpenBrace();}; -TemplateCloseBrace: {this.IsInTemplateString()}? '}' -> popMode; -CloseBrace: '}' {this.ProcessCloseBrace();}; -SemiColon: ';'; -Comma: ','; -Assign: '='; -QuestionMark: '?'; -Colon: ':'; -Ellipsis: '...'; -Dot: '.'; -PlusPlus: '++'; -MinusMinus: '--'; -Plus: '+'; -Minus: '-'; -BitNot: '~'; -Not: '!'; -Multiply: '*'; -Divide: '/'; -Modulus: '%'; -RightShiftArithmetic: '>>'; -LeftShiftArithmetic: '<<'; -RightShiftLogical: '>>>'; -LessThan: '<'; -MoreThan: '>'; -LessThanEquals: '<='; -GreaterThanEquals: '>='; -Equals_: '=='; -NotEquals: '!='; -IdentityEquals: '==='; -IdentityNotEquals: '!=='; -BitAnd: '&'; -BitXOr: '^'; -BitOr: '|'; -And: '&&'; -Or: '||'; -MultiplyAssign: '*='; -DivideAssign: '/='; -ModulusAssign: '%='; -PlusAssign: '+='; -MinusAssign: '-='; -LeftShiftArithmeticAssign: '<<='; -RightShiftArithmeticAssign: '>>='; -RightShiftLogicalAssign: '>>>='; -BitAndAssign: '&='; -BitXorAssign: '^='; -BitOrAssign: '|='; -ARROW: '=>'; - -/// Null Literals - -NullLiteral: 'null'; - -/// Boolean Literals - -BooleanLiteral: 'true' - | 'false'; - -/// Numeric Literals - -DecimalLiteral: DecimalIntegerLiteral '.' [0-9]* ExponentPart? - | '.' [0-9]+ ExponentPart? - | DecimalIntegerLiteral ExponentPart? - ; - -/// Numeric Literals - -HexIntegerLiteral: '0' [xX] HexDigit+; -OctalIntegerLiteral: '0' [0-7]+ {!this.IsStrictMode()}?; -OctalIntegerLiteral2: '0' [oO] [0-7]+; -BinaryIntegerLiteral: '0' [bB] [01]+; - -/// Keywords - -Break: 'break'; -Do: 'do'; -Instanceof: 'instanceof'; -Typeof: 'typeof'; -Case: 'case'; -Else: 'else'; -New: 'new'; -Var: 'var'; -Catch: 'catch'; -Finally: 'finally'; -Return: 'return'; -Void: 'void'; -Continue: 'continue'; -For: 'for'; -Switch: 'switch'; -While: 'while'; -Debugger: 'debugger'; -Function_: 'function'; -This: 'this'; -With: 'with'; -Default: 'default'; -If: 'if'; -Throw: 'throw'; -Delete: 'delete'; -In: 'in'; -Try: 'try'; -As: 'as'; -From: 'from'; -ReadOnly: 'readonly'; -Async: 'async'; - -/// Future Reserved Words - -Class: 'class'; -Enum: 'enum'; -Extends: 'extends'; -Super: 'super'; -Const: 'const'; -Export: 'export'; -Import: 'import'; - -/// The following tokens are also considered to be FutureReservedWords -/// when parsing strict mode - -Implements: 'implements' ; -Let: 'let' ; -Private: 'private' ; -Public: 'public' ; -Interface: 'interface' ; -Package: 'package' ; -Protected: 'protected' ; -Static: 'static' ; -Yield: 'yield' ; - -//keywords: - -Any : 'any'; -Number: 'number'; -Boolean: 'boolean'; -String: 'string'; -Symbol: 'symbol'; - - -TypeAlias : 'type'; - -Get: 'get'; -Set: 'set'; - -Constructor: 'constructor'; -Namespace: 'namespace'; -Require: 'require'; -Module: 'module'; -Declare: 'declare'; - -Abstract: 'abstract'; - -Is: 'is'; - -// -// Ext.2 Additions to 1.8: Decorators -// -At: '@'; - -/// Identifier Names and Identifiers - -Identifier: IdentifierStart IdentifierPart*; - -/// String Literals -StringLiteral: ('"' DoubleStringCharacter* '"' - | '\'' SingleStringCharacter* '\'') {this.ProcessStringLiteral();} - ; - -BackTick: '`' {this.IncreaseTemplateDepth();} -> pushMode(TEMPLATE); - -WhiteSpaces: [\t\u000B\u000C\u0020\u00A0]+ -> channel(HIDDEN); - -LineTerminator: [\r\n\u2028\u2029] -> channel(HIDDEN); - -/// Comments - - -HtmlComment: '' -> channel(HIDDEN); -CDataComment: '' -> channel(HIDDEN); -UnexpectedCharacter: . -> channel(ERROR); - -mode TEMPLATE; - -TemplateStringEscapeAtom: '\\' .; -BackTickInside: '`' {this.DecreaseTemplateDepth();} -> type(BackTick), popMode; -TemplateStringStartExpression: '${' {this.StartTemplateString();} -> pushMode(DEFAULT_MODE); -TemplateStringAtom: ~[`\\]; - -// Fragment rules - -fragment DoubleStringCharacter - : ~["\\\r\n] - | '\\' EscapeSequence - | LineContinuation - ; - -fragment SingleStringCharacter - : ~['\\\r\n] - | '\\' EscapeSequence - | LineContinuation - ; - -fragment EscapeSequence - : CharacterEscapeSequence - | '0' // no digit ahead! TODO - | HexEscapeSequence - | UnicodeEscapeSequence - | ExtendedUnicodeEscapeSequence - ; - -fragment CharacterEscapeSequence - : SingleEscapeCharacter - | NonEscapeCharacter - ; - -fragment HexEscapeSequence - : 'x' HexDigit HexDigit - ; - -fragment UnicodeEscapeSequence - : 'u' HexDigit HexDigit HexDigit HexDigit - ; - -fragment ExtendedUnicodeEscapeSequence - : 'u' '{' HexDigit+ '}' - ; - -fragment SingleEscapeCharacter - : ['"\\bfnrtv] - ; - -fragment NonEscapeCharacter - : ~['"\\bfnrtv0-9xu\r\n] - ; - -fragment EscapeCharacter - : SingleEscapeCharacter - | [0-9] - | [xu] - ; - -fragment LineContinuation - : '\\' [\r\n\u2028\u2029] - ; - -fragment HexDigit - : [0-9a-fA-F] - ; - -fragment DecimalIntegerLiteral - : '0' - | [1-9] [0-9]* - ; - -fragment ExponentPart - : [eE] [+-]? [0-9]+ - ; - -fragment IdentifierPart - : IdentifierStart - | [\p{Mn}] - | [\p{Nd}] - | [\p{Pc}] - | '\u200C' - | '\u200D' - ; - -fragment IdentifierStart - : [\p{L}] - | [$_] - | '\\' UnicodeEscapeSequence - ; - -fragment RegularExpressionFirstChar - : ~[*\r\n\u2028\u2029\\/[] - | RegularExpressionBackslashSequence - | '[' RegularExpressionClassChar* ']' - ; - -fragment RegularExpressionChar - : ~[\r\n\u2028\u2029\\/[] - | RegularExpressionBackslashSequence - | '[' RegularExpressionClassChar* ']' - ; - -fragment RegularExpressionClassChar - : ~[\r\n\u2028\u2029\]\\] - | RegularExpressionBackslashSequence - ; - -fragment RegularExpressionBackslashSequence - : '\\' ~[\r\n\u2028\u2029] - ; - diff --git a/codegpt-core/src/main/antlr/TypeScriptParser.g4 b/codegpt-core/src/main/antlr/TypeScriptParser.g4 deleted file mode 100644 index 6e81b63e..00000000 --- a/codegpt-core/src/main/antlr/TypeScriptParser.g4 +++ /dev/null @@ -1,854 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2014 by Bart Kiers (original author) and Alexandre Vitorelli (contributor -> ported to CSharp) - * Copyright (c) 2017 by Ivan Kochurkin (Positive Technologies): - added ECMAScript 6 support, cleared and transformed to the universal grammar. - * Copyright (c) 2018 by Juan Alvarez (contributor -> ported to Go) - * Copyright (c) 2019 by Andrii Artiushok (contributor -> added TypeScript support) - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -parser grammar TypeScriptParser; - -options { - tokenVocab=TypeScriptLexer; - superClass=TypeScriptParserBase; -} - -// SupportSyntax - -initializer - : '=' singleExpression - ; - -bindingPattern - : (arrayLiteral | objectLiteral) - ; - -// TypeScript SPart -// A.1 Types - -typeParameters - : '<' typeParameterList? '>' - ; - -typeParameterList - : typeParameter (',' typeParameter)* - ; - -typeParameter - : Identifier constraint? - | typeParameters - ; - -constraint - : 'extends' type_ - ; - -typeArguments - : '<' typeArgumentList? '>' - ; - -typeArgumentList - : typeArgument (',' typeArgument)* - ; - -typeArgument - : type_ - ; - -type_ - : unionOrIntersectionOrPrimaryType - | functionType - | constructorType - | typeGeneric - | StringLiteral - ; - -unionOrIntersectionOrPrimaryType - : unionOrIntersectionOrPrimaryType '|' unionOrIntersectionOrPrimaryType #Union - | unionOrIntersectionOrPrimaryType '&' unionOrIntersectionOrPrimaryType #Intersection - | primaryType #Primary - ; - -primaryType - : '(' type_ ')' #ParenthesizedPrimType - | predefinedType #PredefinedPrimType - | typeReference #ReferencePrimType - | objectType #ObjectPrimType - | primaryType {notLineTerminator()}? '[' ']' #ArrayPrimType - | '[' tupleElementTypes ']' #TuplePrimType - | typeQuery #QueryPrimType - | This #ThisPrimType - | typeReference Is primaryType #RedefinitionOfType - ; - -predefinedType - : Any - | Number - | Boolean - | String - | Symbol - | Void - ; - -typeReference - : typeName nestedTypeGeneric? - ; - -nestedTypeGeneric - : typeIncludeGeneric - | typeGeneric - ; - -// I tried recursive include, but it's not working. -// typeGeneric -// : '<' typeArgumentList typeGeneric?'>' -// ; -// -// TODO: Fix recursive -// -typeGeneric - : '<' typeArgumentList '>' - ; - -typeIncludeGeneric - :'<' typeArgumentList '<' typeArgumentList ('>' bindingPattern '>' | '>>') - ; - -typeName - : Identifier - | namespaceName - ; - -objectType - : '{' typeBody? '}' - ; - -typeBody - : typeMemberList (SemiColon | ',')? - ; - -typeMemberList - : typeMember ((SemiColon | ',') typeMember)* - ; - -typeMember - : propertySignatur - | callSignature - | constructSignature - | indexSignature - | methodSignature ('=>' type_)? - ; - -arrayType - : primaryType {notLineTerminator()}? '[' ']' - ; - -tupleType - : '[' tupleElementTypes ']' - ; - -tupleElementTypes - : type_ (',' type_)* - ; - -functionType - : typeParameters? '(' parameterList? ')' '=>' type_ - ; - -constructorType - : 'new' typeParameters? '(' parameterList? ')' '=>' type_ - ; - -typeQuery - : 'typeof' typeQueryExpression - ; - -typeQueryExpression - : Identifier - | (identifierName '.')+ identifierName - ; - -propertySignatur - : ReadOnly? propertyName '?'? typeAnnotation? ('=>' type_)? - ; - -typeAnnotation - : ':' type_ - ; - -callSignature - : typeParameters? '(' parameterList? ')' typeAnnotation? - ; - -parameterList - : restParameter - | parameter (',' parameter)* (',' restParameter)? - ; - -requiredParameterList - : requiredParameter (',' requiredParameter)* - ; - -parameter - : requiredParameter - | optionalParameter - ; - -optionalParameter - : decoratorList? ( accessibilityModifier? identifierOrPattern ('?' typeAnnotation? | typeAnnotation? initializer)) - ; - -restParameter - : '...' singleExpression typeAnnotation? - ; - -requiredParameter - : decoratorList? accessibilityModifier? identifierOrPattern typeAnnotation? - ; - -accessibilityModifier - : Public - | Private - | Protected - ; - -identifierOrPattern - : identifierName - | bindingPattern - ; - -constructSignature - : 'new' typeParameters? '(' parameterList? ')' typeAnnotation? - ; - -indexSignature - : '[' Identifier ':' (Number|String) ']' typeAnnotation - ; - -methodSignature - : propertyName '?'? callSignature - ; - -typeAliasDeclaration - : 'type' Identifier typeParameters? '=' type_ SemiColon - ; - -constructorDeclaration - : accessibilityModifier? Constructor '(' formalParameterList? ')' ( ('{' functionBody '}') | SemiColon)? - ; - -// A.5 Interface - -interfaceDeclaration - : Export? Declare? Interface Identifier typeParameters? interfaceExtendsClause? objectType SemiColon? - ; - -interfaceExtendsClause - : Extends classOrInterfaceTypeList - ; - -classOrInterfaceTypeList - : typeReference (',' typeReference)* - ; - -// A.7 Interface - -enumDeclaration - : Const? Enum Identifier '{' enumBody? '}' - ; - -enumBody - : enumMemberList ','? - ; - -enumMemberList - : enumMember (',' enumMember)* - ; - -enumMember - : propertyName ('=' singleExpression)? - ; - -// A.8 Namespaces - -namespaceDeclaration - : Namespace namespaceName '{' statementList? '}' - ; - -namespaceName - : Identifier ('.'+ Identifier)* - ; - -importAliasDeclaration - : Identifier '=' namespaceName SemiColon - ; - -// Ext.2 Additions to 1.8: Decorators - -decoratorList - : decorator+ ; - -decorator - : '@' (decoratorMemberExpression | decoratorCallExpression) - ; - -decoratorMemberExpression - : Identifier - | decoratorMemberExpression '.' identifierName - | '(' singleExpression ')' - ; - -decoratorCallExpression - : decoratorMemberExpression arguments; - -// ECMAPart -program - : sourceElements? EOF - ; - -sourceElement - : Export? statement - ; - -statement - : block - | importStatement - | exportStatement - | emptyStatement_ - | abstractDeclaration //ADDED - | classDeclaration - | interfaceDeclaration //ADDED - | namespaceDeclaration //ADDED - | ifStatement - | iterationStatement - | continueStatement - | breakStatement - | returnStatement - | yieldStatement - | withStatement - | labelledStatement - | switchStatement - | throwStatement - | tryStatement - | debuggerStatement - | functionDeclaration - | arrowFunctionDeclaration - | generatorFunctionDeclaration - | variableStatement - | typeAliasDeclaration //ADDED - | enumDeclaration //ADDED - | expressionStatement - | Export statement - ; - -block - : '{' statementList? '}' - ; - -statementList - : statement+ - ; - -abstractDeclaration - : Abstract (Identifier callSignature | variableStatement) eos - ; - -importStatement - : Import (fromBlock | importAliasDeclaration) - ; - -fromBlock - : (Multiply | multipleImportStatement) (As identifierName)? From StringLiteral eos - ; - -multipleImportStatement - : (identifierName ',')? '{' identifierName (',' identifierName)* '}' - ; - -exportStatement - : Export Default? (fromBlock | statement) - ; - -variableStatement - : bindingPattern typeAnnotation? initializer SemiColon? - | accessibilityModifier? varModifier? ReadOnly? variableDeclarationList SemiColon? - | Declare varModifier? variableDeclarationList SemiColon? - ; - -variableDeclarationList - : variableDeclaration (',' variableDeclaration)* - ; - -variableDeclaration - : ( identifierOrKeyWord | arrayLiteral | objectLiteral) typeAnnotation? singleExpression? ('=' typeParameters? singleExpression)? // ECMAScript 6: Array & Object Matching - ; - -emptyStatement_ - : SemiColon - ; - -expressionStatement - : {this.notOpenBraceAndNotFunction()}? expressionSequence SemiColon? - ; - -ifStatement - : If '(' expressionSequence ')' statement (Else statement)? - ; - - -iterationStatement - : Do statement While '(' expressionSequence ')' eos # DoStatement - | While '(' expressionSequence ')' statement # WhileStatement - | For '(' expressionSequence? SemiColon expressionSequence? SemiColon expressionSequence? ')' statement # ForStatement - | For '(' varModifier variableDeclarationList SemiColon expressionSequence? SemiColon expressionSequence? ')' - statement # ForVarStatement - | For '(' singleExpression (In | Identifier{this.p("of")}?) expressionSequence ')' statement # ForInStatement - | For '(' varModifier variableDeclaration (In | Identifier{this.p("of")}?) expressionSequence ')' statement # ForVarInStatement - ; - -varModifier - : Var - | Let - | Const - ; - -continueStatement - : Continue ({this.notLineTerminator()}? Identifier)? eos - ; - -breakStatement - : Break ({this.notLineTerminator()}? Identifier)? eos - ; - -returnStatement - : Return ({this.notLineTerminator()}? expressionSequence)? eos - ; - -yieldStatement - : Yield ({this.notLineTerminator()}? expressionSequence)? eos - ; - -withStatement - : With '(' expressionSequence ')' statement - ; - -switchStatement - : Switch '(' expressionSequence ')' caseBlock - ; - -caseBlock - : '{' caseClauses? (defaultClause caseClauses?)? '}' - ; - -caseClauses - : caseClause+ - ; - -caseClause - : Case expressionSequence ':' statementList? - ; - -defaultClause - : Default ':' statementList? - ; - -labelledStatement - : Identifier ':' statement - ; - -throwStatement - : Throw {this.notLineTerminator()}? expressionSequence eos - ; - -tryStatement - : Try block (catchProduction finallyProduction? | finallyProduction) - ; - -catchProduction - : Catch '(' Identifier ')' block - ; - -finallyProduction - : Finally block - ; - -debuggerStatement - : Debugger eos - ; - -functionDeclaration - : Function_ Identifier callSignature ( ('{' functionBody '}') | SemiColon) - ; - -//Ovveride ECMA -classDeclaration - : decoratorList? (Export Default?)? Abstract? Class Identifier typeParameters? classHeritage classTail - ; - -classHeritage - : classExtendsClause? implementsClause? - ; - -classTail - : '{' classElement* '}' - ; - -classExtendsClause - : Extends typeReference - ; - -implementsClause - : Implements classOrInterfaceTypeList - ; - -// Classes modified -classElement - : constructorDeclaration - | decoratorList? propertyMemberDeclaration - | indexMemberDeclaration - | statement - ; - -propertyMemberDeclaration - : propertyMemberBase propertyName '?'? typeAnnotation? initializer? SemiColon # PropertyDeclarationExpression - | propertyMemberBase propertyName callSignature ( ('{' functionBody '}') | SemiColon) # MethodDeclarationExpression - | propertyMemberBase (getAccessor | setAccessor) # GetterSetterDeclarationExpression - | abstractDeclaration # AbstractMemberDeclaration - ; - -propertyMemberBase - : accessibilityModifier? Async? Static? ReadOnly? - ; - -indexMemberDeclaration - : indexSignature SemiColon - ; - -generatorMethod - : '*'? Identifier '(' formalParameterList? ')' '{' functionBody '}' - ; - -generatorFunctionDeclaration - : Function_ '*' Identifier? '(' formalParameterList? ')' '{' functionBody '}' - ; - -generatorBlock - : '{' generatorDefinition (',' generatorDefinition)* ','? '}' - ; - -generatorDefinition - : '*' iteratorDefinition - ; - -iteratorBlock - : '{' iteratorDefinition (',' iteratorDefinition)* ','? '}' - ; - -iteratorDefinition - : '[' singleExpression ']' '(' formalParameterList? ')' '{' functionBody '}' - ; - -formalParameterList - : formalParameterArg (',' formalParameterArg)* (',' lastFormalParameterArg)? - | lastFormalParameterArg - | arrayLiteral // ECMAScript 6: Parameter Context Matching - | objectLiteral (':' formalParameterList)? // ECMAScript 6: Parameter Context Matching - ; - -formalParameterArg - : decorator? accessibilityModifier? identifierOrKeyWord '?'? typeAnnotation? ('=' singleExpression)? // ECMAScript 6: Initialization - ; - -lastFormalParameterArg // ECMAScript 6: Rest Parameter - : Ellipsis Identifier typeAnnotation? - ; - -functionBody - : sourceElements? - ; - -sourceElements - : sourceElement+ - ; - -arrayLiteral - : ('[' elementList? ']') - ; - -elementList - : arrayElement (','+ arrayElement)* - ; - -arrayElement // ECMAScript 6: Spread Operator - : Ellipsis? (singleExpression | Identifier) ','? - ; - -objectLiteral - : '{' (propertyAssignment (',' propertyAssignment)* ','?)? '}' - ; - -// MODIFIED -propertyAssignment - : propertyName (':' |'=') singleExpression # PropertyExpressionAssignment - | '[' singleExpression ']' ':' singleExpression # ComputedPropertyExpressionAssignment - | getAccessor # PropertyGetter - | setAccessor # PropertySetter - | generatorMethod # MethodProperty - | identifierOrKeyWord # PropertyShorthand - | restParameter # RestParameterInObject - ; - -getAccessor - : getter '(' ')' typeAnnotation? '{' functionBody '}' - ; - -setAccessor - : setter '(' ( Identifier | bindingPattern) typeAnnotation? ')' '{' functionBody '}' - ; - -propertyName - : identifierName - | StringLiteral - | numericLiteral - ; - -arguments - : '(' (argumentList ','?)? ')' - ; - -argumentList - : argument (',' argument)* - ; - -argument // ECMAScript 6: Spread Operator - : Ellipsis? (singleExpression | Identifier) - ; - -expressionSequence - : singleExpression (',' singleExpression)* - ; - -functionExpressionDeclaration - : Function_ Identifier? '(' formalParameterList? ')' typeAnnotation? '{' functionBody '}' - ; - -singleExpression - : functionExpressionDeclaration # FunctionExpression - | arrowFunctionDeclaration # ArrowFunctionExpression // ECMAScript 6 - | singleExpression '[' expressionSequence ']' # MemberIndexExpression - | singleExpression '!'? '.' identifierName nestedTypeGeneric? # MemberDotExpression - // Split to try `new Date()` first, then `new Date`. - | New singleExpression typeArguments? arguments # NewExpression - | New singleExpression typeArguments? # NewExpression - | singleExpression arguments # ArgumentsExpression - | singleExpression {this.notLineTerminator()}? '++' # PostIncrementExpression - | singleExpression {this.notLineTerminator()}? '--' # PostDecreaseExpression - | Delete singleExpression # DeleteExpression - | Void singleExpression # VoidExpression - | Typeof singleExpression # TypeofExpression - | '++' singleExpression # PreIncrementExpression - | '--' singleExpression # PreDecreaseExpression - | '+' singleExpression # UnaryPlusExpression - | '-' singleExpression # UnaryMinusExpression - | '~' singleExpression # BitNotExpression - | '!' singleExpression # NotExpression - | singleExpression ('*' | '/' | '%') singleExpression # MultiplicativeExpression - | singleExpression ('+' | '-') singleExpression # AdditiveExpression - | singleExpression ('<<' | '>>' | '>>>') singleExpression # BitShiftExpression - | singleExpression ('<' | '>' | '<=' | '>=') singleExpression # RelationalExpression - | singleExpression Instanceof singleExpression # InstanceofExpression - | singleExpression In singleExpression # InExpression - | singleExpression ('==' | '!=' | '===' | '!==') singleExpression # EqualityExpression - | singleExpression '&' singleExpression # BitAndExpression - | singleExpression '^' singleExpression # BitXOrExpression - | singleExpression '|' singleExpression # BitOrExpression - | singleExpression '&&' singleExpression # LogicalAndExpression - | singleExpression '||' singleExpression # LogicalOrExpression - | singleExpression '?' singleExpression ':' singleExpression # TernaryExpression - | singleExpression '=' singleExpression # AssignmentExpression - | singleExpression assignmentOperator singleExpression # AssignmentOperatorExpression - | singleExpression templateStringLiteral # TemplateStringExpression // ECMAScript 6 - | iteratorBlock # IteratorsExpression // ECMAScript 6 - | generatorBlock # GeneratorsExpression // ECMAScript 6 - | generatorFunctionDeclaration # GeneratorsFunctionExpression // ECMAScript 6 - | yieldStatement # YieldExpression // ECMAScript 6 - | This # ThisExpression - | identifierName singleExpression? # IdentifierExpression - | Super # SuperExpression - | literal # LiteralExpression - | arrayLiteral # ArrayLiteralExpression - | objectLiteral # ObjectLiteralExpression - | '(' expressionSequence ')' # ParenthesizedExpression - | typeArguments expressionSequence? # GenericTypes - | singleExpression As asExpression # CastAsExpression - ; - -asExpression - : predefinedType ('[' ']')? - | singleExpression - ; - -arrowFunctionDeclaration - : Async? arrowFunctionParameters typeAnnotation? '=>' arrowFunctionBody - ; - -arrowFunctionParameters - : Identifier - | '(' formalParameterList? ')' - ; - -arrowFunctionBody - : singleExpression - | '{' functionBody '}' - ; - -assignmentOperator - : '*=' - | '/=' - | '%=' - | '+=' - | '-=' - | '<<=' - | '>>=' - | '>>>=' - | '&=' - | '^=' - | '|=' - ; - -literal - : NullLiteral - | BooleanLiteral - | StringLiteral - | templateStringLiteral - | RegularExpressionLiteral - | numericLiteral - ; - -templateStringLiteral - : BackTick templateStringAtom* BackTick - ; - -templateStringAtom - : TemplateStringAtom - | TemplateStringStartExpression singleExpression TemplateCloseBrace - | TemplateStringEscapeAtom - ; - -numericLiteral - : DecimalLiteral - | HexIntegerLiteral - | OctalIntegerLiteral - | OctalIntegerLiteral2 - | BinaryIntegerLiteral - ; - -identifierName - : Identifier - | reservedWord - ; - -identifierOrKeyWord - : Identifier - | TypeAlias - | Require - ; - -reservedWord - : keyword - | NullLiteral - | BooleanLiteral - ; - -keyword - : Break - | Do - | Instanceof - | Typeof - | Case - | Else - | New - | Var - | Catch - | Finally - | Return - | Void - | Continue - | For - | Switch - | While - | Debugger - | Function_ - | This - | With - | Default - | If - | Throw - | Delete - | In - | Try - | ReadOnly - | Async - | From - | Class - | Enum - | Extends - | Super - | Const - | Export - | Import - | Implements - | Let - | Private - | Public - | Interface - | Package - | Protected - | Static - | Yield - | Get - | Set - | Require - | TypeAlias - | String - | Boolean - | Number - | Module - ; - -getter - : Get propertyName - ; - -setter - : Set propertyName - ; - -eos - : SemiColon - | EOF - | {this.lineTerminatorAhead()}? - | {this.closeBrace()}? - ; diff --git a/codegpt-core/src/main/antlr/UnicodeClasses.g4 b/codegpt-core/src/main/antlr/UnicodeClasses.g4 deleted file mode 100644 index 14d1caeb..00000000 --- a/codegpt-core/src/main/antlr/UnicodeClasses.g4 +++ /dev/null @@ -1,1647 +0,0 @@ -/** - * Taken from http://www.antlr3.org/grammar/1345144569663/AntlrUnicode.txt - */ - -lexer grammar UnicodeClasses; - -UNICODE_CLASS_LL: - '\u0061'..'\u007A' | - '\u00B5' | - '\u00DF'..'\u00F6' | - '\u00F8'..'\u00FF' | - '\u0101' | - '\u0103' | - '\u0105' | - '\u0107' | - '\u0109' | - '\u010B' | - '\u010D' | - '\u010F' | - '\u0111' | - '\u0113' | - '\u0115' | - '\u0117' | - '\u0119' | - '\u011B' | - '\u011D' | - '\u011F' | - '\u0121' | - '\u0123' | - '\u0125' | - '\u0127' | - '\u0129' | - '\u012B' | - '\u012D' | - '\u012F' | - '\u0131' | - '\u0133' | - '\u0135' | - '\u0137' | - '\u0138' | - '\u013A' | - '\u013C' | - '\u013E' | - '\u0140' | - '\u0142' | - '\u0144' | - '\u0146' | - '\u0148' | - '\u0149' | - '\u014B' | - '\u014D' | - '\u014F' | - '\u0151' | - '\u0153' | - '\u0155' | - '\u0157' | - '\u0159' | - '\u015B' | - '\u015D' | - '\u015F' | - '\u0161' | - '\u0163' | - '\u0165' | - '\u0167' | - '\u0169' | - '\u016B' | - '\u016D' | - '\u016F' | - '\u0171' | - '\u0173' | - '\u0175' | - '\u0177' | - '\u017A' | - '\u017C' | - '\u017E'..'\u0180' | - '\u0183' | - '\u0185' | - '\u0188' | - '\u018C' | - '\u018D' | - '\u0192' | - '\u0195' | - '\u0199'..'\u019B' | - '\u019E' | - '\u01A1' | - '\u01A3' | - '\u01A5' | - '\u01A8' | - '\u01AA' | - '\u01AB' | - '\u01AD' | - '\u01B0' | - '\u01B4' | - '\u01B6' | - '\u01B9' | - '\u01BA' | - '\u01BD'..'\u01BF' | - '\u01C6' | - '\u01C9' | - '\u01CC' | - '\u01CE' | - '\u01D0' | - '\u01D2' | - '\u01D4' | - '\u01D6' | - '\u01D8' | - '\u01DA' | - '\u01DC' | - '\u01DD' | - '\u01DF' | - '\u01E1' | - '\u01E3' | - '\u01E5' | - '\u01E7' | - '\u01E9' | - '\u01EB' | - '\u01ED' | - '\u01EF' | - '\u01F0' | - '\u01F3' | - '\u01F5' | - '\u01F9' | - '\u01FB' | - '\u01FD' | - '\u01FF' | - '\u0201' | - '\u0203' | - '\u0205' | - '\u0207' | - '\u0209' | - '\u020B' | - '\u020D' | - '\u020F' | - '\u0211' | - '\u0213' | - '\u0215' | - '\u0217' | - '\u0219' | - '\u021B' | - '\u021D' | - '\u021F' | - '\u0221' | - '\u0223' | - '\u0225' | - '\u0227' | - '\u0229' | - '\u022B' | - '\u022D' | - '\u022F' | - '\u0231' | - '\u0233'..'\u0239' | - '\u023C' | - '\u023F' | - '\u0240' | - '\u0242' | - '\u0247' | - '\u0249' | - '\u024B' | - '\u024D' | - '\u024F'..'\u0293' | - '\u0295'..'\u02AF' | - '\u0371' | - '\u0373' | - '\u0377' | - '\u037B'..'\u037D' | - '\u0390' | - '\u03AC'..'\u03CE' | - '\u03D0' | - '\u03D1' | - '\u03D5'..'\u03D7' | - '\u03D9' | - '\u03DB' | - '\u03DD' | - '\u03DF' | - '\u03E1' | - '\u03E3' | - '\u03E5' | - '\u03E7' | - '\u03E9' | - '\u03EB' | - '\u03ED' | - '\u03EF'..'\u03F3' | - '\u03F5' | - '\u03F8' | - '\u03FB' | - '\u03FC' | - '\u0430'..'\u045F' | - '\u0461' | - '\u0463' | - '\u0465' | - '\u0467' | - '\u0469' | - '\u046B' | - '\u046D' | - '\u046F' | - '\u0471' | - '\u0473' | - '\u0475' | - '\u0477' | - '\u0479' | - '\u047B' | - '\u047D' | - '\u047F' | - '\u0481' | - '\u048B' | - '\u048D' | - '\u048F' | - '\u0491' | - '\u0493' | - '\u0495' | - '\u0497' | - '\u0499' | - '\u049B' | - '\u049D' | - '\u049F' | - '\u04A1' | - '\u04A3' | - '\u04A5' | - '\u04A7' | - '\u04A9' | - '\u04AB' | - '\u04AD' | - '\u04AF' | - '\u04B1' | - '\u04B3' | - '\u04B5' | - '\u04B7' | - '\u04B9' | - '\u04BB' | - '\u04BD' | - '\u04BF' | - '\u04C2' | - '\u04C4' | - '\u04C6' | - '\u04C8' | - '\u04CA' | - '\u04CC' | - '\u04CE' | - '\u04CF' | - '\u04D1' | - '\u04D3' | - '\u04D5' | - '\u04D7' | - '\u04D9' | - '\u04DB' | - '\u04DD' | - '\u04DF' | - '\u04E1' | - '\u04E3' | - '\u04E5' | - '\u04E7' | - '\u04E9' | - '\u04EB' | - '\u04ED' | - '\u04EF' | - '\u04F1' | - '\u04F3' | - '\u04F5' | - '\u04F7' | - '\u04F9' | - '\u04FB' | - '\u04FD' | - '\u04FF' | - '\u0501' | - '\u0503' | - '\u0505' | - '\u0507' | - '\u0509' | - '\u050B' | - '\u050D' | - '\u050F' | - '\u0511' | - '\u0513' | - '\u0515' | - '\u0517' | - '\u0519' | - '\u051B' | - '\u051D' | - '\u051F' | - '\u0521' | - '\u0523' | - '\u0525' | - '\u0527' | - '\u0561'..'\u0587' | - '\u1D00'..'\u1D2B' | - '\u1D6B'..'\u1D77' | - '\u1D79'..'\u1D9A' | - '\u1E01' | - '\u1E03' | - '\u1E05' | - '\u1E07' | - '\u1E09' | - '\u1E0B' | - '\u1E0D' | - '\u1E0F' | - '\u1E11' | - '\u1E13' | - '\u1E15' | - '\u1E17' | - '\u1E19' | - '\u1E1B' | - '\u1E1D' | - '\u1E1F' | - '\u1E21' | - '\u1E23' | - '\u1E25' | - '\u1E27' | - '\u1E29' | - '\u1E2B' | - '\u1E2D' | - '\u1E2F' | - '\u1E31' | - '\u1E33' | - '\u1E35' | - '\u1E37' | - '\u1E39' | - '\u1E3B' | - '\u1E3D' | - '\u1E3F' | - '\u1E41' | - '\u1E43' | - '\u1E45' | - '\u1E47' | - '\u1E49' | - '\u1E4B' | - '\u1E4D' | - '\u1E4F' | - '\u1E51' | - '\u1E53' | - '\u1E55' | - '\u1E57' | - '\u1E59' | - '\u1E5B' | - '\u1E5D' | - '\u1E5F' | - '\u1E61' | - '\u1E63' | - '\u1E65' | - '\u1E67' | - '\u1E69' | - '\u1E6B' | - '\u1E6D' | - '\u1E6F' | - '\u1E71' | - '\u1E73' | - '\u1E75' | - '\u1E77' | - '\u1E79' | - '\u1E7B' | - '\u1E7D' | - '\u1E7F' | - '\u1E81' | - '\u1E83' | - '\u1E85' | - '\u1E87' | - '\u1E89' | - '\u1E8B' | - '\u1E8D' | - '\u1E8F' | - '\u1E91' | - '\u1E93' | - '\u1E95'..'\u1E9D' | - '\u1E9F' | - '\u1EA1' | - '\u1EA3' | - '\u1EA5' | - '\u1EA7' | - '\u1EA9' | - '\u1EAB' | - '\u1EAD' | - '\u1EAF' | - '\u1EB1' | - '\u1EB3' | - '\u1EB5' | - '\u1EB7' | - '\u1EB9' | - '\u1EBB' | - '\u1EBD' | - '\u1EBF' | - '\u1EC1' | - '\u1EC3' | - '\u1EC5' | - '\u1EC7' | - '\u1EC9' | - '\u1ECB' | - '\u1ECD' | - '\u1ECF' | - '\u1ED1' | - '\u1ED3' | - '\u1ED5' | - '\u1ED7' | - '\u1ED9' | - '\u1EDB' | - '\u1EDD' | - '\u1EDF' | - '\u1EE1' | - '\u1EE3' | - '\u1EE5' | - '\u1EE7' | - '\u1EE9' | - '\u1EEB' | - '\u1EED' | - '\u1EEF' | - '\u1EF1' | - '\u1EF3' | - '\u1EF5' | - '\u1EF7' | - '\u1EF9' | - '\u1EFB' | - '\u1EFD' | - '\u1EFF'..'\u1F07' | - '\u1F10'..'\u1F15' | - '\u1F20'..'\u1F27' | - '\u1F30'..'\u1F37' | - '\u1F40'..'\u1F45' | - '\u1F50'..'\u1F57' | - '\u1F60'..'\u1F67' | - '\u1F70'..'\u1F7D' | - '\u1F80'..'\u1F87' | - '\u1F90'..'\u1F97' | - '\u1FA0'..'\u1FA7' | - '\u1FB0'..'\u1FB4' | - '\u1FB6' | - '\u1FB7' | - '\u1FBE' | - '\u1FC2'..'\u1FC4' | - '\u1FC6' | - '\u1FC7' | - '\u1FD0'..'\u1FD3' | - '\u1FD6' | - '\u1FD7' | - '\u1FE0'..'\u1FE7' | - '\u1FF2'..'\u1FF4' | - '\u1FF6' | - '\u1FF7' | - '\u210A' | - '\u210E' | - '\u210F' | - '\u2113' | - '\u212F' | - '\u2134' | - '\u2139' | - '\u213C' | - '\u213D' | - '\u2146'..'\u2149' | - '\u214E' | - '\u2184' | - '\u2C30'..'\u2C5E' | - '\u2C61' | - '\u2C65' | - '\u2C66' | - '\u2C68' | - '\u2C6A' | - '\u2C6C' | - '\u2C71' | - '\u2C73' | - '\u2C74' | - '\u2C76'..'\u2C7B' | - '\u2C81' | - '\u2C83' | - '\u2C85' | - '\u2C87' | - '\u2C89' | - '\u2C8B' | - '\u2C8D' | - '\u2C8F' | - '\u2C91' | - '\u2C93' | - '\u2C95' | - '\u2C97' | - '\u2C99' | - '\u2C9B' | - '\u2C9D' | - '\u2C9F' | - '\u2CA1' | - '\u2CA3' | - '\u2CA5' | - '\u2CA7' | - '\u2CA9' | - '\u2CAB' | - '\u2CAD' | - '\u2CAF' | - '\u2CB1' | - '\u2CB3' | - '\u2CB5' | - '\u2CB7' | - '\u2CB9' | - '\u2CBB' | - '\u2CBD' | - '\u2CBF' | - '\u2CC1' | - '\u2CC3' | - '\u2CC5' | - '\u2CC7' | - '\u2CC9' | - '\u2CCB' | - '\u2CCD' | - '\u2CCF' | - '\u2CD1' | - '\u2CD3' | - '\u2CD5' | - '\u2CD7' | - '\u2CD9' | - '\u2CDB' | - '\u2CDD' | - '\u2CDF' | - '\u2CE1' | - '\u2CE3' | - '\u2CE4' | - '\u2CEC' | - '\u2CEE' | - '\u2CF3' | - '\u2D00'..'\u2D25' | - '\u2D27' | - '\u2D2D' | - '\uA641' | - '\uA643' | - '\uA645' | - '\uA647' | - '\uA649' | - '\uA64B' | - '\uA64D' | - '\uA64F' | - '\uA651' | - '\uA653' | - '\uA655' | - '\uA657' | - '\uA659' | - '\uA65B' | - '\uA65D' | - '\uA65F' | - '\uA661' | - '\uA663' | - '\uA665' | - '\uA667' | - '\uA669' | - '\uA66B' | - '\uA66D' | - '\uA681' | - '\uA683' | - '\uA685' | - '\uA687' | - '\uA689' | - '\uA68B' | - '\uA68D' | - '\uA68F' | - '\uA691' | - '\uA693' | - '\uA695' | - '\uA697' | - '\uA723' | - '\uA725' | - '\uA727' | - '\uA729' | - '\uA72B' | - '\uA72D' | - '\uA72F'..'\uA731' | - '\uA733' | - '\uA735' | - '\uA737' | - '\uA739' | - '\uA73B' | - '\uA73D' | - '\uA73F' | - '\uA741' | - '\uA743' | - '\uA745' | - '\uA747' | - '\uA749' | - '\uA74B' | - '\uA74D' | - '\uA74F' | - '\uA751' | - '\uA753' | - '\uA755' | - '\uA757' | - '\uA759' | - '\uA75B' | - '\uA75D' | - '\uA75F' | - '\uA761' | - '\uA763' | - '\uA765' | - '\uA767' | - '\uA769' | - '\uA76B' | - '\uA76D' | - '\uA76F' | - '\uA771'..'\uA778' | - '\uA77A' | - '\uA77C' | - '\uA77F' | - '\uA781' | - '\uA783' | - '\uA785' | - '\uA787' | - '\uA78C' | - '\uA78E' | - '\uA791' | - '\uA793' | - '\uA7A1' | - '\uA7A3' | - '\uA7A5' | - '\uA7A7' | - '\uA7A9' | - '\uA7FA' | - '\uFB00'..'\uFB06' | - '\uFB13'..'\uFB17' | - '\uFF41'..'\uFF5A'; - -UNICODE_CLASS_LM: - '\u02B0'..'\u02C1' | - '\u02C6'..'\u02D1' | - '\u02E0'..'\u02E4' | - '\u02EC' | - '\u02EE' | - '\u0374' | - '\u037A' | - '\u0559' | - '\u0640' | - '\u06E5' | - '\u06E6' | - '\u07F4' | - '\u07F5' | - '\u07FA' | - '\u081A' | - '\u0824' | - '\u0828' | - '\u0971' | - '\u0E46' | - '\u0EC6' | - '\u10FC' | - '\u17D7' | - '\u1843' | - '\u1AA7' | - '\u1C78'..'\u1C7D' | - '\u1D2C'..'\u1D6A' | - '\u1D78' | - '\u1D9B'..'\u1DBF' | - '\u2071' | - '\u207F' | - '\u2090'..'\u209C' | - '\u2C7C' | - '\u2C7D' | - '\u2D6F' | - '\u2E2F' | - '\u3005' | - '\u3031'..'\u3035' | - '\u303B' | - '\u309D' | - '\u309E' | - '\u30FC'..'\u30FE' | - '\uA015' | - '\uA4F8'..'\uA4FD' | - '\uA60C' | - '\uA67F' | - '\uA717'..'\uA71F' | - '\uA770' | - '\uA788' | - '\uA7F8' | - '\uA7F9' | - '\uA9CF' | - '\uAA70' | - '\uAADD' | - '\uAAF3' | - '\uAAF4' | - '\uFF70' | - '\uFF9E' | - '\uFF9F'; - -UNICODE_CLASS_LO: - '\u00AA' | - '\u00BA' | - '\u01BB' | - '\u01C0'..'\u01C3' | - '\u0294' | - '\u05D0'..'\u05EA' | - '\u05F0'..'\u05F2' | - '\u0620'..'\u063F' | - '\u0641'..'\u064A' | - '\u066E' | - '\u066F' | - '\u0671'..'\u06D3' | - '\u06D5' | - '\u06EE' | - '\u06EF' | - '\u06FA'..'\u06FC' | - '\u06FF' | - '\u0710' | - '\u0712'..'\u072F' | - '\u074D'..'\u07A5' | - '\u07B1' | - '\u07CA'..'\u07EA' | - '\u0800'..'\u0815' | - '\u0840'..'\u0858' | - '\u08A0' | - '\u08A2'..'\u08AC' | - '\u0904'..'\u0939' | - '\u093D' | - '\u0950' | - '\u0958'..'\u0961' | - '\u0972'..'\u0977' | - '\u0979'..'\u097F' | - '\u0985'..'\u098C' | - '\u098F' | - '\u0990' | - '\u0993'..'\u09A8' | - '\u09AA'..'\u09B0' | - '\u09B2' | - '\u09B6'..'\u09B9' | - '\u09BD' | - '\u09CE' | - '\u09DC' | - '\u09DD' | - '\u09DF'..'\u09E1' | - '\u09F0' | - '\u09F1' | - '\u0A05'..'\u0A0A' | - '\u0A0F' | - '\u0A10' | - '\u0A13'..'\u0A28' | - '\u0A2A'..'\u0A30' | - '\u0A32' | - '\u0A33' | - '\u0A35' | - '\u0A36' | - '\u0A38' | - '\u0A39' | - '\u0A59'..'\u0A5C' | - '\u0A5E' | - '\u0A72'..'\u0A74' | - '\u0A85'..'\u0A8D' | - '\u0A8F'..'\u0A91' | - '\u0A93'..'\u0AA8' | - '\u0AAA'..'\u0AB0' | - '\u0AB2' | - '\u0AB3' | - '\u0AB5'..'\u0AB9' | - '\u0ABD' | - '\u0AD0' | - '\u0AE0' | - '\u0AE1' | - '\u0B05'..'\u0B0C' | - '\u0B0F' | - '\u0B10' | - '\u0B13'..'\u0B28' | - '\u0B2A'..'\u0B30' | - '\u0B32' | - '\u0B33' | - '\u0B35'..'\u0B39' | - '\u0B3D' | - '\u0B5C' | - '\u0B5D' | - '\u0B5F'..'\u0B61' | - '\u0B71' | - '\u0B83' | - '\u0B85'..'\u0B8A' | - '\u0B8E'..'\u0B90' | - '\u0B92'..'\u0B95' | - '\u0B99' | - '\u0B9A' | - '\u0B9C' | - '\u0B9E' | - '\u0B9F' | - '\u0BA3' | - '\u0BA4' | - '\u0BA8'..'\u0BAA' | - '\u0BAE'..'\u0BB9' | - '\u0BD0' | - '\u0C05'..'\u0C0C' | - '\u0C0E'..'\u0C10' | - '\u0C12'..'\u0C28' | - '\u0C2A'..'\u0C33' | - '\u0C35'..'\u0C39' | - '\u0C3D' | - '\u0C58' | - '\u0C59' | - '\u0C60' | - '\u0C61' | - '\u0C85'..'\u0C8C' | - '\u0C8E'..'\u0C90' | - '\u0C92'..'\u0CA8' | - '\u0CAA'..'\u0CB3' | - '\u0CB5'..'\u0CB9' | - '\u0CBD' | - '\u0CDE' | - '\u0CE0' | - '\u0CE1' | - '\u0CF1' | - '\u0CF2' | - '\u0D05'..'\u0D0C' | - '\u0D0E'..'\u0D10' | - '\u0D12'..'\u0D3A' | - '\u0D3D' | - '\u0D4E' | - '\u0D60' | - '\u0D61' | - '\u0D7A'..'\u0D7F' | - '\u0D85'..'\u0D96' | - '\u0D9A'..'\u0DB1' | - '\u0DB3'..'\u0DBB' | - '\u0DBD' | - '\u0DC0'..'\u0DC6' | - '\u0E01'..'\u0E30' | - '\u0E32' | - '\u0E33' | - '\u0E40'..'\u0E45' | - '\u0E81' | - '\u0E82' | - '\u0E84' | - '\u0E87' | - '\u0E88' | - '\u0E8A' | - '\u0E8D' | - '\u0E94'..'\u0E97' | - '\u0E99'..'\u0E9F' | - '\u0EA1'..'\u0EA3' | - '\u0EA5' | - '\u0EA7' | - '\u0EAA' | - '\u0EAB' | - '\u0EAD'..'\u0EB0' | - '\u0EB2' | - '\u0EB3' | - '\u0EBD' | - '\u0EC0'..'\u0EC4' | - '\u0EDC'..'\u0EDF' | - '\u0F00' | - '\u0F40'..'\u0F47' | - '\u0F49'..'\u0F6C' | - '\u0F88'..'\u0F8C' | - '\u1000'..'\u102A' | - '\u103F' | - '\u1050'..'\u1055' | - '\u105A'..'\u105D' | - '\u1061' | - '\u1065' | - '\u1066' | - '\u106E'..'\u1070' | - '\u1075'..'\u1081' | - '\u108E' | - '\u10D0'..'\u10FA' | - '\u10FD'..'\u1248' | - '\u124A'..'\u124D' | - '\u1250'..'\u1256' | - '\u1258' | - '\u125A'..'\u125D' | - '\u1260'..'\u1288' | - '\u128A'..'\u128D' | - '\u1290'..'\u12B0' | - '\u12B2'..'\u12B5' | - '\u12B8'..'\u12BE' | - '\u12C0' | - '\u12C2'..'\u12C5' | - '\u12C8'..'\u12D6' | - '\u12D8'..'\u1310' | - '\u1312'..'\u1315' | - '\u1318'..'\u135A' | - '\u1380'..'\u138F' | - '\u13A0'..'\u13F4' | - '\u1401'..'\u166C' | - '\u166F'..'\u167F' | - '\u1681'..'\u169A' | - '\u16A0'..'\u16EA' | - '\u1700'..'\u170C' | - '\u170E'..'\u1711' | - '\u1720'..'\u1731' | - '\u1740'..'\u1751' | - '\u1760'..'\u176C' | - '\u176E'..'\u1770' | - '\u1780'..'\u17B3' | - '\u17DC' | - '\u1820'..'\u1842' | - '\u1844'..'\u1877' | - '\u1880'..'\u18A8' | - '\u18AA' | - '\u18B0'..'\u18F5' | - '\u1900'..'\u191C' | - '\u1950'..'\u196D' | - '\u1970'..'\u1974' | - '\u1980'..'\u19AB' | - '\u19C1'..'\u19C7' | - '\u1A00'..'\u1A16' | - '\u1A20'..'\u1A54' | - '\u1B05'..'\u1B33' | - '\u1B45'..'\u1B4B' | - '\u1B83'..'\u1BA0' | - '\u1BAE' | - '\u1BAF' | - '\u1BBA'..'\u1BE5' | - '\u1C00'..'\u1C23' | - '\u1C4D'..'\u1C4F' | - '\u1C5A'..'\u1C77' | - '\u1CE9'..'\u1CEC' | - '\u1CEE'..'\u1CF1' | - '\u1CF5' | - '\u1CF6' | - '\u2135'..'\u2138' | - '\u2D30'..'\u2D67' | - '\u2D80'..'\u2D96' | - '\u2DA0'..'\u2DA6' | - '\u2DA8'..'\u2DAE' | - '\u2DB0'..'\u2DB6' | - '\u2DB8'..'\u2DBE' | - '\u2DC0'..'\u2DC6' | - '\u2DC8'..'\u2DCE' | - '\u2DD0'..'\u2DD6' | - '\u2DD8'..'\u2DDE' | - '\u3006' | - '\u303C' | - '\u3041'..'\u3096' | - '\u309F' | - '\u30A1'..'\u30FA' | - '\u30FF' | - '\u3105'..'\u312D' | - '\u3131'..'\u318E' | - '\u31A0'..'\u31BA' | - '\u31F0'..'\u31FF' | - '\u3400' | - '\u4DB5' | - '\u4E00' | - '\u9FCC' | - '\uA000'..'\uA014' | - '\uA016'..'\uA48C' | - '\uA4D0'..'\uA4F7' | - '\uA500'..'\uA60B' | - '\uA610'..'\uA61F' | - '\uA62A' | - '\uA62B' | - '\uA66E' | - '\uA6A0'..'\uA6E5' | - '\uA7FB'..'\uA801' | - '\uA803'..'\uA805' | - '\uA807'..'\uA80A' | - '\uA80C'..'\uA822' | - '\uA840'..'\uA873' | - '\uA882'..'\uA8B3' | - '\uA8F2'..'\uA8F7' | - '\uA8FB' | - '\uA90A'..'\uA925' | - '\uA930'..'\uA946' | - '\uA960'..'\uA97C' | - '\uA984'..'\uA9B2' | - '\uAA00'..'\uAA28' | - '\uAA40'..'\uAA42' | - '\uAA44'..'\uAA4B' | - '\uAA60'..'\uAA6F' | - '\uAA71'..'\uAA76' | - '\uAA7A' | - '\uAA80'..'\uAAAF' | - '\uAAB1' | - '\uAAB5' | - '\uAAB6' | - '\uAAB9'..'\uAABD' | - '\uAAC0' | - '\uAAC2' | - '\uAADB' | - '\uAADC' | - '\uAAE0'..'\uAAEA' | - '\uAAF2' | - '\uAB01'..'\uAB06' | - '\uAB09'..'\uAB0E' | - '\uAB11'..'\uAB16' | - '\uAB20'..'\uAB26' | - '\uAB28'..'\uAB2E' | - '\uABC0'..'\uABE2' | - '\uAC00' | - '\uD7A3' | - '\uD7B0'..'\uD7C6' | - '\uD7CB'..'\uD7FB' | - '\uF900'..'\uFA6D' | - '\uFA70'..'\uFAD9' | - '\uFB1D' | - '\uFB1F'..'\uFB28' | - '\uFB2A'..'\uFB36' | - '\uFB38'..'\uFB3C' | - '\uFB3E' | - '\uFB40' | - '\uFB41' | - '\uFB43' | - '\uFB44' | - '\uFB46'..'\uFBB1' | - '\uFBD3'..'\uFD3D' | - '\uFD50'..'\uFD8F' | - '\uFD92'..'\uFDC7' | - '\uFDF0'..'\uFDFB' | - '\uFE70'..'\uFE74' | - '\uFE76'..'\uFEFC' | - '\uFF66'..'\uFF6F' | - '\uFF71'..'\uFF9D' | - '\uFFA0'..'\uFFBE' | - '\uFFC2'..'\uFFC7' | - '\uFFCA'..'\uFFCF' | - '\uFFD2'..'\uFFD7' | - '\uFFDA'..'\uFFDC'; - -UNICODE_CLASS_LT: - '\u01C5' | - '\u01C8' | - '\u01CB' | - '\u01F2' | - '\u1F88'..'\u1F8F' | - '\u1F98'..'\u1F9F' | - '\u1FA8'..'\u1FAF' | - '\u1FBC' | - '\u1FCC' | - '\u1FFC'; - -UNICODE_CLASS_LU: - '\u0041'..'\u005A' | - '\u00C0'..'\u00D6' | - '\u00D8'..'\u00DE' | - '\u0100' | - '\u0102' | - '\u0104' | - '\u0106' | - '\u0108' | - '\u010A' | - '\u010C' | - '\u010E' | - '\u0110' | - '\u0112' | - '\u0114' | - '\u0116' | - '\u0118' | - '\u011A' | - '\u011C' | - '\u011E' | - '\u0120' | - '\u0122' | - '\u0124' | - '\u0126' | - '\u0128' | - '\u012A' | - '\u012C' | - '\u012E' | - '\u0130' | - '\u0132' | - '\u0134' | - '\u0136' | - '\u0139' | - '\u013B' | - '\u013D' | - '\u013F' | - '\u0141' | - '\u0143' | - '\u0145' | - '\u0147' | - '\u014A' | - '\u014C' | - '\u014E' | - '\u0150' | - '\u0152' | - '\u0154' | - '\u0156' | - '\u0158' | - '\u015A' | - '\u015C' | - '\u015E' | - '\u0160' | - '\u0162' | - '\u0164' | - '\u0166' | - '\u0168' | - '\u016A' | - '\u016C' | - '\u016E' | - '\u0170' | - '\u0172' | - '\u0174' | - '\u0176' | - '\u0178' | - '\u0179' | - '\u017B' | - '\u017D' | - '\u0181' | - '\u0182' | - '\u0184' | - '\u0186' | - '\u0187' | - '\u0189'..'\u018B' | - '\u018E'..'\u0191' | - '\u0193' | - '\u0194' | - '\u0196'..'\u0198' | - '\u019C' | - '\u019D' | - '\u019F' | - '\u01A0' | - '\u01A2' | - '\u01A4' | - '\u01A6' | - '\u01A7' | - '\u01A9' | - '\u01AC' | - '\u01AE' | - '\u01AF' | - '\u01B1'..'\u01B3' | - '\u01B5' | - '\u01B7' | - '\u01B8' | - '\u01BC' | - '\u01C4' | - '\u01C7' | - '\u01CA' | - '\u01CD' | - '\u01CF' | - '\u01D1' | - '\u01D3' | - '\u01D5' | - '\u01D7' | - '\u01D9' | - '\u01DB' | - '\u01DE' | - '\u01E0' | - '\u01E2' | - '\u01E4' | - '\u01E6' | - '\u01E8' | - '\u01EA' | - '\u01EC' | - '\u01EE' | - '\u01F1' | - '\u01F4' | - '\u01F6'..'\u01F8' | - '\u01FA' | - '\u01FC' | - '\u01FE' | - '\u0200' | - '\u0202' | - '\u0204' | - '\u0206' | - '\u0208' | - '\u020A' | - '\u020C' | - '\u020E' | - '\u0210' | - '\u0212' | - '\u0214' | - '\u0216' | - '\u0218' | - '\u021A' | - '\u021C' | - '\u021E' | - '\u0220' | - '\u0222' | - '\u0224' | - '\u0226' | - '\u0228' | - '\u022A' | - '\u022C' | - '\u022E' | - '\u0230' | - '\u0232' | - '\u023A' | - '\u023B' | - '\u023D' | - '\u023E' | - '\u0241' | - '\u0243'..'\u0246' | - '\u0248' | - '\u024A' | - '\u024C' | - '\u024E' | - '\u0370' | - '\u0372' | - '\u0376' | - '\u0386' | - '\u0388'..'\u038A' | - '\u038C' | - '\u038E' | - '\u038F' | - '\u0391'..'\u03A1' | - '\u03A3'..'\u03AB' | - '\u03CF' | - '\u03D2'..'\u03D4' | - '\u03D8' | - '\u03DA' | - '\u03DC' | - '\u03DE' | - '\u03E0' | - '\u03E2' | - '\u03E4' | - '\u03E6' | - '\u03E8' | - '\u03EA' | - '\u03EC' | - '\u03EE' | - '\u03F4' | - '\u03F7' | - '\u03F9' | - '\u03FA' | - '\u03FD'..'\u042F' | - '\u0460' | - '\u0462' | - '\u0464' | - '\u0466' | - '\u0468' | - '\u046A' | - '\u046C' | - '\u046E' | - '\u0470' | - '\u0472' | - '\u0474' | - '\u0476' | - '\u0478' | - '\u047A' | - '\u047C' | - '\u047E' | - '\u0480' | - '\u048A' | - '\u048C' | - '\u048E' | - '\u0490' | - '\u0492' | - '\u0494' | - '\u0496' | - '\u0498' | - '\u049A' | - '\u049C' | - '\u049E' | - '\u04A0' | - '\u04A2' | - '\u04A4' | - '\u04A6' | - '\u04A8' | - '\u04AA' | - '\u04AC' | - '\u04AE' | - '\u04B0' | - '\u04B2' | - '\u04B4' | - '\u04B6' | - '\u04B8' | - '\u04BA' | - '\u04BC' | - '\u04BE' | - '\u04C0' | - '\u04C1' | - '\u04C3' | - '\u04C5' | - '\u04C7' | - '\u04C9' | - '\u04CB' | - '\u04CD' | - '\u04D0' | - '\u04D2' | - '\u04D4' | - '\u04D6' | - '\u04D8' | - '\u04DA' | - '\u04DC' | - '\u04DE' | - '\u04E0' | - '\u04E2' | - '\u04E4' | - '\u04E6' | - '\u04E8' | - '\u04EA' | - '\u04EC' | - '\u04EE' | - '\u04F0' | - '\u04F2' | - '\u04F4' | - '\u04F6' | - '\u04F8' | - '\u04FA' | - '\u04FC' | - '\u04FE' | - '\u0500' | - '\u0502' | - '\u0504' | - '\u0506' | - '\u0508' | - '\u050A' | - '\u050C' | - '\u050E' | - '\u0510' | - '\u0512' | - '\u0514' | - '\u0516' | - '\u0518' | - '\u051A' | - '\u051C' | - '\u051E' | - '\u0520' | - '\u0522' | - '\u0524' | - '\u0526' | - '\u0531'..'\u0556' | - '\u10A0'..'\u10C5' | - '\u10C7' | - '\u10CD' | - '\u1E00' | - '\u1E02' | - '\u1E04' | - '\u1E06' | - '\u1E08' | - '\u1E0A' | - '\u1E0C' | - '\u1E0E' | - '\u1E10' | - '\u1E12' | - '\u1E14' | - '\u1E16' | - '\u1E18' | - '\u1E1A' | - '\u1E1C' | - '\u1E1E' | - '\u1E20' | - '\u1E22' | - '\u1E24' | - '\u1E26' | - '\u1E28' | - '\u1E2A' | - '\u1E2C' | - '\u1E2E' | - '\u1E30' | - '\u1E32' | - '\u1E34' | - '\u1E36' | - '\u1E38' | - '\u1E3A' | - '\u1E3C' | - '\u1E3E' | - '\u1E40' | - '\u1E42' | - '\u1E44' | - '\u1E46' | - '\u1E48' | - '\u1E4A' | - '\u1E4C' | - '\u1E4E' | - '\u1E50' | - '\u1E52' | - '\u1E54' | - '\u1E56' | - '\u1E58' | - '\u1E5A' | - '\u1E5C' | - '\u1E5E' | - '\u1E60' | - '\u1E62' | - '\u1E64' | - '\u1E66' | - '\u1E68' | - '\u1E6A' | - '\u1E6C' | - '\u1E6E' | - '\u1E70' | - '\u1E72' | - '\u1E74' | - '\u1E76' | - '\u1E78' | - '\u1E7A' | - '\u1E7C' | - '\u1E7E' | - '\u1E80' | - '\u1E82' | - '\u1E84' | - '\u1E86' | - '\u1E88' | - '\u1E8A' | - '\u1E8C' | - '\u1E8E' | - '\u1E90' | - '\u1E92' | - '\u1E94' | - '\u1E9E' | - '\u1EA0' | - '\u1EA2' | - '\u1EA4' | - '\u1EA6' | - '\u1EA8' | - '\u1EAA' | - '\u1EAC' | - '\u1EAE' | - '\u1EB0' | - '\u1EB2' | - '\u1EB4' | - '\u1EB6' | - '\u1EB8' | - '\u1EBA' | - '\u1EBC' | - '\u1EBE' | - '\u1EC0' | - '\u1EC2' | - '\u1EC4' | - '\u1EC6' | - '\u1EC8' | - '\u1ECA' | - '\u1ECC' | - '\u1ECE' | - '\u1ED0' | - '\u1ED2' | - '\u1ED4' | - '\u1ED6' | - '\u1ED8' | - '\u1EDA' | - '\u1EDC' | - '\u1EDE' | - '\u1EE0' | - '\u1EE2' | - '\u1EE4' | - '\u1EE6' | - '\u1EE8' | - '\u1EEA' | - '\u1EEC' | - '\u1EEE' | - '\u1EF0' | - '\u1EF2' | - '\u1EF4' | - '\u1EF6' | - '\u1EF8' | - '\u1EFA' | - '\u1EFC' | - '\u1EFE' | - '\u1F08'..'\u1F0F' | - '\u1F18'..'\u1F1D' | - '\u1F28'..'\u1F2F' | - '\u1F38'..'\u1F3F' | - '\u1F48'..'\u1F4D' | - '\u1F59' | - '\u1F5B' | - '\u1F5D' | - '\u1F5F' | - '\u1F68'..'\u1F6F' | - '\u1FB8'..'\u1FBB' | - '\u1FC8'..'\u1FCB' | - '\u1FD8'..'\u1FDB' | - '\u1FE8'..'\u1FEC' | - '\u1FF8'..'\u1FFB' | - '\u2102' | - '\u2107' | - '\u210B'..'\u210D' | - '\u2110'..'\u2112' | - '\u2115' | - '\u2119'..'\u211D' | - '\u2124' | - '\u2126' | - '\u2128' | - '\u212A'..'\u212D' | - '\u2130'..'\u2133' | - '\u213E' | - '\u213F' | - '\u2145' | - '\u2183' | - '\u2C00'..'\u2C2E' | - '\u2C60' | - '\u2C62'..'\u2C64' | - '\u2C67' | - '\u2C69' | - '\u2C6B' | - '\u2C6D'..'\u2C70' | - '\u2C72' | - '\u2C75' | - '\u2C7E'..'\u2C80' | - '\u2C82' | - '\u2C84' | - '\u2C86' | - '\u2C88' | - '\u2C8A' | - '\u2C8C' | - '\u2C8E' | - '\u2C90' | - '\u2C92' | - '\u2C94' | - '\u2C96' | - '\u2C98' | - '\u2C9A' | - '\u2C9C' | - '\u2C9E' | - '\u2CA0' | - '\u2CA2' | - '\u2CA4' | - '\u2CA6' | - '\u2CA8' | - '\u2CAA' | - '\u2CAC' | - '\u2CAE' | - '\u2CB0' | - '\u2CB2' | - '\u2CB4' | - '\u2CB6' | - '\u2CB8' | - '\u2CBA' | - '\u2CBC' | - '\u2CBE' | - '\u2CC0' | - '\u2CC2' | - '\u2CC4' | - '\u2CC6' | - '\u2CC8' | - '\u2CCA' | - '\u2CCC' | - '\u2CCE' | - '\u2CD0' | - '\u2CD2' | - '\u2CD4' | - '\u2CD6' | - '\u2CD8' | - '\u2CDA' | - '\u2CDC' | - '\u2CDE' | - '\u2CE0' | - '\u2CE2' | - '\u2CEB' | - '\u2CED' | - '\u2CF2' | - '\uA640' | - '\uA642' | - '\uA644' | - '\uA646' | - '\uA648' | - '\uA64A' | - '\uA64C' | - '\uA64E' | - '\uA650' | - '\uA652' | - '\uA654' | - '\uA656' | - '\uA658' | - '\uA65A' | - '\uA65C' | - '\uA65E' | - '\uA660' | - '\uA662' | - '\uA664' | - '\uA666' | - '\uA668' | - '\uA66A' | - '\uA66C' | - '\uA680' | - '\uA682' | - '\uA684' | - '\uA686' | - '\uA688' | - '\uA68A' | - '\uA68C' | - '\uA68E' | - '\uA690' | - '\uA692' | - '\uA694' | - '\uA696' | - '\uA722' | - '\uA724' | - '\uA726' | - '\uA728' | - '\uA72A' | - '\uA72C' | - '\uA72E' | - '\uA732' | - '\uA734' | - '\uA736' | - '\uA738' | - '\uA73A' | - '\uA73C' | - '\uA73E' | - '\uA740' | - '\uA742' | - '\uA744' | - '\uA746' | - '\uA748' | - '\uA74A' | - '\uA74C' | - '\uA74E' | - '\uA750' | - '\uA752' | - '\uA754' | - '\uA756' | - '\uA758' | - '\uA75A' | - '\uA75C' | - '\uA75E' | - '\uA760' | - '\uA762' | - '\uA764' | - '\uA766' | - '\uA768' | - '\uA76A' | - '\uA76C' | - '\uA76E' | - '\uA779' | - '\uA77B' | - '\uA77D' | - '\uA77E' | - '\uA780' | - '\uA782' | - '\uA784' | - '\uA786' | - '\uA78B' | - '\uA78D' | - '\uA790' | - '\uA792' | - '\uA7A0' | - '\uA7A2' | - '\uA7A4' | - '\uA7A6' | - '\uA7A8' | - '\uA7AA' | - '\uFF21'..'\uFF3A'; - -UNICODE_CLASS_ND: - '\u0030'..'\u0039' | - '\u0660'..'\u0669' | - '\u06F0'..'\u06F9' | - '\u07C0'..'\u07C9' | - '\u0966'..'\u096F' | - '\u09E6'..'\u09EF' | - '\u0A66'..'\u0A6F' | - '\u0AE6'..'\u0AEF' | - '\u0B66'..'\u0B6F' | - '\u0BE6'..'\u0BEF' | - '\u0C66'..'\u0C6F' | - '\u0CE6'..'\u0CEF' | - '\u0D66'..'\u0D6F' | - '\u0E50'..'\u0E59' | - '\u0ED0'..'\u0ED9' | - '\u0F20'..'\u0F29' | - '\u1040'..'\u1049' | - '\u1090'..'\u1099' | - '\u17E0'..'\u17E9' | - '\u1810'..'\u1819' | - '\u1946'..'\u194F' | - '\u19D0'..'\u19D9' | - '\u1A80'..'\u1A89' | - '\u1A90'..'\u1A99' | - '\u1B50'..'\u1B59' | - '\u1BB0'..'\u1BB9' | - '\u1C40'..'\u1C49' | - '\u1C50'..'\u1C59' | - '\uA620'..'\uA629' | - '\uA8D0'..'\uA8D9' | - '\uA900'..'\uA909' | - '\uA9D0'..'\uA9D9' | - '\uAA50'..'\uAA59' | - '\uABF0'..'\uABF9' | - '\uFF10'..'\uFF19'; - -UNICODE_CLASS_NL: - '\u16EE'..'\u16F0' | - '\u2160'..'\u2182' | - '\u2185'..'\u2188' | - '\u3007' | - '\u3021'..'\u3029' | - '\u3038'..'\u303A' | - '\uA6E6'..'\uA6EF'; \ No newline at end of file diff --git a/codegpt-core/src/main/java/ee/carlrobert/embedding/EmbeddingsService.java b/codegpt-core/src/main/java/ee/carlrobert/embedding/EmbeddingsService.java deleted file mode 100644 index 1d77beb2..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/embedding/EmbeddingsService.java +++ /dev/null @@ -1,146 +0,0 @@ -package ee.carlrobert.embedding; - -import static com.github.jelmerk.knn.util.VectorUtils.normalize; -import static java.util.stream.Collectors.toList; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.github.jelmerk.knn.Item; -import com.github.jelmerk.knn.SearchResult; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.progress.ProgressIndicator; -import ee.carlrobert.llm.client.openai.OpenAIClient; -import ee.carlrobert.llm.client.openai.completion.OpenAIChatCompletionModel; -import ee.carlrobert.llm.client.openai.completion.request.OpenAIChatCompletionMessage; -import ee.carlrobert.llm.client.openai.completion.request.OpenAIChatCompletionRequest; -import ee.carlrobert.splitter.SplitterFactory; -import ee.carlrobert.vector.VectorStore; -import ee.carlrobert.vector.Word; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import org.jetbrains.annotations.Nullable; - -public class EmbeddingsService { - - private static final Logger LOG = Logger.getInstance(EmbeddingsService.class); - - private final VectorStore vectorStore; - private final OpenAIClient openAIClient; - - public EmbeddingsService(OpenAIClient openAIClient, Path pluginBasePath) { - this.openAIClient = openAIClient; - this.vectorStore = VectorStore.getInstance(pluginBasePath); - } - - public List getEmbeddings(List chunks) { - return openAIClient.getEmbeddings(chunks); - } - - public String buildPromptWithContext(String prompt) { - try { - var inputEmbedding = openAIClient.getEmbedding(getSearchQuery(prompt)); - var sortedResult = vectorStore.loadIndex() - .findNearest(normalize(inputEmbedding), 10) - .stream() - .map(SearchResult::item) - .sorted(Comparator.comparing(Word::getMeta)) - .collect(toList()); - - var context = sortedResult.stream().map(Word::id).collect(Collectors.joining()); - var fileNames = sortedResult.stream().map(Word::getMeta).collect(Collectors.toSet()); - - return getResourceContent("/prompts/prompt-with-context.txt") - .replace("{prompt}", prompt) - .replace("{context}", new GeneratedContextDetails(context, fileNames).getContext()); - } catch (IOException e) { - LOG.error("Unable to load vector index", e); - return prompt; - } - } - - public List> createEmbeddings( - List referencedFiles, - @Nullable ProgressIndicator indicator) { - var words = new ArrayList>(); - for (int i = 0; i < referencedFiles.size(); i++) { - try { - var referencedFile = referencedFiles.get(i); - addEmbeddings(referencedFile, words); - - if (indicator != null) { - indicator.setFraction((double) i / referencedFiles.size()); - } - } catch (Throwable t) { - // ignore - } - } - return words; - } - - private String getSearchQuery(String userPrompt) throws JsonProcessingException { - var message = new OpenAIChatCompletionMessage( - "user", - getResourceContent("/prompts/text-generator.txt") - .replace("{prompt}", userPrompt)); - var request = new OpenAIChatCompletionRequest.Builder(List.of(message)) - .setModel(OpenAIChatCompletionModel.GPT_4) - .setMaxTokens(400) - .setTemperature(0.1) - .setStream(false) - .build(); - - return openAIClient.getChatCompletion(request) - .getChoices() - .get(0) - .getMessage() - .getContent(); - } - - private void addEmbeddings( - ReferencedFile referencedFile, - List> prevEmbeddings) { - var fileExtension = referencedFile.getFileExtension(); - var codeSplitter = SplitterFactory.getCodeSplitter(fileExtension); - if (codeSplitter != null) { - var chunks = codeSplitter.split( - referencedFile.getFileName(), - referencedFile.getFileContent()); - var embeddings = openAIClient.getEmbeddings(chunks); - for (int i = 0; i < chunks.size(); i++) { - prevEmbeddings.add( - new Word(chunks.get(i), referencedFile.getFileName(), normalize(embeddings.get(i)))); - } - } else { - var chunks = splitText(referencedFile.getFileContent(), 400); - var embeddings = getEmbeddings(chunks); - for (int i = 0; i < chunks.size(); i++) { - prevEmbeddings.add( - new Word(chunks.get(i), referencedFile.getFileName(), normalize(embeddings.get(i)))); - } - } - } - - private static List splitText(String str, int chunkSize) { - int len = str.length(); - var chunks = new ArrayList(); - for (int i = 0; i < len; i += chunkSize) { - chunks.add(str.substring(i, Math.min(len, i + chunkSize))); - } - return chunks; - } - - // TODO: Move to shared module - private static String getResourceContent(String name) { - try (var stream = - Objects.requireNonNull(EmbeddingsService.class.getResourceAsStream(name))) { - return new String(stream.readAllBytes(), StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException("Unable to read resource", e); - } - } -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/embedding/GeneratedContextDetails.java b/codegpt-core/src/main/java/ee/carlrobert/embedding/GeneratedContextDetails.java deleted file mode 100644 index 2ef1d0b9..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/embedding/GeneratedContextDetails.java +++ /dev/null @@ -1,22 +0,0 @@ -package ee.carlrobert.embedding; - -import java.util.Set; - -public class GeneratedContextDetails { - - private final String context; - private final Set fileNames; - - public GeneratedContextDetails(String context, Set fileNames) { - this.context = context; - this.fileNames = fileNames; - } - - public String getContext() { - return context; - } - - public Set getFileNames() { - return fileNames; - } -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/splitter/CodeSplitter.java b/codegpt-core/src/main/java/ee/carlrobert/splitter/CodeSplitter.java deleted file mode 100644 index b018548a..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/splitter/CodeSplitter.java +++ /dev/null @@ -1,34 +0,0 @@ -package ee.carlrobert.splitter; - -import java.util.ArrayList; -import java.util.List; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CodePointCharStream; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.misc.Interval; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeListener; -import org.antlr.v4.runtime.tree.ParseTreeWalker; - -abstract class CodeSplitter implements Splitter { - - protected List chunks = new ArrayList<>(); - - protected abstract ParseTree getParseTree(CodePointCharStream charStream); - - protected abstract ParseTreeListener getParseTreeListener(); - - protected String parseContext(ParserRuleContext ctx) { - return ctx.start.getInputStream().getText( - new Interval(ctx.start.getStartIndex(), ctx.stop.getStopIndex())); - } - - @Override - public List split(String fileName, String content) { - chunks = new ArrayList<>(); - ParseTreeWalker.DEFAULT.walk( - getParseTreeListener(), - getParseTree(CharStreams.fromString(content))); - return chunks; - } -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/splitter/JavaCodeSplitter.java b/codegpt-core/src/main/java/ee/carlrobert/splitter/JavaCodeSplitter.java deleted file mode 100644 index 99e409fe..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/splitter/JavaCodeSplitter.java +++ /dev/null @@ -1,32 +0,0 @@ -package ee.carlrobert.splitter; - -import grammar.JavaLexer; -import grammar.JavaParser; -import grammar.JavaParserBaseListener; -import org.antlr.v4.runtime.CodePointCharStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeListener; - -public class JavaCodeSplitter extends CodeSplitter { - - @Override - protected ParseTree getParseTree(CodePointCharStream charStream) { - return new JavaParser(new CommonTokenStream(new JavaLexer(charStream))).compilationUnit(); - } - - @Override - protected ParseTreeListener getParseTreeListener() { - return new JavaParserBaseListener() { - @Override - public void enterConstructorDeclaration(JavaParser.ConstructorDeclarationContext ctx) { - chunks.add(parseContext(ctx)); - } - - @Override - public void enterMethodDeclaration(JavaParser.MethodDeclarationContext ctx) { - chunks.add(parseContext(ctx)); - } - }; - } -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/splitter/JsonSplitter.java b/codegpt-core/src/main/java/ee/carlrobert/splitter/JsonSplitter.java deleted file mode 100644 index 475ecdfd..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/splitter/JsonSplitter.java +++ /dev/null @@ -1,28 +0,0 @@ -package ee.carlrobert.splitter; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.intellij.openapi.diagnostic.Logger; -import java.util.ArrayList; -import java.util.List; -import org.json.JSONObject; - -public class JsonSplitter implements Splitter { - - private static final Logger LOG = Logger.getInstance(JsonSplitter.class); - - @Override - public List split(String fileName, String content) { - var chunks = new ArrayList(); - - try { - // TODO: Switch to ObjectMapper - for (var entry : new JSONObject(content).toMap().entrySet()) { - chunks.add(new ObjectMapper().writeValueAsString(entry)); - } - } catch (JsonProcessingException e) { - LOG.error("Something went wrong while chunking the json", e); - } - return chunks; - } -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/splitter/PythonCodeSplitter.java b/codegpt-core/src/main/java/ee/carlrobert/splitter/PythonCodeSplitter.java deleted file mode 100644 index 0a766d4a..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/splitter/PythonCodeSplitter.java +++ /dev/null @@ -1,27 +0,0 @@ -package ee.carlrobert.splitter; - -import grammar.PythonLexer; -import grammar.PythonParser; -import grammar.PythonParserBaseListener; -import org.antlr.v4.runtime.CodePointCharStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeListener; - -public class PythonCodeSplitter extends CodeSplitter { - - @Override - protected ParseTree getParseTree(CodePointCharStream charStream) { - return new PythonParser(new CommonTokenStream(new PythonLexer(charStream))).file_input(); - } - - @Override - protected ParseTreeListener getParseTreeListener() { - return new PythonParserBaseListener() { - @Override - public void enterClass_or_func_def_stmt(PythonParser.Class_or_func_def_stmtContext ctx) { - chunks.add(parseContext(ctx)); - } - }; - } -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/splitter/Splitter.java b/codegpt-core/src/main/java/ee/carlrobert/splitter/Splitter.java deleted file mode 100644 index 7e545dc8..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/splitter/Splitter.java +++ /dev/null @@ -1,8 +0,0 @@ -package ee.carlrobert.splitter; - -import java.util.List; - -public interface Splitter { - - List split(String fileName, String content); -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/splitter/SplitterFactory.java b/codegpt-core/src/main/java/ee/carlrobert/splitter/SplitterFactory.java deleted file mode 100644 index 092cd419..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/splitter/SplitterFactory.java +++ /dev/null @@ -1,22 +0,0 @@ -package ee.carlrobert.splitter; - -import org.jetbrains.annotations.Nullable; - -public class SplitterFactory { - - public static @Nullable Splitter getCodeSplitter(String fileExtension) { - switch (fileExtension) { - case "java": - return new JavaCodeSplitter(); - case "py": - return new PythonCodeSplitter(); - case "json": - return new JsonSplitter(); - case "ts": - case "tsx": - return new TypeScriptCodeSplitter(); - default: - return null; - } - } -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/splitter/TypeScriptCodeSplitter.java b/codegpt-core/src/main/java/ee/carlrobert/splitter/TypeScriptCodeSplitter.java deleted file mode 100644 index e14ebf76..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/splitter/TypeScriptCodeSplitter.java +++ /dev/null @@ -1,22 +0,0 @@ -package ee.carlrobert.splitter; - -import grammar.TypeScriptLexer; -import grammar.TypeScriptParser; -import grammar.TypeScriptParserBaseListener; -import org.antlr.v4.runtime.CodePointCharStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeListener; - -public class TypeScriptCodeSplitter extends CodeSplitter { - - @Override - protected ParseTree getParseTree(CodePointCharStream charStream) { - return new TypeScriptParser(new CommonTokenStream(new TypeScriptLexer(charStream))).program(); - } - - @Override - protected ParseTreeListener getParseTreeListener() { - return new TypeScriptParserBaseListener() {}; - } -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/vector/VectorStore.java b/codegpt-core/src/main/java/ee/carlrobert/vector/VectorStore.java deleted file mode 100644 index 15f3c046..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/vector/VectorStore.java +++ /dev/null @@ -1,63 +0,0 @@ -package ee.carlrobert.vector; - -import com.github.jelmerk.knn.DistanceFunctions; -import com.github.jelmerk.knn.Item; -import com.github.jelmerk.knn.hnsw.HnswIndex; -import com.intellij.openapi.application.ApplicationManager; -import com.intellij.openapi.util.io.FileUtil; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; - -public class VectorStore { - - private static VectorStore instance; - - private final String storePath; - - private VectorStore(Path pluginPath) { - this.storePath = getIndexStorePath(pluginPath.toString()); - } - - public static VectorStore getInstance(Path pluginPath) { - if (instance == null) { - instance = new VectorStore(pluginPath); - } - return instance; - } - - public HnswIndex loadIndex() throws IOException { - return loadIndex(storePath); - } - - public HnswIndex loadIndex(String path) throws IOException { - return HnswIndex.load(new File(path), this.getClass().getClassLoader()); - } - - public void save(List> words) { - var hnswIndex = HnswIndex - .newBuilder( - words.get(0).vector().length, - DistanceFunctions.DOUBLE_COSINE_DISTANCE, - words.size()) - .build(); - try { - hnswIndex.addAll(words); - hnswIndex.save(new File(storePath)); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - public boolean isIndexExists() { - return FileUtil.exists(storePath); - } - - private String getIndexStorePath(String pluginBasePath) { - if (ApplicationManager.getApplication().isUnitTestMode()) { - pluginBasePath = new File("src/test/resources/indexes").getAbsolutePath(); - } - return pluginBasePath + File.separator + "hnsw.index"; - } -} diff --git a/codegpt-core/src/main/java/ee/carlrobert/vector/Word.java b/codegpt-core/src/main/java/ee/carlrobert/vector/Word.java deleted file mode 100644 index d27ca20d..00000000 --- a/codegpt-core/src/main/java/ee/carlrobert/vector/Word.java +++ /dev/null @@ -1,46 +0,0 @@ -package ee.carlrobert.vector; - -import com.github.jelmerk.knn.Item; -import java.util.Arrays; - -public class Word implements Item { - - private static final long serialVersionUID = 1L; - - private final String id; - private final String meta; - private final double[] vector; - - public Word(String id, String meta, double[] vector) { - this.id = id; - this.meta = meta; - this.vector = vector; - } - - @Override - public String id() { - return id; - } - - @Override - public double[] vector() { - return vector; - } - - @Override - public int dimensions() { - return vector.length; - } - - @Override - public String toString() { - return "Word{" - + "id='" + id + '\'' - + ", vector=" + Arrays.toString(vector) - + '}'; - } - - public String getMeta() { - return meta; - } -} \ No newline at end of file diff --git a/codegpt-core/src/main/java/grammar/PythonLexerBase.java b/codegpt-core/src/main/java/grammar/PythonLexerBase.java deleted file mode 100644 index f73e9f30..00000000 --- a/codegpt-core/src/main/java/grammar/PythonLexerBase.java +++ /dev/null @@ -1,184 +0,0 @@ -package grammar; - -import java.util.ArrayDeque; -import java.util.Deque; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CommonToken; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Token; - -public abstract class PythonLexerBase extends Lexer { - public static int TabSize = 8; - - // The amount of opened braces, brackets and parenthesis. - private int _opened; - - // The stack that keeps track of the indentation level. - private final Deque _indents = new ArrayDeque<>(); - - // A circular buffer where extra tokens are pushed on (see the NEWLINE and WS lexer rules). - private int _firstTokensInd; - private int _lastTokenInd; - private Token[] _buffer = new Token[32]; - private Token _lastToken; - - protected PythonLexerBase(CharStream input) { - super(input); - } - - @Override - public void emit(Token token) { - super.setToken(token); - - if (_buffer[_firstTokensInd] != null) - { - _lastTokenInd = IncTokenInd(_lastTokenInd); - - if (_lastTokenInd == _firstTokensInd) - { - // Enlarge buffer - Token[] newArray = new Token[_buffer.length * 2]; - int destInd = newArray.length - (_buffer.length - _firstTokensInd); - - System.arraycopy(_buffer, 0, newArray, 0, _firstTokensInd); - System.arraycopy(_buffer, _firstTokensInd, newArray, destInd, _buffer.length - _firstTokensInd); - - _firstTokensInd = destInd; - _buffer = newArray; - } - } - - _buffer[_lastTokenInd] = token; - _lastToken = token; - } - - @Override - public Token nextToken() { - // Check if the end-of-file is ahead and there are still some DEDENTS expected. - if (_input.LA(1) == EOF && _indents.size() > 0) - { - if (_buffer[_lastTokenInd] == null || _buffer[_lastTokenInd].getType() != PythonLexer.LINE_BREAK) - { - // First emit an extra line break that serves as the end of the statement. - emit(PythonLexer.LINE_BREAK); - } - - // Now emit as much DEDENT tokens as needed. - while (_indents.size() != 0) - { - emit(PythonLexer.DEDENT); - _indents.pop(); - } - } - - Token next = super.nextToken(); - - if (_buffer[_firstTokensInd] == null) - { - return next; - } - - Token result = _buffer[_firstTokensInd]; - _buffer[_firstTokensInd] = null; - - if (_firstTokensInd != _lastTokenInd) - { - _firstTokensInd = IncTokenInd(_firstTokensInd); - } - - return result; - } - - protected void HandleNewLine() { - emit(PythonLexer.NEWLINE, HIDDEN, getText()); - - char next = (char) _input.LA(1); - - // Process whitespaces in HandleSpaces - if (next != ' ' && next != '\t' && IsNotNewLineOrComment(next)) - { - ProcessNewLine(0); - } - } - - protected void HandleSpaces() { - char next = (char) _input.LA(1); - - if ((_lastToken == null || _lastToken.getType() == PythonLexer.NEWLINE) && IsNotNewLineOrComment(next)) - { - // Calculates the indentation of the provided spaces, taking the - // following rules into account: - // - // "Tabs are replaced (from left to right) by one to eight spaces - // such that the total number of characters up to and including - // the replacement is a multiple of eight [...]" - // - // -- https://docs.python.org/3.1/reference/lexical_analysis.html#indentation - - int indent = 0; - String text = getText(); - - for (int i = 0; i < text.length(); i++) { - indent += text.charAt(i) == '\t' ? TabSize - indent % TabSize : 1; - } - - ProcessNewLine(indent); - } - - emit(PythonLexer.WS, HIDDEN, getText()); - } - - protected void IncIndentLevel() { - _opened++; - } - - protected void DecIndentLevel() { - if (_opened > 0) { - --_opened; - } - } - - private boolean IsNotNewLineOrComment(char next) { - return _opened == 0 && next != '\r' && next != '\n' && next != '\f' && next != '#'; - } - - private void ProcessNewLine(int indent) { - emit(PythonLexer.LINE_BREAK); - - int previous = _indents.size() == 0 ? 0 : _indents.peek(); - - if (indent > previous) - { - _indents.push(indent); - emit(PythonLexer.INDENT); - } - else - { - // Possibly emit more than 1 DEDENT token. - while (_indents.size() != 0 && _indents.peek() > indent) - { - emit(PythonLexer.DEDENT); - _indents.pop(); - } - } - } - - private int IncTokenInd(int ind) { - return (ind + 1) % _buffer.length; - } - - private void emit(int tokenType) { - emit(tokenType, DEFAULT_TOKEN_CHANNEL, ""); - } - - private void emit(int tokenType, int channel, String text) { - int charIndex = getCharIndex(); - CommonToken token = new CommonToken(_tokenFactorySourcePair, tokenType, channel, charIndex - text.length(), charIndex - 1); - token.setLine(getLine()); - token.setCharPositionInLine(getCharPositionInLine()); - token.setText(text); - - emit(token); - } -} - diff --git a/codegpt-core/src/main/java/grammar/PythonParserBase.java b/codegpt-core/src/main/java/grammar/PythonParserBase.java deleted file mode 100644 index 84790864..00000000 --- a/codegpt-core/src/main/java/grammar/PythonParserBase.java +++ /dev/null @@ -1,26 +0,0 @@ -package grammar; - -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.TokenStream; - -public abstract class PythonParserBase extends Parser -{ - public PythonVersion Version = PythonVersion.Autodetect; - - protected PythonParserBase(TokenStream input) { - super(input); - } - - protected boolean CheckVersion(int version) { - return Version == PythonVersion.Autodetect || version == Version.getValue(); - } - - protected void SetVersion(int requiredVersion) { - if (requiredVersion == 2) { - Version = PythonVersion.Python2; - } else if (requiredVersion == 3) { - Version = PythonVersion.Python3; - } - } -} - diff --git a/codegpt-core/src/main/java/grammar/PythonVersion.java b/codegpt-core/src/main/java/grammar/PythonVersion.java deleted file mode 100644 index 50fae9e1..00000000 --- a/codegpt-core/src/main/java/grammar/PythonVersion.java +++ /dev/null @@ -1,17 +0,0 @@ -package grammar; - -public enum PythonVersion { - Autodetect(0), - Python2(2), - Python3(3); - - private final int value; - - PythonVersion(int value) { - this.value = value; - } - - public int getValue() { - return value; - } -} diff --git a/codegpt-core/src/main/java/grammar/TypeScriptLexerBase.java b/codegpt-core/src/main/java/grammar/TypeScriptLexerBase.java deleted file mode 100644 index ad763b7a..00000000 --- a/codegpt-core/src/main/java/grammar/TypeScriptLexerBase.java +++ /dev/null @@ -1,166 +0,0 @@ -package grammar; - -import java.util.ArrayDeque; -import java.util.Deque; -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Token; - -/** - * All lexer methods that used in grammar (IsStrictMode) - * should start with Upper Case Char similar to Lexer rules. - */ -public abstract class TypeScriptLexerBase extends Lexer -{ - /** - * Stores values of nested modes. By default mode is strict or - * defined externally (useStrictDefault) - */ - private final Deque scopeStrictModes = new ArrayDeque<>(); - - private Token lastToken = null; - /** - * Default value of strict mode - * Can be defined externally by setUseStrictDefault - */ - private boolean useStrictDefault = false; - /** - * Current value of strict mode - * Can be defined during parsing, see StringFunctions.js and StringGlobal.js samples - */ - private boolean useStrictCurrent = false; - /** - * Keeps track of the current depth of nested template string backticks. - * E.g. after the X in: - * - * `${a ? `${X - * - * templateDepth will be 2. This variable is needed to determine if a `}` is a - * plain CloseBrace, or one that closes an expression inside a template string. - */ - private int templateDepth = 0; - - /** - * Keeps track of the depth of open- and close-braces. Used for expressions like: - * - * `${[1, 2, 3].map(x => { return x * 2;}).join("")}` - * - * where the '}' from `return x * 2;}` should not become a `TemplateCloseBrace` - * token but rather a `CloseBrace` token. - */ - private int bracesDepth = 0; - - public TypeScriptLexerBase(CharStream input) { - super(input); - } - - public boolean getStrictDefault() { - return useStrictDefault; - } - - public void setUseStrictDefault(boolean value) { - useStrictDefault = value; - useStrictCurrent = value; - } - - public boolean IsStrictMode() { - return useStrictCurrent; - } - - public void StartTemplateString() { - this.bracesDepth = 0; - } - - public boolean IsInTemplateString() { - return this.templateDepth > 0 && this.bracesDepth == 0; - } - - /** - * Return the next token from the character stream and records this last - * token in case it resides on the default channel. This recorded token - * is used to determine when the lexer could possibly match a regex - * literal. Also changes scopeStrictModes stack if tokenize special - * string 'use strict'; - * - * @return the next token from the character stream. - */ - @Override - public Token nextToken() { - Token next = super.nextToken(); - - if (next.getChannel() == Token.DEFAULT_CHANNEL) { - // Keep track of the last token on the default channel. - this.lastToken = next; - } - - return next; - } - - protected void ProcessOpenBrace() - { - bracesDepth++; - useStrictCurrent = scopeStrictModes.size() > 0 && scopeStrictModes.peek() ? true : useStrictDefault; - scopeStrictModes.push(useStrictCurrent); - } - - protected void ProcessCloseBrace() - { - bracesDepth--; - useStrictCurrent = scopeStrictModes.size() > 0 ? scopeStrictModes.pop() : useStrictDefault; - } - - protected void ProcessStringLiteral() - { - if (lastToken == null || lastToken.getType() == TypeScriptLexer.OpenBrace) - { - String text = getText(); - if (text.equals("\"use strict\"") || text.equals("'use strict'")) - { - if (scopeStrictModes.size() > 0) - scopeStrictModes.pop(); - useStrictCurrent = true; - scopeStrictModes.push(useStrictCurrent); - } - } - } - - protected void IncreaseTemplateDepth() { - this.templateDepth++; - } - - protected void DecreaseTemplateDepth() { - this.templateDepth--; - } - - /** - * Returns {@code true} if the lexer can match a regex literal. - */ - protected boolean IsRegexPossible() { - - if (this.lastToken == null) { - // No token has been produced yet: at the start of the input, - // no division is possible, so a regex literal _is_ possible. - return true; - } - - switch (this.lastToken.getType()) { - case TypeScriptLexer.Identifier: - case TypeScriptLexer.NullLiteral: - case TypeScriptLexer.BooleanLiteral: - case TypeScriptLexer.This: - case TypeScriptLexer.CloseBracket: - case TypeScriptLexer.CloseParen: - case TypeScriptLexer.OctalIntegerLiteral: - case TypeScriptLexer.DecimalLiteral: - case TypeScriptLexer.HexIntegerLiteral: - case TypeScriptLexer.StringLiteral: - case TypeScriptLexer.PlusPlus: - case TypeScriptLexer.MinusMinus: - // After any of the tokens above, no regex literal can follow. - return false; - default: - // In all other cases, a regex literal _is_ possible. - return true; - } - } -} \ No newline at end of file diff --git a/codegpt-core/src/main/java/grammar/TypeScriptParserBase.java b/codegpt-core/src/main/java/grammar/TypeScriptParserBase.java deleted file mode 100644 index 01f5859f..00000000 --- a/codegpt-core/src/main/java/grammar/TypeScriptParserBase.java +++ /dev/null @@ -1,124 +0,0 @@ -package grammar; - -import org.antlr.v4.runtime.Lexer; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; - -/** - * All parser methods that used in grammar (p, prev, notLineTerminator, etc.) - * should start with lower case char similar to parser rules. - */ -public abstract class TypeScriptParserBase extends Parser -{ - public TypeScriptParserBase(TokenStream input) { - super(input); - } - - /** - * Short form for prev(String str) - */ - protected boolean p(String str) { - return prev(str); - } - - /** - * Whether the previous token value equals to @param str - */ - protected boolean prev(String str) { - return _input.LT(-1).getText().equals(str); - } - - /** - * Short form for next(String str) - */ - protected boolean n(String str) { - return next(str); - } - - /** - * Whether the next token value equals to @param str - */ - protected boolean next(String str) { - return _input.LT(1).getText().equals(str); - } - - protected boolean notLineTerminator() { - return !here(TypeScriptParser.LineTerminator); - } - - protected boolean notOpenBraceAndNotFunction() { - int nextTokenType = _input.LT(1).getType(); - return nextTokenType != TypeScriptParser.OpenBrace && nextTokenType != TypeScriptParser.Function_; - } - - protected boolean closeBrace() { - return _input.LT(1).getType() == TypeScriptParser.CloseBrace; - } - - /** - * Returns {@code true} iff on the current index of the parser's - * token stream a token of the given {@code type} exists on the - * {@code HIDDEN} channel. - * - * @param type - * the type of the token on the {@code HIDDEN} channel - * to check. - * - * @return {@code true} iff on the current index of the parser's - * token stream a token of the given {@code type} exists on the - * {@code HIDDEN} channel. - */ - private boolean here(final int type) { - - // Get the token ahead of the current index. - int possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 1; - Token ahead = _input.get(possibleIndexEosToken); - - // Check if the token resides on the HIDDEN channel and if it's of the - // provided type. - return (ahead.getChannel() == Lexer.HIDDEN) && (ahead.getType() == type); - } - - /** - * Returns {@code true} iff on the current index of the parser's - * token stream a token exists on the {@code HIDDEN} channel which - * either is a line terminator, or is a multi line comment that - * contains a line terminator. - * - * @return {@code true} iff on the current index of the parser's - * token stream a token exists on the {@code HIDDEN} channel which - * either is a line terminator, or is a multi line comment that - * contains a line terminator. - */ - protected boolean lineTerminatorAhead() { - - // Get the token ahead of the current index. - int possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 1; - Token ahead = _input.get(possibleIndexEosToken); - - if (ahead.getChannel() != Lexer.HIDDEN) { - // We're only interested in tokens on the HIDDEN channel. - return false; - } - - if (ahead.getType() == TypeScriptParser.LineTerminator) { - // There is definitely a line terminator ahead. - return true; - } - - if (ahead.getType() == TypeScriptParser.WhiteSpaces) { - // Get the token ahead of the current whitespaces. - possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 2; - ahead = _input.get(possibleIndexEosToken); - } - - // Get the token's text and type. - String text = ahead.getText(); - int type = ahead.getType(); - - // Check if the token is, or contains a line terminator. - return (type == TypeScriptParser.MultiLineComment && (text.contains("\r") || text.contains("\n"))) || - (type == TypeScriptParser.LineTerminator); - } -} \ No newline at end of file diff --git a/codegpt-core/src/main/resources/prompts/inline-completion-prompt.txt b/codegpt-core/src/main/resources/prompts/inline-completion-prompt.txt deleted file mode 100644 index 391ad2c2..00000000 --- a/codegpt-core/src/main/resources/prompts/inline-completion-prompt.txt +++ /dev/null @@ -1 +0,0 @@ -{pre}{codeBefore}{suf}{codeAfter}{mid} \ No newline at end of file diff --git a/codegpt-core/src/main/resources/prompts/prompt-with-context.txt b/codegpt-core/src/main/resources/prompts/prompt-with-context.txt deleted file mode 100644 index b0090787..00000000 --- a/codegpt-core/src/main/resources/prompts/prompt-with-context.txt +++ /dev/null @@ -1,10 +0,0 @@ -Use the following pieces of context to answer the question at the end. -If you don't know the answer, just say that you don't know, don't try to make up an answer. - -Context: - -{context} - -Question: {prompt} - -Helpful answer in Markdown format: \ No newline at end of file diff --git a/codegpt-core/src/main/resources/prompts/text-generator.txt b/codegpt-core/src/main/resources/prompts/text-generator.txt deleted file mode 100644 index 827c1eea..00000000 --- a/codegpt-core/src/main/resources/prompts/text-generator.txt +++ /dev/null @@ -1,10 +0,0 @@ -You are Text Generator, a helpful expert of generating natural language into semantically comparable search query. - -Text: List all the dependencies that the project uses -AI: project dependencies, development dependencies, versions, libraries, frameworks, packages - -Text: Are there any scheduled tasks or background jobs running in our codebase, and if so, what are they responsible for? -AI: scheduled tasks, background jobs, cron jobs, task schedules, codebase tasks - -Text: {prompt} -AI: \ No newline at end of file diff --git a/codegpt-core/src/test/resources/indexes/hnsw.index b/codegpt-core/src/test/resources/indexes/hnsw.index deleted file mode 100644 index ef622134..00000000 Binary files a/codegpt-core/src/test/resources/indexes/hnsw.index and /dev/null differ diff --git a/settings.gradle.kts b/settings.gradle.kts index 2921dd58..214088a2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,4 +1,3 @@ rootProject.name = "CodeGPT" -include("codegpt-core") -include("codegpt-treesitter") +include(":codegpt-treesitter") include(":codegpt-telemetry") diff --git a/src/main/java/ee/carlrobert/codegpt/CodeGPTKeys.java b/src/main/java/ee/carlrobert/codegpt/CodeGPTKeys.java index 57083705..d87a8e54 100644 --- a/src/main/java/ee/carlrobert/codegpt/CodeGPTKeys.java +++ b/src/main/java/ee/carlrobert/codegpt/CodeGPTKeys.java @@ -3,7 +3,6 @@ package ee.carlrobert.codegpt; import com.intellij.openapi.editor.EditorCustomElementRenderer; import com.intellij.openapi.editor.Inlay; import com.intellij.openapi.util.Key; -import ee.carlrobert.embedding.ReferencedFile; import java.util.List; public class CodeGPTKeys { diff --git a/src/main/java/ee/carlrobert/codegpt/ProjectCompilationStatusListener.java b/src/main/java/ee/carlrobert/codegpt/ProjectCompilationStatusListener.java index 857c9d98..73d292c2 100644 --- a/src/main/java/ee/carlrobert/codegpt/ProjectCompilationStatusListener.java +++ b/src/main/java/ee/carlrobert/codegpt/ProjectCompilationStatusListener.java @@ -18,7 +18,6 @@ import ee.carlrobert.codegpt.conversations.message.Message; import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings; import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowContentManager; import ee.carlrobert.codegpt.ui.OverlayUtil; -import ee.carlrobert.embedding.ReferencedFile; import java.io.File; import java.util.ArrayList; import java.util.Collection; diff --git a/codegpt-core/src/main/java/ee/carlrobert/embedding/ReferencedFile.java b/src/main/java/ee/carlrobert/codegpt/ReferencedFile.java similarity index 97% rename from codegpt-core/src/main/java/ee/carlrobert/embedding/ReferencedFile.java rename to src/main/java/ee/carlrobert/codegpt/ReferencedFile.java index e4ce8e29..126edde0 100644 --- a/codegpt-core/src/main/java/ee/carlrobert/embedding/ReferencedFile.java +++ b/src/main/java/ee/carlrobert/codegpt/ReferencedFile.java @@ -1,4 +1,4 @@ -package ee.carlrobert.embedding; +package ee.carlrobert.codegpt; import java.io.File; import java.io.IOException; diff --git a/src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextAction.java b/src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextAction.java index e4b9c6af..89b79c3d 100644 --- a/src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextAction.java +++ b/src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextAction.java @@ -26,13 +26,13 @@ import com.intellij.util.ui.UI.PanelFactory; import ee.carlrobert.codegpt.CodeGPTBundle; import ee.carlrobert.codegpt.CodeGPTKeys; import ee.carlrobert.codegpt.EncodingManager; +import ee.carlrobert.codegpt.ReferencedFile; import ee.carlrobert.codegpt.settings.IncludedFilesSettings; import ee.carlrobert.codegpt.ui.UIUtil; import ee.carlrobert.codegpt.ui.checkbox.FileCheckboxTree; import ee.carlrobert.codegpt.ui.checkbox.PsiElementCheckboxTree; import ee.carlrobert.codegpt.ui.checkbox.VirtualFileCheckboxTree; import ee.carlrobert.codegpt.util.file.FileUtil; -import ee.carlrobert.embedding.ReferencedFile; import java.awt.Dimension; import java.io.IOException; import java.nio.file.Files; diff --git a/src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextNotifier.java b/src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextNotifier.java index 0bd22934..796a0090 100644 --- a/src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextNotifier.java +++ b/src/main/java/ee/carlrobert/codegpt/actions/IncludeFilesInContextNotifier.java @@ -1,7 +1,7 @@ package ee.carlrobert.codegpt.actions; import com.intellij.util.messages.Topic; -import ee.carlrobert.embedding.ReferencedFile; +import ee.carlrobert.codegpt.ReferencedFile; import java.util.List; public interface IncludeFilesInContextNotifier { diff --git a/src/main/java/ee/carlrobert/codegpt/actions/editor/EditorActionsUtil.java b/src/main/java/ee/carlrobert/codegpt/actions/editor/EditorActionsUtil.java index 7fe96922..06d02345 100644 --- a/src/main/java/ee/carlrobert/codegpt/actions/editor/EditorActionsUtil.java +++ b/src/main/java/ee/carlrobert/codegpt/actions/editor/EditorActionsUtil.java @@ -11,11 +11,11 @@ import com.intellij.openapi.editor.impl.EditorImpl; import com.intellij.openapi.extensions.PluginId; import com.intellij.openapi.project.Project; import ee.carlrobert.codegpt.CodeGPTKeys; +import ee.carlrobert.codegpt.ReferencedFile; import ee.carlrobert.codegpt.conversations.message.Message; import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings; import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowContentManager; import ee.carlrobert.codegpt.util.file.FileUtil; -import ee.carlrobert.embedding.ReferencedFile; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; diff --git a/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestHandler.java b/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestHandler.java index 4a3c057f..d6de2bb6 100644 --- a/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestHandler.java +++ b/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestHandler.java @@ -17,15 +17,11 @@ public class CompletionRequestHandler { private static final Logger LOG = Logger.getInstance(CompletionRequestHandler.class); private final StringBuilder messageBuilder = new StringBuilder(); - private final boolean useContextualSearch; private final CompletionResponseEventListener completionResponseEventListener; private SwingWorker swingWorker; private EventSource eventSource; - public CompletionRequestHandler( - boolean useContextualSearch, - CompletionResponseEventListener completionResponseEventListener) { - this.useContextualSearch = useContextualSearch; + public CompletionRequestHandler(CompletionResponseEventListener completionResponseEventListener) { this.completionResponseEventListener = completionResponseEventListener; } @@ -46,7 +42,7 @@ public class CompletionRequestHandler { CompletionEventListener eventListener) { try { return CompletionRequestService.getInstance() - .getChatCompletionAsync(callParameters, useContextualSearch, eventListener); + .getChatCompletionAsync(callParameters, eventListener); } catch (Throwable ex) { handleCallException(ex); throw ex; diff --git a/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestProvider.java b/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestProvider.java index b2e4a826..1454df16 100644 --- a/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestProvider.java +++ b/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestProvider.java @@ -9,8 +9,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.diagnostic.Logger; -import ee.carlrobert.codegpt.CodeGPTPlugin; import ee.carlrobert.codegpt.EncodingManager; +import ee.carlrobert.codegpt.ReferencedFile; import ee.carlrobert.codegpt.completions.llama.LlamaModel; import ee.carlrobert.codegpt.completions.llama.PromptTemplate; import ee.carlrobert.codegpt.conversations.Conversation; @@ -29,8 +29,6 @@ import ee.carlrobert.codegpt.settings.service.openai.OpenAISettings; import ee.carlrobert.codegpt.settings.service.you.YouSettings; import ee.carlrobert.codegpt.telemetry.core.configuration.TelemetryConfiguration; import ee.carlrobert.codegpt.telemetry.core.service.UserId; -import ee.carlrobert.embedding.EmbeddingsService; -import ee.carlrobert.embedding.ReferencedFile; import ee.carlrobert.llm.client.anthropic.completion.ClaudeCompletionRequest; import ee.carlrobert.llm.client.anthropic.completion.ClaudeCompletionRequestMessage; import ee.carlrobert.llm.client.llama.completion.LlamaCompletionRequest; @@ -66,18 +64,14 @@ public class CompletionRequestProvider { "/prompts/fix-compile-errors.txt"); private final EncodingManager encodingManager = EncodingManager.getInstance(); - private final EmbeddingsService embeddingsService; private final Conversation conversation; public CompletionRequestProvider(Conversation conversation) { - this.embeddingsService = new EmbeddingsService( - CompletionClientProvider.getOpenAIClient(), - CodeGPTPlugin.getPluginBasePath()); this.conversation = conversation; } public static String getPromptWithContext(List referencedFiles, - String userPrompt) { + String userPrompt) { var includedFilesSettings = IncludedFilesSettings.getCurrentState(); var repeatableContext = referencedFiles.stream() .map(item -> includedFilesSettings.getRepeatableContext() @@ -184,11 +178,10 @@ public class CompletionRequestProvider { public OpenAIChatCompletionRequest buildOpenAIChatCompletionRequest( @Nullable String model, CallParameters callParameters, - boolean useContextualSearch, @Nullable String overriddenPath) { var configuration = ConfigurationSettings.getCurrentState(); var builder = new OpenAIChatCompletionRequest.Builder( - buildMessages(model, callParameters, useContextualSearch)) + buildMessages(model, callParameters)) .setModel(model) .setMaxTokens(configuration.getMaxTokens()) .setStream(true) @@ -206,7 +199,7 @@ public class CompletionRequestProvider { CallParameters callParameters) { return buildCustomOpenAIChatCompletionRequest( customConfiguration, - buildMessages(callParameters, false), + buildMessages(callParameters), true); } @@ -273,43 +266,33 @@ public class CompletionRequestProvider { return request; } - private List buildMessages( - CallParameters callParameters, - boolean useContextualSearch) { + private List buildMessages(CallParameters callParameters) { var message = callParameters.getMessage(); var messages = new ArrayList(); - if (useContextualSearch) { - var prompt = embeddingsService.buildPromptWithContext( - message.getPrompt()); - LOG.info("Retrieved context:\n" + prompt); - messages.add(new OpenAIChatCompletionMessage("user", prompt)); - } else { - if (callParameters.getConversationType() == ConversationType.DEFAULT) { - messages.add(new OpenAIChatCompletionMessage( - "system", - ConfigurationSettings.getCurrentState().getSystemPrompt())); - } - if (callParameters.getConversationType() == ConversationType.FIX_COMPILE_ERRORS) { - messages.add(new OpenAIChatCompletionMessage("system", FIX_COMPILE_ERRORS_SYSTEM_PROMPT)); - } - - for (var prevMessage : conversation.getMessages()) { - if (callParameters.isRetry() && prevMessage.getId().equals(message.getId())) { - break; - } - messages.add(new OpenAIChatCompletionMessage("user", prevMessage.getPrompt())); - messages.add(new OpenAIChatCompletionMessage("assistant", prevMessage.getResponse())); - } - messages.add(new OpenAIChatCompletionMessage("user", message.getPrompt())); + if (callParameters.getConversationType() == ConversationType.DEFAULT) { + messages.add(new OpenAIChatCompletionMessage( + "system", + ConfigurationSettings.getCurrentState().getSystemPrompt())); } + if (callParameters.getConversationType() == ConversationType.FIX_COMPILE_ERRORS) { + messages.add(new OpenAIChatCompletionMessage("system", FIX_COMPILE_ERRORS_SYSTEM_PROMPT)); + } + + for (var prevMessage : conversation.getMessages()) { + if (callParameters.isRetry() && prevMessage.getId().equals(message.getId())) { + break; + } + messages.add(new OpenAIChatCompletionMessage("user", prevMessage.getPrompt())); + messages.add(new OpenAIChatCompletionMessage("assistant", prevMessage.getResponse())); + } + messages.add(new OpenAIChatCompletionMessage("user", message.getPrompt())); return messages; } private List buildMessages( @Nullable String model, - CallParameters callParameters, - boolean useContextualSearch) { - var messages = buildMessages(callParameters, useContextualSearch); + CallParameters callParameters) { + var messages = buildMessages(callParameters); if (model == null || GeneralSettings.getCurrentState().getSelectedService() == ServiceType.YOU) { diff --git a/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestService.java b/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestService.java index cdd42790..14eed71e 100644 --- a/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestService.java +++ b/src/main/java/ee/carlrobert/codegpt/completions/CompletionRequestService.java @@ -61,68 +61,62 @@ public final class CompletionRequestService { public EventSource getChatCompletionAsync( CallParameters callParameters, - boolean useContextualSearch, CompletionEventListener eventListener) { var requestProvider = new CompletionRequestProvider(callParameters.getConversation()); - switch (GeneralSettings.getCurrentState().getSelectedService()) { - case OPENAI: + return switch (GeneralSettings.getCurrentState().getSelectedService()) { + case OPENAI -> { var openAISettings = OpenAISettings.getCurrentState(); - return CompletionClientProvider.getOpenAIClient().getChatCompletionAsync( + yield CompletionClientProvider.getOpenAIClient().getChatCompletionAsync( requestProvider.buildOpenAIChatCompletionRequest( openAISettings.getModel(), callParameters, - useContextualSearch, null), eventListener); - case CUSTOM_OPENAI: + } + case CUSTOM_OPENAI -> { var customConfiguration = CustomServiceSettings.getCurrentState(); - return getCustomOpenAIChatCompletionAsync( + yield getCustomOpenAIChatCompletionAsync( requestProvider.buildCustomOpenAIChatCompletionRequest( customConfiguration, callParameters), eventListener); - case ANTHROPIC: - return CompletionClientProvider.getClaudeClient().getCompletionAsync( - requestProvider.buildAnthropicChatCompletionRequest(callParameters), - eventListener); - case AZURE: + } + case ANTHROPIC -> CompletionClientProvider.getClaudeClient().getCompletionAsync( + requestProvider.buildAnthropicChatCompletionRequest(callParameters), + eventListener); + case AZURE -> { var azureSettings = AzureSettings.getCurrentState(); - return CompletionClientProvider.getAzureClient().getChatCompletionAsync( + yield CompletionClientProvider.getAzureClient().getChatCompletionAsync( requestProvider.buildOpenAIChatCompletionRequest( null, callParameters, - useContextualSearch, azureSettings.isUsingCustomPath() ? azureSettings.getPath() : null), eventListener); - case YOU: - return CompletionClientProvider.getYouClient().getChatCompletionAsync( - requestProvider.buildYouCompletionRequest(callParameters.getMessage()), - eventListener); - case LLAMA_CPP: - return CompletionClientProvider.getLlamaClient().getChatCompletionAsync( - requestProvider.buildLlamaCompletionRequest( - callParameters.getMessage(), - callParameters.getConversationType()), - eventListener); - default: - throw new IllegalArgumentException(); - } + } + case YOU -> CompletionClientProvider.getYouClient().getChatCompletionAsync( + requestProvider.buildYouCompletionRequest(callParameters.getMessage()), + eventListener); + case LLAMA_CPP -> CompletionClientProvider.getLlamaClient().getChatCompletionAsync( + requestProvider.buildLlamaCompletionRequest( + callParameters.getMessage(), + callParameters.getConversationType()), + eventListener); + default -> throw new IllegalArgumentException(); + }; } public EventSource getCodeCompletionAsync( InfillRequestDetails requestDetails, CompletionEventListener eventListener) { var requestProvider = new CodeCompletionRequestProvider(requestDetails); - switch (GeneralSettings.getCurrentState().getSelectedService()) { - case OPENAI: - return CompletionClientProvider.getOpenAIClient() - .getCompletionAsync(requestProvider.buildOpenAIRequest(), eventListener); - case LLAMA_CPP: - return CompletionClientProvider.getLlamaClient() - .getChatCompletionAsync(requestProvider.buildLlamaRequest(), eventListener); - default: - throw new IllegalArgumentException("Code completion not supported for selected service"); - } + return switch (GeneralSettings.getCurrentState().getSelectedService()) { + case OPENAI -> CompletionClientProvider.getOpenAIClient() + .getCompletionAsync(requestProvider.buildOpenAIRequest(), eventListener); + case LLAMA_CPP -> CompletionClientProvider.getLlamaClient() + .getChatCompletionAsync(requestProvider.buildLlamaRequest(), eventListener); + default -> + throw new IllegalArgumentException("Code completion not supported for selected service"); + }; } public void generateCommitMessageAsync( diff --git a/src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingAction.java b/src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingAction.java deleted file mode 100644 index ceeb8333..00000000 --- a/src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingAction.java +++ /dev/null @@ -1,28 +0,0 @@ -package ee.carlrobert.codegpt.indexes; - -import static com.intellij.openapi.ui.DialogWrapper.OK_EXIT_CODE; - -import com.intellij.icons.AllIcons; -import com.intellij.openapi.actionSystem.AnAction; -import com.intellij.openapi.actionSystem.AnActionEvent; -import ee.carlrobert.codegpt.ui.OverlayUtil; -import org.jetbrains.annotations.NotNull; - -public class CodebaseIndexingAction extends AnAction { - - public CodebaseIndexingAction() { - super("Update Indexes", "Update indexes", AllIcons.Actions.Refresh); - } - - @Override - public void actionPerformed(@NotNull AnActionEvent event) { - var project = event.getProject(); - if (project != null) { - var folderStructureTreePanel = new FolderStructureTreePanel(project); - var show = OverlayUtil.showFileStructureDialog(project, folderStructureTreePanel); - if (show == OK_EXIT_CODE) { - new CodebaseIndexingTask(project, folderStructureTreePanel.getReferencedFiles()).run(); - } - } - } -} diff --git a/src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingCompletedNotifier.java b/src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingCompletedNotifier.java deleted file mode 100644 index 19ed5c99..00000000 --- a/src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingCompletedNotifier.java +++ /dev/null @@ -1,11 +0,0 @@ -package ee.carlrobert.codegpt.indexes; - -import com.intellij.util.messages.Topic; - -public interface CodebaseIndexingCompletedNotifier { - - Topic INDEXING_COMPLETED_TOPIC = - Topic.create("codebaseIndexingCompleted", CodebaseIndexingCompletedNotifier.class); - - void indexingCompleted(); -} diff --git a/src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingTask.java b/src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingTask.java deleted file mode 100644 index a1bfb748..00000000 --- a/src/main/java/ee/carlrobert/codegpt/indexes/CodebaseIndexingTask.java +++ /dev/null @@ -1,81 +0,0 @@ -package ee.carlrobert.codegpt.indexes; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.jelmerk.knn.Item; -import com.intellij.notification.NotificationType; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.progress.ProgressIndicator; -import com.intellij.openapi.progress.ProgressManager; -import com.intellij.openapi.progress.Task; -import com.intellij.openapi.progress.impl.BackgroundableProcessIndicator; -import com.intellij.openapi.project.Project; -import ee.carlrobert.codegpt.CodeGPTBundle; -import ee.carlrobert.codegpt.CodeGPTPlugin; -import ee.carlrobert.codegpt.completions.CompletionClientProvider; -import ee.carlrobert.codegpt.ui.OverlayUtil; -import ee.carlrobert.codegpt.util.file.FileUtil; -import ee.carlrobert.embedding.EmbeddingsService; -import ee.carlrobert.embedding.ReferencedFile; -import ee.carlrobert.vector.VectorStore; -import java.util.List; -import java.util.Map; -import org.jetbrains.annotations.NotNull; - -public class CodebaseIndexingTask extends Task.Backgroundable { - - private static final Logger LOG = Logger.getInstance(CodebaseIndexingTask.class); - private final Project project; - private final List referencedFiles; - private final EmbeddingsService embeddingsService; - - public CodebaseIndexingTask(Project project, List referencedFiles) { - super(project, CodeGPTBundle.get("codebaseIndexing.task.title"), true); - this.project = project; - this.referencedFiles = referencedFiles; - this.embeddingsService = new EmbeddingsService( - CompletionClientProvider.getOpenAIClient(), - CodeGPTPlugin.getPluginBasePath()); - } - - public void run() { - ProgressManager.getInstance() - .runProcessWithProgressAsynchronously(this, new BackgroundableProcessIndicator(this)); - } - - @Override - public void run(@NotNull ProgressIndicator indicator) { - LOG.info("Indexing started"); - - String fileContent; - try { - fileContent = new ObjectMapper().writeValueAsString(Map.of("content", referencedFiles)); - } catch (JsonProcessingException e) { - throw new RuntimeException("Unable to serialize json file"); - } - - if (!com.intellij.openapi.util.io.FileUtil.exists(CodeGPTPlugin.getIndexStorePath())) { - FileUtil.tryCreateDirectory(CodeGPTPlugin.getIndexStorePath()); - } - FileUtil.createFile( - CodeGPTPlugin.getProjectIndexStorePath(project), "index.json", fileContent); - - try { - indicator.setFraction(0); - List> embeddings = - embeddingsService.createEmbeddings(referencedFiles, indicator); - VectorStore.getInstance(CodeGPTPlugin.getPluginBasePath()).save(embeddings); - OverlayUtil.showNotification("Indexing completed", NotificationType.INFORMATION); - - project.getMessageBus() - .syncPublisher(CodebaseIndexingCompletedNotifier.INDEXING_COMPLETED_TOPIC) - .indexingCompleted(); - } catch (RuntimeException e) { - LOG.error("Something went wrong while indexing the codebase", e); - } finally { - if (indicator.isRunning()) { - indicator.stop(); - } - } - } -} diff --git a/src/main/java/ee/carlrobert/codegpt/indexes/FolderStructureTreePanel.java b/src/main/java/ee/carlrobert/codegpt/indexes/FolderStructureTreePanel.java deleted file mode 100644 index d6e13dba..00000000 --- a/src/main/java/ee/carlrobert/codegpt/indexes/FolderStructureTreePanel.java +++ /dev/null @@ -1,242 +0,0 @@ -package ee.carlrobert.codegpt.indexes; - -import static java.util.stream.Collectors.toList; - -import com.intellij.icons.AllIcons; -import com.intellij.openapi.fileTypes.FileTypeManager; -import com.intellij.openapi.project.Project; -import com.intellij.openapi.project.ProjectUtil; -import com.intellij.openapi.vcs.changes.ChangeListManager; -import com.intellij.openapi.vcs.changes.VcsIgnoreManager; -import com.intellij.openapi.vfs.VirtualFile; -import com.intellij.openapi.vfs.newvfs.impl.VirtualDirectoryImpl; -import com.intellij.openapi.vfs.newvfs.impl.VirtualFileImpl; -import com.intellij.openapi.vfs.newvfs.impl.VirtualFileSystemEntry; -import com.intellij.ui.CheckboxTree; -import com.intellij.ui.CheckboxTreeListener; -import com.intellij.ui.CheckedTreeNode; -import com.intellij.ui.ScrollPaneFactory; -import com.intellij.ui.components.JBLabel; -import com.intellij.util.ui.AsyncProcessIcon; -import com.intellij.util.ui.JBUI; -import ee.carlrobert.codegpt.util.file.FileUtil; -import ee.carlrobert.embedding.ReferencedFile; -import java.awt.BorderLayout; -import java.awt.FlowLayout; -import java.awt.event.MouseAdapter; -import java.awt.event.MouseEvent; -import java.io.File; -import java.text.DecimalFormat; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import javax.swing.JPanel; -import javax.swing.JTree; -import javax.swing.SwingUtilities; -import org.jetbrains.annotations.NotNull; - -public class FolderStructureTreePanel { - - private CheckboxTree checkboxTree; - private final AsyncProcessIcon loadingFilesSpinner; - private final JPanel rootPanelContainer; - private final VcsIgnoreManager ignoreManager; - private final ChangeListManager changeListManager; - private long totalSize = 0; - - public FolderStructureTreePanel(@NotNull Project project) { - this.ignoreManager = VcsIgnoreManager.getInstance(project); - this.loadingFilesSpinner = new AsyncProcessIcon("loading_files"); - this.changeListManager = ChangeListManager.getInstance(project); - - var projectDirectory = ProjectUtil.guessProjectDir(project); - if (projectDirectory == null) { - throw new RuntimeException("Couldn't find project directory"); - } - - var rootNode = new CheckedTreeNode(projectDirectory); - rootPanelContainer = new JPanel(new BorderLayout()); - - CompletableFuture - .runAsync(() -> traverseDirectory(rootNode, projectDirectory)) - .thenRun(() -> SwingUtilities.invokeLater(() -> { - checkboxTree = createCheckboxTree(rootNode, true); - loadingFilesSpinner.setVisible(false); - updatePanel(); - })); - - checkboxTree = createCheckboxTree(rootNode, false); - - rootPanelContainer.add(createRootPanel()); - rootPanelContainer.add(createFooterPanel(), BorderLayout.SOUTH); - } - - private CheckboxTree createCheckboxTree(CheckedTreeNode rootNode, boolean enabled) { - checkboxTree = new CheckboxTree(createFileTypesRenderer(), rootNode); - checkboxTree.setEditable(enabled); - checkboxTree.setEnabled(enabled); - checkboxTree.addCheckboxTreeListener(new CheckboxTreeListener() { - @Override - public void nodeStateChanged(@NotNull CheckedTreeNode node) { - try { - var length = ((VirtualFileImpl) node.getUserObject()).getLength(); - if (node.isChecked()) { - totalSize += length; - } else { - totalSize -= length; - } - } catch (Throwable ignored) { - // ignore - } - } - }); - checkboxTree.addMouseListener(new MouseAdapter() { - @Override - public void mouseClicked(MouseEvent e) { - updatePanel(); - } - }); - return checkboxTree; - } - - private void updatePanel() { - rootPanelContainer.removeAll(); - rootPanelContainer.add(createRootPanel()); - rootPanelContainer.add(createFooterPanel(), BorderLayout.SOUTH); - rootPanelContainer.revalidate(); - rootPanelContainer.repaint(); - } - - public JPanel getPanel() { - return rootPanelContainer; - } - - private JPanel createRootPanel() { - var scrollPane = ScrollPaneFactory.createScrollPane(checkboxTree); - scrollPane.setPreferredSize(JBUI.size(250, 375)); - return JBUI.Panels.simplePanel().addToCenter(scrollPane); - } - - private JPanel createFooterPanel() { - var panel = new JPanel(new FlowLayout(FlowLayout.LEFT, 0, 8)); - if (loadingFilesSpinner.isVisible()) { - panel.add(new JBLabel("Total size:")); - panel.add(loadingFilesSpinner); - } else { - panel.add(new JBLabel("Total size: " - + FileUtil.convertFileSize(totalSize) + " ~ " - + (FileUtil.convertLongValue(totalSize / 4)) + " tokens " + " ~ " - + new DecimalFormat("#.##") - .format(((double) (totalSize / 4) / 1000) * 0.0001) + " $")); - } - return panel; - } - - public List getReferencedFiles() { - return getCheckedVirtualFiles().stream() - .map(item -> new ReferencedFile(new File(item.getPath()))) - .collect(toList()); - } - - private List getCheckedVirtualFiles() { - return Arrays.stream(checkboxTree.getCheckedNodes( - VirtualFileSystemEntry.class, - node -> node instanceof VirtualFileImpl)) - .map(entry -> (VirtualFileImpl) entry) - .collect(toList()); - } - - private void processChildFile(@NotNull CheckedTreeNode node, @NotNull VirtualFile file) { - long fileSize = file.getLength(); - - try { - if (node.isChecked()) { - node.setChecked(!changeListManager.isIgnoredFile(file) - && !ignoreManager.isPotentiallyIgnoredFile(file) - && FileUtil.isUtf8File(file.getPath()) - && fileSize < Math.pow(1024, 2)); - } - - if (node.isChecked()) { - totalSize += fileSize; - } - } catch (RuntimeException ignored) { - // ignore - } - } - - private void traverseDirectory(@NotNull CheckedTreeNode parentNode, - @NotNull VirtualFile projectDirectory) { - for (VirtualFile childFile : projectDirectory.getChildren()) { - var node = new CheckedTreeNode(childFile); - parentNode.add(node); - - var potentiallyIgnored = ignoredFileDirectories.parallelStream() - .anyMatch(it -> it.equalsIgnoreCase(childFile.getName())); - if (!parentNode.isChecked() || potentiallyIgnored) { - node.setChecked(false); - } - - if (childFile.isDirectory()) { - traverseDirectory(node, childFile); - } else { - processChildFile(node, childFile); - } - } - } - - private @NotNull CheckboxTree.CheckboxTreeCellRenderer createFileTypesRenderer() { - return new CheckboxTree.CheckboxTreeCellRenderer() { - @Override - public void customizeRenderer(JTree t, - Object value, - boolean selected, - boolean expanded, - boolean leaf, - int row, - boolean focus) { - if (!(value instanceof CheckedTreeNode)) { - return; - } - - var node = (CheckedTreeNode) value; - var userObject = node.getUserObject(); - - if (userObject instanceof VirtualFileSystemEntry) { - getTextRenderer().append(((VirtualFileSystemEntry) userObject).getName()); - - if (userObject instanceof VirtualDirectoryImpl) { - getTextRenderer().setIcon(AllIcons.Nodes.Folder); - } else { - var fileType = FileTypeManager.getInstance() - .getFileTypeByFile((VirtualFileSystemEntry) userObject); - getTextRenderer().setIcon(fileType.getIcon()); - getTextRenderer().append( - " - " + FileUtil.convertFileSize( - ((VirtualFileSystemEntry) userObject).getLength())); - } - } - } - }; - } - - private static final List ignoredFileDirectories = List.of( - "node_modules", - ".git", - ".svn", - ".bzr", - ".cvs", - ".m2", - ".idea", - ".vscode", - ".project", - ".settings", - "node_modules", - "vendor", - "lib", - "build", - "target", - "media", - "logs", - "uploads"); -} diff --git a/src/main/java/ee/carlrobert/codegpt/toolwindow/ProjectToolWindowFactory.java b/src/main/java/ee/carlrobert/codegpt/toolwindow/ProjectToolWindowFactory.java index 93d56d27..e3c43b2c 100644 --- a/src/main/java/ee/carlrobert/codegpt/toolwindow/ProjectToolWindowFactory.java +++ b/src/main/java/ee/carlrobert/codegpt/toolwindow/ProjectToolWindowFactory.java @@ -6,7 +6,6 @@ import com.intellij.openapi.wm.ToolWindow; import com.intellij.openapi.wm.ToolWindowFactory; import com.intellij.ui.content.ContentManagerEvent; import com.intellij.ui.content.ContentManagerListener; -import ee.carlrobert.codegpt.toolwindow.chat.contextual.ContextualChatToolWindowPanel; import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowPanel; import ee.carlrobert.codegpt.toolwindow.conversations.ConversationsToolWindow; import javax.swing.JComponent; @@ -16,12 +15,9 @@ public class ProjectToolWindowFactory implements ToolWindowFactory, DumbAware { public void createToolWindowContent(@NotNull Project project, @NotNull ToolWindow toolWindow) { var chatToolWindowPanel = new StandardChatToolWindowPanel(project, toolWindow.getDisposable()); - // var contextualChatToolWindowPanel = - // new ContextualChatToolWindowPanel(project, toolWindow.getDisposable()); var conversationsToolWindow = new ConversationsToolWindow(project); addContent(toolWindow, chatToolWindowPanel, "Chat"); - // addContent(toolWindow, contextualChatToolWindowPanel, "Contextual Chat"); addContent(toolWindow, conversationsToolWindow.getContent(), "Chat History"); toolWindow.addContentManagerListener(new ContentManagerListener() { public void selectionChanged(@NotNull ContentManagerEvent event) { diff --git a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ChatToolWindowTabPanel.java b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ChatToolWindowTabPanel.java index 58afb1f2..b59867ed 100644 --- a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ChatToolWindowTabPanel.java +++ b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ChatToolWindowTabPanel.java @@ -14,6 +14,7 @@ import com.intellij.ui.JBColor; import com.intellij.util.ui.JBUI; import ee.carlrobert.codegpt.CodeGPTKeys; import ee.carlrobert.codegpt.EncodingManager; +import ee.carlrobert.codegpt.ReferencedFile; import ee.carlrobert.codegpt.actions.ActionType; import ee.carlrobert.codegpt.completions.CallParameters; import ee.carlrobert.codegpt.completions.CompletionRequestHandler; @@ -37,7 +38,6 @@ import ee.carlrobert.codegpt.toolwindow.chat.ui.textarea.UserPromptTextArea; import ee.carlrobert.codegpt.toolwindow.chat.ui.textarea.UserPromptTextAreaHeader; import ee.carlrobert.codegpt.util.EditorUtil; import ee.carlrobert.codegpt.util.file.FileUtil; -import ee.carlrobert.embedding.ReferencedFile; import java.awt.BorderLayout; import java.awt.GridBagConstraints; import java.awt.GridBagLayout; @@ -51,7 +51,6 @@ public abstract class ChatToolWindowTabPanel implements Disposable { private static final Logger LOG = Logger.getInstance(ChatToolWindowTabPanel.class); - private final boolean useContextualSearch; private final JPanel rootPanel; private final Conversation conversation; private final UserPromptTextArea userPromptTextArea; @@ -69,7 +68,6 @@ public abstract class ChatToolWindowTabPanel implements Disposable { boolean useContextualSearch) { this.project = project; this.conversation = conversation; - this.useContextualSearch = useContextualSearch; conversationService = ConversationService.getInstance(); toolWindowScrollablePanel = new ChatToolWindowScrollablePanel(); totalTokensPanel = new TotalTokensPanel( @@ -217,7 +215,6 @@ public abstract class ChatToolWindowTabPanel implements Disposable { } var requestHandler = new CompletionRequestHandler( - useContextualSearch, new ToolWindowCompletionResponseEventListener( conversationService, responsePanel, diff --git a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowLandingPanel.java b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowLandingPanel.java deleted file mode 100644 index 2398c90e..00000000 --- a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowLandingPanel.java +++ /dev/null @@ -1,113 +0,0 @@ -package ee.carlrobert.codegpt.toolwindow.chat.contextual; - -import static com.intellij.openapi.ui.DialogWrapper.OK_EXIT_CODE; -import static javax.swing.event.HyperlinkEvent.EventType.ACTIVATED; - -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.options.ShowSettingsUtil; -import com.intellij.openapi.project.Project; -import ee.carlrobert.codegpt.CodeGPTPlugin; -import ee.carlrobert.codegpt.indexes.CodebaseIndexingCompletedNotifier; -import ee.carlrobert.codegpt.indexes.CodebaseIndexingTask; -import ee.carlrobert.codegpt.indexes.FolderStructureTreePanel; -import ee.carlrobert.codegpt.settings.GeneralSettingsConfigurable; -import ee.carlrobert.codegpt.toolwindow.chat.ui.ResponsePanel; -import ee.carlrobert.codegpt.ui.OverlayUtil; -import ee.carlrobert.codegpt.ui.UIUtil; -import ee.carlrobert.vector.VectorStore; -import javax.swing.JTextPane; -import javax.swing.event.HyperlinkEvent; - -class ContextualChatToolWindowLandingPanel extends ResponsePanel { - - private static final Logger LOG = Logger.getInstance(ContextualChatToolWindowLandingPanel.class); - private final Project project; - private final EditorActionEvent actionEvent; - - ContextualChatToolWindowLandingPanel(Project project, EditorActionEvent actionEvent) { - this.project = project; - this.actionEvent = actionEvent; - addContent(createContent()); - - project.getMessageBus() - .connect() - .subscribe(CodebaseIndexingCompletedNotifier.INDEXING_COMPLETED_TOPIC, - (CodebaseIndexingCompletedNotifier) () -> updateContent(createContent())); - } - - private JTextPane createContent() { - var description = UIUtil.createTextPane("", false, this::handleHyperlinkClicked); - if (VectorStore.getInstance(CodeGPTPlugin.getPluginBasePath()).isIndexExists()) { - description.setText("" - + "

" - + "Feel free to ask me anything about your codebase, and I'll be your helpful guide, " - + "dedicated to providing you with the best answers possible!" - + "

" - + "

" - + "Here are a few examples of how I might be helpful:" - + "

" - + "" - + ""); - } else { - description.setText("" - + "

" - + "It looks like you haven't indexed your codebase yet." - + "

" - + "

" - + "Start indexing your codebase to get " - + "access to contextual chat experience." - + "

" - + ""); - } - - return description; - } - - private void handleHyperlinkClicked(HyperlinkEvent event) { - if (ACTIVATED.equals(event.getEventType())) { - if (event.getURL() == null) { - switch (event.getDescription()) { - case "LOGIN": - ShowSettingsUtil.getInstance() - .showSettingsDialog(project, GeneralSettingsConfigurable.class); - break; - case "LIST_DEPENDENCIES": - actionEvent.handleAction("List all the dependencies that the project uses"); - break; - case "SCHEDULED_TASKS": - actionEvent.handleAction("Are there any scheduled tasks or background " - + "jobs running in our codebase, and if so, what are they responsible for?"); - break; - case "AUTHENTICATION_MECHANISM": - actionEvent.handleAction("Can you provide an overview of the authentication " - + "and authorization mechanism implemented in our application?"); - break; - case "START_INDEXING": - var folderStructureTreePanel = new FolderStructureTreePanel(project); - var show = OverlayUtil.showFileStructureDialog(project, folderStructureTreePanel); - if (show == OK_EXIT_CODE) { - new CodebaseIndexingTask(project, folderStructureTreePanel.getReferencedFiles()) - .run(); - } - break; - default: - LOG.error("Could not trigger action {}", event.getDescription()); - } - } else { - UIUtil.handleHyperlinkClicked(event); - } - } - } -} diff --git a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowPanel.java b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowPanel.java deleted file mode 100644 index 022e66c7..00000000 --- a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowPanel.java +++ /dev/null @@ -1,40 +0,0 @@ -package ee.carlrobert.codegpt.toolwindow.chat.contextual; - -import com.intellij.openapi.Disposable; -import com.intellij.openapi.actionSystem.ActionManager; -import com.intellij.openapi.actionSystem.ActionToolbar; -import com.intellij.openapi.actionSystem.DefaultActionGroup; -import com.intellij.openapi.project.Project; -import com.intellij.openapi.ui.SimpleToolWindowPanel; -import com.intellij.openapi.util.Disposer; -import ee.carlrobert.codegpt.actions.toolwindow.ClearChatWindowAction; -import ee.carlrobert.codegpt.conversations.Conversation; -import ee.carlrobert.codegpt.indexes.CodebaseIndexingAction; -import org.jetbrains.annotations.NotNull; - -public class ContextualChatToolWindowPanel extends SimpleToolWindowPanel { - - public ContextualChatToolWindowPanel( - @NotNull Project project, - @NotNull Conversation conversation, - @NotNull Disposable parentDisposable) { - super(true); - - var tabPanel = new ContextualChatToolWindowTabPanel(project, conversation); - setToolbar(createActionToolbar(tabPanel).getComponent()); - setContent(tabPanel.getContent()); - Disposer.register(parentDisposable, tabPanel); - } - - private ActionToolbar createActionToolbar(ContextualChatToolWindowTabPanel tabPanel) { - var actionGroup = new DefaultActionGroup("TOOLBAR_ACTION_GROUP", false); - actionGroup.add(new ClearChatWindowAction(tabPanel::displayLandingView)); - actionGroup.addSeparator(); - actionGroup.add(new CodebaseIndexingAction()); - - var toolbar = ActionManager.getInstance() - .createActionToolbar("NAVIGATION_BAR_TOOLBAR", actionGroup, false); - toolbar.setTargetComponent(this); - return toolbar; - } -} diff --git a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowTabPanel.java b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowTabPanel.java deleted file mode 100644 index 7baced65..00000000 --- a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/ContextualChatToolWindowTabPanel.java +++ /dev/null @@ -1,26 +0,0 @@ -package ee.carlrobert.codegpt.toolwindow.chat.contextual; - -import com.intellij.openapi.project.Project; -import ee.carlrobert.codegpt.completions.ConversationType; -import ee.carlrobert.codegpt.conversations.Conversation; -import ee.carlrobert.codegpt.conversations.message.Message; -import ee.carlrobert.codegpt.toolwindow.chat.ChatToolWindowTabPanel; -import javax.swing.JComponent; -import org.jetbrains.annotations.NotNull; - -public class ContextualChatToolWindowTabPanel extends ChatToolWindowTabPanel { - - public ContextualChatToolWindowTabPanel( - @NotNull Project project, - @NotNull Conversation conversation) { - super(project, conversation, true); - displayLandingView(); - } - - @Override - protected JComponent getLandingView() { - return new ContextualChatToolWindowLandingPanel( - project, - (prompt) -> sendMessage(new Message(prompt), ConversationType.DEFAULT)); - } -} diff --git a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/EditorActionEvent.java b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/EditorActionEvent.java deleted file mode 100644 index b8fdecc9..00000000 --- a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/contextual/EditorActionEvent.java +++ /dev/null @@ -1,7 +0,0 @@ -package ee.carlrobert.codegpt.toolwindow.chat.contextual; - -@FunctionalInterface -public interface EditorActionEvent { - - void handleAction(String prompt); -} diff --git a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/standard/StandardChatToolWindowPanel.java b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/standard/StandardChatToolWindowPanel.java index 537c06fa..18e1d43b 100644 --- a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/standard/StandardChatToolWindowPanel.java +++ b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/standard/StandardChatToolWindowPanel.java @@ -8,6 +8,7 @@ import com.intellij.openapi.project.Project; import com.intellij.openapi.ui.SimpleToolWindowPanel; import com.intellij.openapi.util.Disposer; import com.intellij.util.ui.JBUI; +import ee.carlrobert.codegpt.ReferencedFile; import ee.carlrobert.codegpt.actions.IncludeFilesInContextNotifier; import ee.carlrobert.codegpt.actions.toolwindow.ClearChatWindowAction; import ee.carlrobert.codegpt.actions.toolwindow.CreateNewConversationAction; @@ -15,7 +16,6 @@ import ee.carlrobert.codegpt.actions.toolwindow.OpenInEditorAction; import ee.carlrobert.codegpt.conversations.ConversationService; import ee.carlrobert.codegpt.conversations.ConversationsState; import ee.carlrobert.codegpt.toolwindow.chat.ui.SelectedFilesNotification; -import ee.carlrobert.embedding.ReferencedFile; import java.awt.BorderLayout; import java.util.List; import javax.swing.JPanel; diff --git a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/SelectedFilesNotification.java b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/SelectedFilesNotification.java index 6ff2c1ed..0b0072b2 100644 --- a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/SelectedFilesNotification.java +++ b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/SelectedFilesNotification.java @@ -11,8 +11,8 @@ import com.intellij.ui.components.JBLabel; import com.intellij.util.ui.JBUI; import com.intellij.util.ui.JBUI.CurrentTheme.NotificationInfo; import ee.carlrobert.codegpt.CodeGPTKeys; +import ee.carlrobert.codegpt.ReferencedFile; import ee.carlrobert.codegpt.actions.IncludeFilesInContextNotifier; -import ee.carlrobert.embedding.ReferencedFile; import java.awt.BorderLayout; import java.nio.file.Paths; import java.util.List; diff --git a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/textarea/TotalTokensPanel.java b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/textarea/TotalTokensPanel.java index fab7e156..88763678 100644 --- a/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/textarea/TotalTokensPanel.java +++ b/src/main/java/ee/carlrobert/codegpt/toolwindow/chat/ui/textarea/TotalTokensPanel.java @@ -14,9 +14,9 @@ import com.intellij.ui.components.JBLabel; import com.intellij.util.ui.JBUI; import ee.carlrobert.codegpt.CodeGPTKeys; import ee.carlrobert.codegpt.EncodingManager; +import ee.carlrobert.codegpt.ReferencedFile; import ee.carlrobert.codegpt.actions.IncludeFilesInContextNotifier; import ee.carlrobert.codegpt.conversations.Conversation; -import ee.carlrobert.embedding.ReferencedFile; import java.awt.FlowLayout; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; diff --git a/src/main/java/ee/carlrobert/codegpt/ui/OverlayUtil.java b/src/main/java/ee/carlrobert/codegpt/ui/OverlayUtil.java index 3b94f81c..8284fd4a 100644 --- a/src/main/java/ee/carlrobert/codegpt/ui/OverlayUtil.java +++ b/src/main/java/ee/carlrobert/codegpt/ui/OverlayUtil.java @@ -12,7 +12,6 @@ import com.intellij.notification.NotificationType; import com.intellij.notification.Notifications; import com.intellij.openapi.actionSystem.AnActionEvent; import com.intellij.openapi.project.Project; -import com.intellij.openapi.ui.DialogBuilder; import com.intellij.openapi.ui.DoNotAskOption; import com.intellij.openapi.ui.MessageDialogBuilder; import com.intellij.openapi.ui.MessageType; @@ -21,12 +20,8 @@ import com.intellij.openapi.ui.popup.Balloon; import com.intellij.openapi.ui.popup.Balloon.Position; import com.intellij.openapi.ui.popup.JBPopupFactory; import com.intellij.ui.awt.RelativePoint; -import com.intellij.ui.components.JBLabel; -import com.intellij.util.ui.JBFont; -import com.intellij.util.ui.JBUI; import ee.carlrobert.codegpt.CodeGPTBundle; import ee.carlrobert.codegpt.conversations.ConversationsState; -import ee.carlrobert.codegpt.indexes.FolderStructureTreePanel; import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings; import ee.carlrobert.codegpt.util.EditorUtil; import java.awt.Point; @@ -44,30 +39,6 @@ public class OverlayUtil { Notifications.Bus.notify(getDefaultNotification(content, type)); } - public static int showFileStructureDialog( - Project project, - FolderStructureTreePanel folderStructureTreePanel) { - var dialogBuilder = new DialogBuilder(project); - dialogBuilder.setNorthPanel(JBUI.Panels.simplePanel(new JBLabel( - "" - + "

Indexing files enables direct queries related to your codebase.

" - + "
" - + "

File indexing occurs locally on your computer; " - + "no files are sent to any 3rd party services.

" - + "

For additional information, refer to the " - + "CodeGPT documentation.

" - + "") - .setCopyable(true) - .setAllowAutoWrapping(true) - .withFont(JBFont.medium())) - .withBorder(JBUI.Borders.emptyBottom(12))); - dialogBuilder.setCenterPanel(folderStructureTreePanel.getPanel()); - dialogBuilder.addOkAction().setText("Start Indexing"); - dialogBuilder.addCancelAction(); - dialogBuilder.setTitle("Choose Files for Indexing"); - return dialogBuilder.show(); - } - public static int showDeleteConversationDialog() { return Messages.showYesNoDialog( CodeGPTBundle.get("dialog.deleteConversation.description"), diff --git a/src/main/java/ee/carlrobert/codegpt/ui/checkbox/FileCheckboxTree.java b/src/main/java/ee/carlrobert/codegpt/ui/checkbox/FileCheckboxTree.java index 546f91a9..f88a0d0d 100644 --- a/src/main/java/ee/carlrobert/codegpt/ui/checkbox/FileCheckboxTree.java +++ b/src/main/java/ee/carlrobert/codegpt/ui/checkbox/FileCheckboxTree.java @@ -5,8 +5,8 @@ import com.intellij.openapi.vfs.VirtualFile; import com.intellij.ui.CheckboxTree; import com.intellij.ui.CheckedTreeNode; import com.intellij.ui.ColoredTreeCellRenderer; +import ee.carlrobert.codegpt.ReferencedFile; import ee.carlrobert.codegpt.util.file.FileUtil; -import ee.carlrobert.embedding.ReferencedFile; import java.util.List; import org.jetbrains.annotations.NotNull; diff --git a/src/main/java/ee/carlrobert/codegpt/ui/checkbox/PsiElementCheckboxTree.java b/src/main/java/ee/carlrobert/codegpt/ui/checkbox/PsiElementCheckboxTree.java index 6b5197ab..32140d0a 100644 --- a/src/main/java/ee/carlrobert/codegpt/ui/checkbox/PsiElementCheckboxTree.java +++ b/src/main/java/ee/carlrobert/codegpt/ui/checkbox/PsiElementCheckboxTree.java @@ -9,7 +9,7 @@ import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; import com.intellij.psi.impl.file.PsiDirectoryImpl; import com.intellij.ui.CheckedTreeNode; -import ee.carlrobert.embedding.ReferencedFile; +import ee.carlrobert.codegpt.ReferencedFile; import java.io.File; import java.util.Arrays; import java.util.List; diff --git a/src/main/java/ee/carlrobert/codegpt/ui/checkbox/VirtualFileCheckboxTree.java b/src/main/java/ee/carlrobert/codegpt/ui/checkbox/VirtualFileCheckboxTree.java index da16b6fa..496d1731 100644 --- a/src/main/java/ee/carlrobert/codegpt/ui/checkbox/VirtualFileCheckboxTree.java +++ b/src/main/java/ee/carlrobert/codegpt/ui/checkbox/VirtualFileCheckboxTree.java @@ -6,7 +6,7 @@ import com.intellij.icons.AllIcons; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.ui.CheckedTreeNode; import com.intellij.util.PlatformIcons; -import ee.carlrobert.embedding.ReferencedFile; +import ee.carlrobert.codegpt.ReferencedFile; import java.io.File; import java.util.Arrays; import java.util.List; @@ -53,8 +53,7 @@ public class VirtualFileCheckboxTree extends FileCheckboxTree { return new FileCheckboxTreeCellRenderer() { @Override void updatePresentation(Object userObject) { - if (userObject instanceof VirtualFile) { - VirtualFile virtualFile = (VirtualFile) userObject; + if (userObject instanceof VirtualFile virtualFile) { if (virtualFile.isDirectory()) { getTextRenderer().append(virtualFile.getName()); getTextRenderer().setIcon(PlatformIcons.FOLDER_ICON); diff --git a/codegpt-core/src/main/resources/prompts/default-completion-system-prompt.txt b/src/main/resources/prompts/default-completion-system-prompt.txt similarity index 100% rename from codegpt-core/src/main/resources/prompts/default-completion-system-prompt.txt rename to src/main/resources/prompts/default-completion-system-prompt.txt diff --git a/codegpt-core/src/main/resources/prompts/fix-compile-errors.txt b/src/main/resources/prompts/fix-compile-errors.txt similarity index 100% rename from codegpt-core/src/main/resources/prompts/fix-compile-errors.txt rename to src/main/resources/prompts/fix-compile-errors.txt diff --git a/codegpt-core/src/main/resources/prompts/generate-commit-message-system-prompt.txt b/src/main/resources/prompts/generate-commit-message-system-prompt.txt similarity index 100% rename from codegpt-core/src/main/resources/prompts/generate-commit-message-system-prompt.txt rename to src/main/resources/prompts/generate-commit-message-system-prompt.txt diff --git a/codegpt-core/src/main/resources/prompts/method-name-generator.txt b/src/main/resources/prompts/method-name-generator.txt similarity index 100% rename from codegpt-core/src/main/resources/prompts/method-name-generator.txt rename to src/main/resources/prompts/method-name-generator.txt diff --git a/src/test/java/ee/carlrobert/codegpt/completions/CompletionRequestProviderTest.java b/src/test/java/ee/carlrobert/codegpt/completions/CompletionRequestProviderTest.java index 9b6872ad..ccd2d7d8 100644 --- a/src/test/java/ee/carlrobert/codegpt/completions/CompletionRequestProviderTest.java +++ b/src/test/java/ee/carlrobert/codegpt/completions/CompletionRequestProviderTest.java @@ -1,11 +1,6 @@ package ee.carlrobert.codegpt.completions; import static ee.carlrobert.codegpt.completions.CompletionRequestProvider.COMPLETION_SYSTEM_PROMPT; -import static ee.carlrobert.llm.client.util.JSONUtil.e; -import static ee.carlrobert.llm.client.util.JSONUtil.jsonArray; -import static ee.carlrobert.llm.client.util.JSONUtil.jsonMap; -import static ee.carlrobert.llm.client.util.JSONUtil.jsonMapResponse; -import static org.apache.http.HttpHeaders.AUTHORIZATION; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.groups.Tuple.tuple; @@ -13,11 +8,7 @@ import ee.carlrobert.codegpt.conversations.ConversationService; import ee.carlrobert.codegpt.conversations.message.Message; import ee.carlrobert.codegpt.credentials.OpenAICredentialManager; import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings; -import ee.carlrobert.llm.client.http.ResponseEntity; -import ee.carlrobert.llm.client.http.exchange.BasicHttpExchange; import ee.carlrobert.llm.client.openai.completion.OpenAIChatCompletionModel; -import java.util.List; -import java.util.Map; import testsupport.IntegrationTest; public class CompletionRequestProviderTest extends IntegrationTest { @@ -39,7 +30,6 @@ public class CompletionRequestProviderTest extends IntegrationTest { ConversationType.DEFAULT, new Message("TEST_CHAT_COMPLETION_PROMPT"), false), - false, null); assertThat(request.getMessages()) @@ -68,7 +58,6 @@ public class CompletionRequestProviderTest extends IntegrationTest { ConversationType.DEFAULT, new Message("TEST_CHAT_COMPLETION_PROMPT"), false), - false, null); assertThat(request.getMessages()) @@ -98,7 +87,6 @@ public class CompletionRequestProviderTest extends IntegrationTest { ConversationType.DEFAULT, secondMessage, true), - false, null); assertThat(request.getMessages()) @@ -128,7 +116,6 @@ public class CompletionRequestProviderTest extends IntegrationTest { ConversationType.DEFAULT, new Message("TEST_CHAT_COMPLETION_PROMPT"), false), - false, null); assertThat(request.getMessages()) @@ -155,82 +142,9 @@ public class CompletionRequestProviderTest extends IntegrationTest { ConversationType.DEFAULT, createDummyMessage(100), false), - false, null)); } - public void testContextualSearch() { - useOpenAIService(); - var conversation = ConversationService.getInstance().startConversation(); - expectOpenAI((BasicHttpExchange) request -> { - assertThat(request.getUri().getPath()).isEqualTo("/v1/chat/completions"); - assertThat(request.getMethod()).isEqualTo("POST"); - assertThat(request.getHeaders().get(AUTHORIZATION).get(0)).isEqualTo("Bearer TEST_API_KEY"); - assertThat(request.getBody()) - .extracting("model", "messages") - .containsExactly("gpt-4", - List.of(Map.of( - "role", "user", - "content", - "You are Text Generator, a helpful expert of generating natural " - + "language into semantically comparable search query.\n" - + "\n" - + "Text: List all the dependencies that the project uses\n" - + "AI: project dependencies, development dependencies, " - + "versions, libraries, frameworks, packages\n" - + "\n" - + "Text: Are there any scheduled tasks or background jobs " - + "running in our codebase, and if so, what are they responsible for?\n" - + "AI: scheduled tasks, background jobs, cron jobs, " - + "task schedules, codebase tasks\n" - + "\n" - + "Text: TEST_CHAT_COMPLETION_PROMPT\n" - + "AI:"))); - - return new ResponseEntity(200, - jsonMapResponse("choices", jsonArray(jsonMap("message", - jsonMap(e("role", "assistant"), e("content", "TEST_CHAT_COMPLETION_RESPONSE")))))); - }); - expectOpenAI((BasicHttpExchange) request -> { - assertThat(request.getUri().getPath()).isEqualTo("/v1/embeddings"); - var headers = request.getHeaders(); - assertThat(headers.get("Authorization").get(0)).isEqualTo("Bearer TEST_API_KEY"); - assertThat(request.getBody()) - .extracting("model", "input") - .containsExactly("text-embedding-ada-002", List.of("TEST_CHAT_COMPLETION_RESPONSE")); - return new ResponseEntity(200, jsonMapResponse("data", - jsonArray(jsonMap("embedding", List.of(-0.00692, -0.0053, -4.5471, -0.0240))))); - }); - - var request = new CompletionRequestProvider(conversation) - .buildOpenAIChatCompletionRequest( - OpenAIChatCompletionModel.GPT_3_5.getCode(), - new CallParameters( - conversation, - ConversationType.DEFAULT, - new Message("TEST_CHAT_COMPLETION_PROMPT"), - false), - true, - null); - - assertThat(request.getModel()).isEqualTo("gpt-3.5-turbo"); - assertThat(request.getMessages().size()).isEqualTo(1); - assertThat(request.getMessages().get(0)) - .extracting("role", "content") - .containsExactly("user", - "Use the following pieces of context to answer the question at the end.\n" - + "If you don't know the answer, just say that you don't know, " - + "don't try to make up an answer.\n" - + "\n" - + "Context:\n" - + "\n" - + "TEST_CONTEXT\n" - + "\n" - + "Question: TEST_CHAT_COMPLETION_PROMPT\n" - + "\n" - + "Helpful answer in Markdown format:"); - } - private Message createDummyMessage(int tokenSize) { return createDummyMessage("TEST_PROMPT", tokenSize); } diff --git a/src/test/java/ee/carlrobert/codegpt/completions/DefaultCompletionRequestHandlerTest.java b/src/test/java/ee/carlrobert/codegpt/completions/DefaultCompletionRequestHandlerTest.java index 955b298c..bee9e647 100644 --- a/src/test/java/ee/carlrobert/codegpt/completions/DefaultCompletionRequestHandlerTest.java +++ b/src/test/java/ee/carlrobert/codegpt/completions/DefaultCompletionRequestHandlerTest.java @@ -27,7 +27,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest { useOpenAIService(); var message = new Message("TEST_PROMPT"); var conversation = ConversationService.getInstance().startConversation(); - var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message)); + var requestHandler = new CompletionRequestHandler(getRequestEventListener(message)); expectOpenAI((StreamHttpExchange) request -> { assertThat(request.getUri().getPath()).isEqualTo("/v1/chat/completions"); assertThat(request.getMethod()).isEqualTo("POST"); @@ -83,7 +83,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest { jsonMapResponse("choices", jsonArray(jsonMap("delta", jsonMap("content", "!"))))); }); var message = new Message("TEST_PROMPT"); - var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message)); + var requestHandler = new CompletionRequestHandler(getRequestEventListener(message)); requestHandler.call(new CallParameters(conversation, ConversationType.DEFAULT, message, false)); @@ -122,7 +122,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest { jsonMapResponse("choices", jsonArray(jsonMap("delta", jsonMap("content", "!"))))); }); var message = new Message("TEST_PROMPT"); - var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message)); + var requestHandler = new CompletionRequestHandler(getRequestEventListener(message)); requestHandler.call(new CallParameters(conversation, ConversationType.DEFAULT, message, false)); @@ -134,7 +134,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest { var message = new Message("TEST_PROMPT"); var conversation = ConversationService.getInstance().startConversation(); conversation.addMessage(new Message("Ping", "Pong")); - var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message)); + var requestHandler = new CompletionRequestHandler(getRequestEventListener(message)); expectYou((StreamHttpExchange) request -> { assertThat(request.getUri().getPath()).isEqualTo("/api/streamingSearch"); assertThat(request.getMethod()).isEqualTo("GET"); @@ -186,7 +186,7 @@ public class DefaultCompletionRequestHandlerTest extends IntegrationTest { var message = new Message("TEST_PROMPT"); var conversation = ConversationService.getInstance().startConversation(); conversation.addMessage(new Message("Ping", "Pong")); - var requestHandler = new CompletionRequestHandler(false, getRequestEventListener(message)); + var requestHandler = new CompletionRequestHandler(getRequestEventListener(message)); expectLlama((StreamHttpExchange) request -> { assertThat(request.getUri().getPath()).isEqualTo("/completion"); assertThat(request.getBody()) diff --git a/src/test/java/ee/carlrobert/codegpt/toolwindow/chat/StandardChatToolWindowTabPanelTest.java b/src/test/java/ee/carlrobert/codegpt/toolwindow/chat/StandardChatToolWindowTabPanelTest.java index 99e1a9b0..a7ec0ed0 100644 --- a/src/test/java/ee/carlrobert/codegpt/toolwindow/chat/StandardChatToolWindowTabPanelTest.java +++ b/src/test/java/ee/carlrobert/codegpt/toolwindow/chat/StandardChatToolWindowTabPanelTest.java @@ -14,6 +14,7 @@ import static org.awaitility.Awaitility.await; import ee.carlrobert.codegpt.CodeGPTKeys; import ee.carlrobert.codegpt.EncodingManager; +import ee.carlrobert.codegpt.ReferencedFile; import ee.carlrobert.codegpt.completions.ConversationType; import ee.carlrobert.codegpt.completions.HuggingFaceModel; import ee.carlrobert.codegpt.conversations.ConversationService; @@ -21,7 +22,6 @@ import ee.carlrobert.codegpt.conversations.message.Message; import ee.carlrobert.codegpt.settings.configuration.ConfigurationSettings; import ee.carlrobert.codegpt.settings.service.llama.LlamaSettings; import ee.carlrobert.codegpt.toolwindow.chat.standard.StandardChatToolWindowTabPanel; -import ee.carlrobert.embedding.ReferencedFile; import ee.carlrobert.llm.client.http.exchange.StreamHttpExchange; import java.util.List; import java.util.Map;