Commit
·
fe0e2ec
1
Parent(s):
248421d
update tree-sitter
Browse files
app.py
CHANGED
@@ -12,8 +12,7 @@ tokenizer = Tokenizer.from_file("syntax_bpe_tokenizer.json") # New BPE tokenize
|
|
12 |
model = tf.keras.models.load_model("crv3.keras") # CNN model
|
13 |
|
14 |
parser = Parser()
|
15 |
-
|
16 |
-
parser.set_language(java_lang) # Set it for the parser
|
17 |
|
18 |
def syntax_aware_tokenize(code):
|
19 |
"""Tokenizes Java code using Tree-Sitter (AST-based)."""
|
|
|
12 |
model = tf.keras.models.load_model("crv3.keras") # CNN model
|
13 |
|
14 |
parser = Parser()
|
15 |
+
parser.set_language(tree_sitter_languages.get_language("java"))
|
|
|
16 |
|
17 |
def syntax_aware_tokenize(code):
|
18 |
"""Tokenizes Java code using Tree-Sitter (AST-based)."""
|