From bd8728abdd0be694f36555204dad01ecfdc8c306 Mon Sep 17 00:00:00 2001
From: Daniel Bulant <danbulant@gmail.com>
Date: Sun, 18 Feb 2024 11:58:20 +0100
Subject: [PATCH] almost working

---
 src/surrealql.grammar | 12 ++++++++----
 src/tokens.ts         | 17 +++++++----------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/src/surrealql.grammar b/src/surrealql.grammar
index 93f2a89..b74c857 100644
--- a/src/surrealql.grammar
+++ b/src/surrealql.grammar
@@ -43,8 +43,8 @@
     In,
     Then,
     Else,
-    End,
-    identifier
+    End//,
+    // identifier
 }
 @external specialize {identifier} specializeIdent from "./tokens" {
     Return,
@@ -177,8 +177,10 @@ Field {
     expression (As identifier)? |
     Star
 }
-ReferencedFulltextOperator {
-    "@" identifier "@"
+@skip {} {
+    ReferencedFulltextOperator {
+        "@" int+ "@"
+    }
 }
 BinaryExpression {
     expression !and And expression |
@@ -364,6 +366,8 @@ statement[@isGroup=Statement] {
     whitespace { @whitespace+ }
     LineComment { ("//" | "-- ") ![\n]* }
 
+    identifier { $[a-zA-Z_] $[a-zA-Z0-9_]* }
+
     int { @digit+ }
 
     RIDDelim { "`" }
diff --git a/src/tokens.ts b/src/tokens.ts
index 2cb5379..587c3fc 100644
--- a/src/tokens.ts
+++ b/src/tokens.ts
@@ -64,8 +64,7 @@ import {
 
     // extend
     Changes,
-    Table,
-    identifier
+    Table
 } from "./surrealql.grammar.terms"
 
 const enum Ch {
@@ -154,6 +153,7 @@ const kwmap = new Map([
     [["else"], Else],
     [["end"], End],
 ]);
+let allkws = [...kwmap.keys()].reduce((a, b) => a.concat(b), [])
 const specmap = new Map([
     ["return", Return],
     ["transaction", Transaction],
@@ -199,10 +199,10 @@ const cspecmap = new Map([
 ]);
 
 function isAlpha(ch: number) {
-    return ch >= Ch.A && ch <= Ch.Z || ch >= Ch.a && ch <= Ch.z
+    return ch >= Ch.A && ch <= Ch.Z || ch >= Ch.a && ch <= Ch.z || ch == Ch.Underscore
 }
 function isAlphaNum(ch: number) {
-    return ch >= Ch.A && ch <= Ch.Z || ch >= Ch.a && ch <= Ch.z || ch >= Ch._0 && ch <= Ch._9
+    return ch >= Ch.A && ch <= Ch.Z || ch >= Ch.a && ch <= Ch.z || ch >= Ch._0 && ch <= Ch._9 || ch == Ch.Underscore
 }
 
 function readWord(input: InputStream, result?: string) {
@@ -225,7 +225,6 @@ export const tokens = new ExternalTokenizer((input, stack) => {
     if(isAlpha(next)) {
         input.advance()
         let word = readWord(input, String.fromCharCode(next))
-        console.log("Got word", word)
         let word2: string | undefined = undefined
         if (word != null) {
             word = word.toLowerCase()
@@ -242,21 +241,19 @@ export const tokens = new ExternalTokenizer((input, stack) => {
                     return
                 }
             }
-            console.log("accepting identifier", word)
-            input.acceptToken(identifier)
         }
     }
-}, {contextual: true})
+}, {contextual: false})
 
 export const specializeIdent = (text, stack) => {
     if(specmap.has(text)) {
         return specmap.get(text)
     }
-    // console.log("sspec", text)
+    return -1
 }
 export const extendIdent = (text, stack) => {
     if(cspecmap.has(text)) {
         return cspecmap.get(text)
     }
-    console.log("cspec", text)
+    return -1
 }
\ No newline at end of file