From c1f38bace88e980c4c2691ee4b7e4e229cdf2b34 Mon Sep 17 00:00:00 2001
From: Daniel Bulant <danbulant@gmail.com>
Date: Sun, 18 Feb 2024 12:17:09 +0100
Subject: [PATCH] working version

---
 src/tokens.ts | 46 ++++++++++++++++++++++++++++++----------------
 test.js       |  6 +++---
 2 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/src/tokens.ts b/src/tokens.ts
index 587c3fc..d74e58d 100644
--- a/src/tokens.ts
+++ b/src/tokens.ts
@@ -125,7 +125,7 @@ const kwmap = new Map([
         "outside",
         "intersects",
         "<", ">", "<=", ">=",
-        "@@", "@"
+        "@@"
     ], Comparison],
     [["&&", "and"], And],
     [["||", "or"], Or],
@@ -205,7 +205,7 @@ function isAlphaNum(ch: number) {
     return ch >= Ch.A && ch <= Ch.Z || ch >= Ch.a && ch <= Ch.z || ch >= Ch._0 && ch <= Ch._9 || ch == Ch.Underscore
 }
 
-function readWord(input: InputStream, result?: string) {
+function readWord(input: InputStream, result: string) {
     for (;;) {
         if (input.next != Ch.Underscore && !isAlphaNum(input.next)) break
         if (result != null) result += String.fromCharCode(input.next)
@@ -224,23 +224,37 @@ export const tokens = new ExternalTokenizer((input, stack) => {
     let {next} = input;
     if(isAlpha(next)) {
         input.advance()
-        let word = readWord(input, String.fromCharCode(next))
-        let word2: string | undefined = undefined
-        if (word != null) {
-            word = word.toLowerCase()
-            if(["is", "not"].includes(word)) {
-                // needs another word
-                skipSpaces(input)
-                word2 = readWord(input)
-                if(!word2) return;
-                word = word + " " + word2.toLowerCase()
+        let word = readWord(input, String.fromCharCode(next)).toLowerCase()
+        let word2;
+        if(["is", "not"].includes(word)) {
+            // needs another word
+            skipSpaces(input)
+            word2 = readWord(input, "")
+            if(!word2) return;
+            word = word + " " + word2.toLowerCase()
+        }
+        for(let [kws, token] of kwmap) {
+            if(kws.includes(word)) {
+                input.acceptToken(token)
+                return
             }
-            for(let [kws, token] of kwmap) {
-                if(kws.includes(word)) {
-                    input.acceptToken(token)
-                    return
+        }
+    } else {
+        // no idea why this doesn't work generally, it fails the parser weirdly.
+        // so this is used for special character operators only
+        let str = String.fromCharCode(next).toLowerCase();
+        while(allkws.find(kw => kw.startsWith(str))) {
+            if(allkws.includes(str)) {
+                for(let [kws, token] of kwmap) {
+                    if(kws.includes(str)) {
+                        input.advance()
+                        input.acceptToken(token)
+                        return
+                    }
                 }
             }
+            input.advance()
+            str += String.fromCharCode(input.next).toLowerCase()
         }
     }
 }, {contextual: false})
diff --git a/test.js b/test.js
index 33ec2dc..ea06284 100644
--- a/test.js
+++ b/test.js
@@ -1,4 +1,4 @@
 import {baseParser} from "./dist/index.js"
-// console.log(baseParser.parse('select field from type::table($var) where $var > 1 and field2 @@ field1 timeout 10s').toString())
-// console.log(baseParser.parse('select * from documents where contents @@ "test"').toString())
-console.log(baseParser.parse('select * from documents').toString())
\ No newline at end of file
+console.log(baseParser.parse('select field from type::table($var) where $var > 1 and field2 @1@ field1 timeout 10s').toString())
+console.log(baseParser.parse('select * from documents where contents @@ "test"').toString())
+console.log(baseParser.parse('select * from documents where test > 1').toString())
\ No newline at end of file