Fix handling of identifiers preceded by a “.”

Author: JanJakes

custom-parser/parser/MySQLLexer.php

@@ -2035,7 +2035,8 @@ private function nextToken()
 			if ($this->isDigit($la2)) {
 				$this->NUMBER();
 			} else {
-				$this->DOT_IDENTIFIER();
+				$this->consume();
+				$this->type = self::DOT_SYMBOL;
 			}
 		} elseif ($la === '=') {
 			$this->consume();
@@ -2222,11 +2223,16 @@ private function nextToken()
 		} elseif (($la === 'x' || $la === 'X' || $la === 'b' || $la === 'B') && $la2 === "'") {
 			$this->NUMBER();
 		} elseif (preg_match('/\G' . self::PATTERN_UNQUOTED_IDENTIFIER . '/u', $this->input, $matches, 0, $this->position)) {
+			$p = $this->position - 1;
 			$this->text = $matches[0];
 			$this->position += strlen($this->text);
 			$this->c = $this->input[$this->position] ?? null;
 			$this->n = $this->input[$this->position + 1] ?? null;
-			if ($la === '_' && isset(self::UNDERSCORE_CHARSETS[strtolower($this->text)])) {
+
+			// When preceded by a dot, it is always an identifier.
+			if ($p >= 0 && $this->input[$p] === '.') {
+				$this->type = self::IDENTIFIER;
+			} elseif ($la === '_' && isset(self::UNDERSCORE_CHARSETS[strtolower($this->text)])) {
 				$this->type = self::UNDERSCORE_CHARSET;
 			} else {
 				$this->IDENTIFIER_OR_KEYWORD();
@@ -2277,17 +2283,6 @@ protected function checkVersion(string $text): bool
         return false;
     }
 
-    /**
-     * This is a place holder to support features of MySQLBaseLexer which are not yet implemented
-     * in the PHP target.
-     *
-     * @return void
-     */
-    protected function emitDot(): void
-    {
-        return;
-    }
-
     protected function IDENTIFIER_OR_KEYWORD()
     {
 		$text = strtoupper($this->getText());
@@ -2409,13 +2404,6 @@ protected function sqlCodeInComment(): void
         $this->VERSION_COMMENT_END();
     }
 
-    protected function DOT_IDENTIFIER()
-    {
-        $this->consume(); // Consume the '.'.
-        $this->type = self::IDENTIFIER;
-        $this->type = self::DOT_SYMBOL;//@TODO: DOT_IDENTIFIER);
-    }
-
     protected function NUMBER()
     {
         if (($this->c === '0' && $this->n === 'x') || (strtolower($this->c) === 'x' && $this->n === "'")) {
@@ -2447,10 +2435,6 @@ protected function NUMBER()
                     } else {
                         $this->type = self::DECIMAL_NUMBER;
                     }
-                } else {
-                    // If there is no digit after the '.', it's a DOT_IDENTIFIER.
-                    $this->emitDot();
-                    $this->type = self::IDENTIFIER;
                 }
             } elseif (($this->c === 'e' || $this->c === 'E') && ($this->n === '+' || $this->n === '-' || $this->isDigit($this->n))) {
                 $this->consume();

tests/parser/data/failures.csv

@@ -1645,8 +1645,6 @@ WHERE (dt.f2 <> ANY (SELECT t1.f1 FROM t1 WHERE t1.f2 = dt.f2))"
 "create table t1 (k int, index (k), index using hash (k)) engine=heap"
 "create table t1 (k int, index using btree (k), index (k)) engine=innodb"
 "create table t1 (k int, index using btree (k), index using hash (k)) engine=innodb"
-"SELECT @@global.default.key_buffer_size"
-"SELECT @@global.default.`key_buffer_size`"
 "create table t3 (like t1)"
 "create table tm (k int, index using btree (k)) charset utf8mb4 engine=myisam"
 "alter table tm add column l int, add index using btree (l)"
@@ -3669,12 +3667,6 @@ end"
 begin
 alter database character set koi8r;
 end"
-"drop function if exists test.database"
-"drop function if exists test.current_user"
-"select test.database(), test.database ()"
-"select test.current_user(), test.current_user ()"
-"drop function test.database"
-"drop function test.current_user"
 "CREATE PROCEDURE p1(IN loops BIGINT(19) UNSIGNED)
 BEGIN
 WHILE loops > 0 DO
@@ -3853,7 +3845,6 @@ end"
 "SELECT IF(e='a', e, NULL) FROM t GROUP BY ROLLUP(e) EXCEPT SELECT 1"
 "SELECT COALESCE(e, NULL) FROM t GROUP BY ROLLUP(e) EXCEPT SELECT 1"
 "SELECT 10,10.0,10.,.1e+2,100.0e-1"
-"SELECT 1e1,1.e1,1.0e1,1e+1,1.e+1,1.0e+1,1e-1,1.e-1,1.0e-1"
 "create table t1 (c nchar varchar(10))"
 "create table t1 (c national character varying(10))"
 "create table t1 (c nchar varying(10))"
@@ -4443,12 +4434,6 @@ ROW_NUMBER() OVER (PARTITION BY j)"
 "EXPLAIN FORMAT=tree SELECT a, SUM(c) OVER (ORDER BY b), AVG(c) OVER (ORDER BY a), SUM(c) OVER (PARTITION BY a) AS x FROM t1 ORDER BY b, x"
 "SELECT k, AVG(DISTINCT j), MIN(k) OVER (ROWS UNBOUNDED PRECEDING) min,
 MAX(k) OVER (ROWS UNBOUNDED PRECEDING) max FROM t GROUP BY (k)"
-"SELECT t.*, MIN(t.rank) OVER (ROWS UNBOUNDED PRECEDING) min,
-MAX(t.rank) OVER (ROWS UNBOUNDED PRECEDING) max FROM
-(SELECT sex, id, date, ROW_NUMBER() OVER w AS row_no, RANK() OVER w AS `rank` FROM t1,t2
-WHERE t1.id=t2.user_id
-WINDOW w AS (PARTITION BY date ORDER BY id)
-) AS t"
 "SELECT AVG(id) OVER w, MIN(id) OVER w min, MAX(id) OVER w max FROM t1
 WINDOW w AS (PARTITION BY sex)"
 "SELECT id, AVG(id) OVER (ROWS UNBOUNDED PRECEDING) avg,

tests/parser/data/stats.txt

@@ -1 +1 @@
-Total: 66241  |  Failures: 1876 /  2%  |  Exceptions:    0 /  0%
+Total: 66241  |  Failures: 1866 /  2%  |  Exceptions:    0 /  0%