Fix number vs identifier matching, add some tests

Author: JanJakes

custom-parser/parser/MySQLLexer.php

@@ -2452,7 +2452,7 @@ protected function NUMBER()
                     $this->emitDot();
                     $this->type = self::IDENTIFIER;
                 }
-            } elseif ($this->c === 'e' || $this->c === 'E') {
+            } elseif (($this->c === 'e' || $this->c === 'E') && ($this->n === '+' || $this->n === '-' || $this->isDigit($this->n))) {
                 $this->consume();
                 if ($this->c === '+' || $this->c === '-') {
                     $this->consume();
@@ -2463,6 +2463,21 @@ protected function NUMBER()
                 $this->type = self::FLOAT_NUMBER;
             }
         }
+
+        // In MySQL, when an input matches both a number and an identifier, the number always wins.
+        // However, when the number is followed by a non-numeric identifier-like character, it is
+        // considered an identifier... unless it's a float number, which ignores subsequent input.
+        $possibleIdentifierPrefix =
+            $this->type === self::INT_NUMBER
+            || ($this->text[0] === '0' && ($this->text[1] === 'b' || $this->text[1] === 'x'));
+
+        if ($possibleIdentifierPrefix && preg_match('/\G' . self::PATTERN_UNQUOTED_IDENTIFIER . '/u', $this->input, $matches, 0, $this->position)) {
+            $this->text .= $matches[0];
+            $this->position += strlen($matches[0]);
+            $this->c = $this->input[$this->position] ?? null;
+            $this->n = $this->input[$this->position + 1] ?? null;
+            $this->type = self::IDENTIFIER;
+        }
     }
 
     protected function SINGLE_QUOTED_TEXT()

tests/parser/data/failures.csv

@@ -328,10 +328,7 @@ ALTER TABLE t1 ALTER CONSTRAINT f2_check NOT ENFORCED"
 ALTER TABLE t1 ALTER CONSTRAINT constraint_name NOT ENFORCED"
 "ALTER TABLE tmp ALTER CONSTRAINT f2_check NOT ENFORCED"
 "create table t1 like `a/a`"
-"create table 1ea10 (1a20 int,1e int)"
-"insert into 1ea10 values(1,1)"
 "select 1ea10.1a20,1e+ 1e+10 from 1ea10"
-"drop table 1ea10"
 "create table t3 like t1"
 "create table if not exists t3 like t1"
 "create temporary table t3 like t2"
@@ -594,8 +591,6 @@ WHERE dt1.x = dt2.x"
 "SHOW REPLICA STATUS"
 "SHOW BINARY LOG STATUS"
 "SHOW REPLICAS"
-"create table t(id int, $id int, $id2 int, `$$id` int, $ int, $1 int,
-`$$$` int, id$$$ int, 1$ int, `$$` int, _$ int, b$$lit$$ int)"
 "EOF
 
 DROP DATABASE bug19573998"
@@ -757,18 +752,6 @@ WHERE innr1.col_int_key= 4)"
 gc INTEGER GENERATED ALWAYS AS (x LIKE 'abba' ESCAPE 'b'))"
 exit
 "SELECT r FROM t LEFT JOIN LATERAL (SELECT i, RAND(0) AS r) AS dt ON TRUE"
-"SELECT TIMEDIFF(TIME('17:00:00'),TIME('17:00:00'))=TIME('00:00:00') AS 1Eq,
-TIMEDIFF(TIME('17:59:00'),TIME('17:00:00'))=TIME('00:00:00') AS 1NEq1,
-TIMEDIFF(TIME('18:00:00'),TIME('17:00:00'))=TIME('00:00:00') AS 1NEq2,
-TIMEDIFF(TIME('17:00:00'),TIME('17:00:00'))=     '00:00:00'  AS 2Eq,
-TIMEDIFF(TIME('17:59:00'),TIME('17:00:00'))=     '00:00:00'  AS 2NEq1,
-TIMEDIFF(TIME('18:00:00'),TIME('17:00:00'))=     '00:00:00'  AS 2NEq2,
-TIMEDIFF(TIME('17:00:00'),TIME('17:00:00'))=TIME(0)          AS 3Eq,
-TIMEDIFF(TIME('17:59:00'),TIME('17:00:00'))=TIME(0)          AS 3NEq1,
-TIMEDIFF(TIME('18:00:00'),TIME('17:00:00'))=TIME(0)          AS 3NEq2,
-TIME(0) AS Time0, TIME('00:00:00') AS Time00, '00:00:00' AS Literal0000,
-TIMEDIFF(TIME('17:59:00'),TIME('17:00:00')),
-TIMEDIFF(TIME('17:00:00'),TIME('17:59:00'))"
 """',repeat('a',60),repeat('b',60),repeat('c',60),repeat('d',100)), '""');
 
 select insert('txs',2,1,'hi'),insert('is ',4,0,'a'),insert('txxxxt',2,4,'es');
@@ -1060,14 +1043,6 @@ boundary POLYGON)"
 "create table test.t_duplicated like mysqltest_db1.t_select_priv"
 "flush table mysqltest_db1.t1"
 "ALTER USER user() IDENTIFIED BY 'abc'"
-"CREATE USER 20553132_u1@localhost"
-"CREATE USER 20553132_u2@localhost"
-"GRANT ALL ON *.* TO 20553132_u1@localhost"
-"ALTER USER 20553132_u1@localhost PASSWORD EXPIRE"
-"ALTER USER 20553132_u2@localhost IDENTIFIED BY 'abcd', 20553132_u1@localhost IDENTIFIED BY 'defg' PASSWORD EXPIRE NEVER"
-"ALTER USER 20553132_u2@localhost IDENTIFIED BY 'abcd', 20553132_u1@localhost IDENTIFIED WITH 'caching_sha2_password' BY 'hijk' PASSWORD EXPIRE DEFAULT"
-"DROP USER 20553132_u1@localhost"
-"DROP USER 20553132_u2@localhost"
 "EOF
 
 
@@ -1508,10 +1483,6 @@ KEY k4 (f,g) -- 'f' and 'g' are independet.
 "EXPLAIN FORMAT=JSON INTO @explain_output
 SELECT a FROM t1 x2 WHERE x2.b IN
 (SELECT a FROM t1 x1 WHERE x1.b IN (SELECT a FROM t1 x0))"
-"INSERT INTO t (a)
-WITH RECURSIVE 150tup(n) AS
-(SELECT 1 UNION ALL SELECT n + 1 FROM 150tup WHERE n < 150)
-SELECT n FROM 150tup"
 "EXPLAIN FORMAT=TREE
 SELECT /*+ SET_VAR(optimizer_max_subgraph_pairs = 1) */ COUNT(*)
 FROM t AS t1 LEFT JOIN t AS t2 ON TRUE,
@@ -3735,12 +3706,6 @@ DECLARE a int;
 SELECT 1 INTO a;
 RETURN a;
 END "
-"create procedure 15298_1 () sql security definer show grants for current_user"
-"create procedure 15298_2 () sql security definer show grants"
-"call 15298_1()"
-"call 15298_2()"
-"drop procedure 15298_1"
-"drop procedure 15298_2"
 "CREATE EVENT e1 ON SCHEDULE EVERY 1 SECOND STARTS NOW() DO
 BEGIN
 DECLARE EXIT HANDLER FOR 1136 BEGIN

tests/parser/data/stats.txt

@@ -1 +1 @@
-Total: 66241  |  Failures: 1896 /  2%  |  Exceptions:    0 /  0%
+Total: 66241  |  Failures: 1876 /  2%  |  Exceptions:    0 /  0%

tests/parser/run-lexer-tests.php

@@ -23,3 +23,62 @@
 }
 
 echo "Tokenized $i queries in ",  microtime(true) - $start, 's', PHP_EOL;
+
+// Add some manual tests
+$tests = [
+	/**
+	 * Numbers vs. identifiers:
+	 *
+	 * In MySQL, when an input matches both a number and an identifier, the number always wins.
+	 * However, when the number is followed by a non-numeric identifier-like character, it is
+	 * considered an identifier... unless it's a float number, which ignores subsequent input.
+	 */
+
+	// INT numbers vs. identifiers
+	'123' => ['INT_NUMBER', 'EOF'],
+	'123abc' => ['IDENTIFIER', 'EOF'], // identifier
+
+	// BIN numbers vs. identifiers
+	'0b01' => ['BIN_NUMBER', 'EOF'],
+	'0b01xyz' => ['IDENTIFIER', 'EOF'], // identifier
+	"b'01'" => ['BIN_NUMBER', 'EOF'],
+	"b'01xyz'" => ['BIN_NUMBER', 'IDENTIFIER', 'INVALID_INPUT', 'EOF'],
+
+	// HEX numbers vs. identifiers
+	'0xab01' => ['HEX_NUMBER', 'EOF'],
+	'0xab01xyz' => ['IDENTIFIER', 'EOF'], // identifier
+	"x'ab01'" => ['HEX_NUMBER', 'EOF'],
+	"x'ab01xyz'" => ['HEX_NUMBER', 'IDENTIFIER', 'INVALID_INPUT', 'EOF'],
+
+	// DECIMAL numbers vs. identifiers
+	'123.456' => ['DECIMAL_NUMBER', 'EOF'],
+	'.123' => ['DECIMAL_NUMBER', 'EOF'],
+	'123.456abc' => ['DECIMAL_NUMBER', 'IDENTIFIER', 'EOF'], // not identifier
+	'.123abc' => ['DECIMAL_NUMBER', 'IDENTIFIER', 'EOF'], // not identifier
+
+	// FLOAT numbers vs. identifiers
+	'1e10' => ['FLOAT_NUMBER', 'EOF'],
+	'1e+10' => ['FLOAT_NUMBER', 'EOF'],
+	'1e-10' => ['FLOAT_NUMBER', 'EOF'],
+	'1e10abc' => ['FLOAT_NUMBER', 'IDENTIFIER', 'EOF'], // not identifier (this differs from INT/BIN/HEX numbers)
+	'1e+10abc' => ['FLOAT_NUMBER', 'IDENTIFIER', 'EOF'], // not identifier
+	'1e-10abc' => ['FLOAT_NUMBER', 'IDENTIFIER', 'EOF'], // not identifier
+];
+
+$failures = 0;
+foreach ($tests as $input => $expected) {
+	$tokens = tokenizeQuery($input);
+	$token_names = array_map(function ($token) {
+		return $token->getName();
+	}, $tokens);
+	if ($token_names !== $expected) {
+		$failures += 1;
+		echo "\nFailed test for input: $input\n";
+		echo "  Expected: ", implode(', ', $expected), "\n";
+		echo "  Actual:   ", implode(', ', $token_names), "\n";
+	}
+}
+if ($failures > 0) {
+	echo "\n$failures tests failed!\n";
+}
+