commit

Author: 钱宇

src/ClickHouseSQLParser.php

@@ -643,7 +643,19 @@ public static function token_get_all($str, $options = array())
             return array();
         }
         $tokens = array();
-        \preg_match_all("{[a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*|\\`(?:[^\\`\\\\]|\\\\.)*\\`|\\\"(?:[^\\\"\\\\]|\\\\.)*\\\"|\\'(?:[^\\'\\\\]|\\\\.)*\\'|\\s+|\\d+(?:\\.\\d*)?(?:[Ee][\\+\\-]?\\d+)?|\\<\\=\\>|\\!\\=|\\>\\=|\\<\\=|\\<\\>|\\<\\<|\\>\\>|\\:\\=|&&|\\|\\||@@|\\-\\>|\\-\\-.*?(?=[\\r\\n]|\$)|\\/\\*.*?(?:\\*\\/|\$)|.}s", $str, $m);
+        $pattern = '{'
+            . '[a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*'  // Identifier
+            . '|\`[^\`\\\\]*(?:\\\\.[^\`\\\\]*)*\`'       // Optimization: Backtick string
+            . '|\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*\"'       // Optimization: Double-quoted string
+            . '|\'[^\'\\\\]*(?:\\\\.[^\'\\\\]*)*\''       // Optimization: Single-quoted string
+            . '|\s+'                                      // Whitespace
+            . '|\d+(?:\.\d*)?(?:[Ee][\+\-]?\d+)?'        // Number
+            . '|<=>|!=|>=|<=|<>|<<|>>|:=|&&|\|\||@@|->'  // Operator
+            . '|--[^\r\n]*?(?=[\r\n]|$)'                  // Single-line comment
+            . '|/\*.*?(?:\*/|$)'                          // Multi-line comment
+            . '|.'                                        // Other characters
+            . '}s';
+        \preg_match_all($pattern, $str, $m);
         foreach ($m[0] as $token) {
             switch (isset($map[$token[0]]) ? $map[$token[0]] : -1) {
                 case 1: