tokenize.diff

--- tokenize-old.lisp	2012-08-23 16:27:07.000000000 +0300
+++ tokenize.lisp	2012-09-07 20:25:10.559952524 +0300
@@ -1,3 +1,38 @@
+
+;; Copyright (c) Marijn Haverbeke, marijnh@gmail.com
+
+;; This software is provided 'as-is', without any express or implied
+;; warranty. In no event will the authors be held liable for any
+;; damages arising from the use of this software.
+
+;; Permission is granted to anyone to use this software for any
+;; purpose, including commercial applications, and to alter it and
+;; redistribute it freely, subject to the following restrictions:
+
+;; 1. The origin of this software must not be misrepresented; you must
+;;    not claim that you wrote the original software. If you use this
+;;    software in a product, an acknowledgment in the product
+;;    documentation would be appreciated but is not required.
+
+;; 2. Altered source versions must be plainly marked as such, and must
+;;    not be misrepresented as being the original software.
+
+;; 3. This notice may not be removed or altered from any source
+;;    distribution.
+
+;; ## Important Note
+
+;; Cheat-JS includes a modified version of
+;; [`parse-js`](http://marijnhaverbeke.nl/parse-js/), written by Marijn
+;; Haverbeke. This is necessary because I (Miron Brezuleanu) needed to
+;; modify `parse-js` a little. The license of `parse-js` is in the
+;; `LICENSE-parse-js.txt` file. The modified files from `parse-js`
+;; included in Cheat-JS are `parse.lisp`, `tokenize.lisp` and
+;; `util.lisp`. The modifications were permitted by the `parse-js`
+;; license. This is not an official copy of `parse-js` and is not
+;; supported by Marijn Haverbeke. If the modified parsing code in
+;; Cheat-JS breaks, it's exclusively my fault - I messed up the code.
+
 (in-package #:parse-js)
 
 (defstruct token type value line char pos newline-before comments-before)
@@ -108,6 +143,8 @@
               ((equal body "") (ret nil))
               (t (ret (parse-integer body))))))))
 
+(defvar *allow-at-signs* t)
+
 (defun/defs lex-js (stream &key include-comments)
   (def expression-allowed t)
   (def newline-before nil)
@@ -290,7 +327,7 @@
              (read-regexp)
              (read-operator "/")))))
 
-  (def identifier-char-p (ch) (or (and (alphanumericp ch) (not (find ch *whitespace-chars*))) (eql ch #\$) (eql ch #\_)))
+  (def identifier-char-p (ch) (or (and (alphanumericp ch) (not (find ch *whitespace-chars*))) (eql ch #\$) (eql ch #\_) (and *allow-at-signs* (eql ch #\@))))
   (def read-word ()
     (let* ((unicode-escape nil)
            (word (with-output-to-string (*standard-output*)