The tokenizer shouldn't get into an infinite loop in certain termination conditions

lib/handlebars/handlebars_lexer.js

@@ -8,6 +8,20 @@ Handlebars.HandlebarsLexer = function() {
 };
 Handlebars.HandlebarsLexer.prototype = new Handlebars.Lexer;
 
+// The HandlebarsLexer uses a Lexer interface that is compatible
+// with Jison.
+//
+// setupLex      reset internal state for a new token
+// peek(n)       lookahead n characters and return (default 1)
+// getchar(n)    remove n characters from the input and add
+//               them to the matched text (default 1)
+// readchar(n)   remove n characters from the input, but do not
+//               add them to the matched text (default 1)
+// ignorechar(n) remove n characters from the input, and act
+//               as though they were already matched in a
+//               previous lex. this will ensure that the
+//               pointer in the case of parse errors is in
+//               the right place.
 Handlebars.HandlebarsLexer.prototype.lex = function() {
   if(this.input === "") return;
 
@@ -16,25 +30,50 @@ Handlebars.HandlebarsLexer.prototype.lex = function() {
   var lookahead = this.peek(2);
   var result = '';
 
+  if(lookahead === "") return;
+
   if(this.state == "MUSTACHE") {
     // chomp optional whitespace
-    while(this.peek() === " ") { this.readchar(); }
+    while(this.peek() === " ") { this.ignorechar(); }
 
-    if(this.peek(2) === "}}") {
+    var lookahead = this.peek(2);
+
+    // in a mustache, but less than 2 characters left => error
+    if(lookahead.length != 2) { return; }
+
+    // if the next characters are '}}', the mustache is done
+    if(lookahead === "}}") {
       this.state = "CONTENT"
       this.getchar(2);
 
+      // handle the case of {{{ foo }}} by always chomping
+      // a final }. TODO: Track escape state and handle the
+      // error condition here
       if(this.peek() == "}") this.getchar();
       return "CLOSE";
+
+    // if the next character is a quote => enter a String
     } else if(this.peek() === '"') {
       this.readchar();
+
+      // scan the String until another quote is reached, skipping over escaped quotes
       while(this.peek() !== '"') { if(this.peek(2) === '\\"') { this.readchar() }; this.getchar() }
       this.readchar();
       return "STRING";
+
+    // All other cases are IDs or errors
     } else {
-      while(this.peek().match(/[A-Za-z]/)) { this.getchar() }
-      return "ID"
+      // grab alphanumeric characters
+      while(this.peek().match(/[0-9A-Za-z]/)) { this.getchar() }
+
+      // if any characters were grabbed => ID
+      if(this.yytext.length) { return "ID" }
+
+      // Otherwise => Error
+      else return;
     }
+
+  // Next chars are {{ => Open mustache
   } else if(lookahead == "{{") {
     this.state = "MUSTACHE";
     this.getchar(2);
@@ -63,6 +102,8 @@ Handlebars.HandlebarsLexer.prototype.lex = function() {
     } else {
       return "OPEN";
     }
+
+  // Otherwise => content section
   } else {
     while(this.peek(2) !== "{{" && this.peek(2) !== "") { result = result + this.getchar(); }
     return "CONTENT"