The tokenizer shouldn't get into an infinite loop in certain termination conditions

lib/handlebars/handlebars_lexer.js

@@ -8,6 +8,20 @@ Handlebars.HandlebarsLexer = function() {
 };
 Handlebars.HandlebarsLexer.prototype = new Handlebars.Lexer;
 
+// The HandlebarsLexer uses a Lexer interface that is compatible
+// with Jison.
+//
+// setupLex      reset internal state for a new token
+// peek(n)       lookahead n characters and return (default 1)
+// getchar(n)    remove n characters from the input and add
+//               them to the matched text (default 1)
+// readchar(n)   remove n characters from the input, but do not
+//               add them to the matched text (default 1)
+// ignorechar(n) remove n characters from the input, and act
+//               as though they were already matched in a
+//               previous lex. this will ensure that the
+//               pointer in the case of parse errors is in
+//               the right place.
 Handlebars.HandlebarsLexer.prototype.lex = function() {
   if(this.input === "") return;
 
@@ -16,25 +30,50 @@ Handlebars.HandlebarsLexer.prototype.lex = function() {
   var lookahead = this.peek(2);
   var result = '';
 
+  if(lookahead === "") return;
+
   if(this.state == "MUSTACHE") {
     // chomp optional whitespace
-    while(this.peek() === " ") { this.readchar(); }
+    while(this.peek() === " ") { this.ignorechar(); }
 
-    if(this.peek(2) === "}}") {
+    var lookahead = this.peek(2);
+
+    // in a mustache, but less than 2 characters left => error
+    if(lookahead.length != 2) { return; }
+
+    // if the next characters are '}}', the mustache is done
+    if(lookahead === "}}") {
       this.state = "CONTENT"
       this.getchar(2);
 
+      // handle the case of {{{ foo }}} by always chomping
+      // a final }. TODO: Track escape state and handle the
+      // error condition here
       if(this.peek() == "}") this.getchar();
       return "CLOSE";
+
+    // if the next character is a quote => enter a String
     } else if(this.peek() === '"') {
       this.readchar();
+
+      // scan the String until another quote is reached, skipping over escaped quotes
       while(this.peek() !== '"') { if(this.peek(2) === '\\"') { this.readchar() }; this.getchar() }
       this.readchar();
       return "STRING";
+
+    // All other cases are IDs or errors
     } else {
-      while(this.peek().match(/[A-Za-z]/)) { this.getchar() }
+      // grab alphanumeric characters
-      return "ID"
+      while(this.peek().match(/[0-9A-Za-z]/)) { this.getchar() }
+
+      // if any characters were grabbed => ID
+      if(this.yytext.length) { return "ID" }
+
+      // Otherwise => Error
+      else return;
     }
+
+  // Next chars are {{ => Open mustache
   } else if(lookahead == "{{") {
     this.state = "MUSTACHE";
     this.getchar(2);
@@ -63,6 +102,8 @@ Handlebars.HandlebarsLexer.prototype.lex = function() {
     } else {
       return "OPEN";
     }
+
+  // Otherwise => content section
   } else {
     while(this.peek(2) !== "{{" && this.peek(2) !== "") { result = result + this.getchar(); }
     return "CONTENT"

lib/handlebars/jison_ext.js

@@ -5,23 +5,30 @@ Handlebars.Lexer = function() {};
 Handlebars.Lexer.prototype = {
   setInput: function(input) {
     this.input = input;
+    this.matched = this.match = '';
     this.yylineno = 0;
   },
 
   setupLex: function() {
     this.yyleng = 0;
     this.yytext = '';
+    this.match = '';
+    this.readchars = 0;
   },
 
   getchar: function(n) {
     n = n || 1;
-    var char = "";
+    var chars = "", char = "";
 
     for(var i=0; i<n; i++) {
-      char += this.input[0];
+      char = this.input[0];
-      this.yytext += this.input[0];
+      chars += char;
+      this.yytext += char;
       this.yyleng++;
 
+      this.matched += char;
+      this.match += char;
+
       if(char === "\n") this.yylineno++;
 
       this.input = this.input.slice(1);
@@ -29,21 +36,48 @@ Handlebars.Lexer.prototype = {
     return char;
   },
 
-  readchar: function(n) {
+  readchar: function(n, ignore) {
     n = n || 1;
     var char;
 
     for(var i=0; i<n; i++) {
       char = this.input[i];
       if(char === "\n") this.yylineno++;
+
+      this.matched += char;
+      this.match += char;
+      if(ignore) { this.readchars++; }
     }
 
     this.input = this.input.slice(n);
   },
 
+  ignorechar: function(n) {
+    this.readchar(n, true);
+  },
+
   peek: function(n) {
     return this.input.slice(0, n || 1);
-  }
+  },
+
+  pastInput:function () {
+    var past = this.matched.substr(0, this.matched.length - this.match.length);
+    return (past.length > 20 ? '...':'') + past.substr(-20).replace(/\n/g, "");
+  },
+
+  upcomingInput:function () {
+    var next = this.match;
+    if (next.length < 20) {
+      next += this.input.substr(0, 20-next.length);
+    }
+    return (next.substr(0,20)+(next.length > 20 ? '...':'')).replace(/\n/g, "");
+  },
+
+  showPosition:function () {
+    var pre = this.pastInput();
+    var c = new Array(pre.length + 1 + this.readchars).join("-");
+    return pre + this.upcomingInput() + "\n" + c+"^";
+  },
 };
 
 Handlebars.Visitor = function() {};

spec/tokenizer_spec.rb

@@ -116,4 +116,12 @@ describe "Tokenizer" do
     result.should match_tokens(%w(OPEN ID STRING CLOSE))
     result[2].should be_token("STRING", %{bar"baz})
   end
+
+  it "does not time out with broken input" do
+    lambda do
+      Timeout.timeout(1) do
+        tokenize("{{foo}")
+      end
+    end.should_not raise_error(Timeout::Error)
+  end
 end