Switch to Jison's lexer.

Rakefile

@@ -1,8 +1,8 @@
 require "rubygems"
 require "bundler/setup"
 
-file "lib/handlebars/parser.js" => "src/handlebars.yy" do
+file "lib/handlebars/parser.js" => ["src/handlebars.yy","src/handlebars.l"] do
-  system "jison src/handlebars.yy"
+  system "jison src/handlebars.yy src/handlebars.l"
   sh "mv handlebars.js lib/handlebars/parser.js"
 end
 
@@ -20,11 +20,11 @@ def remove_exports(string)
   match ? match[1] : string
 end
 
-minimal_deps = %w(parser compiler ast jison_ext handlebars_lexer runtime utils).map do |file|
+minimal_deps = %w(parser compiler ast runtime utils).map do |file|
   "lib/handlebars/#{file}.js"
 end
 
-debug_deps = %w(parser compiler ast jison_ext handlebars_lexer printer runtime utils).map do |file|
+debug_deps = %w(parser compiler ast printer runtime utils).map do |file|
   "lib/handlebars/#{file}.js"
 end
 

lib/handlebars/compiler.js

@@ -7,7 +7,6 @@ Handlebars.Parser = handlebars;
 
 Handlebars.parse = function(string) {
   Handlebars.Parser.yy = Handlebars.AST;
-  Handlebars.Parser.lexer = new Handlebars.HandlebarsLexer();
   return Handlebars.Parser.parse(string);
 };
 

lib/handlebars/handlebars_lexer.js

@@ -1,127 +0,0 @@
-var Handlebars = {};
-Handlebars.Lexer = require("handlebars/jison_ext").Lexer;
-
-// BEGIN(BROWSER)
-Handlebars.HandlebarsLexer = function() {
-  this.state = "CONTENT";
-};
-Handlebars.HandlebarsLexer.prototype = new Handlebars.Lexer();
-
-// The HandlebarsLexer uses a Lexer interface that is compatible
-// with Jison.
-//
-// setupLex      reset internal state for a new token
-// peek(n)       lookahead n characters and return (default 1)
-// getchar(n)    remove n characters from the input and add
-//               them to the matched text (default 1)
-// readchar(n)   remove n characters from the input, but do not
-//               add them to the matched text (default 1)
-// ignorechar(n) remove n characters from the input, and act
-//               as though they were already matched in a
-//               previous lex. this will ensure that the
-//               pointer in the case of parse errors is in
-//               the right place.
-Handlebars.HandlebarsLexer.prototype.lex = function() {
-  if(this.input === "") { return; }
-
-  this.setupLex();
-
-  var lookahead = this.peek(2);
-  var result = '';
-  var peek;
-
-  if(lookahead === "") { return; }
-
-  if(this.state == "MUSTACHE") {
-    if(this.peek() === "/") {
-      this.getchar();
-      return "SEP";
-    }
-
-    // chomp optional whitespace
-    while(this.peek() === " ") { this.ignorechar(); }
-
-    lookahead = this.peek(2);
-
-    // in a mustache, but less than 2 characters left => error
-    if(lookahead.length != 2) { return; }
-
-    // if the next characters are '}}', the mustache is done
-    if(lookahead === "}}") {
-      this.state = "CONTENT";
-      this.getchar(2);
-
-      // handle the case of {{{ foo }}} by always chomping
-      // a final }. TODO: Track escape state and handle the
-      // error condition here
-      if(this.peek() == "}") { this.getchar(); }
-      return "CLOSE";
-
-    // if the next character is a quote => enter a String
-    } else if(this.peek() === '"') {
-      this.readchar();
-
-      // scan the String until another quote is reached, skipping over escaped quotes
-      while(this.peek() !== '"') { if(this.peek(2) === '\\"') { this.readchar(); } this.getchar(); }
-      this.readchar();
-      return "STRING";
-
-    // All other cases are IDs or errors
-    } else {
-      // grab alphanumeric characters
-      while(this.peek().match(/[_0-9A-Za-z\.]/)) { this.getchar(); }
-
-      peek = this.peek();
-      if(peek !== "}" && peek !== " " && peek !== "/") {
-        return;
-      }
-
-      // if any characters were grabbed => ID
-      if(this.yytext.length) { return "ID"; }
-
-      // Otherwise => Error
-      else { return; }
-    }
-
-  // Next chars are {{ => Open mustache
-  } else if(lookahead == "{{") {
-    this.state = "MUSTACHE";
-    this.getchar(2);
-
-    peek = this.peek();
-
-    if(peek === ">") {
-      this.getchar();
-      return "OPEN_PARTIAL";
-    } else if(peek === "#") {
-      this.getchar();
-      return "OPEN_BLOCK";
-    } else if(peek === "/") {
-      this.getchar();
-      return "OPEN_ENDBLOCK";
-    } else if(peek === "^") {
-      this.getchar();
-      return "OPEN_INVERSE";
-    } else if(peek === "{" || peek === "&") {
-      this.getchar();
-      return "OPEN_UNESCAPED";
-    } else if(peek === "!") {
-      this.readchar();
-      this.setupLex(); // reset the lexer state so the yytext is the comment only
-      while(this.peek(2) !== "}}") { this.getchar(); }
-      this.readchar(2);
-      this.state = "CONTENT";
-      return "COMMENT";
-    } else {
-      return "OPEN";
-    }
-
-  // Otherwise => content section
-  } else {
-    while(this.peek(2) !== "{{" && this.peek(2) !== "") { result = result + this.getchar(); }
-    return "CONTENT";
-  }
-};
-// END(BROWSER)
-
-exports.Lexer = Handlebars.HandlebarsLexer;

lib/handlebars/jison_ext.js

@@ -1,94 +0,0 @@
-var Handlebars = {};
-
-// BEGIN(BROWSER)
-Handlebars.Lexer = function() {};
-
-Handlebars.Lexer.prototype = {
-  setInput: function(input) {
-    this.input = input;
-    this.matched = this.match = '';
-    this.yylineno = 0;
-  },
-
-  setupLex: function() {
-    this.yyleng = 0;
-    this.yytext = '';
-    this.match = '';
-    this.readchars = 0;
-  },
-
-  getchar: function(n) {
-    n = n || 1;
-    var chars = "", chr = "";
-
-    for(var i=0; i<n; i++) {
-      chr = this.input[0];
-      chars += chr;
-      this.yytext += chr;
-      this.yyleng++;
-
-      this.matched += chr;
-      this.match += chr;
-
-      if(chr === "\n") { this.yylineno++; }
-
-      this.input = this.input.slice(1);
-    }
-    return chr;
-  },
-
-  readchar: function(n, ignore) {
-    n = n || 1;
-    var chr;
-
-    for(var i=0; i<n; i++) {
-      chr = this.input[i];
-      if(chr === "\n") { this.yylineno++; }
-
-      this.matched += chr;
-      this.match += chr;
-      if(ignore) { this.readchars++; }
-    }
-
-    this.input = this.input.slice(n);
-  },
-
-  ignorechar: function(n) {
-    this.readchar(n, true);
-  },
-
-  peek: function(n) {
-    return this.input.slice(0, n || 1);
-  },
-
-  pastInput:function () {
-    var past = this.matched.substr(0, this.matched.length - this.match.length);
-    return (past.length > 20 ? '...':'') + past.substr(-20).replace(/\n/g, "");
-  },
-
-  upcomingInput:function () {
-    var next = this.match;
-    if (next.length < 20) {
-      next += this.input.substr(0, 20-next.length);
-    }
-    return (next.substr(0,20)+(next.length > 20 ? '...':'')).replace(/\n/g, "");
-  },
-
-  showPosition:function () {
-    var pre = this.pastInput();
-    var c = new Array(pre.length + 1 + this.readchars).join("-");
-    return pre + this.upcomingInput() + "\n" + c+"^";
-  }
-};
-
-Handlebars.Visitor = function() {};
-
-Handlebars.Visitor.prototype = {
-  accept: function(object) {
-    return this[object.type](object);
-  }
-};
-// END(BROWSER)
-
-exports.Lexer = Handlebars.Lexer;
-exports.Visitor = Handlebars.Visitor;

lib/handlebars/printer.js

@@ -1,5 +1,5 @@
 var Handlebars = {};
-Handlebars.Visitor = require("handlebars/jison_ext").Visitor;
+Handlebars.Visitor = require("handlebars/visitor").Visitor;
 
 // BEGIN(BROWSER)
 Handlebars.PrintVisitor = function() { this.padding = 0; };

lib/handlebars/visitor.js

@@ -0,0 +1,14 @@
+var Handlebars = {};
+
+// BEGIN(BROWSER)
+
+Handlebars.Visitor = function() {};
+
+Handlebars.Visitor.prototype = {
+  accept: function(object) {
+    return this[object.type](object);
+  }
+};
+// END(BROWSER)
+
+exports.Visitor = Handlebars.Visitor;

spec/spec_helper.rb

@@ -71,8 +71,7 @@ module Handlebars
       Handlebars::Spec.js_load('lib/handlebars/parser.js')
       Handlebars::Spec.js_load('lib/handlebars/compiler.js');
       Handlebars::Spec.js_load('lib/handlebars/ast.js');
-      Handlebars::Spec.js_load('lib/handlebars/jison_ext.js');
+      Handlebars::Spec.js_load('lib/handlebars/visitor.js');
-      Handlebars::Spec.js_load('lib/handlebars/handlebars_lexer.js')
       Handlebars::Spec.js_load('lib/handlebars/printer.js')
       Handlebars::Spec.js_load('lib/handlebars/runtime.js')
       Handlebars::Spec.js_load('lib/handlebars/utils.js')

spec/tokenizer_spec.rb

@@ -1,8 +1,8 @@
 require "spec_helper"
 
 describe "Tokenizer" do
-  let(:handlebars) { @context["Handlebars"] }
+  let(:parser) { @context["handlebars"] }
-  let(:lexer) { handlebars["HandlebarsLexer"].new }
+  let(:lexer) { @context["handlebars"]["lexer"] }
 
   Token = Struct.new(:name, :text)
 
@@ -10,7 +10,7 @@ describe "Tokenizer" do
     lexer.setInput(string)
     out = []
 
-    while result = lexer.lex
+    while result = parser.terminals_[lexer.lex] and result != "EOF"
       out << Token.new(result, lexer.yytext)
     end
 

src/handlebars.l

@@ -0,0 +1,27 @@
+
+%x mu
+
+%%
+
+(.|\s)*?/("{{")              { this.begin("mu"); if (yytext) return 'CONTENT'; }
+(.|\s)+                      { return 'CONTENT'; }
+
+<mu>"{{>"                    { return 'OPEN_PARTIAL'; }
+<mu>"{{#"                    { return 'OPEN_BLOCK'; }
+<mu>"{{/"                    { return 'OPEN_ENDBLOCK'; }
+<mu>"{{^"                    { return 'OPEN_INVERSE'; }
+<mu>"{{{"                    { return 'OPEN_UNESCAPED'; }
+<mu>"{{&"                    { return 'OPEN_UNESCAPED'; }
+<mu>"{{!".*?"}}"             { yytext = yytext.substr(3,yyleng-5); this.begin("INITIAL"); return 'COMMENT'; }
+<mu>"{{"                     { return 'OPEN'; }
+
+<mu>"/"                      { return 'SEP'; }
+<mu>\s+                      { /*ignore whitespace*/ }
+<mu>"}}}"                    { this.begin("INITIAL"); return 'CLOSE'; }
+<mu>"}}"                     { this.begin("INITIAL"); return 'CLOSE'; }
+<mu>'"'("\\"["]|[^"])*'"'    { yytext = yytext.substr(1,yyleng-2).replace(/\\"/g,'"'); return 'STRING'; }
+<mu>[a-zA-Z0-9_.]+/[} /]     { return 'ID'; }
+<mu>.                        { return 'INVALID'; }
+
+<INITIAL,mu><<EOF>>          { return 'EOF'; }
+

src/handlebars.yy

@@ -3,7 +3,7 @@
 %%
 
 root
-  : program { return $1 }
+  : program EOF { return $1 }
   ;
 
 program