From 97536f897e251440178b9e8207ec90db0d78e3cd Mon Sep 17 00:00:00 2001 From: Zach Carter Date: Tue, 14 Dec 2010 19:07:05 -0500 Subject: [PATCH] Switch to Jison's lexer. --- Rakefile | 8 +- lib/handlebars/compiler.js | 1 - lib/handlebars/handlebars_lexer.js | 127 ----------------------------- lib/handlebars/jison_ext.js | 94 --------------------- lib/handlebars/printer.js | 2 +- lib/handlebars/visitor.js | 14 ++++ spec/spec_helper.rb | 3 +- spec/tokenizer_spec.rb | 6 +- src/handlebars.l | 27 ++++++ src/handlebars.yy | 2 +- 10 files changed, 51 insertions(+), 233 deletions(-) delete mode 100644 lib/handlebars/handlebars_lexer.js delete mode 100644 lib/handlebars/jison_ext.js create mode 100644 lib/handlebars/visitor.js create mode 100644 src/handlebars.l diff --git a/Rakefile b/Rakefile index f15f6190..e10b950d 100644 --- a/Rakefile +++ b/Rakefile @@ -1,8 +1,8 @@ require "rubygems" require "bundler/setup" -file "lib/handlebars/parser.js" => "src/handlebars.yy" do - system "jison src/handlebars.yy" +file "lib/handlebars/parser.js" => ["src/handlebars.yy","src/handlebars.l"] do + system "jison src/handlebars.yy src/handlebars.l" sh "mv handlebars.js lib/handlebars/parser.js" end @@ -20,11 +20,11 @@ def remove_exports(string) match ? match[1] : string end -minimal_deps = %w(parser compiler ast jison_ext handlebars_lexer runtime utils).map do |file| +minimal_deps = %w(parser compiler ast runtime utils).map do |file| "lib/handlebars/#{file}.js" end -debug_deps = %w(parser compiler ast jison_ext handlebars_lexer printer runtime utils).map do |file| +debug_deps = %w(parser compiler ast printer runtime utils).map do |file| "lib/handlebars/#{file}.js" end diff --git a/lib/handlebars/compiler.js b/lib/handlebars/compiler.js index 33e1fdf2..f892a509 100644 --- a/lib/handlebars/compiler.js +++ b/lib/handlebars/compiler.js @@ -7,7 +7,6 @@ Handlebars.Parser = handlebars; Handlebars.parse = function(string) { Handlebars.Parser.yy = Handlebars.AST; - Handlebars.Parser.lexer = new Handlebars.HandlebarsLexer(); return Handlebars.Parser.parse(string); }; diff --git a/lib/handlebars/handlebars_lexer.js b/lib/handlebars/handlebars_lexer.js deleted file mode 100644 index 5f4638c3..00000000 --- a/lib/handlebars/handlebars_lexer.js +++ /dev/null @@ -1,127 +0,0 @@ -var Handlebars = {}; -Handlebars.Lexer = require("handlebars/jison_ext").Lexer; - -// BEGIN(BROWSER) -Handlebars.HandlebarsLexer = function() { - this.state = "CONTENT"; -}; -Handlebars.HandlebarsLexer.prototype = new Handlebars.Lexer(); - -// The HandlebarsLexer uses a Lexer interface that is compatible -// with Jison. -// -// setupLex reset internal state for a new token -// peek(n) lookahead n characters and return (default 1) -// getchar(n) remove n characters from the input and add -// them to the matched text (default 1) -// readchar(n) remove n characters from the input, but do not -// add them to the matched text (default 1) -// ignorechar(n) remove n characters from the input, and act -// as though they were already matched in a -// previous lex. this will ensure that the -// pointer in the case of parse errors is in -// the right place. -Handlebars.HandlebarsLexer.prototype.lex = function() { - if(this.input === "") { return; } - - this.setupLex(); - - var lookahead = this.peek(2); - var result = ''; - var peek; - - if(lookahead === "") { return; } - - if(this.state == "MUSTACHE") { - if(this.peek() === "/") { - this.getchar(); - return "SEP"; - } - - // chomp optional whitespace - while(this.peek() === " ") { this.ignorechar(); } - - lookahead = this.peek(2); - - // in a mustache, but less than 2 characters left => error - if(lookahead.length != 2) { return; } - - // if the next characters are '}}', the mustache is done - if(lookahead === "}}") { - this.state = "CONTENT"; - this.getchar(2); - - // handle the case of {{{ foo }}} by always chomping - // a final }. TODO: Track escape state and handle the - // error condition here - if(this.peek() == "}") { this.getchar(); } - return "CLOSE"; - - // if the next character is a quote => enter a String - } else if(this.peek() === '"') { - this.readchar(); - - // scan the String until another quote is reached, skipping over escaped quotes - while(this.peek() !== '"') { if(this.peek(2) === '\\"') { this.readchar(); } this.getchar(); } - this.readchar(); - return "STRING"; - - // All other cases are IDs or errors - } else { - // grab alphanumeric characters - while(this.peek().match(/[_0-9A-Za-z\.]/)) { this.getchar(); } - - peek = this.peek(); - if(peek !== "}" && peek !== " " && peek !== "/") { - return; - } - - // if any characters were grabbed => ID - if(this.yytext.length) { return "ID"; } - - // Otherwise => Error - else { return; } - } - - // Next chars are {{ => Open mustache - } else if(lookahead == "{{") { - this.state = "MUSTACHE"; - this.getchar(2); - - peek = this.peek(); - - if(peek === ">") { - this.getchar(); - return "OPEN_PARTIAL"; - } else if(peek === "#") { - this.getchar(); - return "OPEN_BLOCK"; - } else if(peek === "/") { - this.getchar(); - return "OPEN_ENDBLOCK"; - } else if(peek === "^") { - this.getchar(); - return "OPEN_INVERSE"; - } else if(peek === "{" || peek === "&") { - this.getchar(); - return "OPEN_UNESCAPED"; - } else if(peek === "!") { - this.readchar(); - this.setupLex(); // reset the lexer state so the yytext is the comment only - while(this.peek(2) !== "}}") { this.getchar(); } - this.readchar(2); - this.state = "CONTENT"; - return "COMMENT"; - } else { - return "OPEN"; - } - - // Otherwise => content section - } else { - while(this.peek(2) !== "{{" && this.peek(2) !== "") { result = result + this.getchar(); } - return "CONTENT"; - } -}; -// END(BROWSER) - -exports.Lexer = Handlebars.HandlebarsLexer; diff --git a/lib/handlebars/jison_ext.js b/lib/handlebars/jison_ext.js deleted file mode 100644 index 068d80e1..00000000 --- a/lib/handlebars/jison_ext.js +++ /dev/null @@ -1,94 +0,0 @@ -var Handlebars = {}; - -// BEGIN(BROWSER) -Handlebars.Lexer = function() {}; - -Handlebars.Lexer.prototype = { - setInput: function(input) { - this.input = input; - this.matched = this.match = ''; - this.yylineno = 0; - }, - - setupLex: function() { - this.yyleng = 0; - this.yytext = ''; - this.match = ''; - this.readchars = 0; - }, - - getchar: function(n) { - n = n || 1; - var chars = "", chr = ""; - - for(var i=0; i 20 ? '...':'') + past.substr(-20).replace(/\n/g, ""); - }, - - upcomingInput:function () { - var next = this.match; - if (next.length < 20) { - next += this.input.substr(0, 20-next.length); - } - return (next.substr(0,20)+(next.length > 20 ? '...':'')).replace(/\n/g, ""); - }, - - showPosition:function () { - var pre = this.pastInput(); - var c = new Array(pre.length + 1 + this.readchars).join("-"); - return pre + this.upcomingInput() + "\n" + c+"^"; - } -}; - -Handlebars.Visitor = function() {}; - -Handlebars.Visitor.prototype = { - accept: function(object) { - return this[object.type](object); - } -}; -// END(BROWSER) - -exports.Lexer = Handlebars.Lexer; -exports.Visitor = Handlebars.Visitor; diff --git a/lib/handlebars/printer.js b/lib/handlebars/printer.js index 08dfa2f7..2f663e3a 100644 --- a/lib/handlebars/printer.js +++ b/lib/handlebars/printer.js @@ -1,5 +1,5 @@ var Handlebars = {}; -Handlebars.Visitor = require("handlebars/jison_ext").Visitor; +Handlebars.Visitor = require("handlebars/visitor").Visitor; // BEGIN(BROWSER) Handlebars.PrintVisitor = function() { this.padding = 0; }; diff --git a/lib/handlebars/visitor.js b/lib/handlebars/visitor.js new file mode 100644 index 00000000..04f025d7 --- /dev/null +++ b/lib/handlebars/visitor.js @@ -0,0 +1,14 @@ +var Handlebars = {}; + +// BEGIN(BROWSER) + +Handlebars.Visitor = function() {}; + +Handlebars.Visitor.prototype = { + accept: function(object) { + return this[object.type](object); + } +}; +// END(BROWSER) + +exports.Visitor = Handlebars.Visitor; diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 5394cb00..9ddde768 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -71,8 +71,7 @@ module Handlebars Handlebars::Spec.js_load('lib/handlebars/parser.js') Handlebars::Spec.js_load('lib/handlebars/compiler.js'); Handlebars::Spec.js_load('lib/handlebars/ast.js'); - Handlebars::Spec.js_load('lib/handlebars/jison_ext.js'); - Handlebars::Spec.js_load('lib/handlebars/handlebars_lexer.js') + Handlebars::Spec.js_load('lib/handlebars/visitor.js'); Handlebars::Spec.js_load('lib/handlebars/printer.js') Handlebars::Spec.js_load('lib/handlebars/runtime.js') Handlebars::Spec.js_load('lib/handlebars/utils.js') diff --git a/spec/tokenizer_spec.rb b/spec/tokenizer_spec.rb index ddecec5f..e2f84ceb 100644 --- a/spec/tokenizer_spec.rb +++ b/spec/tokenizer_spec.rb @@ -1,8 +1,8 @@ require "spec_helper" describe "Tokenizer" do - let(:handlebars) { @context["Handlebars"] } - let(:lexer) { handlebars["HandlebarsLexer"].new } + let(:parser) { @context["handlebars"] } + let(:lexer) { @context["handlebars"]["lexer"] } Token = Struct.new(:name, :text) @@ -10,7 +10,7 @@ describe "Tokenizer" do lexer.setInput(string) out = [] - while result = lexer.lex + while result = parser.terminals_[lexer.lex] and result != "EOF" out << Token.new(result, lexer.yytext) end diff --git a/src/handlebars.l b/src/handlebars.l new file mode 100644 index 00000000..e29864d5 --- /dev/null +++ b/src/handlebars.l @@ -0,0 +1,27 @@ + +%x mu + +%% + +(.|\s)*?/("{{") { this.begin("mu"); if (yytext) return 'CONTENT'; } +(.|\s)+ { return 'CONTENT'; } + +"{{>" { return 'OPEN_PARTIAL'; } +"{{#" { return 'OPEN_BLOCK'; } +"{{/" { return 'OPEN_ENDBLOCK'; } +"{{^" { return 'OPEN_INVERSE'; } +"{{{" { return 'OPEN_UNESCAPED'; } +"{{&" { return 'OPEN_UNESCAPED'; } +"{{!".*?"}}" { yytext = yytext.substr(3,yyleng-5); this.begin("INITIAL"); return 'COMMENT'; } +"{{" { return 'OPEN'; } + +"/" { return 'SEP'; } +\s+ { /*ignore whitespace*/ } +"}}}" { this.begin("INITIAL"); return 'CLOSE'; } +"}}" { this.begin("INITIAL"); return 'CLOSE'; } +'"'("\\"["]|[^"])*'"' { yytext = yytext.substr(1,yyleng-2).replace(/\\"/g,'"'); return 'STRING'; } +[a-zA-Z0-9_.]+/[} /] { return 'ID'; } +. { return 'INVALID'; } + +<> { return 'EOF'; } + diff --git a/src/handlebars.yy b/src/handlebars.yy index ac178ba5..7d2dcb01 100644 --- a/src/handlebars.yy +++ b/src/handlebars.yy @@ -3,7 +3,7 @@ %% root - : program { return $1 } + : program EOF { return $1 } ; program