psimsa/handlebars.js
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch to Jison's lexer.

Browse Source
This commit is contained in:
Zach Carter
2010-12-14 19:07:05 -05:00
parent ec948c7382
commit 97536f897e
10 changed files with 51 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Rakefile
+4 -4
View File
@@ -1,8 +1,8 @@
require "rubygems"
require "bundler/setup"
file "lib/handlebars/parser.js" => "src/handlebars.yy" do
system "jison src/handlebars.yy"
file "lib/handlebars/parser.js" => ["src/handlebars.yy","src/handlebars.l"] do
system "jison src/handlebars.yy src/handlebars.l"
sh "mv handlebars.js lib/handlebars/parser.js"
end
@@ -20,11 +20,11 @@ def remove_exports(string)
match ? match[1] : string
end
minimal_deps = %w(parser compiler ast jison_ext handlebars_lexer runtime utils).map do |file|
minimal_deps = %w(parser compiler ast runtime utils).map do |file|
"lib/handlebars/#{file}.js"
end
debug_deps = %w(parser compiler ast jison_ext handlebars_lexer printer runtime utils).map do |file|
debug_deps = %w(parser compiler ast printer runtime utils).map do |file|
"lib/handlebars/#{file}.js"
end
lib/handlebars/compiler.js
-1
View File
@@ -7,7 +7,6 @@ Handlebars.Parser = handlebars;
Handlebars.parse = function(string) {
Handlebars.Parser.yy = Handlebars.AST;
Handlebars.Parser.lexer = new Handlebars.HandlebarsLexer();
return Handlebars.Parser.parse(string);
};
lib/handlebars/handlebars_lexer.js
-127
View File
@@ -1,127 +0,0 @@
var Handlebars = {};
Handlebars.Lexer = require("handlebars/jison_ext").Lexer;
// BEGIN(BROWSER)
Handlebars.HandlebarsLexer = function() {
this.state = "CONTENT";
};
Handlebars.HandlebarsLexer.prototype = new Handlebars.Lexer();
// The HandlebarsLexer uses a Lexer interface that is compatible
// with Jison.
//
// setupLex reset internal state for a new token
// peek(n) lookahead n characters and return (default 1)
// getchar(n) remove n characters from the input and add
// them to the matched text (default 1)
// readchar(n) remove n characters from the input, but do not
// add them to the matched text (default 1)
// ignorechar(n) remove n characters from the input, and act
// as though they were already matched in a
// previous lex. this will ensure that the
// pointer in the case of parse errors is in
// the right place.
Handlebars.HandlebarsLexer.prototype.lex = function() {
if(this.input === "") { return; }
this.setupLex();
var lookahead = this.peek(2);
var result = '';
var peek;
if(lookahead === "") { return; }
if(this.state == "MUSTACHE") {
if(this.peek() === "/") {
this.getchar();
return "SEP";
}
// chomp optional whitespace
while(this.peek() === " ") { this.ignorechar(); }
lookahead = this.peek(2);
// in a mustache, but less than 2 characters left => error
if(lookahead.length != 2) { return; }
// if the next characters are '}}', the mustache is done
if(lookahead === "}}") {
this.state = "CONTENT";
this.getchar(2);
// handle the case of {{{ foo }}} by always chomping
// a final }. TODO: Track escape state and handle the
// error condition here
if(this.peek() == "}") { this.getchar(); }
return "CLOSE";
// if the next character is a quote => enter a String
} else if(this.peek() === '"') {
this.readchar();
// scan the String until another quote is reached, skipping over escaped quotes
while(this.peek() !== '"') { if(this.peek(2) === '\\"') { this.readchar(); } this.getchar(); }
this.readchar();
return "STRING";
// All other cases are IDs or errors
} else {
// grab alphanumeric characters
while(this.peek().match(/[_0-9A-Za-z\.]/)) { this.getchar(); }
peek = this.peek();
if(peek !== "}" && peek !== " " && peek !== "/") {
return;
}
// if any characters were grabbed => ID
if(this.yytext.length) { return "ID"; }
// Otherwise => Error
else { return; }
}
// Next chars are {{ => Open mustache
} else if(lookahead == "{{") {
this.state = "MUSTACHE";
this.getchar(2);
peek = this.peek();
if(peek === ">") {
this.getchar();
return "OPEN_PARTIAL";
} else if(peek === "#") {
this.getchar();
return "OPEN_BLOCK";
} else if(peek === "/") {
this.getchar();
return "OPEN_ENDBLOCK";
} else if(peek === "^") {
this.getchar();
return "OPEN_INVERSE";
} else if(peek === "{" || peek === "&") {
this.getchar();
return "OPEN_UNESCAPED";
} else if(peek === "!") {
this.readchar();
this.setupLex(); // reset the lexer state so the yytext is the comment only
while(this.peek(2) !== "}}") { this.getchar(); }
this.readchar(2);
this.state = "CONTENT";
return "COMMENT";
} else {
return "OPEN";
}
// Otherwise => content section
} else {
while(this.peek(2) !== "{{" && this.peek(2) !== "") { result = result + this.getchar(); }
return "CONTENT";
}
};
// END(BROWSER)
exports.Lexer = Handlebars.HandlebarsLexer;
lib/handlebars/jison_ext.js
-94
View File
@@ -1,94 +0,0 @@
var Handlebars = {};
// BEGIN(BROWSER)
Handlebars.Lexer = function() {};
Handlebars.Lexer.prototype = {
setInput: function(input) {
this.input = input;
this.matched = this.match = '';
this.yylineno = 0;
},
setupLex: function() {
this.yyleng = 0;
this.yytext = '';
this.match = '';
this.readchars = 0;
},
getchar: function(n) {
n = n || 1;
var chars = "", chr = "";
for(var i=0; i<n; i++) {
chr = this.input[0];
chars += chr;
this.yytext += chr;
this.yyleng++;
this.matched += chr;
this.match += chr;
if(chr === "\n") { this.yylineno++; }
this.input = this.input.slice(1);
}
return chr;
},
readchar: function(n, ignore) {
n = n || 1;
var chr;
for(var i=0; i<n; i++) {
chr = this.input[i];
if(chr === "\n") { this.yylineno++; }
this.matched += chr;
this.match += chr;
if(ignore) { this.readchars++; }
}
this.input = this.input.slice(n);
},
ignorechar: function(n) {
this.readchar(n, true);
},
peek: function(n) {
return this.input.slice(0, n || 1);
},
pastInput:function () {
var past = this.matched.substr(0, this.matched.length - this.match.length);
return (past.length > 20 ? '...':'') + past.substr(-20).replace(/\n/g, "");
},
upcomingInput:function () {
var next = this.match;
if (next.length < 20) {
next += this.input.substr(0, 20-next.length);
}
return (next.substr(0,20)+(next.length > 20 ? '...':'')).replace(/\n/g, "");
},
showPosition:function () {
var pre = this.pastInput();
var c = new Array(pre.length + 1 + this.readchars).join("-");
return pre + this.upcomingInput() + "\n" + c+"^";
}
};
Handlebars.Visitor = function() {};
Handlebars.Visitor.prototype = {
accept: function(object) {
return this[object.type](object);
}
};
// END(BROWSER)
exports.Lexer = Handlebars.Lexer;
exports.Visitor = Handlebars.Visitor;
lib/handlebars/printer.js
+1 -1
View File
@@ -1,5 +1,5 @@
var Handlebars = {};
Handlebars.Visitor = require("handlebars/jison_ext").Visitor;
Handlebars.Visitor = require("handlebars/visitor").Visitor;
// BEGIN(BROWSER)
Handlebars.PrintVisitor = function() { this.padding = 0; };
lib/handlebars/visitor.js
+14
View File
@@ -0,0 +1,14 @@
var Handlebars = {};
// BEGIN(BROWSER)
Handlebars.Visitor = function() {};
Handlebars.Visitor.prototype = {
accept: function(object) {
return this[object.type](object);
}
};
// END(BROWSER)
exports.Visitor = Handlebars.Visitor;
spec/spec_helper.rb
+1 -2
View File
@@ -71,8 +71,7 @@ module Handlebars
Handlebars::Spec.js_load('lib/handlebars/parser.js')
Handlebars::Spec.js_load('lib/handlebars/compiler.js');
Handlebars::Spec.js_load('lib/handlebars/ast.js');
Handlebars::Spec.js_load('lib/handlebars/jison_ext.js');
Handlebars::Spec.js_load('lib/handlebars/handlebars_lexer.js')
Handlebars::Spec.js_load('lib/handlebars/visitor.js');
Handlebars::Spec.js_load('lib/handlebars/printer.js')
Handlebars::Spec.js_load('lib/handlebars/runtime.js')
Handlebars::Spec.js_load('lib/handlebars/utils.js')
spec/tokenizer_spec.rb
+3 -3
View File
@@ -1,8 +1,8 @@
require "spec_helper"
describe "Tokenizer" do
let(:handlebars) { @context["Handlebars"] }
let(:lexer) { handlebars["HandlebarsLexer"].new }
let(:parser) { @context["handlebars"] }
let(:lexer) { @context["handlebars"]["lexer"] }
Token = Struct.new(:name, :text)
@@ -10,7 +10,7 @@ describe "Tokenizer" do
lexer.setInput(string)
out = []
while result = lexer.lex
while result = parser.terminals_[lexer.lex] and result != "EOF"
out << Token.new(result, lexer.yytext)
end
src/handlebars.l
+27
View File
@@ -0,0 +1,27 @@
%x mu
%%
(.|\s)*?/("{{") { this.begin("mu"); if (yytext) return 'CONTENT'; }
(.|\s)+ { return 'CONTENT'; }
<mu>"{{>" { return 'OPEN_PARTIAL'; }
<mu>"{{#" { return 'OPEN_BLOCK'; }
<mu>"{{/" { return 'OPEN_ENDBLOCK'; }
<mu>"{{^" { return 'OPEN_INVERSE'; }
<mu>"{{{" { return 'OPEN_UNESCAPED'; }
<mu>"{{&" { return 'OPEN_UNESCAPED'; }
<mu>"{{!".*?"}}" { yytext = yytext.substr(3,yyleng-5); this.begin("INITIAL"); return 'COMMENT'; }
<mu>"{{" { return 'OPEN'; }
<mu>"/" { return 'SEP'; }
<mu>\s+ { /*ignore whitespace*/ }
<mu>"}}}" { this.begin("INITIAL"); return 'CLOSE'; }
<mu>"}}" { this.begin("INITIAL"); return 'CLOSE'; }
<mu>'"'("\\"["]|[^"])*'"' { yytext = yytext.substr(1,yyleng-2).replace(/\\"/g,'"'); return 'STRING'; }
<mu>[a-zA-Z0-9_.]+/[} /] { return 'ID'; }
<mu>. { return 'INVALID'; }
<INITIAL,mu><<EOF>> { return 'EOF'; }
src/handlebars.yy
+1 -1
View File
@@ -3,7 +3,7 @@
%%
root
: program { return $1 }
: program EOF { return $1 }
;
program