Add tests for the tokenizer using RubyRacer

Rakefile

@@ -1,6 +1,15 @@
+require "rubygems"
+require "bundler/setup"
+
 file "lib/handlebars/parser.js" => "src/handlebars.yy" do
   system "jison src/handlebars.yy"
   sh "mv handlebars.js lib/handlebars/parser.js"
 end
 
-task :default => "lib/handlebars/parser.js"
+task :build => "lib/handlebars/parser.js"
+
+task :test => :build do
+  system "rspec -cfs spec"
+end
+
+task :default => [:build, :test]

spec/spec_helper.rb

@@ -0,0 +1,16 @@
+require "v8"
+
+RSpec.configure do |config|
+  config.before(:all) do
+    @context = V8::Context.new
+    @context['exports'] = nil
+    @context.eval("Handlebars = {}");
+
+    @context.load('lib/handlebars/ast.js')
+    @context.load('lib/handlebars/jison_ext.js')
+    @context.load('lib/handlebars/handlebars_lexer.js')
+    @context.load('lib/handlebars/printer.js')
+    @context.load('lib/handlebars/parser.js')
+    @context.load('lib/handlebars.js')
+  end
+end

spec/tokenizer_spec.rb

@@ -0,0 +1,119 @@
+require "spec_helper"
+
+describe "Tokenizer" do
+  let(:handlebars) { @context["Handlebars"] }
+  let(:lexer) { handlebars["HandlebarsLexer"].new }
+
+  Token = Struct.new(:name, :text)
+
+  def tokenize(string)
+    lexer.setInput(string)
+    out = []
+
+    while result = lexer.lex
+      out << Token.new(result, lexer.yytext)
+    end
+
+    out
+  end
+
+  RSpec::Matchers.define :match_tokens do |tokens|
+    match do |result|
+      result.map(&:name).should == tokens
+    end
+  end
+
+  RSpec::Matchers.define :be_token do |name, string|
+    match do |token|
+      token.name.should == name
+      token.text.should == string
+    end
+  end
+
+  it "tokenizes a simple mustache as 'OPEN ID CLOSE'" do
+    result = tokenize("{{foo}}")
+    result.should match_tokens(%w(OPEN ID CLOSE))
+    result[1].should be_token("ID", "foo")
+  end
+
+  it "tokenizes a simple mustahe with spaces as 'OPEN ID CLOSE'" do
+    result = tokenize("{{  foo  }}")
+    result.should match_tokens(%w(OPEN ID CLOSE))
+    result[1].should be_token("ID", "foo")
+  end
+
+  it "tokenizes raw content as 'CONTENT'" do
+    result = tokenize("foo {{ bar }} baz")
+    result.should match_tokens(%w(CONTENT OPEN ID CLOSE CONTENT))
+    result[0].should be_token("CONTENT", "foo ")
+    result[4].should be_token("CONTENT", " baz")
+  end
+
+  it "tokenizes a partial as 'OPEN_PARTIAL ID CLOSE'" do
+    result = tokenize("{{> foo}}")
+    result.should match_tokens(%w(OPEN_PARTIAL ID CLOSE))
+  end
+
+  it "tokenizes a partial without spaces as 'OPEN_PARTIAL ID CLOSE'" do
+    result = tokenize("{{>foo}}")
+    result.should match_tokens(%w(OPEN_PARTIAL ID CLOSE))
+  end
+
+  it "tokenizes a partial space at the end as 'OPEN_PARTIAL ID CLOSE'" do
+    result = tokenize("{{>foo  }}")
+    result.should match_tokens(%w(OPEN_PARTIAL ID CLOSE))
+  end
+
+  it "tokenizes a comment as 'COMMENT'" do
+    result = tokenize("foo {{! this is a comment }} bar {{ baz }}")
+    result.should match_tokens(%w(CONTENT COMMENT CONTENT OPEN ID CLOSE))
+    result[1].should be_token("COMMENT", " this is a comment ")
+  end
+
+  it "tokenizes open and closing blocks as 'OPEN_BLOCK ID CLOSE ... OPEN_ENDBLOCK ID CLOSE'" do
+    result = tokenize("{{#foo}}content{{/foo}}")
+    result.should match_tokens(%w(OPEN_BLOCK ID CLOSE CONTENT OPEN_ENDBLOCK ID CLOSE))
+  end
+
+  it "tokenizes inverse sections as 'OPEN_INVERSE CLOSE'" do
+    tokenize("{{^}}").should match_tokens(%w(OPEN_INVERSE CLOSE))
+  end
+
+  it "tokenizes inverse sections with ID as 'OPEN_INVERSE ID CLOSE'" do
+    result = tokenize("{{^foo}}")
+    result.should match_tokens(%w(OPEN_INVERSE ID CLOSE))
+    result[1].should be_token("ID", "foo")
+  end
+
+  it "tokenizes inverse sections with ID and spaces as 'OPEN_INVERSE ID CLOSE'" do
+    result = tokenize("{{^ foo  }}")
+    result.should match_tokens(%w(OPEN_INVERSE ID CLOSE))
+    result[1].should be_token("ID", "foo")
+  end
+
+  it "tokenizes mustaches with params as 'OPEN ID ID ID CLOSE'" do
+    result = tokenize("{{ foo bar baz }}")
+    result.should match_tokens(%w(OPEN ID ID ID CLOSE))
+    result[1].should be_token("ID", "foo")
+    result[2].should be_token("ID", "bar")
+    result[3].should be_token("ID", "baz")
+  end
+
+  it "tokenizes mustaches with String params as 'OPEN ID ID STRING CLOSE'" do
+    result = tokenize("{{ foo bar \"baz\" }}")
+    result.should match_tokens(%w(OPEN ID ID STRING CLOSE))
+    result[3].should be_token("STRING", "baz")
+  end
+
+  it "tokenizes String params with spaces inside as 'STRING'" do
+    result = tokenize("{{ foo bar \"baz bat\" }}")
+    result.should match_tokens(%w(OPEN ID ID STRING CLOSE))
+    result[3].should be_token("STRING", "baz bat")
+  end
+
+  it "tokenizes String params with escapes quotes as 'STRING'" do
+    result = tokenize(%|{{ foo "bar\\"baz" }}|)
+    result.should match_tokens(%w(OPEN ID STRING CLOSE))
+    result[2].should be_token("STRING", %{bar"baz})
+  end
+end