Inline HtmlTokenizer in this gem

This gem is the only user of that library, so it is better to inline
here than to depend on a separate gem that we also have to maintain.

.gitignore

@@ -34,3 +34,5 @@ bower.json
 
 # Ignore pow environment settings
 .powenv
+
+*.so

Gemfile

@@ -4,8 +4,8 @@ gemspec
 
 gem 'actionview'
 gem 'rake'
+gem 'rake-compiler'
 gem 'minitest'
-gem 'html_tokenizer'
 gem 'mocha'
 
 group :deployment, :test do

Gemfile.lock

@@ -6,7 +6,6 @@ PATH
       activesupport (>= 4.0)
       ast (~> 2.0)
       erubi (~> 1.4)
-      html_tokenizer (~> 0.0.6)
       parser (>= 2.4)
       smart_properties
 
@@ -31,7 +30,6 @@ GEM
     concurrent-ruby (1.0.5)
     crass (1.0.6)
     erubi (1.6.1)
-    html_tokenizer (0.0.6)
     i18n (0.8.6)
     loofah (2.18.0)
       crass (~> 1.0.2)
@@ -60,6 +58,8 @@ GEM
     rails-html-sanitizer (1.4.3)
       loofah (~> 2.3)
     rake (13.0.6)
+    rake-compiler (1.2.0)
+      rake
     smart_properties (1.15.0)
     thread_safe (0.3.6)
     tzinfo (1.2.10)
@@ -71,11 +71,11 @@ PLATFORMS
 DEPENDENCIES
   actionview
   better_html!
-  html_tokenizer
   minitest
   mocha
   pry-byebug
   rake
+  rake-compiler
 
 BUNDLED WITH
    2.2.22

MIT-LICENSE

@@ -1,3 +1,4 @@
+Copyright 2022-present Shopify Inc.
 Copyright 2016 Francois Chagnon
 
 Permission is hereby granted, free of charge, to any person obtaining

Rakefile

@@ -5,6 +5,10 @@ rescue LoadError
   puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
 end
 
+require 'rake/extensiontask'
+
+Rake::ExtensionTask.new("better_html_ext")
+
 require 'rdoc/task'
 
 RDoc::Task.new(:rdoc) do |rdoc|
@@ -25,4 +29,4 @@ Rake::TestTask.new(:test) do |t|
 end
 
 
-task default: :test
+task default: [:compile, :test]

better_html.gemspec

@@ -21,17 +21,17 @@ Gem::Specification.new do |s|
     "allowed_push_host" => "https://rubygems.org"
   }
 
+  s.extensions    = ['ext/better_html_ext/extconf.rb']
   s.files = Dir["{app,config,db,lib,ext}/**/*", "MIT-LICENSE", "Rakefile", "README.rdoc"]
   s.test_files = Dir["test/**/*"]
-  s.require_paths = ["lib"]
+  s.require_paths = ["lib", "ext"]
 
   s.add_dependency 'ast', '~> 2.0'
   s.add_dependency 'erubi', '~> 1.4'
   s.add_dependency 'activesupport', '>= 4.0'
   s.add_dependency 'actionview', '>= 4.0'
   s.add_dependency 'parser', '>= 2.4'
   s.add_dependency 'smart_properties'
-  s.add_dependency 'html_tokenizer', '~> 0.0.6'
 
   s.add_development_dependency 'rake', '~> 13'
 end

ext/better_html_ext/better_html.h

@@ -0,0 +1,2 @@
+#include <ruby.h>
+#include "html_tokenizer.h"

ext/better_html_ext/extconf.rb

@@ -0,0 +1,12 @@
+require 'mkmf'
+
+$CXXFLAGS += " -std=c++11 "
+$CXXFLAGS += " -g -O1 -ggdb "
+$CFLAGS += " -g -O1 -ggdb "
+
+if ENV['DEBUG']
+  $CXXFLAGS += "  -DDEBUG "
+  $CFLAGS += "  -DDEBUG "
+end
+
+create_makefile('better_html_ext')

ext/better_html_ext/html_tokenizer.c

@@ -0,0 +1,12 @@
+#include <ruby.h>
+#include "tokenizer.h"
+#include "parser.h"
+
+static VALUE mHtmlTokenizer = Qnil;
+
+void Init_better_html_ext()
+{
+  mHtmlTokenizer = rb_define_module("HtmlTokenizer");
+  Init_html_tokenizer_tokenizer(mHtmlTokenizer);
+  Init_html_tokenizer_parser(mHtmlTokenizer);
+}

ext/better_html_ext/html_tokenizer.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#ifdef DEBUG
+#define DBG_PRINT(msg, arg...) printf("%s:%u: " msg "\n", __FUNCTION__, __LINE__, arg);
+#else
+#define DBG_PRINT(msg, arg...) ((void)0);
+#endif