Merge pull request #1873 from Shopify/bp/tokenizer-fix

Fix bug in tokenizer with nil source value
Shopify · Dec 17, 2024 · c3ac0e0 · c3ac0e0
2 parents fdd8c71 + a0a4307
commit c3ac0e0
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/lib/liquid/tokenizer.rb b/lib/liquid/tokenizer.rb
@@ -5,7 +5,7 @@ class Tokenizer
     attr_reader :line_number, :for_liquid_tag
 
     def initialize(source, line_numbers = false, line_number: nil, for_liquid_tag: false)
-      @source         = source
+      @source         = source.to_s
       @line_number    = line_number || (line_numbers ? 1 : nil)
       @for_liquid_tag = for_liquid_tag
       @offset         = 0

diff --git a/test/unit/tokenizer_unit_test.rb b/test/unit/tokenizer_unit_test.rb
@@ -30,6 +30,10 @@ def test_calculate_line_numbers_per_token_with_profiling
     assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} "))
   end
 
+  def test_tokenize_with_nil_source_returns_empty_array
+    assert_equal([], tokenize(nil))
+  end
+
   private
 
   def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)