FIX: OpenAI Tokenizer was failing to truncate mid emojis (#91)

* FIX: OpenAI Tokenizer was failing to truncate mid emojis

* Update spec/shared/tokenizer.rb

Co-authored-by: Joffrey JAFFEUX <[email protected]>

---------

Co-authored-by: Joffrey JAFFEUX <[email protected]>

lib/shared/tokenizer/tokenizer.rb

@@ -49,6 +49,9 @@ module DiscourseAi
         return text if text.size < max_length
 
         tokenizer.decode(tokenize(text).take(max_length))
+      rescue Tiktoken::UnicodeError
+        max_length = max_length - 1
+        retry
       end
     end
   end

spec/shared/tokenizer.rb

@@ -76,5 +76,10 @@ describe DiscourseAi::Tokenizer::OpenAiTokenizer do
       sentence = "foo bar baz qux quux corge grault garply waldo fred plugh xyzzy thud"
       expect(described_class.truncate(sentence, 3)).to eq("foo bar baz")
     end
+
+    it "truncates a sentence successfully at a multibyte unicode character" do
+      sentence = "foo bar 👨🏿‍👩🏿‍👧🏿‍👧🏿 baz qux quux corge grault garply waldo fred plugh xyzzy thud"
+      expect(described_class.truncate(sentence, 7)).to eq("foo bar 👨🏿")
+    end
   end
 end