tweaks to open llm implementation

testing mistral at the moment and did the minimum to get it going

assets/javascripts/initializers/ai-bot-replies.js

@@ -8,7 +8,7 @@ import { registerWidgetShim } from "discourse/widgets/render-glimmer";
 import { hbs } from "ember-cli-htmlbars";
 
 function isGPTBot(user) {
-  return user && [-110, -111, -112].includes(user.id);
+  return user && [-110, -111, -112, -113].includes(user.id);
 }
 
 function attachHeaderIcon(api) {

config/locales/client.en.yml

@@ -88,6 +88,7 @@ en:
           gpt-3:
             5-turbo: "GPT-3.5"
           claude-2: "Claude 2"
+          open-llm: "Open LLM"
 
     review:
       types:

config/settings.yml

@@ -228,6 +228,7 @@ discourse_ai:
       - gpt-3.5-turbo
       - gpt-4
       - claude-2
+      - open-llm
   ai_bot_enabled_chat_commands:
     type: list
     default: "categories|google|image|search|tags|time|read"

lib/modules/ai_bot/bot.rb

@@ -73,7 +73,11 @@ module DiscourseAi
       MAX_COMPLETIONS = 5
 
       def self.as(bot_user)
-        available_bots = [DiscourseAi::AiBot::OpenAiBot, DiscourseAi::AiBot::AnthropicBot]
+        available_bots = [
+          DiscourseAi::AiBot::OpenAiBot,
+          DiscourseAi::AiBot::AnthropicBot,
+          DiscourseAi::AiBot::OpenLlmBot,
+        ]
 
         bot =
           available_bots.detect(-> { raise BOT_NOT_FOUND }) do |bot_klass|

lib/modules/ai_bot/entry_point.rb

@@ -8,7 +8,14 @@ module DiscourseAi
       GPT4_ID = -110
       GPT3_5_TURBO_ID = -111
       CLAUDE_V2_ID = -112
-      BOTS = [[GPT4_ID, "gpt4_bot"], [GPT3_5_TURBO_ID, "gpt3.5_bot"], [CLAUDE_V2_ID, "claude_bot"]]
+      OPEN_LLM_ID = -113
+
+      BOTS = [
+        [GPT4_ID, "gpt4_bot"],
+        [GPT3_5_TURBO_ID, "gpt3.5_bot"],
+        [CLAUDE_V2_ID, "claude_bot"],
+        [OPEN_LLM_ID, "open_llm_bot"],
+      ]
 
       def self.map_bot_model_to_user_id(model_name)
         case model_name
@@ -18,6 +25,8 @@ module DiscourseAi
           GPT4_ID
         in "claude-2"
           CLAUDE_V2_ID
+        in "open-llm"
+          OPEN_LLM_ID
         else
           nil
         end
@@ -29,6 +38,7 @@ module DiscourseAi
         require_relative "bot"
         require_relative "anthropic_bot"
         require_relative "open_ai_bot"
+        require_relative "open_llm_bot"
         require_relative "commands/command"
         require_relative "commands/search_command"
         require_relative "commands/categories_command"

lib/modules/ai_bot/open_llm_bot.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module AiBot
+    class OpenLlmBot < Bot
+      # format of thebloke chat models is:
+      # [INST] <<SYS>> sys message <</SYS>> {prompt} [/INST] {model_reply} [INST] {prompt} [/INST]
+
+      def self.can_reply_as?(bot_user)
+        bot_user.id == DiscourseAi::AiBot::EntryPoint::OPEN_LLM_ID
+      end
+
+      def bot_prompt_with_topic_context(post)
+        messages = super(post)
+
+        # start with system
+        result = +""
+        result << "<s>[INST] <<SYS>>\n #{messages.shift[:content]} <</SYS>>\n\n #{messages.shift[:content]} [/INST]"
+
+        messages.each do |message|
+          if message[:bot]
+            result << message[:content]
+          else
+            result << "</s><s>[INST]#{message[:bot] ? "" : message[:username] + ":"} #{message[:content]} [/INST]"
+          end
+        end
+
+        result
+      end
+
+      def prompt_limit
+        2000
+      end
+
+      def title_prompt(post)
+        super(post).join("\n\n") + "\n\nAssistant:"
+      end
+
+      def get_delta(partial, context)
+        partial.dig(:token, :text) || ""
+      end
+
+      private
+
+      def build_message(poster_username, content, system: false, function: nil)
+        { bot: poster_username == bot_user.username, username: poster_username, content: content }
+      end
+
+      def model_for
+        # we only support single model hosting for huggingface api for now
+        "random-string-for-now"
+      end
+
+      def get_updated_title(prompt)
+        DiscourseAi::Inference::HuggingFaceTextGeneration.perform!(
+          prompt,
+          model_for,
+          temperature: 0.7,
+          max_tokens: 40,
+        ).dig(:completion)
+      end
+
+      def submit_prompt(prompt, prefer_low_cost: false, &blk)
+        DiscourseAi::Inference::HuggingFaceTextGeneration.perform!(
+          prompt,
+          model_for,
+          temperature: 0.4,
+          max_tokens: 1000,
+          &blk
+        )
+      end
+
+      def tokenize(text)
+        DiscourseAi::Tokenizer::AnthropicTokenizer.tokenize(text)
+      end
+    end
+  end
+end

lib/modules/summarization/models/llama2.rb

@@ -20,9 +20,9 @@ module DiscourseAi
           )
         end
 
-        def concatenate_summaries(summaries, &on_partial_blk)
+        def concatenate_summaries(summaries)
-          prompt = <<~TEXT
+          completion(<<~TEXT)
-            [INST] <<SYS>>
+            <s>[INST] <<SYS>>
             You are a helpful bot
             <</SYS>>