DEV: Update megapixel requirement for small image

When navigating between topic we were not correctly resetting
internal state for summarization. This leads to a situation where
incorrect summaries can be displayed to users and wrong summaries
can be displayed.

Additionally our controller for grabbing summaries was always
streaming results via message bus, which could be delayed when
sidekiq is overloaded. We now will return the cached summary
right away if it is available direct from REST endpoint.

.discourse-compatibility

@@ -1,8 +1,6 @@
+< 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
+< 3.3.0.beta5-dev: 4d8090002f6dcd8e34d41033606bf131fa221475
 < 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
-< 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
+< 3.3.0.beta1-dev: d94234d89f50733678a5c62809445198a9fda74b
-< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
+3.2.999: d94234d89f50733678a5c62809445198a9fda74b
-< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
-< 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
 3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
-< 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
-3.1.0.beta3: 057fbe1ce6

Gemfile.lock

@@ -32,7 +32,7 @@ GEM
     rack (3.1.6)
     rainbow (3.1.1)
     regexp_parser (2.9.2)
-    rexml (3.3.1)
+    rexml (3.3.3)
       strscan
     rubocop (1.64.1)
       json (~> 2.3)

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-new.js

@@ -3,6 +3,7 @@ import DiscourseRoute from "discourse/routes/discourse";
 export default DiscourseRoute.extend({
   async model() {
     const record = this.store.createRecord("ai-llm");
+    record.provider_params = {};
     return record;
   },
 

app/controllers/discourse_ai/admin/ai_llms_controller.rb

@@ -38,7 +38,7 @@ module DiscourseAi
           llm_model.toggle_companion_user
           render json: { ai_persona: LlmModelSerializer.new(llm_model) }, status: :created
         else
-          render_json_error LlmModelSerializer.new(llm_model)
+          render_json_error llm_model
         end
       end
 
@@ -49,7 +49,7 @@ module DiscourseAi
           llm_model.toggle_companion_user
           render json: LlmModelSerializer.new(llm_model)
         else
-          render_json_error LlmModelSerializer.new(llm_model)
+          render_json_error llm_model
         end
       end
 
@@ -97,6 +97,8 @@ module DiscourseAi
       private
 
       def ai_llm_params(updating: nil)
+        return {} if params[:ai_llm].blank?
+
         permitted =
           params.require(:ai_llm).permit(
             :display_name,
@@ -106,14 +108,14 @@ module DiscourseAi
             :max_prompt_tokens,
             :api_key,
             :enabled_chat_bot,
+            :vision_enabled,
           )
 
         provider = updating ? updating.provider : permitted[:provider]
-        permit_url =
+        permit_url = provider != LlmModel::BEDROCK_PROVIDER_NAME
-          (updating && updating.url != LlmModel::RESERVED_VLLM_SRV_URL) ||
-            provider != LlmModel::BEDROCK_PROVIDER_NAME
 
-        permitted[:url] = params.dig(:ai_llm, :url) if permit_url
+        new_url = params.dig(:ai_llm, :url)
+        permitted[:url] = new_url if permit_url && new_url
 
         extra_field_names = LlmModel.provider_params.dig(provider&.to_sym, :fields).to_a
         received_prov_params = params.dig(:ai_llm, :provider_params)

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -63,7 +63,7 @@ module DiscourseAi
 
           render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
         else
-          render_json_error LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
+          render_json_error @ai_persona
         end
       end
 
@@ -71,7 +71,7 @@ module DiscourseAi
         if @ai_persona.destroy
           head :no_content
         else
-          render_json_error LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
+          render_json_error @ai_persona
         end
       end
 

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -81,7 +81,7 @@ module DiscourseAi
           :description,
           :script,
           :summary,
-          parameters: [:name, :type, :description, :required, :enum, enum_values: []],
+          parameters: [:name, :type, :description, :required, enum: []],
         )
       end
     end

app/controllers/discourse_ai/ai_helper/assistant_controller.rb

@@ -89,18 +89,26 @@ module DiscourseAi
         end
       end
 
-      def explain
+      def stream_suggestion
         post_id = get_post_param!
-        term_to_explain = get_text_param!
+        text = get_text_param!
         post = Post.includes(:topic).find_by(id: post_id)
+        prompt = CompletionPrompt.find_by(id: params[:mode])
 
+        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
         raise Discourse::InvalidParameters.new(:post_id) unless post
 
+        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+          raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
+        end
+
         Jobs.enqueue(
           :stream_post_helper,
           post_id: post.id,
           user_id: current_user.id,
-          term_to_explain: term_to_explain,
+          text: text,
+          prompt: prompt.name,
+          custom_prompt: params[:custom_prompt],
         )
 
         render json: { success: true }, status: 200
@@ -157,9 +165,13 @@ module DiscourseAi
       end
 
       def ensure_can_request_suggestions
-        if !current_user.in_any_groups?(SiteSetting.ai_helper_allowed_groups_map)
+        allowed_groups =
-          raise Discourse::InvalidAccess
+          (
-        end
+            SiteSetting.composer_ai_helper_allowed_groups_map |
+              SiteSetting.post_ai_helper_allowed_groups_map
+          )
+
+        raise Discourse::InvalidAccess if !current_user.in_any_groups?(allowed_groups)
       end
     end
   end

app/controllers/discourse_ai/summarization/summary_controller.rb

@@ -14,19 +14,32 @@ module DiscourseAi
         RateLimiter.new(current_user, "summary", 6, 5.minutes).performed! if current_user
 
         opts = params.permit(:skip_age_check)
+        skip_age_check = opts[:skip_age_check] == "true"
 
         if params[:stream] && current_user
+          cached_summary = DiscourseAi::TopicSummarization.cached_summary(topic, current_user)
+
+          if cached_summary && !skip_age_check
+            render_serialized(cached_summary, AiTopicSummarySerializer)
+            return
+          end
+
           Jobs.enqueue(
             :stream_topic_ai_summary,
             topic_id: topic.id,
             user_id: current_user.id,
-            opts: opts.as_json,
+            skip_age_check: skip_age_check,
           )
 
           render json: success_json
         else
           hijack do
-            summary = DiscourseAi::TopicSummarization.summarize(topic, current_user, opts)
+            summary =
+              DiscourseAi::TopicSummarization.summarize(
+                topic,
+                current_user,
+                skip_age_check: skip_age_check,
+              )
             render_serialized(summary, AiTopicSummarySerializer)
           end
         end

app/jobs/regular/stream_post_helper.rb

@@ -7,27 +7,35 @@ module Jobs
     def execute(args)
       return unless post = Post.includes(:topic).find_by(id: args[:post_id])
       return unless user = User.find_by(id: args[:user_id])
-      return unless args[:term_to_explain]
+      return unless args[:text]
 
       topic = post.topic
       reply_to = post.reply_to_post
 
       return unless user.guardian.can_see?(post)
 
-      prompt = CompletionPrompt.enabled_by_name("explain")
+      prompt = CompletionPrompt.enabled_by_name(args[:prompt])
 
-      input = <<~TEXT
+      if prompt.id == CompletionPrompt::CUSTOM_PROMPT
-      <term>#{args[:term_to_explain]}</term>
+        prompt.custom_instruction = args[:custom_prompt]
+      end
+
+      if prompt.name == "explain"
+        input = <<~TEXT
+      <term>#{args[:text]}</term>
       <context>#{post.raw}</context>
           <topic>#{topic.title}</topic>
           #{reply_to ? "<replyTo>#{reply_to.raw}</replyTo>" : nil}
         TEXT
+      else
+        input = args[:text]
+      end
 
       DiscourseAi::AiHelper::Assistant.new.stream_prompt(
         prompt,
         input,
         user,
-        "/discourse-ai/ai-helper/explain/#{post.id}",
+        "/discourse-ai/ai-helper/stream_suggestion/#{post.id}",
       )
     end
   end

app/jobs/regular/stream_topic_ai_summary.rb

@@ -14,15 +14,15 @@ module Jobs
       guardian = Guardian.new(user)
       return unless guardian.can_see?(topic)
 
-      opts = args[:opts] || {}
+      skip_age_check = !!args[:skip_age_check]
 
       streamed_summary = +""
       start = Time.now
 
       summary =
         DiscourseAi::TopicSummarization
-          .new(strategy)
+          .new(strategy, topic, user)
-          .summarize(topic, user, opts) do |partial_summary|
+          .summarize(skip_age_check: skip_age_check) do |partial_summary|
             streamed_summary << partial_summary
 
             # Throttle updates.

app/models/llm_model.rb

@@ -2,43 +2,17 @@
 
 class LlmModel < ActiveRecord::Base
   FIRST_BOT_USER_ID = -1200
-  RESERVED_VLLM_SRV_URL = "https://vllm.shadowed-by-srv.invalid"
   BEDROCK_PROVIDER_NAME = "aws_bedrock"
 
   belongs_to :user
 
-  validates :url, exclusion: { in: [RESERVED_VLLM_SRV_URL] }, if: :url_changed?
+  validates :display_name, presence: true, length: { maximum: 100 }
-
+  validates :tokenizer, presence: true, inclusion: DiscourseAi::Completions::Llm.tokenizer_names
-  def self.seed_srv_backed_model
+  validates :provider, presence: true, inclusion: DiscourseAi::Completions::Llm.provider_names
-    srv = SiteSetting.ai_vllm_endpoint_srv
+  validates :url, presence: true, unless: -> { provider == BEDROCK_PROVIDER_NAME }
-    srv_model = find_by(url: RESERVED_VLLM_SRV_URL)
+  validates_presence_of :name, :api_key
-
+  validates :max_prompt_tokens, numericality: { greater_than: 0 }
-    if srv.present?
+  validate :required_provider_params
-      if srv_model.present?
-        current_key = SiteSetting.ai_vllm_api_key
-        srv_model.update!(api_key: current_key) if current_key != srv_model.api_key
-      else
-        record =
-          new(
-            display_name: "vLLM SRV LLM",
-            name: "mistralai/Mixtral-8x7B-Instruct-v0.1",
-            provider: "vllm",
-            tokenizer: "DiscourseAi::Tokenizer::MixtralTokenizer",
-            url: RESERVED_VLLM_SRV_URL,
-            max_prompt_tokens: 8000,
-            api_key: SiteSetting.ai_vllm_api_key,
-          )
-
-        record.save(validate: false) # Ignore reserved URL validation
-      end
-    else
-      # Clean up companion users
-      srv_model&.enabled_chat_bot = false
-      srv_model&.toggle_companion_user
-
-      srv_model&.destroy!
-    end
-  end
 
   def self.provider_params
     {
@@ -54,7 +28,7 @@ class LlmModel < ActiveRecord::Base
   end
 
   def to_llm
-    DiscourseAi::Completions::Llm.proxy_from_obj(self)
+    DiscourseAi::Completions::Llm.proxy("custom:#{id}")
   end
 
   def toggle_companion_user
@@ -106,6 +80,18 @@ class LlmModel < ActiveRecord::Base
   def lookup_custom_param(key)
     provider_params&.dig(key)
   end
+
+  private
+
+  def required_provider_params
+    return if provider != BEDROCK_PROVIDER_NAME
+
+    %w[access_key_id region].each do |field|
+      if lookup_custom_param(field).blank?
+        errors.add(:base, I18n.t("discourse_ai.llm_models.missing_provider_param", param: field))
+      end
+    end
+  end
 end
 
 # == Schema Information
@@ -124,4 +110,6 @@ end
 #  api_key           :string
 #  user_id           :integer
 #  enabled_chat_bot  :boolean          default(FALSE), not null
+#  provider_params   :jsonb
+#  vision_enabled    :boolean          default(FALSE), not null
 #

app/serializers/ai_topic_summary_serializer.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
 class AiTopicSummarySerializer < ApplicationSerializer
-  attributes :summarized_text, :algorithm, :outdated, :can_regenerate, :new_posts_since_summary
+  attributes :summarized_text,
+             :algorithm,
+             :outdated,
+             :can_regenerate,
+             :new_posts_since_summary,
+             :updated_at
 
   def can_regenerate
     scope.can_request_summary?

app/serializers/llm_model_serializer.rb

@@ -12,12 +12,13 @@ class LlmModelSerializer < ApplicationSerializer
              :api_key,
              :url,
              :enabled_chat_bot,
-             :shadowed_by_srv,
+             :provider_params,
-             :provider_params
+             :vision_enabled,
+             :used_by
 
   has_one :user, serializer: BasicUserSerializer, embed: :object
 
-  def shadowed_by_srv
+  def used_by
-    object.url == LlmModel::RESERVED_VLLM_SRV_URL
+    DiscourseAi::Configuration::LlmValidator.new.modules_using(object)
   end
 end

app/services/discourse_ai/topic_summarization.rb

@@ -2,33 +2,36 @@
 
 module DiscourseAi
   class TopicSummarization
-    def self.summarize(topic, user, opts = {}, &on_partial_blk)
+    def self.summarize(topic, user, skip_age_check: false, &on_partial_blk)
-      new(DiscourseAi::Summarization.default_strategy).summarize(topic, user, opts, &on_partial_blk)
+      new(DiscourseAi::Summarization.default_strategy, topic, user).summarize(
+        skip_age_check: skip_age_check,
+        &on_partial_blk
+      )
     end
 
-    def initialize(strategy)
+    def self.cached_summary(topic, user)
+      new(DiscourseAi::Summarization.default_strategy, topic, user).cached_summary
+    end
+
+    def initialize(strategy, topic, user)
       @strategy = strategy
+      @topic = topic
+      @user = user
     end
 
-    def summarize(topic, user, opts = {}, &on_partial_blk)
+    attr_reader :strategy, :topic, :user
-      existing_summary = AiSummary.find_by(target: topic)
 
+    def cached_summary
+      existing_summary
+    end
+
+    def summarize(skip_age_check: false, &on_partial_blk)
       # Existing summary shouldn't be nil in this scenario because the controller checks its existence.
       return if !user && !existing_summary
 
-      targets_data = summary_targets(topic).pluck(:post_number, :raw, :username)
+      return existing_summary if use_cached?(skip_age_check)
 
-      current_topic_sha = build_sha(targets_data.map(&:first))
+      delete_cached_summaries! if existing_summary
-      can_summarize = Guardian.new(user).can_request_summary?
-
-      if use_cached?(existing_summary, can_summarize, current_topic_sha, !!opts[:skip_age_check])
-        # It's important that we signal a cached summary is outdated
-        existing_summary.mark_as_outdated if new_targets?(existing_summary, current_topic_sha)
-
-        return existing_summary
-      end
-
-      delete_cached_summaries_of(topic) if existing_summary
 
       content = {
         resource_path: "#{Discourse.base_path}/t/-/#{topic.id}",
@@ -36,7 +39,7 @@ module DiscourseAi
         contents: [],
       }
 
-      targets_data.map do |(pn, raw, username)|
+      summary_targets_data.map do |(pn, raw, username)|
         raw_text = raw
 
         if pn == 1 && topic.topic_embed&.embed_content_cache.present?
@@ -47,19 +50,34 @@ module DiscourseAi
       end
 
       summarization_result = strategy.summarize(content, user, &on_partial_blk)
-
+      cache_summary(summarization_result)
-      cache_summary(summarization_result, targets_data.map(&:first), topic)
     end
 
-    def summary_targets(topic)
+    def summary_targets
-      topic.has_summary? ? best_replies(topic) : pick_selection(topic)
+      topic.has_summary? ? best_replies : pick_selection
     end
 
     private
 
-    attr_reader :strategy
+    def summary_sha
+      @summary_sha ||= build_sha(summary_targets_data.map(&:first))
+    end
 
-    def best_replies(topic)
+    def summary_targets_data
+      @summary_targets_data ||= summary_targets.pluck(:post_number, :raw, :username)
+    end
+
+    def existing_summary
+      if !defined?(@existing_summary)
+        @existing_summary = AiSummary.find_by(target: topic)
+        if @existing_summary && existing_summary.original_content_sha != summary_sha
+          @existing_summary.mark_as_outdated
+        end
+      end
+      @existing_summary
+    end
+
+    def best_replies
       Post
         .summary(topic.id)
         .where("post_type = ?", Post.types[:regular])
@@ -68,7 +86,7 @@ module DiscourseAi
         .order(:post_number)
     end
 
-    def pick_selection(topic)
+    def pick_selection
       posts =
         Post
           .where(topic_id: topic.id)
@@ -87,31 +105,34 @@ module DiscourseAi
         .order(:post_number)
     end
 
-    def delete_cached_summaries_of(topic)
+    def delete_cached_summaries!
       AiSummary.where(target: topic).destroy_all
     end
 
-    # For users without permissions to generate a summary or fresh summaries, we return what we have cached.
+    def use_cached?(skip_age_check)
-    def use_cached?(existing_summary, can_summarize, current_sha, skip_age_check)
+      can_summarize = Guardian.new(user).can_request_summary?
+
       existing_summary &&
         !(
-          can_summarize && new_targets?(existing_summary, current_sha) &&
+          can_summarize && new_targets? &&
             (skip_age_check || existing_summary.created_at < 1.hour.ago)
         )
     end
 
-    def new_targets?(summary, current_sha)
+    def new_targets?
-      summary.original_content_sha != current_sha
+      existing_summary&.original_content_sha != summary_sha
     end
 
-    def cache_summary(result, post_numbers, topic)
+    def cache_summary(result)
+      post_numbers = summary_targets_data.map(&:first)
+
       cached_summary =
         AiSummary.create!(
           target: topic,
           algorithm: strategy.display_name,
           content_range: (post_numbers.first..post_numbers.last),
           summarized_text: result[:summary],
-          original_content_sha: build_sha(post_numbers),
+          original_content_sha: summary_sha,
         )
 
       cached_summary

assets/javascripts/discourse/admin/models/ai-llm.js

@@ -13,7 +13,8 @@ export default class AiLlm extends RestModel {
       "url",
       "api_key",
       "enabled_chat_bot",
-      "provider_params"
+      "provider_params",
+      "vision_enabled"
     );
   }
 

assets/javascripts/discourse/admin/models/ai-tool.js

@@ -27,9 +27,13 @@ export default class AiTool extends RestModel {
       attrs.parameters?.map((p) => {
         const parameter = new TrackedObject(p);
 
+        //Backwards-compatibility code.
+        // TODO(roman): Remove aug 2024. Leave only else clause.
         if (parameter.enum_values) {
-          parameter.enumValues = new TrackedArray(parameter.enum_values);
+          parameter.enum = new TrackedArray(parameter.enum_values);
           delete parameter.enum_values;
+        } else {
+          parameter.enum = new TrackedArray(parameter.enum);
         }
 
         return parameter;

assets/javascripts/discourse/components/ai-helper-button-group.gjs

@@ -0,0 +1,19 @@
+import DButton from "discourse/components/d-button";
+import concatClass from "discourse/helpers/concat-class";
+
+const AiHelperButtonGroup = <template>
+  <ul class="ai-helper-button-group" ...attributes>
+    {{#each @buttons as |button|}}
+      <li>
+        <DButton
+          @icon={{button.icon}}
+          @label={{button.label}}
+          @action={{button.action}}
+          class={{concatClass "btn-flat" button.classes}}
+        />
+      </li>
+    {{/each}}
+  </ul>
+</template>;
+
+export default AiHelperButtonGroup;

assets/javascripts/discourse/components/ai-helper-custom-prompt.gjs

@@ -1,38 +1,40 @@
 import Component from "@glimmer/component";
-import { tracked } from "@glimmer/tracking";
-import { Input } from "@ember/component";
 import { fn } from "@ember/helper";
+import { on } from "@ember/modifier";
 import { action } from "@ember/object";
-import didInsert from "@ember/render-modifiers/modifiers/did-insert";
 import DButton from "discourse/components/d-button";
+import withEventValue from "discourse/helpers/with-event-value";
+import autoFocus from "discourse/modifiers/auto-focus";
 import i18n from "discourse-common/helpers/i18n";
 import not from "truth-helpers/helpers/not";
 
 export default class AiHelperCustomPrompt extends Component {
-  @tracked _customPromptInput;
-
   @action
-  setupCustomPrompt() {
+  sendInput(event) {
-    this._customPromptInput = document.querySelector(
+    if (event.key !== "Enter") {
-      ".ai-custom-prompt__input"
+      return;
-    );
+    }
-    this._customPromptInput.focus();
-  }
-
-  @action
-  sendInput() {
     return this.args.submit(this.args.promptArgs);
   }
 
   <template>
-    <div class="ai-custom-prompt" {{didInsert this.setupCustomPrompt}}>
+    <div class="ai-custom-prompt">
-      <Input
+
-        @value={{@value}}
+      <input
+        {{on "input" (withEventValue (fn (mut @value)))}}
+        {{on "keydown" this.sendInput}}
+        value={{@value}}
         placeholder={{i18n
           "discourse_ai.ai_helper.context_menu.custom_prompt.placeholder"
         }}
         class="ai-custom-prompt__input"
-        @enter={{this.sendInput}}
+        type="text"
+        {{!-- Using {{autoFocus}} helper instead of built in autofocus="autofocus"
+            because built in autofocus doesn't work consistently when component is
+            invoked twice separetly without a page refresh.
+            (i.e. trigger in post AI helper followed by trigger in composer AI helper)
+      --}}
+        {{autoFocus}}
       />
 
       <DButton

assets/javascripts/discourse/components/ai-helper-loading.gjs

@@ -2,7 +2,7 @@ import DButton from "discourse/components/d-button";
 import i18n from "discourse-common/helpers/i18n";
 
 const AiHelperLoading = <template>
-  <div class="ai-helper-context-menu__loading">
+  <div class="ai-helper-loading">
     <div class="dot-falling"></div>
     <span>
       {{i18n "discourse_ai.ai_helper.context_menu.loading"}}

assets/javascripts/discourse/components/ai-helper-options-list.gjs

@@ -0,0 +1,31 @@
+import { fn } from "@ember/helper";
+import DButton from "discourse/components/d-button";
+import eq from "truth-helpers/helpers/eq";
+import AiHelperCustomPrompt from "../components/ai-helper-custom-prompt";
+
+const AiHelperOptionsList = <template>
+  <ul class="ai-helper-options">
+    {{#each @options as |option|}}
+      {{#if (eq option.name "custom_prompt")}}
+        <AiHelperCustomPrompt
+          @value={{@customPromptValue}}
+          @promptArgs={{option}}
+          @submit={{@performAction}}
+        />
+      {{else}}
+        <li data-name={{option.translated_name}} data-value={{option.id}}>
+          <DButton
+            @icon={{option.icon}}
+            @translatedLabel={{option.translated_name}}
+            @action={{fn @performAction option}}
+            data-name={{option.name}}
+            data-value={{option.id}}
+            class="btn-flat ai-helper-options__button"
+          />
+        </li>
+      {{/if}}
+    {{/each}}
+  </ul>
+</template>;
+
+export default AiHelperOptionsList;

assets/javascripts/discourse/components/ai-llm-editor-form.gjs

@@ -30,14 +30,6 @@ export default class AiLlmEditorForm extends Component {
   @tracked testError = null;
   @tracked apiKeySecret = true;
 
-  didReceiveAttrs() {
-    super.didReceiveAttrs(...arguments);
-
-    if (!this.args.model.provider_params) {
-      this.populateProviderParams(this.args.model.provider);
-    }
-  }
-
   get selectedProviders() {
     const t = (provName) => {
       return I18n.t(`discourse_ai.llms.providers.${provName}`);
@@ -60,17 +52,20 @@ export default class AiLlmEditorForm extends Component {
     return this.testRunning || this.testResult !== null;
   }
 
-  get displaySRVWarning() {
-    return this.args.model.shadowed_by_srv && !this.args.model.isNew;
-  }
-
   get canEditURL() {
     // Explicitly false.
-    if (this.metaProviderParams.url_editable === false) {
+    return this.metaProviderParams.url_editable !== false;
-      return false;
+  }
-    }
 
-    return !this.args.model.shadowed_by_srv || this.args.model.isNew;
+  get modulesUsingModel() {
+    return this.args.model.used_by?.join(", ");
+  }
+
+  get inUseWarning() {
+    return I18n.t("discourse_ai.llms.in_use_warning", {
+      settings: this.modulesUsingModel,
+      count: this.args.model.used_by.length,
+    });
   }
 
   @computed("args.model.provider")
@@ -174,10 +169,10 @@ export default class AiLlmEditorForm extends Component {
   }
 
   <template>
-    {{#if this.displaySRVWarning}}
+    {{#if this.modulesUsingModel}}
       <div class="alert alert-info">
         {{icon "exclamation-circle"}}
-        {{I18n.t "discourse_ai.llms.srv_warning"}}
+        {{this.inUseWarning}}
       </div>
     {{/if}}
     <form class="form-horizontal ai-llm-editor">
@@ -267,6 +262,14 @@ export default class AiLlmEditorForm extends Component {
           @content={{I18n.t "discourse_ai.llms.hints.max_prompt_tokens"}}
         />
       </div>
+      <div class="control-group ai-llm-editor__vision-enabled">
+        <Input @type="checkbox" @checked={{@model.vision_enabled}} />
+        <label>{{I18n.t "discourse_ai.llms.vision_enabled"}}</label>
+        <DTooltip
+          @icon="question-circle"
+          @content={{I18n.t "discourse_ai.llms.hints.vision_enabled"}}
+        />
+      </div>
       <div class="control-group">
         <DToggleSwitch
           class="ai-llm-editor__enabled-chat-bot"

assets/javascripts/discourse/components/ai-tool-editor.gjs

@@ -6,13 +6,13 @@ import { action } from "@ember/object";
 import didInsert from "@ember/render-modifiers/modifiers/did-insert";
 import didUpdate from "@ember/render-modifiers/modifiers/did-update";
 import { service } from "@ember/service";
+import AceEditor from "discourse/components/ace-editor";
 import BackButton from "discourse/components/back-button";
 import DButton from "discourse/components/d-button";
 import DTooltip from "discourse/components/d-tooltip";
 import withEventValue from "discourse/helpers/with-event-value";
 import { popupAjaxError } from "discourse/lib/ajax-error";
 import I18n from "discourse-i18n";
-import AceEditor from "admin/components/ace-editor";
 import ComboBox from "select-kit/components/combo-box";
 import AiToolParameterEditor from "./ai-tool-parameter-editor";
 import AiToolTestModal from "./modal/ai-tool-test-modal";
@@ -71,13 +71,6 @@ export default class AiToolEditor extends Component {
         "summary"
       );
 
-      for (const p of data.parameters) {
-        if (p.enumValues) {
-          p.enum_values = p.enumValues;
-          delete p.enumValues;
-        }
-      }
-
       await this.args.model.save(data);
 
       this.toasts.success({
@@ -124,7 +117,7 @@ export default class AiToolEditor extends Component {
   <template>
     <BackButton
       @route="adminPlugins.show.discourse-ai-tools"
-      @label="discourse_ai.ai_tool.back"
+      @label="discourse_ai.tools.back"
     />
 
     <form

assets/javascripts/discourse/components/ai-tool-parameter-editor.gjs

@@ -7,6 +7,7 @@ import DButton from "discourse/components/d-button";
 import withEventValue from "discourse/helpers/with-event-value";
 import I18n from "discourse-i18n";
 import ComboBox from "select-kit/components/combo-box";
+import and from "truth-helpers/helpers/and";
 
 const PARAMETER_TYPES = [
   { name: "string", id: "string" },
@@ -24,8 +25,7 @@ export default class AiToolParameterEditor extends Component {
         description: "",
         type: "string",
         required: false,
-        enum: false,
+        enum: null,
-        enumValues: null,
       })
     );
   }
@@ -43,26 +43,27 @@ export default class AiToolParameterEditor extends Component {
 
   @action
   toggleEnum(parameter) {
-    parameter.enum = !parameter.enum;
+    if (parameter.enum) {
-    if (!parameter.enum) {
+      parameter.enum = null;
-      parameter.enumValues = null;
+    } else {
+      this.addEnumValue(parameter);
     }
   }
 
   @action
   addEnumValue(parameter) {
-    parameter.enumValues ||= new TrackedArray();
+    parameter.enum ||= new TrackedArray();
-    parameter.enumValues.push("");
+    parameter.enum.push("");
   }
 
   @action
   removeEnumValue(parameter, index) {
-    parameter.enumValues.splice(index, 1);
+    parameter.enum.splice(index, 1);
   }
 
   @action
   updateEnumValue(parameter, index, event) {
-    parameter.enumValues[index] = event.target.value;
+    parameter.enum[index] = event.target.value;
   }
 
   <template>
@@ -100,7 +101,7 @@ export default class AiToolParameterEditor extends Component {
           <label>
             <input
               {{on "input" (fn this.toggleEnum parameter)}}
-              checked={{parameter.enum}}
+              checked={{and parameter.enum parameter.enum.length}}
               type="checkbox"
             />
             {{I18n.t "discourse_ai.tools.parameter_enum"}}
@@ -113,9 +114,9 @@ export default class AiToolParameterEditor extends Component {
           />
         </div>
 
-        {{#if parameter.enum}}
+        {{#if (and parameter.enum parameter.enum.length)}}
           <div class="parameter-enum-values">
-            {{#each parameter.enumValues as |enumValue enumIndex|}}
+            {{#each parameter.enum as |enumValue enumIndex|}}
               <div class="enum-value-row">
                 <input
                   {{on "change" (fn this.updateEnumValue parameter enumIndex)}}

assets/javascripts/discourse/connectors/after-d-editor/ai-composer-helper.gjs

@@ -0,0 +1,204 @@
+import Component from "@glimmer/component";
+import { tracked } from "@glimmer/tracking";
+import { inject as service } from "@ember/service";
+import { modifier } from "ember-modifier";
+import { caretPosition, getCaretPosition } from "discourse/lib/utilities";
+import { INPUT_DELAY } from "discourse-common/config/environment";
+import { afterRender, bind, debounce } from "discourse-common/utils/decorators";
+import AiComposerHelperMenu from "../../components/ai-composer-helper-menu";
+import { showComposerAIHelper } from "../../lib/show-ai-helper";
+import virtualElementFromCaretCoords from "../../lib/virtual-element-from-caret-coords";
+
+export default class AiComposerHelper extends Component {
+  static shouldRender(outletArgs, helper) {
+    return showComposerAIHelper(outletArgs, helper, "context_menu");
+  }
+
+  @service menu;
+  @service aiComposerHelper;
+  @tracked caretCoords;
+  @tracked menuPlacement = "bottom-start";
+  @tracked menuOffset = [9, 21];
+  @tracked selectedText = "";
+  @tracked isSelecting = false;
+  @tracked menuElement = null;
+  @tracked menuInstance = null;
+  @tracked dEditorInput;
+  @tracked selectionRange = { x: 0, y: 0 };
+  minSelectionChars = 3;
+
+  documentListeners = modifier(() => {
+    document.addEventListener("mousedown", this.onMouseDown, { passive: true });
+    document.addEventListener("mouseup", this.onMouseUp, { passive: true });
+    document.addEventListener("selectionchange", this.onSelectionChanged);
+
+    this.dEditorInput = document.querySelector(".d-editor-input");
+
+    if (this.dEditorInput) {
+      this.dEditorInput.addEventListener("scroll", this.updatePosition);
+    }
+
+    return () => {
+      document.removeEventListener("mousedown", this.onMouseDown);
+      document.removeEventListener("mouseup", this.onMouseUp);
+      document.removeEventListener("selectionchange", this.onSelectionChanged);
+
+      if (this.dEditorInput) {
+        this.dEditorInput.removeEventListener("scroll", this.updatePosition);
+      }
+    };
+  });
+
+  willDestroy() {
+    super.willDestroy(...arguments);
+    this.menuInstance?.close();
+  }
+
+  @bind
+  onSelectionChanged() {
+    if (
+      this.aiComposerHelper.menuState !==
+      this.aiComposerHelper.MENU_STATES.triggers
+    ) {
+      return;
+    }
+
+    if (this.isSelecting) {
+      return;
+    }
+
+    if (document.activeElement !== this.dEditorInput) {
+      return;
+    }
+
+    const canSelect = Boolean(
+      window.getSelection() &&
+        document.activeElement &&
+        document.activeElement.value
+    );
+
+    this.selectedText = canSelect
+      ? document.activeElement.value.substring(
+          document.activeElement.selectionStart,
+          document.activeElement.selectionEnd
+        )
+      : "";
+
+    this.selectionRange = canSelect
+      ? {
+          x: document.activeElement.selectionStart,
+          y: document.activeElement.selectionEnd,
+        }
+      : { x: 0, y: 0 };
+
+    if (this.selectedText?.length === 0) {
+      this.menuInstance?.close();
+      return;
+    }
+    if (this.selectedText?.length < this.minSelectionChars) {
+      return;
+    }
+
+    this.selectionChanged();
+  }
+
+  @debounce(INPUT_DELAY)
+  selectionChanged() {
+    this.positionMenu();
+  }
+
+  @bind
+  onMouseDown() {
+    this.isSelecting = true;
+  }
+
+  @bind
+  onMouseUp() {
+    this.isSelecting = false;
+    this.onSelectionChanged();
+  }
+
+  @bind
+  updatePosition() {
+    if (!this.menuInstance) {
+      return;
+    }
+
+    this.positionMenu();
+  }
+
+  @afterRender
+  async positionMenu() {
+    this.caretCoords = getCaretPosition(this.dEditorInput, {
+      pos: caretPosition(this.dEditorInput),
+    });
+    const virtualElement = virtualElementFromCaretCoords(
+      this.caretCoords,
+      this.menuOffset
+    );
+
+    if (this.handleBoundaries(virtualElement)) {
+      return;
+    }
+
+    // Position context menu at based on if interfering with button bar
+    this.menuInstance = await this.menu.show(virtualElement, {
+      identifier: "ai-composer-helper-menu",
+      placement: this.menuPlacement,
+      component: AiComposerHelperMenu,
+      inline: true,
+      modalForMobile: false,
+      data: {
+        selectedText: this.selectedText,
+        dEditorInput: this.dEditorInput,
+        selectionRange: this.selectionRange,
+      },
+      interactive: true,
+      onClose: () => {
+        this.aiComposerHelper.menuState =
+          this.aiComposerHelper.MENU_STATES.triggers;
+      },
+    });
+    this.menuElement = this.menuInstance.content;
+  }
+
+  handleBoundaries(virtualElement) {
+    const textAreaWrapper = document
+      .querySelector(".d-editor-textarea-wrapper")
+      .getBoundingClientRect();
+    const buttonBar = document
+      .querySelector(".d-editor-button-bar")
+      .getBoundingClientRect();
+    const boundaryElement = {
+      top: buttonBar.bottom,
+      bottom: textAreaWrapper.bottom,
+    };
+
+    const menuHeightBuffer = 35; // rough estimate of menu height since we can't get actual in this context.
+    if (this.caretCoords.y - menuHeightBuffer < boundaryElement.top) {
+      this.menuPlacement = "bottom-start";
+    } else {
+      this.menuPlacement = "top-start";
+    }
+
+    if (this.isScrolledOutOfBounds(boundaryElement, virtualElement)) {
+      this.menuInstance?.close();
+      return true;
+    }
+  }
+
+  isScrolledOutOfBounds(boundaryElement, virtualElement) {
+    // Hide context menu if it's scrolled out of bounds:
+    if (virtualElement.rect.top < boundaryElement.top) {
+      return true;
+    } else if (virtualElement.rect.bottom > boundaryElement.bottom) {
+      return true;
+    }
+
+    return false;
+  }
+
+  <template>
+    <div {{this.documentListeners}}></div>
+  </template>
+}

assets/javascripts/discourse/connectors/after-d-editor/ai-helper-context-menu.hbs

@@ -1,109 +0,0 @@
-<div {{did-insert this.setupContextMenu}}>
-  {{#if this.showContextMenu}}
-    <div class="ai-helper-context-menu">
-      {{#if (eq this.menuState this.CONTEXT_MENU_STATES.triggers)}}
-        <ul class="ai-helper-context-menu__trigger">
-          <li>
-            <DButton
-              @icon="discourse-sparkles"
-              @action={{this.toggleAiHelperOptions}}
-              @label="discourse_ai.ai_helper.context_menu.trigger"
-              class="btn-flat"
-            />
-          </li>
-        </ul>
-
-      {{else if (eq this.menuState this.CONTEXT_MENU_STATES.options)}}
-        <ul class="ai-helper-context-menu__options">
-          {{#each this.helperOptions as |option|}}
-            {{#if (eq option.name "custom_prompt")}}
-              <AiHelperCustomPrompt
-                @value={{this.customPromptValue}}
-                @promptArgs={{option}}
-                @submit={{this.updateSelected}}
-              />
-            {{else}}
-              <li data-name={{option.translated_name}} data-value={{option.id}}>
-                <DButton
-                  @icon={{option.icon}}
-                  @translatedLabel={{option.translated_name}}
-                  @action={{fn this.updateSelected option}}
-                  class="btn-flat"
-                />
-              </li>
-            {{/if}}
-          {{/each}}
-        </ul>
-
-      {{else if (eq this.menuState this.CONTEXT_MENU_STATES.loading)}}
-        <AiHelperLoading @cancel={{this.cancelAIAction}} />
-
-      {{else if (eq this.menuState this.CONTEXT_MENU_STATES.review)}}
-        <ul class="ai-helper-context-menu__review">
-          <li>
-            <DButton
-              @icon="exchange-alt"
-              @label="discourse_ai.ai_helper.context_menu.view_changes"
-              @action={{this.viewChanges}}
-              class="btn-flat view-changes"
-            />
-          </li>
-          <li>
-            <DButton
-              @icon="undo"
-              @label="discourse_ai.ai_helper.context_menu.revert"
-              @action={{this.undoAIAction}}
-              class="btn-flat revert"
-            />
-          </li>
-          <li>
-            <DButton
-              @icon="check"
-              @label="discourse_ai.ai_helper.context_menu.confirm"
-              @action={{this.confirmChanges}}
-              class="btn-flat confirm"
-            />
-          </li>
-        </ul>
-
-      {{else if (eq this.menuState this.CONTEXT_MENU_STATES.resets)}}
-        <ul class="ai-helper-context-menu__resets">
-          <li>
-            <DButton
-              @icon="undo"
-              @label="discourse_ai.ai_helper.context_menu.undo"
-              @action={{this.undoAIAction}}
-              class="btn-flat undo"
-            />
-          </li>
-          <li>
-            <DButton
-              @icon="discourse-sparkles"
-              @label="discourse_ai.ai_helper.context_menu.regen"
-              @action={{fn this.updateSelected this.lastUsedOption}}
-              class="btn-flat"
-            />
-          </li>
-        </ul>
-      {{/if}}
-    </div>
-  {{/if}}
-</div>
-
-{{#if this.showDiffModal}}
-  <Modal::DiffModal
-    @confirm={{this.confirmChanges}}
-    @diff={{this.diff}}
-    @oldValue={{this.initialValue}}
-    @newValue={{this.newSelectedText}}
-    @revert={{this.undoAIAction}}
-    @closeModal={{fn (mut this.showDiffModal) false}}
-  />
-{{/if}}
-
-{{#if this.showThumbnailModal}}
-  <Modal::ThumbnailSuggestions
-    @thumbnails={{this.thumbnailSuggestions}}
-    @closeModal={{fn (mut this.showThumbnailModal) false}}
-  />
-{{/if}}

assets/javascripts/discourse/connectors/post-text-buttons/ai-post-helper-trigger.gjs

@@ -0,0 +1,190 @@
+import Component from "@glimmer/component";
+import { tracked } from "@glimmer/tracking";
+import { action } from "@ember/object";
+import { inject as service } from "@ember/service";
+import DButton from "discourse/components/d-button";
+import virtualElementFromTextRange from "discourse/lib/virtual-element-from-text-range";
+import eq from "truth-helpers/helpers/eq";
+import AiPostHelperMenu from "../../components/ai-post-helper-menu";
+import { showPostAIHelper } from "../../lib/show-ai-helper";
+
+export default class AiPostHelperTrigger extends Component {
+  static shouldRender(outletArgs, helper) {
+    return showPostAIHelper(outletArgs, helper);
+  }
+
+  @service site;
+  @service menu;
+
+  @tracked menuState = this.MENU_STATES.triggers;
+  @tracked showMainButtons = true;
+  @tracked showAiButtons = true;
+  @tracked originalPostHTML = null;
+  @tracked postHighlighted = false;
+  @tracked currentMenu = this.menu.getByIdentifier(
+    "post-text-selection-toolbar"
+  );
+
+  MENU_STATES = {
+    triggers: "TRIGGERS",
+    options: "OPTIONS",
+  };
+
+  highlightSelectedText() {
+    const postId = this.args.outletArgs.data.quoteState.postId;
+    const postElement = document.querySelector(
+      `article[data-post-id='${postId}'] .cooked`
+    );
+
+    if (!postElement) {
+      return;
+    }
+
+    this.originalPostHTML = postElement.innerHTML;
+    this.selectedText = this.args.outletArgs.data.quoteState.buffer;
+
+    const selection = window.getSelection();
+    if (!selection.rangeCount) {
+      return;
+    }
+
+    const range = selection.getRangeAt(0);
+
+    // Split start/end text nodes at their range boundary
+    if (
+      range.startContainer.nodeType === Node.TEXT_NODE &&
+      range.startOffset > 0
+    ) {
+      const newStartNode = range.startContainer.splitText(range.startOffset);
+      range.setStart(newStartNode, 0);
+    }
+    if (
+      range.endContainer.nodeType === Node.TEXT_NODE &&
+      range.endOffset < range.endContainer.length
+    ) {
+      range.endContainer.splitText(range.endOffset);
+    }
+
+    // Create a Walker to traverse text nodes within range
+    const walker = document.createTreeWalker(
+      range.commonAncestorContainer,
+      NodeFilter.SHOW_TEXT,
+      {
+        acceptNode: (node) =>
+          range.intersectsNode(node)
+            ? NodeFilter.FILTER_ACCEPT
+            : NodeFilter.FILTER_REJECT,
+      }
+    );
+
+    const textNodes = [];
+
+    if (walker.currentNode?.nodeType === Node.TEXT_NODE) {
+      textNodes.push(walker.currentNode);
+    } else {
+      while (walker.nextNode()) {
+        textNodes.push(walker.currentNode);
+      }
+    }
+
+    for (let textNode of textNodes) {
+      const highlight = document.createElement("span");
+      highlight.classList.add("ai-helper-highlighted-selection");
+
+      // Replace textNode with highlighted clone
+      const clone = textNode.cloneNode(true);
+      highlight.appendChild(clone);
+
+      textNode.parentNode.replaceChild(highlight, textNode);
+    }
+
+    selection.removeAllRanges();
+    this.postHighlighted = true;
+  }
+
+  removeHighlightedText() {
+    if (!this.postHighlighted) {
+      return;
+    }
+
+    const postId = this.args.outletArgs.data.quoteState.postId;
+    const postElement = document.querySelector(
+      `article[data-post-id='${postId}'] .cooked`
+    );
+
+    if (!postElement) {
+      return;
+    }
+
+    postElement.innerHTML = this.originalPostHTML;
+    this.postHighlighted = false;
+  }
+
+  willDestroy() {
+    super.willDestroy(...arguments);
+    this.removeHighlightedText();
+  }
+
+  @action
+  async showAiPostHelperMenu() {
+    this.highlightSelectedText();
+    if (this.site.mobileView) {
+      this.currentMenu.close();
+
+      await this.menu.show(virtualElementFromTextRange(), {
+        identifier: "ai-post-helper-menu",
+        component: AiPostHelperMenu,
+        inline: true,
+        placement: this.shouldRenderUnder ? "bottom-start" : "top-start",
+        fallbackPlacements: this.shouldRenderUnder
+          ? ["bottom-end", "top-start"]
+          : ["bottom-start"],
+        trapTab: false,
+        closeOnScroll: false,
+        modalForMobile: true,
+        data: this.menuData,
+      });
+    }
+
+    this.showMainButtons = false;
+    this.menuState = this.MENU_STATES.options;
+  }
+
+  get menuData() {
+    // Streamline of data model to be passed to the component when
+    // instantiated as a DMenu or a simple component in the template
+    return {
+      ...this.args.outletArgs.data,
+      quoteState: {
+        buffer: this.args.outletArgs.data.quoteState.buffer,
+        opts: this.args.outletArgs.data.quoteState.opts,
+        postId: this.args.outletArgs.data.quoteState.postId,
+      },
+      post: this.args.outletArgs.post,
+      selectedText: this.selectedText,
+    };
+  }
+
+  <template>
+    {{#if this.showMainButtons}}
+      {{yield}}
+    {{/if}}
+
+    {{#if this.showAiButtons}}
+      <div class="ai-post-helper">
+        {{#if (eq this.menuState this.MENU_STATES.triggers)}}
+          <DButton
+            @icon="discourse-sparkles"
+            @title="discourse_ai.ai_helper.post_options_menu.title"
+            @label="discourse_ai.ai_helper.post_options_menu.trigger"
+            @action={{this.showAiPostHelperMenu}}
+            class="btn-flat ai-post-helper__trigger"
+          />
+
+        {{else if (eq this.menuState this.MENU_STATES.options)}}
+          <AiPostHelperMenu @data={{this.menuData}} />
+        {{/if}}
+      </div>
+    {{/if}}
+  </template>
+}