DEV: Decorate boosted and AI results

2023-11-27 11:01:14 -08:00
694 changed files with 102193 additions and 1545426 deletions
@@ -1,6 +1,3 @@
 < 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
 < 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
 < 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
 < 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
 < 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
 3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
@@ -1,88 +1,58 @@
 GEM
  remote: https://rubygems.org/
  specs:
    activesupport (7.1.3.4)
      base64
      bigdecimal
      concurrent-ruby (~> 1.0, >= 1.0.2)
      connection_pool (>= 2.2.5)
      drb
      i18n (>= 1.6, < 2)
      minitest (>= 5.1)
      mutex_m
      tzinfo (~> 2.0)
    ast (2.4.2)
-    base64 (0.2.0)
+    json (2.6.3)
    bigdecimal (3.1.8)
    concurrent-ruby (1.3.3)
    connection_pool (2.4.1)
    drb (2.2.1)
    i18n (1.14.5)
      concurrent-ruby (~> 1.0)
    json (2.7.2)
    language_server-protocol (3.17.0.3)
-    minitest (5.24.1)
+    parallel (1.23.0)
-    mutex_m (0.2.0)
+    parser (3.2.2.4)
    parallel (1.25.1)
    parser (3.3.3.0)
      ast (~> 2.4.1)
      racc
    prettier_print (1.2.1)
-    racc (1.8.0)
+    racc (1.7.2)
    rack (3.1.6)
    rainbow (3.1.1)
-    regexp_parser (2.9.2)
+    regexp_parser (2.8.2)
-    rexml (3.3.1)
+    rexml (3.2.6)
-      strscan
+    rubocop (1.57.2)
    rubocop (1.64.1)
      json (~> 2.3)
      language_server-protocol (>= 3.17.0)
      parallel (~> 1.10)
-      parser (>= 3.3.0.2)
+      parser (>= 3.2.2.4)
      rainbow (>= 2.2.2, < 4.0)
      regexp_parser (>= 1.8, < 3.0)
      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
      ruby-progressbar (~> 1.7)
      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.3)
+    rubocop-ast (1.30.0)
-      parser (>= 3.3.1.0)
+      parser (>= 3.2.1.0)
-    rubocop-capybara (2.21.0)
+    rubocop-capybara (2.19.0)
      rubocop (~> 1.41)
-    rubocop-discourse (3.8.1)
+    rubocop-discourse (3.4.0)
-      activesupport (>= 6.1)
+      rubocop (>= 1.1.0)
-      rubocop (>= 1.59.0)
+      rubocop-rspec (>= 2.0.0)
-      rubocop-capybara (>= 2.0.0)
+    rubocop-factory_bot (2.24.0)
-      rubocop-factory_bot (>= 2.0.0)
+      rubocop (~> 1.33)
-      rubocop-rails (>= 2.25.0)
+    rubocop-rspec (2.25.0)
-      rubocop-rspec (>= 3.0.1)
+      rubocop (~> 1.40)
-      rubocop-rspec_rails (>= 2.30.0)
+      rubocop-capybara (~> 2.17)
-    rubocop-factory_bot (2.26.1)
+      rubocop-factory_bot (~> 2.22)
      rubocop (~> 1.61)
    rubocop-rails (2.25.1)
      activesupport (>= 4.2.0)
      rack (>= 1.1)
      rubocop (>= 1.33.0, < 2.0)
      rubocop-ast (>= 1.31.1, < 2.0)
    rubocop-rspec (3.0.2)
      rubocop (~> 1.61)
    rubocop-rspec_rails (2.30.0)
      rubocop (~> 1.61)
      rubocop-rspec (~> 3, >= 3.0.1)
    ruby-progressbar (1.13.0)
    strscan (3.1.0)
    syntax_tree (6.2.0)
      prettier_print (>= 1.2.0)
    tzinfo (2.0.6)
      concurrent-ruby (~> 1.0)
    unicode-display_width (2.5.0)
 PLATFORMS
  arm64-darwin-20
  ruby
  x86_64-darwin-18
  x86_64-darwin-19
  x86_64-darwin-20
  x86_64-linux
 DEPENDENCIES
  rubocop-discourse
  syntax_tree
 BUNDLED WITH
-   2.5.10
+   2.4.21
@@ -1,17 +0,0 @@
 import DiscourseRoute from "discourse/routes/discourse";
 export default DiscourseRoute.extend({
  async model(params) {
    const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
    const id = parseInt(params.id, 10);
    return allLlms.findBy("id", id);
  },
  setupController(controller, model) {
    this._super(controller, model);
    controller.set(
      "allLlms",
      this.modelFor("adminPlugins.show.discourse-ai-llms")
    );
  },
 });
@@ -1,7 +0,0 @@
 import DiscourseRoute from "discourse/routes/discourse";
 export default class DiscourseAiAiLlmsRoute extends DiscourseRoute {
  model() {
    return this.store.findAll("ai-llm");
  }
 }
@@ -1,23 +0,0 @@
 import { AUTO_GROUPS } from "discourse/lib/constants";
 import DiscourseRoute from "discourse/routes/discourse";
 export default DiscourseRoute.extend({
  async model() {
    const record = this.store.createRecord("ai-persona");
    record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
    record.set("rag_uploads", []);
    // these match the defaults on the table
    record.set("rag_chunk_tokens", 374);
    record.set("rag_chunk_overlap_tokens", 10);
    record.set("rag_conversation_chunks", 10);
    return record;
  },
  setupController(controller, model) {
    this._super(controller, model);
    controller.set(
      "allPersonas",
      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
  },
 });
@@ -1,15 +0,0 @@
 import DiscourseRoute from "discourse/routes/discourse";
 export default class DiscourseAiToolsNewRoute extends DiscourseRoute {
  async model() {
    return this.store.createRecord("ai-tool");
  }
  setupController(controller) {
    super.setupController(...arguments);
    const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
    controller.set("allTools", toolsModel);
    controller.set("presets", toolsModel.resultSetMeta.presets);
  }
 }
@@ -1,18 +0,0 @@
 import DiscourseRoute from "discourse/routes/discourse";
 export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
  async model(params) {
    const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
    const id = parseInt(params.id, 10);
    return allTools.find((tool) => tool.id === id);
  }
  setupController(controller) {
    super.setupController(...arguments);
    const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
    controller.set("allTools", toolsModel);
    controller.set("presets", toolsModel.resultSetMeta.presets);
  }
 }
@@ -1,10 +0,0 @@
 import { service } from "@ember/service";
 import DiscourseRoute from "discourse/routes/discourse";
 export default class DiscourseAiToolsRoute extends DiscourseRoute {
  @service store;
  model() {
    return this.store.findAll("ai-tool");
  }
 }
@@ -1 +0,0 @@
 <AiLlmsListEditor @llms={{this.model}} />
@@ -1 +0,0 @@
 <AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
 <AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
 <AiToolListEditor @tools={{this.model}} />
@@ -1,5 +0,0 @@
 <AiToolEditor
  @tools={{this.allTools}}
  @model={{this.model}}
  @presets={{this.presets}}
 />
@@ -1,5 +0,0 @@
 <AiToolEditor
  @tools={{this.allTools}}
  @model={{this.model}}
  @presets={{this.presets}}
 />
@@ -1,128 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module Admin
    class AiLlmsController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      def index
        llms = LlmModel.all.order(:display_name)
        render json: {
                 ai_llms:
                   ActiveModel::ArraySerializer.new(
                     llms,
                     each_serializer: LlmModelSerializer,
                     root: false,
                   ).as_json,
                 meta: {
                   provider_params: LlmModel.provider_params,
                   presets: DiscourseAi::Completions::Llm.presets,
                   providers: DiscourseAi::Completions::Llm.provider_names,
                   tokenizers:
                     DiscourseAi::Completions::Llm.tokenizer_names.map { |tn|
                       { id: tn, name: tn.split("::").last }
                     },
                 },
               }
      end
      def show
        llm_model = LlmModel.find(params[:id])
        render json: LlmModelSerializer.new(llm_model)
      end
      def create
        llm_model = LlmModel.new(ai_llm_params)
        if llm_model.save
          llm_model.toggle_companion_user
          render json: { ai_persona: LlmModelSerializer.new(llm_model) }, status: :created
        else
          render_json_error LlmModelSerializer.new(llm_model)
        end
      end
      def update
        llm_model = LlmModel.find(params[:id])
        if llm_model.update(ai_llm_params(updating: llm_model))
          llm_model.toggle_companion_user
          render json: LlmModelSerializer.new(llm_model)
        else
          render_json_error LlmModelSerializer.new(llm_model)
        end
      end
      def destroy
        llm_model = LlmModel.find(params[:id])
        in_use_by = DiscourseAi::Configuration::LlmValidator.new.modules_using(llm_model)
        if !in_use_by.empty?
          return(
            render_json_error(
              I18n.t(
                "discourse_ai.llm.delete_failed",
                settings: in_use_by.join(", "),
                count: in_use_by.length,
              ),
              status: 409,
            )
          )
        end
        # Clean up companion users
        llm_model.enabled_chat_bot = false
        llm_model.toggle_companion_user
        if llm_model.destroy
          head :no_content
        else
          render_json_error llm_model
        end
      end
      def test
        RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!
        llm_model = LlmModel.new(ai_llm_params)
        DiscourseAi::Configuration::LlmValidator.new.run_test(llm_model)
        render json: { success: true }
      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
        render json: { success: false, error: e.message }
      end
      private
      def ai_llm_params(updating: nil)
        permitted =
          params.require(:ai_llm).permit(
            :display_name,
            :name,
            :provider,
            :tokenizer,
            :max_prompt_tokens,
            :api_key,
            :enabled_chat_bot,
          )
        provider = updating ? updating.provider : permitted[:provider]
        permit_url =
          (updating && updating.url != LlmModel::RESERVED_VLLM_SRV_URL) ||
            provider != LlmModel::BEDROCK_PROVIDER_NAME
        permitted[:url] = params.dig(:ai_llm, :url) if permit_url
        extra_field_names = LlmModel.provider_params.dig(provider&.to_sym, :fields).to_a
        received_prov_params = params.dig(:ai_llm, :provider_params)
        permitted[:provider_params] = received_prov_params.slice(
          *extra_field_names,
        ).permit! if !extra_field_names.empty? && received_prov_params.present?
        permitted
      end
    end
  end
 end
@@ -3,10 +3,7 @@
 module DiscourseAi
  module Admin
    class AiPersonasController < ::Admin::AdminController
-      requires_plugin ::DiscourseAi::PLUGIN_NAME
+      before_action :find_ai_persona, only: %i[show update destroy]
      before_action :find_ai_persona,
                    only: %i[show update destroy create_user indexing_status_check]
      def index
        ai_personas =
@@ -15,23 +12,11 @@ module DiscourseAi
            # localized for system personas
            LocalizedAiPersonaSerializer.new(persona, root: false)
          end
-        tools =
+        commands =
-          DiscourseAi::AiBot::Personas::Persona.all_available_tools.map do |tool|
+          DiscourseAi::AiBot::Personas::Persona.all_available_commands.map do |command|
-            AiToolSerializer.new(tool, root: false)
+            { id: command.to_s.split("::").last, name: command.name.humanize.titleize }
          end
-        AiTool
+        render json: { ai_personas: ai_personas, meta: { commands: commands } }
          .where(enabled: true)
          .each do |tool|
            tools << {
              id: "custom-#{tool.id}",
              name: I18n.t("discourse_ai.tools.custom_name", name: tool.name.capitalize),
            }
          end
        llms =
          DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
            { id: hash[:value], name: hash[:name] }
          end
        render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
      end
      def show
@@ -39,31 +24,19 @@ module DiscourseAi
      end
      def create
-        ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
+        ai_persona = AiPersona.new(ai_persona_params)
        if ai_persona.save
-          RagDocumentFragment.link_persona_and_uploads(ai_persona, attached_upload_ids)
+          render json: { ai_persona: ai_persona }, status: :created
          render json: {
                   ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
                 },
                 status: :created
        else
          render_json_error ai_persona
        end
      end
      def create_user
        user = @ai_persona.create_user!
        render json: BasicUserSerializer.new(user, root: "user")
      end
      def update
-        if @ai_persona.update(ai_persona_params.except(:rag_uploads))
+        if @ai_persona.update(ai_persona_params)
-          RagDocumentFragment.update_persona_uploads(@ai_persona, attached_upload_ids)
+          render json: @ai_persona
          render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
        else
-          render_json_error LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
+          render_json_error @ai_persona
        end
      end
@@ -71,122 +44,27 @@ module DiscourseAi
        if @ai_persona.destroy
          head :no_content
        else
-          render_json_error LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
+          render_json_error @ai_persona
        end
      end
      def upload_file
        file = params[:file] || params[:files].first
        if !SiteSetting.ai_embeddings_enabled?
          raise Discourse::InvalidAccess.new("Embeddings not enabled")
        end
        validate_extension!(file.original_filename)
        validate_file_size!(file.tempfile.size)
        hijack do
          upload =
            UploadCreator.new(
              file.tempfile,
              file.original_filename,
              type: "discourse_ai_rag_upload",
              skip_validations: true,
            ).create_for(current_user.id)
          if upload.persisted?
            render json: UploadSerializer.new(upload)
          else
            render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
          end
        end
      end
      def indexing_status_check
        render json: RagDocumentFragment.indexing_status(@ai_persona, @ai_persona.uploads)
      end
      private
      def find_ai_persona
        @ai_persona = AiPersona.find(params[:id])
      end
      def attached_upload_ids
        ai_persona_params[:rag_uploads].to_a.map { |h| h[:id] }
      end
      def ai_persona_params
-        permitted =
+        params.require(:ai_persona).permit(
-          params.require(:ai_persona).permit(
+          :name,
-            :name,
+          :description,
-            :description,
+          :enabled,
-            :enabled,
+          :system_prompt,
-            :system_prompt,
+          :enabled,
-            :priority,
+          :priority,
-            :top_p,
+          allowed_group_ids: [],
-            :temperature,
+          commands: [],
-            :default_llm,
+        )
            :user_id,
            :mentionable,
            :max_context_posts,
            :vision_enabled,
            :vision_max_pixels,
            :rag_chunk_tokens,
            :rag_chunk_overlap_tokens,
            :rag_conversation_chunks,
            :question_consolidator_llm,
            :allow_chat,
            :tool_details,
            allowed_group_ids: [],
            rag_uploads: [:id],
          )
        if tools = params.dig(:ai_persona, :tools)
          permitted[:tools] = permit_tools(tools)
        end
        permitted
      end
      def permit_tools(tools)
        return [] if !tools.is_a?(Array)
        tools.filter_map do |tool, options|
          break nil if !tool.is_a?(String)
          options&.permit! if options && options.is_a?(ActionController::Parameters)
          if options
            [tool, options]
          else
            tool
          end
        end
      end
      def validate_extension!(filename)
        extension = File.extname(filename)[1..-1] || ""
        authorized_extensions = %w[txt md]
        if !authorized_extensions.include?(extension)
          raise Discourse::InvalidParameters.new(
                  I18n.t(
                    "upload.unauthorized",
                    authorized_extensions: authorized_extensions.join(" "),
                  ),
                )
        end
      end
      def validate_file_size!(filesize)
        max_size_bytes = 20.megabytes
        if filesize > max_size_bytes
          raise Discourse::InvalidParameters.new(
                  I18n.t(
                    "upload.attachments.too_large_humanized",
                    max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
                  ),
                )
        end
      end
    end
  end
@@ -1,89 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module Admin
    class AiToolsController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      before_action :find_ai_tool, only: %i[show update destroy]
      def index
        ai_tools = AiTool.all
        render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
      end
      def show
        render_serialized(@ai_tool, AiCustomToolSerializer)
      end
      def create
        ai_tool = AiTool.new(ai_tool_params)
        ai_tool.created_by_id = current_user.id
        if ai_tool.save
          render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
        else
          render_json_error ai_tool
        end
      end
      def update
        if @ai_tool.update(ai_tool_params)
          render_serialized(@ai_tool, AiCustomToolSerializer)
        else
          render_json_error @ai_tool
        end
      end
      def destroy
        if @ai_tool.destroy
          head :no_content
        else
          render_json_error @ai_tool
        end
      end
      def test
        if params[:id].present?
          ai_tool = AiTool.find(params[:id])
        else
          ai_tool = AiTool.new(ai_tool_params)
        end
        parameters = params[:parameters].to_unsafe_h
        # we need an llm so we have a tokenizer
        # but will do without if none is available
        llm = LlmModel.first&.to_llm
        runner = ai_tool.runner(parameters, llm: llm, bot_user: current_user, context: {})
        result = runner.invoke
        if result.is_a?(Hash) && result[:error]
          render_json_error result[:error]
        else
          render json: { output: result }
        end
      rescue ActiveRecord::RecordNotFound => e
        render_json_error e.message, status: 400
      rescue => e
        render_json_error "Error executing the tool: #{e.message}", status: 400
      end
      private
      def find_ai_tool
        @ai_tool = AiTool.find(params[:id])
      end
      def ai_tool_params
        params.require(:ai_tool).permit(
          :name,
          :description,
          :script,
          :summary,
          parameters: [:name, :type, :description, :required, :enum, enum_values: []],
        )
      end
    end
  end
 end
@@ -6,21 +6,6 @@ module DiscourseAi
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      requires_login
      def show_debug_info
        post = Post.find(params[:post_id])
        guardian.ensure_can_debug_ai_bot_conversation!(post)
        posts =
          Post
            .where("post_number <= ?", post.post_number)
            .where(topic_id: post.topic_id)
            .order("post_number DESC")
        debug_info = AiApiAuditLog.where(post: posts).order(created_at: :desc).first
        render json: AiApiAuditLogSerializer.new(debug_info, root: false), status: 200
      end
      def stop_streaming_response
        post = Post.find(params[:post_id])
        guardian.ensure_can_see!(post)
@@ -31,10 +16,12 @@ module DiscourseAi
      end
      def show_bot_username
-        bot_user = DiscourseAi::AiBot::EntryPoint.find_user_from_model(params[:username])
+        bot_user_id = DiscourseAi::AiBot::EntryPoint.map_bot_model_to_user_id(params[:username])
-        raise Discourse::InvalidParameters.new(:username) if !bot_user
+        raise Discourse::InvalidParameters.new(:username) if !bot_user_id
-        render json: { bot_username: bot_user.username_lower }, status: 200
+        bot_username_lower = User.find(bot_user_id).username_lower
        render json: { bot_username: bot_username_lower }, status: 200
      end
    end
  end
@@ -1,111 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module AiBot
    class SharedAiConversationsController < ::ApplicationController
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      requires_login only: %i[create update destroy]
      before_action :require_site_settings!
      skip_before_action :preload_json, :check_xhr, only: %i[show]
      def create
        ensure_allowed_create!
        RateLimiter.new(current_user, "share-ai-conversation", 10, 1.minute).performed!
        shared_conversation = SharedAiConversation.share_conversation(current_user, @topic)
        if shared_conversation.persisted?
          render json: success_json.merge(share_key: shared_conversation.share_key)
        else
          render json:
                   failed_json.merge(error: I18n.t("discourse_ai.share_ai.errors.failed_to_share")),
                 status: :unprocessable_entity
        end
      end
      def destroy
        ensure_allowed_destroy!
        SharedAiConversation.destroy_conversation(@shared_conversation)
        render json:
                 success_json.merge(
                   message: I18n.t("discourse_ai.share_ai.errors.conversation_deleted"),
                 )
      end
      def show
        @shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
        raise Discourse::NotFound if @shared_conversation.blank?
        expires_in 1.minute, public: true
        response.headers["X-Robots-Tag"] = "noindex"
        if request.format.json?
          render json: success_json.merge(@shared_conversation.to_json)
        else
          render "show", layout: false
        end
      end
      def preview
        ensure_allowed_preview!
        data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
        data[:error] = @error if @error
        data[:share_key] = @shared_conversation.share_key if @shared_conversation
        data[:topic_id] = @topic.id
        render json: data
      end
      private
      def require_site_settings!
        if !SiteSetting.discourse_ai_enabled ||
             !SiteSetting.ai_bot_public_sharing_allowed_groups_map.any? ||
             !SiteSetting.ai_bot_enabled
          raise Discourse::NotFound
        end
      end
      def ensure_allowed_preview!
        @topic = Topic.find_by(id: params[:topic_id])
        raise Discourse::NotFound if !@topic
        @shared_conversation = SharedAiConversation.find_by(target: @topic)
        @error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
        if @error == :not_allowed
          raise Discourse::InvalidAccess.new(
                  nil,
                  nil,
                  custom_message: "discourse_ai.share_ai.errors.#{@error}",
                )
        end
      end
      def ensure_allowed_destroy!
        @shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
        raise Discourse::InvalidAccess if @shared_conversation.blank?
        guardian.ensure_can_destroy_shared_ai_bot_conversation!(@shared_conversation)
      end
      def ensure_allowed_create!
        @topic = Topic.find_by(id: params[:topic_id])
        raise Discourse::NotFound if !@topic
        error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
        if error
          raise Discourse::InvalidAccess.new(
                  nil,
                  nil,
                  custom_message: "discourse_ai.share_ai.errors.#{error}",
                )
        end
      end
    end
  end
 end
@@ -8,35 +8,35 @@ module DiscourseAi
      before_action :ensure_can_request_suggestions
      before_action :rate_limiter_performed!, except: %i[prompts]
-      include SecureUploadEndpointHelpers
+      def prompts
        render json:
                 ActiveModel::ArraySerializer.new(
                   DiscourseAi::AiHelper::Assistant.new.available_prompts,
                   root: false,
                 ),
               status: 200
      end
      def suggest
        input = get_text_param!
        force_default_locale = params[:force_default_locale] || false
        prompt = CompletionPrompt.find_by(id: params[:mode])
        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
-
+        if prompt.prompt_type == "custom_prompt" && params[:custom_prompt].blank?
-        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+          raise Discourse::InvalidParameters.new(:custom_prompt)
          raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
          prompt.custom_instruction = params[:custom_prompt]
        end
        suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
        hijack do
          render json:
                   DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
                     prompt,
                     input,
                     current_user,
                     force_default_locale,
                   ),
                 status: 200
        end
-      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                          status: 502
      end
@@ -56,7 +56,7 @@ module DiscourseAi
                   ),
                 status: 200
        end
-      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                          status: 502
      end
@@ -81,7 +81,9 @@ module DiscourseAi
               status: 200
      end
-      def suggest_thumbnails(input)
+      def suggest_thumbnails
        input = get_text_param!
        hijack do
          thumbnails = DiscourseAi::AiHelper::Painter.new.commission_thumbnails(input, current_user)
@@ -96,48 +98,13 @@ module DiscourseAi
        raise Discourse::InvalidParameters.new(:post_id) unless post
-        Jobs.enqueue(
+        render json:
-          :stream_post_helper,
+                 DiscourseAi::AiHelper::TopicHelper.new(current_user).explain(
-          post_id: post.id,
+                   term_to_explain,
-          user_id: current_user.id,
+                   post,
-          term_to_explain: term_to_explain,
+                 ),
-        )
+               status: 200
-
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
        render json: { success: true }, status: 200
      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                          status: 502
      end
      def caption_image
        image_url = params[:image_url]
        image_url_type = params[:image_url_type]
        raise Discourse::InvalidParameters.new(:image_url) if !image_url
        raise Discourse::InvalidParameters.new(:image_url) if !image_url_type
        if image_url_type == "short_path"
          image = Upload.find_by(sha1: Upload.sha1_from_short_path(image_url))
        elsif image_url_type == "short_url"
          image = Upload.find_by(sha1: Upload.sha1_from_short_url(image_url))
        else
          image = upload_from_full_url(image_url)
        end
        raise Discourse::NotFound if image.blank?
        check_secure_upload_permission(image) if image.secure?
        user = current_user
        hijack do
          caption = DiscourseAi::AiHelper::Assistant.new.generate_image_caption(image, user)
          render json: {
                   caption:
                     "#{caption} (#{I18n.t("discourse_ai.ai_helper.image_caption.attribution")})",
                 },
                 status: 200
        end
      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed, Net::HTTPBadResponse
        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                          status: 502
      end
@@ -157,9 +124,14 @@ module DiscourseAi
      end
      def ensure_can_request_suggestions
-        if !current_user.in_any_groups?(SiteSetting.ai_helper_allowed_groups_map)
+        user_group_ids = current_user.group_ids
-          raise Discourse::InvalidAccess
+
-        end
+        allowed =
          SiteSetting.ai_helper_allowed_groups_map.any? do |group_id|
            user_group_ids.include?(group_id)
          end
        raise Discourse::InvalidAccess if !allowed
      end
    end
  end
@@ -20,7 +20,6 @@ module DiscourseAi
            term: query,
            search_context: guardian,
            use_pg_headlines_for_excerpt: false,
            can_lazy_load_categories: guardian.can_lazy_load_categories?,
          )
        semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
@@ -37,34 +36,6 @@ module DiscourseAi
          render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
        end
      end
      def quick_search
        query = params[:q].to_s
        if query.length < SiteSetting.min_search_term_length
          raise Discourse::InvalidParameters.new(:q)
        end
        grouped_results =
          Search::GroupedSearchResults.new(
            type_filter: SEMANTIC_SEARCH_TYPE,
            term: query,
            search_context: guardian,
            use_pg_headlines_for_excerpt: false,
          )
        semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
        if !semantic_search.cached_query?(query)
          RateLimiter.new(current_user, "semantic-search", 60, 1.minutes).performed!
        end
        hijack do
          semantic_search.quick_search(query).each { |topic_post| grouped_results.add(topic_post) }
          render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
        end
      end
    end
  end
 end
@@ -1,48 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module Summarization
    class ChatSummaryController < ::Chat::ApiController
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      requires_plugin ::Chat::PLUGIN_NAME
      VALID_SINCE_VALUES = [1, 3, 6, 12, 24, 72, 168]
      def show
        since = params[:since].to_i
        raise Discourse::InvalidParameters.new(:since) if !VALID_SINCE_VALUES.include?(since)
        channel = ::Chat::Channel.find(params[:channel_id])
        guardian.ensure_can_join_chat_channel!(channel)
        strategy = DiscourseAi::Summarization.default_strategy
        raise Discourse::NotFound.new unless strategy
        guardian.ensure_can_request_summary!
        RateLimiter.new(current_user, "channel_summary", 6, 5.minutes).performed!
        hijack do
          content = { content_title: channel.name }
          content[:contents] = channel
            .chat_messages
            .where("chat_messages.created_at > ?", since.hours.ago)
            .includes(:user)
            .order(created_at: :asc)
            .pluck(:id, :username_lower, :message)
            .map { { id: _1, poster: _2, text: _3 } }
          summarized_text =
            if content[:contents].empty?
              I18n.t("discourse_ai.summarization.chat.no_targets")
            else
              strategy.summarize(content, current_user).dig(:summary)
            end
          render json: { summary: summarized_text }
        end
      end
    end
  end
 end
@@ -1,36 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module Summarization
    class SummaryController < ::ApplicationController
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      def show
        topic = Topic.find(params[:topic_id])
        guardian.ensure_can_see!(topic)
        raise Discourse::NotFound if !guardian.can_see_summary?(topic)
        RateLimiter.new(current_user, "summary", 6, 5.minutes).performed! if current_user
        opts = params.permit(:skip_age_check)
        if params[:stream] && current_user
          Jobs.enqueue(
            :stream_topic_ai_summary,
            topic_id: topic.id,
            user_id: current_user.id,
            opts: opts.as_json,
          )
          render json: success_json
        else
          hijack do
            summary = DiscourseAi::TopicSummarization.summarize(topic, current_user, opts)
            render_serialized(summary, AiTopicSummarySerializer)
          end
        end
      end
    end
  end
 end
@@ -1,15 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module AiBot
    module SharedAiConversationsHelper
      # bump up version when assets change
      # long term we may want to change this cause it is hard to remember
      # to bump versions, but for now this does the job
      VERSION = "1"
      def share_asset_url(short_path)
        ::UrlHelper.absolute("/plugins/discourse-ai/ai-share/#{short_path}?#{VERSION}")
      end
    end
  end
 end
@@ -1,28 +0,0 @@
 # frozen_string_literal: true
 module ::Jobs
  class CreateAiChatReply < ::Jobs::Base
    sidekiq_options retry: false
    def execute(args)
      channel = ::Chat::Channel.find_by(id: args[:channel_id])
      return if channel.blank?
      message = ::Chat::Message.find_by(id: args[:message_id])
      return if message.blank?
      personaClass =
        DiscourseAi::AiBot::Personas::Persona.find_by(id: args[:persona_id], user: message.user)
      return if personaClass.blank?
      user = User.find_by(id: personaClass.user_id)
      bot = DiscourseAi::AiBot::Bot.as(user, persona: personaClass.new)
      DiscourseAi::AiBot::Playground.new(bot).reply_to_chat_message(
        message,
        channel,
        args[:context_post_ids],
      )
    end
  end
 end
@@ -1,172 +0,0 @@
 # frozen_string_literal: true
 module ::Jobs
  class DigestRagUpload < ::Jobs::Base
    CHUNK_SIZE = 1024
    CHUNK_OVERLAP = 64
    MAX_FRAGMENTS = 100_000
    # TODO(roman): Add a way to automatically recover from errors, resulting in unindexed uploads.
    def execute(args)
      return if (upload = Upload.find_by(id: args[:upload_id])).nil?
      return if (ai_persona = AiPersona.find_by(id: args[:ai_persona_id])).nil?
      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
      vector_rep =
        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
      tokenizer = vector_rep.tokenizer
      chunk_tokens = ai_persona.rag_chunk_tokens
      overlap_tokens = ai_persona.rag_chunk_overlap_tokens
      fragment_ids = RagDocumentFragment.where(ai_persona: ai_persona, upload: upload).pluck(:id)
      # Check if this is the first time we process this upload.
      if fragment_ids.empty?
        document = get_uploaded_file(upload)
        return if document.nil?
        RagDocumentFragment.publish_status(upload, { total: 0, indexed: 0, left: 0 })
        fragment_ids = []
        idx = 0
        ActiveRecord::Base.transaction do
          chunk_document(
            file: document,
            tokenizer: tokenizer,
            chunk_tokens: chunk_tokens,
            overlap_tokens: overlap_tokens,
          ) do |chunk, metadata|
            fragment_ids << RagDocumentFragment.create!(
              ai_persona: ai_persona,
              fragment: chunk,
              fragment_number: idx + 1,
              upload: upload,
              metadata: metadata,
            ).id
            idx += 1
            if idx > MAX_FRAGMENTS
              Rails.logger.warn("Upload #{upload.id} has too many fragments, truncating.")
              break
            end
          end
        end
      end
      fragment_ids.each_slice(50) do |slice|
        Jobs.enqueue(:generate_rag_embeddings, fragment_ids: slice)
      end
    end
    private
    def chunk_document(file:, tokenizer:, chunk_tokens:, overlap_tokens:)
      buffer = +""
      current_metadata = nil
      done = false
      overlap = ""
      # generally this will be plenty
      read_size = chunk_tokens * 10
      while buffer.present? || !done
        if buffer.length < read_size
          read = file.read(read_size)
          done = true if read.nil?
          read = Encodings.to_utf8(read) if read
          buffer << (read || "")
        end
        # at this point we unconditionally have 2x CHUNK_SIZE worth of data in the buffer
        metadata_regex = /\[\[metadata (.*?)\]\]/m
        before_metadata, new_metadata, after_metadata = buffer.split(metadata_regex)
        to_chunk = nil
        if before_metadata.present?
          to_chunk = before_metadata
        elsif after_metadata.present?
          current_metadata = new_metadata
          to_chunk = after_metadata
          buffer = buffer.split(metadata_regex, 2).last
          overlap = ""
        else
          current_metadata = new_metadata
          buffer = buffer.split(metadata_regex, 2).last
          overlap = ""
          next
        end
        chunk, split_char = first_chunk(to_chunk, tokenizer: tokenizer, chunk_tokens: chunk_tokens)
        buffer = buffer[chunk.length..-1]
        processed_chunk = overlap + chunk
        processed_chunk.strip!
        processed_chunk.gsub!(/\n[\n]+/, "\n\n")
        yield processed_chunk, current_metadata
        current_chunk_tokens = tokenizer.encode(chunk)
        overlap_token_ids = current_chunk_tokens[-overlap_tokens..-1] || current_chunk_tokens
        overlap = ""
        while overlap_token_ids.present?
          begin
            overlap = tokenizer.decode(overlap_token_ids) + split_char
            break if overlap.encoding == Encoding::UTF_8
          rescue StandardError
            # it is possible that we truncated mid char
          end
          overlap_token_ids.shift
        end
        # remove first word it is probably truncated
        overlap = overlap.split(" ", 2).last
      end
    end
    def first_chunk(text, chunk_tokens:, tokenizer:, splitters: ["\n\n", "\n", ".", ""])
      return text, " " if tokenizer.tokenize(text).length <= chunk_tokens
      splitters = splitters.find_all { |s| text.include?(s) }.compact
      buffer = +""
      split_char = nil
      splitters.each do |splitter|
        split_char = splitter
        text
          .split(split_char)
          .each do |part|
            break if tokenizer.tokenize(buffer + split_char + part).length > chunk_tokens
            buffer << split_char
            buffer << part
          end
        break if buffer.length > 0
      end
      [buffer, split_char]
    end
    def get_uploaded_file(upload)
      store = Discourse.store
      @file ||=
        if store.external?
          # Upload#filesize could be approximate.
          # add two extra Mbs to make sure that we'll be able to download the upload.
          max_filesize = upload.filesize + 2.megabytes
          store.download(upload, max_file_size_kb: max_filesize)
        else
          File.open(store.path_for(upload))
        end
    end
  end
 end
@@ -1,26 +0,0 @@
 # frozen_string_literal: true
 module Jobs
  class GenerateEmbeddings < ::Jobs::Base
    sidekiq_options queue: "low"
    def execute(args)
      return unless SiteSetting.ai_embeddings_enabled
      return if args[:target_type].blank? || args[:target_id].blank?
      target = args[:target_type].constantize.find_by_id(args[:target_id])
      return if target.nil? || target.deleted_at.present?
      topic = target.is_a?(Topic) ? target : target.topic
      post = target.is_a?(Post) ? target : target.first_post
      return if topic.blank? || post.blank?
      return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
      return if post.raw.blank?
      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
      vector_rep =
        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
      vector_rep.generate_representation_from(target)
    end
  end
 end
@@ -1,27 +0,0 @@
 # frozen_string_literal: true
 module ::Jobs
  class GenerateRagEmbeddings < ::Jobs::Base
    sidekiq_options queue: "ultra_low"
    # we could also restrict concurrency but this takes so long if it is not concurrent
    def execute(args)
      return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
      vector_rep =
        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
      # generate_representation_from checks compares the digest value to make sure
      # the embedding is only generated once per fragment unless something changes.
      fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
      last_fragment = fragments.last
      ai_persona = last_fragment.ai_persona
      upload = last_fragment.upload
      indexing_status = RagDocumentFragment.indexing_status(ai_persona, [upload])[upload.id]
      RagDocumentFragment.publish_status(upload, indexing_status)
    end
  end
 end
@@ -1,51 +0,0 @@
 # frozen_string_literal: true
 module ::Jobs
  class SharedConversationAdjustUploadSecurity < ::Jobs::Base
    def execute(args)
      if args[:conversation_id].present?
        # The conversation context includes post cooked content so this
        # must be updated when target uploads security changes.
        update_conversation(args[:conversation_id])
      elsif args[:target_id].present? && args[:target_type].present?
        # If we deleted the conversation then we just need to update the target's
        # uploads security, no need to update the conversation.
        update_target(args[:target_id], args[:target_type])
      end
    end
    private
    def update_conversation(conversation_id)
      conversation = SharedAiConversation.find_by(id: conversation_id)
      return if conversation.blank?
      # NOTE: Only Topics are supported for now, in future we will need a more flexible
      # way of doing this.
      if conversation.target_type == "Topic"
        rebaked_posts = TopicUploadSecurityManager.new(conversation.target).run
        if rebaked_posts.any?
          new_context =
            conversation.context.map do |context_post|
              rebaked_post = rebaked_posts.find { |p| p.id == context_post["id"] }
              context_post["cooked"] = rebaked_post.cooked if rebaked_post
              context_post
            end
          conversation.update(context: new_context)
        end
      end
    end
    def update_target(target_id, target_type)
      # NOTE: Only Topics are supported for now, in future we will need a more flexible
      # way of doing this.
      if target_type == "Topic"
        topic = target_type.constantize.find_by(id: target_id)
        return if topic.blank?
        TopicUploadSecurityManager.new(topic).run
      end
    end
  end
 end
--- a/Show More
+++ b/Show More
		`@@ -1 +0,0 @@`
			`<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />`