DEV: Test if site setting can be added without translation

.discourse-compatibility

@@ -1,7 +1,8 @@
-< 3.4.0.beta3-dev: decf1bb49d737ea15308400f22f89d1d1e71d13d
-< 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
-< 3.3.0.beta5-dev: 4d8090002f6dcd8e34d41033606bf131fa221475
 < 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
-< 3.3.0.beta1-dev: d94234d89f50733678a5c62809445198a9fda74b
-3.2.999: d94234d89f50733678a5c62809445198a9fda74b
+< 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
+< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
+< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
+< 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
 3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
+< 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
+3.1.0.beta3: 057fbe1ce6

.eslintrc.cjs

@@ -0,0 +1 @@
+module.exports = require("@discourse/lint-configs/eslint");

.gitignore

@@ -1,4 +1,3 @@
 node_modules
 /gems
 /auto_generated
-.env

.npmrc

@@ -1,2 +0,0 @@
-engine-strict = true
-auto-install-peers = false

Gemfile.lock

@@ -32,7 +32,8 @@ GEM
     rack (3.1.6)
     rainbow (3.1.1)
     regexp_parser (2.9.2)
-    rexml (3.3.9)
+    rexml (3.3.1)
+      strscan
     rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
@@ -69,6 +70,7 @@ GEM
       rubocop (~> 1.61)
       rubocop-rspec (~> 3, >= 3.0.1)
     ruby-progressbar (1.13.0)
+    strscan (3.1.0)
     syntax_tree (6.2.0)
       prettier_print (>= 1.2.0)
     tzinfo (2.0.6)

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-new.js

@@ -1,25 +1,16 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
-  queryParams = {
-    llmTemplate: { refreshModel: true },
-  };
-
+export default DiscourseRoute.extend({
   async model() {
     const record = this.store.createRecord("ai-llm");
-    record.provider_params = {};
     return record;
-  }
+  },
 
   setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
     controller.set(
       "allLlms",
       this.modelFor("adminPlugins.show.discourse-ai-llms")
     );
-    controller.set(
-      "llmTemplate",
-      this.paramsFor(this.routeName).llmTemplate || null
-    );
-  }
-}
+  },
+});

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-show.js

@@ -1,19 +1,17 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
+export default DiscourseRoute.extend({
   async model(params) {
     const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
     const id = parseInt(params.id, 10);
-    const record = allLlms.findBy("id", id);
-    record.provider_params = record.provider_params || {};
-    return record;
-  }
+    return allLlms.findBy("id", id);
+  },
 
   setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
     controller.set(
       "allLlms",
       this.modelFor("adminPlugins.show.discourse-ai-llms")
     );
-  }
-}
+  },
+});

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-personas-new.js

@@ -1,7 +1,7 @@
 import { AUTO_GROUPS } from "discourse/lib/constants";
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
+export default DiscourseRoute.extend({
   async model() {
     const record = this.store.createRecord("ai-persona");
     record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
@@ -10,16 +10,14 @@ export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRou
     record.set("rag_chunk_tokens", 374);
     record.set("rag_chunk_overlap_tokens", 10);
     record.set("rag_conversation_chunks", 10);
-    record.set("allow_personal_messages", true);
-    record.set("tool_details", false);
     return record;
-  }
+  },
 
   setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
     controller.set(
       "allPersonas",
       this.modelFor("adminPlugins.show.discourse-ai-personas")
     );
-  }
-}
+  },
+});

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-personas-show.js

@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
+export default DiscourseRoute.extend({
   async model(params) {
     const allPersonas = this.modelFor(
       "adminPlugins.show.discourse-ai-personas"
     );
     const id = parseInt(params.id, 10);
     return allPersonas.findBy("id", id);
-  }
+  },
 
   setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
     controller.set(
       "allPersonas",
       this.modelFor("adminPlugins.show.discourse-ai-personas")
     );
-  }
-}
+  },
+});

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-spam.js

@@ -1,11 +0,0 @@
-import { service } from "@ember/service";
-import { ajax } from "discourse/lib/ajax";
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiSpamRoute extends DiscourseRoute {
-  @service store;
-
-  model() {
-    return ajax("/admin/plugins/discourse-ai/ai-spam.json");
-  }
-}

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-tools-show.js

@@ -1,6 +1,6 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
+export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
   async model(params) {
     const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
     const id = parseInt(params.id, 10);

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-usage.js

@@ -1,11 +0,0 @@
-import { service } from "@ember/service";
-import { ajax } from "discourse/lib/ajax";
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiUsageRoute extends DiscourseRoute {
-  @service store;
-
-  model() {
-    return ajax("/admin/plugins/discourse-ai/ai-usage.json");
-  }
-}

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-llms/new.hbs

@@ -1,5 +1 @@
-<AiLlmsListEditor
-  @llms={{this.allLlms}}
-  @currentLlm={{this.model}}
-  @llmTemplate={{this.llmTemplate}}
-/>
+<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-spam.hbs

@@ -1 +0,0 @@
-<AiSpam @model={{this.model}} />

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-tools/edit.hbs

@@ -1,7 +0,0 @@
-<section class="ai-persona-tool-editor__current admin-detail pull-left">
-  <AiToolEditor
-    @tools={{this.allTools}}
-    @model={{this.model}}
-    @presets={{this.presets}}
-  />
-</section>

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-tools/new.hbs

@@ -1,7 +1,5 @@
-<section class="ai-persona-tool-editor__current admin-detail pull-left">
-  <AiToolEditor
-    @tools={{this.allTools}}
-    @model={{this.model}}
-    @presets={{this.presets}}
-  />
-</section>
+<AiToolEditor
+  @tools={{this.allTools}}
+  @model={{this.model}}
+  @presets={{this.presets}}
+/>

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-tools/show.hbs

@@ -0,0 +1,5 @@
+<AiToolEditor
+  @tools={{this.allTools}}
+  @model={{this.model}}
+  @presets={{this.presets}}
+/>

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-usage.hbs

@@ -1 +0,0 @@
-<AiUsage @model={{this.model}} />

app/controllers/discourse_ai/admin/ai_llms_controller.rb

@@ -14,9 +14,6 @@ module DiscourseAi
                      llms,
                      each_serializer: LlmModelSerializer,
                      root: false,
-                     scope: {
-                       llm_usage: DiscourseAi::Configuration::LlmEnumerator.global_usage,
-                     },
                    ).as_json,
                  meta: {
                    provider_params: LlmModel.provider_params,
@@ -30,10 +27,7 @@ module DiscourseAi
                }
       end
 
-      def new
-      end
-
-      def edit
+      def show
         llm_model = LlmModel.find(params[:id])
         render json: LlmModelSerializer.new(llm_model)
       end
@@ -42,34 +36,26 @@ module DiscourseAi
         llm_model = LlmModel.new(ai_llm_params)
         if llm_model.save
           llm_model.toggle_companion_user
-          render json: LlmModelSerializer.new(llm_model), status: :created
+          render json: { ai_persona: LlmModelSerializer.new(llm_model) }, status: :created
         else
-          render_json_error llm_model
+          render_json_error LlmModelSerializer.new(llm_model)
         end
       end
 
       def update
         llm_model = LlmModel.find(params[:id])
 
-        if llm_model.seeded?
-          return render_json_error(I18n.t("discourse_ai.llm.cannot_edit_builtin"), status: 403)
-        end
-
         if llm_model.update(ai_llm_params(updating: llm_model))
           llm_model.toggle_companion_user
           render json: LlmModelSerializer.new(llm_model)
         else
-          render_json_error llm_model
+          render_json_error LlmModelSerializer.new(llm_model)
         end
       end
 
       def destroy
         llm_model = LlmModel.find(params[:id])
 
-        if llm_model.seeded?
-          return render_json_error(I18n.t("discourse_ai.llm.cannot_delete_builtin"), status: 403)
-        end
-
         in_use_by = DiscourseAi::Configuration::LlmValidator.new.modules_using(llm_model)
 
         if !in_use_by.empty?
@@ -111,8 +97,6 @@ module DiscourseAi
       private
 
       def ai_llm_params(updating: nil)
-        return {} if params[:ai_llm].blank?
-
         permitted =
           params.require(:ai_llm).permit(
             :display_name,
@@ -122,30 +106,20 @@ module DiscourseAi
             :max_prompt_tokens,
             :api_key,
             :enabled_chat_bot,
-            :vision_enabled,
           )
 
         provider = updating ? updating.provider : permitted[:provider]
-        permit_url = provider != LlmModel::BEDROCK_PROVIDER_NAME
+        permit_url =
+          (updating && updating.url != LlmModel::RESERVED_VLLM_SRV_URL) ||
+            provider != LlmModel::BEDROCK_PROVIDER_NAME
 
-        new_url = params.dig(:ai_llm, :url)
-        permitted[:url] = new_url if permit_url && new_url
+        permitted[:url] = params.dig(:ai_llm, :url) if permit_url
 
-        extra_field_names = LlmModel.provider_params.dig(provider&.to_sym)
-        if extra_field_names.present?
-          received_prov_params =
-            params.dig(:ai_llm, :provider_params)&.slice(*extra_field_names.keys)
-
-          if received_prov_params.present?
-            received_prov_params.each do |pname, value|
-              if extra_field_names[pname.to_sym] == :checkbox
-                received_prov_params[pname] = ActiveModel::Type::Boolean.new.cast(value)
-              end
-            end
-
-            permitted[:provider_params] = received_prov_params.permit!
-          end
-        end
+        extra_field_names = LlmModel.provider_params.dig(provider&.to_sym, :fields).to_a
+        received_prov_params = params.dig(:ai_llm, :provider_params)
+        permitted[:provider_params] = received_prov_params.slice(
+          *extra_field_names,
+        ).permit! if !extra_field_names.empty? && received_prov_params.present?
 
         permitted
       end

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -5,7 +5,8 @@ module DiscourseAi
     class AiPersonasController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_persona, only: %i[edit update destroy create_user]
+      before_action :find_ai_persona,
+                    only: %i[show update destroy create_user indexing_status_check]
 
       def index
         ai_personas =
@@ -33,17 +34,14 @@ module DiscourseAi
         render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
       end
 
-      def new
-      end
-
-      def edit
+      def show
         render json: LocalizedAiPersonaSerializer.new(@ai_persona)
       end
 
       def create
         ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
         if ai_persona.save
-          RagDocumentFragment.link_target_and_uploads(ai_persona, attached_upload_ids)
+          RagDocumentFragment.link_persona_and_uploads(ai_persona, attached_upload_ids)
 
           render json: {
                    ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
@@ -61,11 +59,11 @@ module DiscourseAi
 
       def update
         if @ai_persona.update(ai_persona_params.except(:rag_uploads))
-          RagDocumentFragment.update_target_uploads(@ai_persona, attached_upload_ids)
+          RagDocumentFragment.update_persona_uploads(@ai_persona, attached_upload_ids)
 
           render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
         else
-          render_json_error @ai_persona
+          render_json_error LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
         end
       end
 
@@ -73,97 +71,43 @@ module DiscourseAi
         if @ai_persona.destroy
           head :no_content
         else
-          render_json_error @ai_persona
+          render_json_error LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
         end
       end
 
-      def stream_reply
-        persona =
-          AiPersona.find_by(name: params[:persona_name]) ||
-            AiPersona.find_by(id: params[:persona_id])
-        return render_json_error(I18n.t("discourse_ai.errors.persona_not_found")) if persona.nil?
+      def upload_file
+        file = params[:file] || params[:files].first
 
-        return render_json_error(I18n.t("discourse_ai.errors.persona_disabled")) if !persona.enabled
-
-        if persona.default_llm.blank?
-          return render_json_error(I18n.t("discourse_ai.errors.no_default_llm"))
+        if !SiteSetting.ai_embeddings_enabled?
+          raise Discourse::InvalidAccess.new("Embeddings not enabled")
         end
 
-        if params[:query].blank?
-          return render_json_error(I18n.t("discourse_ai.errors.no_query_specified"))
-        end
+        validate_extension!(file.original_filename)
+        validate_file_size!(file.tempfile.size)
 
-        if !persona.user_id
-          return render_json_error(I18n.t("discourse_ai.errors.no_user_for_persona"))
-        end
+        hijack do
+          upload =
+            UploadCreator.new(
+              file.tempfile,
+              file.original_filename,
+              type: "discourse_ai_rag_upload",
+              skip_validations: true,
+            ).create_for(current_user.id)
 
-        if !params[:username] && !params[:user_unique_id]
-          return render_json_error(I18n.t("discourse_ai.errors.no_user_specified"))
-        end
-
-        user = nil
-
-        if params[:username]
-          user = User.find_by_username(params[:username])
-          return render_json_error(I18n.t("discourse_ai.errors.user_not_found")) if user.nil?
-        elsif params[:user_unique_id]
-          user = stage_user
-        end
-
-        raise Discourse::NotFound if user.nil?
-
-        topic_id = params[:topic_id].to_i
-        topic = nil
-
-        if topic_id > 0
-          topic = Topic.find(topic_id)
-
-          if topic.topic_allowed_users.where(user_id: user.id).empty?
-            return render_json_error(I18n.t("discourse_ai.errors.user_not_allowed"))
+          if upload.persisted?
+            render json: UploadSerializer.new(upload)
+          else
+            render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
           end
         end
+      end
 
-        hijack = request.env["rack.hijack"]
-        io = hijack.call
-
-        DiscourseAi::AiBot::ResponseHttpStreamer.queue_streamed_reply(
-          io: io,
-          persona: persona,
-          user: user,
-          topic: topic,
-          query: params[:query].to_s,
-          custom_instructions: params[:custom_instructions].to_s,
-          current_user: current_user,
-        )
+      def indexing_status_check
+        render json: RagDocumentFragment.indexing_status(@ai_persona, @ai_persona.uploads)
       end
 
       private
 
-      AI_STREAM_CONVERSATION_UNIQUE_ID = "ai-stream-conversation-unique-id"
-
-      def stage_user
-        unique_id = params[:user_unique_id].to_s
-        field = UserCustomField.find_by(name: AI_STREAM_CONVERSATION_UNIQUE_ID, value: unique_id)
-
-        if field
-          field.user
-        else
-          preferred_username = params[:preferred_username]
-          username = UserNameSuggester.suggest(preferred_username || unique_id)
-
-          user =
-            User.new(
-              username: username,
-              email: "#{SecureRandom.hex}@invalid.com",
-              staged: true,
-              active: false,
-            )
-          user.custom_fields[AI_STREAM_CONVERSATION_UNIQUE_ID] = unique_id
-          user.save!
-          user
-        end
-      end
-
       def find_ai_persona
         @ai_persona = AiPersona.find(params[:id])
       end
@@ -184,6 +128,7 @@ module DiscourseAi
             :temperature,
             :default_llm,
             :user_id,
+            :mentionable,
             :max_context_posts,
             :vision_enabled,
             :vision_max_pixels,
@@ -191,13 +136,8 @@ module DiscourseAi
             :rag_chunk_overlap_tokens,
             :rag_conversation_chunks,
             :question_consolidator_llm,
-            :allow_chat_channel_mentions,
-            :allow_chat_direct_messages,
-            :allow_topic_mentions,
-            :allow_personal_messages,
+            :allow_chat,
             :tool_details,
-            :forced_tool_count,
-            :force_default_llm,
             allowed_group_ids: [],
             rag_uploads: [:id],
           )
@@ -212,12 +152,40 @@ module DiscourseAi
       def permit_tools(tools)
         return [] if !tools.is_a?(Array)
 
-        tools.filter_map do |tool, options, force_tool|
+        tools.filter_map do |tool, options|
           break nil if !tool.is_a?(String)
           options&.permit! if options && options.is_a?(ActionController::Parameters)
 
-          # this is simpler from a storage perspective, 1 way to store tools
-          [tool, options, !!force_tool]
+          if options
+            [tool, options]
+          else
+            tool
+          end
+        end
+      end
+
+      def validate_extension!(filename)
+        extension = File.extname(filename)[1..-1] || ""
+        authorized_extensions = %w[txt md]
+        if !authorized_extensions.include?(extension)
+          raise Discourse::InvalidParameters.new(
+                  I18n.t(
+                    "upload.unauthorized",
+                    authorized_extensions: authorized_extensions.join(" "),
+                  ),
+                )
+        end
+      end
+
+      def validate_file_size!(filesize)
+        max_size_bytes = 20.megabytes
+        if filesize > max_size_bytes
+          raise Discourse::InvalidParameters.new(
+                  I18n.t(
+                    "upload.attachments.too_large_humanized",
+                    max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
+                  ),
+                )
         end
       end
     end

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Admin
-    class AiSpamController < ::Admin::AdminController
-      requires_plugin "discourse-ai"
-
-      def show
-        render json: AiSpamSerializer.new(spam_config, root: false)
-      end
-
-      def update
-        updated_params = {}
-        if allowed_params.key?(:llm_model_id)
-          llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
-          if llm_model_id.to_i < 0 &&
-               !SiteSetting.ai_spam_detection_model_allowed_seeded_models_map.include?(
-                 "custom:#{llm_model_id}",
-               )
-            return(
-              render_json_error(
-                I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"),
-                status: 422,
-              )
-            )
-          end
-        end
-        updated_params[:data] = {
-          custom_instructions: allowed_params[:custom_instructions],
-        } if allowed_params.key?(:custom_instructions)
-
-        if updated_params.present?
-          # not using upsert cause we will not get the correct validation errors
-          if AiModerationSetting.spam
-            AiModerationSetting.spam.update!(updated_params)
-          else
-            AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
-          end
-        end
-
-        is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
-
-        if allowed_params.key?(:is_enabled)
-          if is_enabled && !AiModerationSetting.spam&.llm_model_id
-            return(
-              render_json_error(
-                I18n.t("discourse_ai.llm.configuration.must_select_model"),
-                status: 422,
-              )
-            )
-          end
-
-          SiteSetting.ai_spam_detection_enabled = is_enabled
-        end
-
-        render json: AiSpamSerializer.new(spam_config, root: false)
-      end
-
-      def test
-        url = params[:post_url].to_s
-        post = nil
-
-        if url.match?(/^\d+$/)
-          post_id = url.to_i
-          post = Post.find_by(id: post_id)
-        end
-
-        route = UrlHelper.rails_route_from_url(url) if !post
-
-        if route
-          if route[:controller] == "topics"
-            post_number = route[:post_number] || 1
-            post = Post.with_deleted.find_by(post_number: post_number, topic_id: route[:topic_id])
-          end
-        end
-
-        raise Discourse::NotFound if !post
-
-        result =
-          DiscourseAi::AiModeration::SpamScanner.test_post(
-            post,
-            custom_instructions: params[:custom_instructions],
-            llm_id: params[:llm_id],
-          )
-
-        render json: result
-      end
-
-      private
-
-      def allowed_params
-        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
-      end
-
-      def spam_config
-        spam_config = {
-          enabled: SiteSetting.ai_spam_detection_enabled,
-          settings: AiModerationSetting.spam,
-        }
-
-        spam_config[:stats] = DiscourseAi::AiModeration::SpamReport.generate(min_date: 1.week.ago)
-
-        if spam_config[:stats].scanned_count > 0
-          spam_config[
-            :flagging_username
-          ] = DiscourseAi::AiModeration::SpamScanner.flagging_user&.username
-        end
-        spam_config
-      end
-    end
-  end
-end

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -5,17 +5,14 @@ module DiscourseAi
     class AiToolsController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_tool, only: %i[test edit update destroy]
+      before_action :find_ai_tool, only: %i[show update destroy]
 
       def index
         ai_tools = AiTool.all
         render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
       end
 
-      def new
-      end
-
-      def edit
+      def show
         render_serialized(@ai_tool, AiCustomToolSerializer)
       end
 
@@ -24,7 +21,6 @@ module DiscourseAi
         ai_tool.created_by_id = current_user.id
 
         if ai_tool.save
-          RagDocumentFragment.link_target_and_uploads(ai_tool, attached_upload_ids)
           render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
         else
           render_json_error ai_tool
@@ -33,7 +29,6 @@ module DiscourseAi
 
       def update
         if @ai_tool.update(ai_tool_params)
-          RagDocumentFragment.update_target_uploads(@ai_tool, attached_upload_ids)
           render_serialized(@ai_tool, AiCustomToolSerializer)
         else
           render_json_error @ai_tool
@@ -49,13 +44,18 @@ module DiscourseAi
       end
 
       def test
-        @ai_tool.assign_attributes(ai_tool_params) if params[:ai_tool]
+        if params[:id].present?
+          ai_tool = AiTool.find(params[:id])
+        else
+          ai_tool = AiTool.new(ai_tool_params)
+        end
+
         parameters = params[:parameters].to_unsafe_h
 
         # we need an llm so we have a tokenizer
         # but will do without if none is available
         llm = LlmModel.first&.to_llm
-        runner = @ai_tool.runner(parameters, llm: llm, bot_user: current_user, context: {})
+        runner = ai_tool.runner(parameters, llm: llm, bot_user: current_user, context: {})
         result = runner.invoke
 
         if result.is_a?(Hash) && result[:error]
@@ -71,28 +71,18 @@ module DiscourseAi
 
       private
 
-      def attached_upload_ids
-        params[:ai_tool][:rag_uploads].to_a.map { |h| h[:id] }
-      end
-
       def find_ai_tool
-        @ai_tool = AiTool.find(params[:id].to_i)
+        @ai_tool = AiTool.find(params[:id])
       end
 
       def ai_tool_params
-        params
-          .require(:ai_tool)
-          .permit(
-            :name,
-            :description,
-            :script,
-            :summary,
-            :rag_chunk_tokens,
-            :rag_chunk_overlap_tokens,
-            rag_uploads: [:id],
-            parameters: [:name, :type, :description, :required, enum: []],
-          )
-          .except(:rag_uploads)
+        params.require(:ai_tool).permit(
+          :name,
+          :description,
+          :script,
+          :summary,
+          parameters: [:name, :type, :description, :required, :enum, enum_values: []],
+        )
       end
     end
   end

app/controllers/discourse_ai/admin/ai_usage_controller.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Admin
-    class AiUsageController < ::Admin::AdminController
-      requires_plugin "discourse-ai"
-
-      def show
-      end
-
-      def report
-        render json: AiUsageSerializer.new(create_report, root: false)
-      end
-
-      private
-
-      def create_report
-        report =
-          DiscourseAi::Completions::Report.new(
-            start_date: params[:start_date]&.to_date || 30.days.ago,
-            end_date: params[:end_date]&.to_date || Time.current,
-          )
-
-        report = report.filter_by_feature(params[:feature]) if params[:feature].present?
-        report = report.filter_by_model(params[:model]) if params[:model].present?
-        report
-      end
-    end
-  end
-end

app/controllers/discourse_ai/admin/rag_document_fragments_controller.rb

@@ -1,75 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Admin
-    class RagDocumentFragmentsController < ::Admin::AdminController
-      requires_plugin ::DiscourseAi::PLUGIN_NAME
-
-      def indexing_status_check
-        if params[:target_type] == "AiPersona"
-          @target = AiPersona.find(params[:target_id])
-        elsif params[:target_type] == "AiTool"
-          @target = AiTool.find(params[:target_id])
-        else
-          raise Discourse::InvalidParameters.new("Invalid target type")
-        end
-
-        render json: RagDocumentFragment.indexing_status(@target, @target.uploads)
-      end
-
-      def upload_file
-        file = params[:file] || params[:files].first
-
-        if !SiteSetting.ai_embeddings_enabled?
-          raise Discourse::InvalidAccess.new("Embeddings not enabled")
-        end
-
-        validate_extension!(file.original_filename)
-        validate_file_size!(file.tempfile.size)
-
-        hijack do
-          upload =
-            UploadCreator.new(
-              file.tempfile,
-              file.original_filename,
-              type: "discourse_ai_rag_upload",
-              skip_validations: true,
-            ).create_for(current_user.id)
-
-          if upload.persisted?
-            render json: UploadSerializer.new(upload)
-          else
-            render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
-          end
-        end
-      end
-
-      private
-
-      def validate_extension!(filename)
-        extension = File.extname(filename)[1..-1] || ""
-        authorized_extensions = %w[txt md]
-        if !authorized_extensions.include?(extension)
-          raise Discourse::InvalidParameters.new(
-                  I18n.t(
-                    "upload.unauthorized",
-                    authorized_extensions: authorized_extensions.join(" "),
-                  ),
-                )
-        end
-      end
-
-      def validate_file_size!(filesize)
-        max_size_bytes = 20.megabytes
-        if filesize > max_size_bytes
-          raise Discourse::InvalidParameters.new(
-                  I18n.t(
-                    "upload.attachments.too_large_humanized",
-                    max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
-                  ),
-                )
-        end
-      end
-    end
-  end
-end

app/controllers/discourse_ai/ai_bot/artifacts_controller.rb

@@ -1,100 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module AiBot
-    class ArtifactsController < ApplicationController
-      requires_plugin DiscourseAi::PLUGIN_NAME
-      before_action :require_site_settings!
-
-      skip_before_action :preload_json, :check_xhr, only: %i[show]
-
-      def show
-        artifact = AiArtifact.find(params[:id])
-
-        post = Post.find_by(id: artifact.post_id)
-        if artifact.metadata&.dig("public")
-          # no guardian needed
-        else
-          raise Discourse::NotFound if !post&.topic&.private_message?
-          raise Discourse::NotFound if !guardian.can_see?(post)
-        end
-
-        name = artifact.name
-
-        if params[:version].present?
-          artifact = artifact.versions.find_by(version_number: params[:version])
-          raise Discourse::NotFound if !artifact
-        end
-
-        js = artifact.js || ""
-        if !js.match?(%r{\A\s*<script.*</script>}mi)
-          mod = ""
-          mod = " type=\"module\"" if js.match?(/\A\s*import.*/)
-          js = "<script#{mod}>\n#{js}\n</script>"
-        end
-        # Prepare the inner (untrusted) HTML document
-        untrusted_html = <<~HTML
-          <!DOCTYPE html>
-          <html>
-            <head>
-              <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(name)}</title>
-              <style>
-                #{artifact.css}
-              </style>
-            </head>
-            <body>
-              #{artifact.html}
-              #{js}
-            </body>
-          </html>
-        HTML
-
-        # Prepare the outer (trusted) HTML document
-        trusted_html = <<~HTML
-          <!DOCTYPE html>
-          <html>
-            <head>
-              <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(name)}</title>
-              <style>
-                html, body, iframe {
-                  margin: 0;
-                  padding: 0;
-                  width: 100%;
-                  height: 100%;
-                  border: 0;
-                  overflow: hidden;
-                }
-                iframe {
-                  overflow: auto;
-                }
-              </style>
-            </head>
-            <body>
-              <iframe sandbox="allow-scripts allow-forms" height="100%" width="100%" srcdoc="#{ERB::Util.html_escape(untrusted_html)}" frameborder="0"></iframe>
-            </body>
-          </html>
-        HTML
-
-        response.headers.delete("X-Frame-Options")
-        response.headers[
-          "Content-Security-Policy"
-        ] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://unpkg.com https://cdnjs.cloudflare.com https://ajax.googleapis.com https://cdn.jsdelivr.net;"
-        response.headers["X-Robots-Tag"] = "noindex"
-
-        # Render the content
-        render html: trusted_html.html_safe, layout: false, content_type: "text/html"
-      end
-
-      private
-
-      def require_site_settings!
-        if !SiteSetting.discourse_ai_enabled ||
-             !SiteSetting.ai_artifact_security.in?(%w[lax strict])
-          raise Discourse::NotFound
-        end
-      end
-    end
-  end
-end

app/controllers/discourse_ai/ai_bot/bot_controller.rb

@@ -6,14 +6,6 @@ module DiscourseAi
       requires_plugin ::DiscourseAi::PLUGIN_NAME
       requires_login
 
-      def show_debug_info_by_id
-        log = AiApiAuditLog.find(params[:id])
-        raise Discourse::NotFound if !log.topic
-
-        guardian.ensure_can_debug_ai_bot_conversation!(log.topic)
-        render json: AiApiAuditLogSerializer.new(log, root: false), status: 200
-      end
-
       def show_debug_info
         post = Post.find(params[:post_id])
         guardian.ensure_can_debug_ai_bot_conversation!(post)

app/controllers/discourse_ai/ai_bot/shared_ai_conversations_controller.rb

@@ -7,8 +7,7 @@ module DiscourseAi
       requires_login only: %i[create update destroy]
       before_action :require_site_settings!
 
-      skip_before_action :preload_json, :check_xhr, only: %i[show asset]
-      skip_before_action :verify_authenticity_token, only: ["asset"]
+      skip_before_action :preload_json, :check_xhr, only: %i[show]
 
       def create
         ensure_allowed_create!
@@ -51,30 +50,6 @@ module DiscourseAi
         end
       end
 
-      def asset
-        no_cookies
-
-        name = params[:name]
-        path, content_type =
-          if name == "share"
-            %w[share.css text/css]
-          elsif name == "highlight"
-            %w[highlight.min.js application/javascript]
-          else
-            raise Discourse::NotFound
-          end
-
-        content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
-
-        # note, path contains a ":version" which automatically busts the cache
-        # based on file content, so this is safe
-        response.headers["Last-Modified"] = 10.years.ago.httpdate
-        response.headers["Content-Length"] = content.bytesize.to_s
-        immutable_for 1.year
-
-        render plain: content, disposition: :nil, content_type: content_type
-      end
-
       def preview
         ensure_allowed_preview!
         data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)

app/controllers/discourse_ai/ai_helper/assistant_controller.rb

@@ -6,21 +6,10 @@ module DiscourseAi
       requires_plugin ::DiscourseAi::PLUGIN_NAME
       requires_login
       before_action :ensure_can_request_suggestions
-      before_action :rate_limiter_performed!
+      before_action :rate_limiter_performed!, except: %i[prompts]
 
       include SecureUploadEndpointHelpers
 
-      RATE_LIMITS = {
-        "default" => {
-          amount: 6,
-          interval: 3.minutes,
-        },
-        "caption_image" => {
-          amount: 20,
-          interval: 1.minute,
-        },
-      }.freeze
-
       def suggest
         input = get_text_param!
         force_default_locale = params[:force_default_locale] || false
@@ -35,7 +24,7 @@ module DiscourseAi
           prompt.custom_instruction = params[:custom_prompt]
         end
 
-        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+        suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
 
         hijack do
           render json:
@@ -100,26 +89,18 @@ module DiscourseAi
         end
       end
 
-      def stream_suggestion
+      def explain
         post_id = get_post_param!
-        text = get_text_param!
+        term_to_explain = get_text_param!
         post = Post.includes(:topic).find_by(id: post_id)
-        prompt = CompletionPrompt.find_by(id: params[:mode])
 
-        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
         raise Discourse::InvalidParameters.new(:post_id) unless post
 
-        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
-          raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
-        end
-
         Jobs.enqueue(
           :stream_post_helper,
           post_id: post.id,
           user_id: current_user.id,
-          text: text,
-          prompt: prompt.name,
-          custom_prompt: params[:custom_prompt],
+          term_to_explain: term_to_explain,
         )
 
         render json: { success: true }, status: 200
@@ -172,23 +153,13 @@ module DiscourseAi
       end
 
       def rate_limiter_performed!
-        action_rate_limit = RATE_LIMITS[action_name] || RATE_LIMITS["default"]
-        RateLimiter.new(
-          current_user,
-          "ai_assistant",
-          action_rate_limit[:amount],
-          action_rate_limit[:interval],
-        ).performed!
+        RateLimiter.new(current_user, "ai_assistant", 6, 3.minutes).performed!
       end
 
       def ensure_can_request_suggestions
-        allowed_groups =
-          (
-            SiteSetting.composer_ai_helper_allowed_groups_map |
-              SiteSetting.post_ai_helper_allowed_groups_map
-          )
-
-        raise Discourse::InvalidAccess if !current_user.in_any_groups?(allowed_groups)
+        if !current_user.in_any_groups?(SiteSetting.ai_helper_allowed_groups_map)
+          raise Discourse::InvalidAccess
+        end
       end
     end
   end