Fix lib directory

DEV: Update engine config to match skeleton
- adds `lib/` autoloading (and removes the plugin.rb workaround) - adds eager loading of scheduled jobs in development (otherwise they don't work at all)
2024-05-02 19:19:09 +01:00 · 2024-05-02 18:53:22 +01:00
751 changed files with 111552 additions and 454118 deletions
@@ -1,7 +1,7 @@
-< 3.4.0.beta3-dev: decf1bb49d737ea15308400f22f89d1d1e71d13d
+< 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
-< 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
+< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
-< 3.3.0.beta5-dev: 4d8090002f6dcd8e34d41033606bf131fa221475
+< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
-< 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
+< 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
 < 3.3.0.beta1-dev: d94234d89f50733678a5c62809445198a9fda74b
 3.2.999: d94234d89f50733678a5c62809445198a9fda74b
 3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
 < 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
 3.1.0.beta3: 057fbe1ce6
@@ -0,0 +1 @@
 module.exports = require("@discourse/lint-configs/eslint");
@@ -1,4 +1,3 @@
 node_modules
 /gems
 /auto_generated
 .env
@@ -1,2 +0,0 @@
 engine-strict = true
 auto-install-peers = false
@@ -1,7 +1,7 @@
 GEM
  remote: https://rubygems.org/
  specs:
-    activesupport (7.1.3.4)
+    activesupport (7.1.3.2)
      base64
      bigdecimal
      concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -13,27 +13,26 @@ GEM
      tzinfo (~> 2.0)
    ast (2.4.2)
    base64 (0.2.0)
-    bigdecimal (3.1.8)
+    bigdecimal (3.1.6)
-    concurrent-ruby (1.3.3)
+    concurrent-ruby (1.2.3)
    connection_pool (2.4.1)
    drb (2.2.1)
-    i18n (1.14.5)
+    i18n (1.14.1)
      concurrent-ruby (~> 1.0)
-    json (2.7.2)
+    json (2.7.1)
    language_server-protocol (3.17.0.3)
-    minitest (5.24.1)
+    minitest (5.22.2)
    mutex_m (0.2.0)
-    parallel (1.25.1)
+    parallel (1.24.0)
-    parser (3.3.3.0)
+    parser (3.3.0.5)
      ast (~> 2.4.1)
      racc
    prettier_print (1.2.1)
-    racc (1.8.0)
+    racc (1.7.3)
    rack (3.1.6)
    rainbow (3.1.1)
-    regexp_parser (2.9.2)
+    regexp_parser (2.9.0)
-    rexml (3.3.9)
+    rexml (3.2.6)
-    rubocop (1.64.1)
+    rubocop (1.61.0)
      json (~> 2.3)
      language_server-protocol (>= 3.17.0)
      parallel (~> 1.10)
@@ -41,33 +40,25 @@ GEM
      rainbow (>= 2.2.2, < 4.0)
      regexp_parser (>= 1.8, < 3.0)
      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
      ruby-progressbar (~> 1.7)
      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.3)
+    rubocop-ast (1.31.1)
-      parser (>= 3.3.1.0)
+      parser (>= 3.3.0.4)
-    rubocop-capybara (2.21.0)
+    rubocop-capybara (2.20.0)
      rubocop (~> 1.41)
-    rubocop-discourse (3.8.1)
+    rubocop-discourse (3.7.1)
      activesupport (>= 6.1)
      rubocop (>= 1.59.0)
      rubocop-capybara (>= 2.0.0)
      rubocop-factory_bot (>= 2.0.0)
-      rubocop-rails (>= 2.25.0)
+      rubocop-rspec (>= 2.25.0)
-      rubocop-rspec (>= 3.0.1)
+    rubocop-factory_bot (2.25.1)
-      rubocop-rspec_rails (>= 2.30.0)
+      rubocop (~> 1.41)
-    rubocop-factory_bot (2.26.1)
+    rubocop-rspec (2.27.1)
-      rubocop (~> 1.61)
+      rubocop (~> 1.40)
-    rubocop-rails (2.25.1)
+      rubocop-capybara (~> 2.17)
-      activesupport (>= 4.2.0)
+      rubocop-factory_bot (~> 2.22)
      rack (>= 1.1)
      rubocop (>= 1.33.0, < 2.0)
      rubocop-ast (>= 1.31.1, < 2.0)
    rubocop-rspec (3.0.2)
      rubocop (~> 1.61)
    rubocop-rspec_rails (2.30.0)
      rubocop (~> 1.61)
      rubocop-rspec (~> 3, >= 3.0.1)
    ruby-progressbar (1.13.0)
    syntax_tree (6.2.0)
      prettier_print (>= 1.2.0)
@@ -83,4 +74,4 @@ DEPENDENCIES
  syntax_tree
 BUNDLED WITH
-   2.5.10
+   2.5.4
@@ -1,25 +0,0 @@
 import DiscourseRoute from "discourse/routes/discourse";
 export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
  queryParams = {
    llmTemplate: { refreshModel: true },
  };
  async model() {
    const record = this.store.createRecord("ai-llm");
    record.provider_params = {};
    return record;
  }
  setupController(controller, model) {
    super.setupController(controller, model);
    controller.set(
      "allLlms",
      this.modelFor("adminPlugins.show.discourse-ai-llms")
    );
    controller.set(
      "llmTemplate",
      this.paramsFor(this.routeName).llmTemplate || null
    );
  }
 }
@@ -1,19 +0,0 @@
 import DiscourseRoute from "discourse/routes/discourse";
 export default class AdminPluginsShowDiscourseAiLlmsShow extends DiscourseRoute {
  async model(params) {
    const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
    const id = parseInt(params.id, 10);
    const record = allLlms.findBy("id", id);
    record.provider_params = record.provider_params || {};
    return record;
  }
  setupController(controller, model) {
    super.setupController(controller, model);
    controller.set(
      "allLlms",
      this.modelFor("adminPlugins.show.discourse-ai-llms")
    );
  }
 }
@@ -1,7 +0,0 @@
 import DiscourseRoute from "discourse/routes/discourse";
 export default class DiscourseAiAiLlmsRoute extends DiscourseRoute {
  model() {
    return this.store.findAll("ai-llm");
  }
 }
@@ -1,7 +1,7 @@
 import { AUTO_GROUPS } from "discourse/lib/constants";
 import DiscourseRoute from "discourse/routes/discourse";
-export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
+export default DiscourseRoute.extend({
  async model() {
    const record = this.store.createRecord("ai-persona");
    record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
@@ -10,16 +10,14 @@ export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRou
    record.set("rag_chunk_tokens", 374);
    record.set("rag_chunk_overlap_tokens", 10);
    record.set("rag_conversation_chunks", 10);
    record.set("allow_personal_messages", true);
    record.set("tool_details", false);
    return record;
-  }
+  },
  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allPersonas",
      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
-  }
+  },
-}
+});
@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";
-export default class AdminPluginsShowDiscourseAiPersonasShow extends DiscourseRoute {
+export default DiscourseRoute.extend({
  async model(params) {
    const allPersonas = this.modelFor(
      "adminPlugins.show.discourse-ai-personas"
    );
    const id = parseInt(params.id, 10);
    return allPersonas.findBy("id", id);
-  }
+  },
  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allPersonas",
      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
-  }
+  },
-}
+});
@@ -1,15 +0,0 @@
 import DiscourseRoute from "discourse/routes/discourse";
 export default class DiscourseAiToolsNewRoute extends DiscourseRoute {
  async model() {
    return this.store.createRecord("ai-tool");
  }
  setupController(controller) {
    super.setupController(...arguments);
    const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
    controller.set("allTools", toolsModel);
    controller.set("presets", toolsModel.resultSetMeta.presets);
  }
 }
@@ -1,18 +0,0 @@
 import DiscourseRoute from "discourse/routes/discourse";
 export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
  async model(params) {
    const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
    const id = parseInt(params.id, 10);
    return allTools.find((tool) => tool.id === id);
  }
  setupController(controller) {
    super.setupController(...arguments);
    const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
    controller.set("allTools", toolsModel);
    controller.set("presets", toolsModel.resultSetMeta.presets);
  }
 }
@@ -1,10 +0,0 @@
 import { service } from "@ember/service";
 import DiscourseRoute from "discourse/routes/discourse";
 export default class DiscourseAiToolsRoute extends DiscourseRoute {
  @service store;
  model() {
    return this.store.findAll("ai-tool");
  }
 }
@@ -1,11 +0,0 @@
 import { service } from "@ember/service";
 import { ajax } from "discourse/lib/ajax";
 import DiscourseRoute from "discourse/routes/discourse";
 export default class DiscourseAiUsageRoute extends DiscourseRoute {
  @service store;
  model() {
    return ajax("/admin/plugins/discourse-ai/ai-usage.json");
  }
 }
@@ -1 +0,0 @@
 <AiLlmsListEditor @llms={{this.model}} />
@@ -1,5 +0,0 @@
 <AiLlmsListEditor
  @llms={{this.allLlms}}
  @currentLlm={{this.model}}
  @llmTemplate={{this.llmTemplate}}
 />
@@ -1 +0,0 @@
 <AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
 <AiToolListEditor @tools={{this.model}} />
@@ -1,7 +0,0 @@
 <section class="ai-persona-tool-editor__current admin-detail pull-left">
  <AiToolEditor
    @tools={{this.allTools}}
    @model={{this.model}}
    @presets={{this.presets}}
  />
 </section>
@@ -1,7 +0,0 @@
 <section class="ai-persona-tool-editor__current admin-detail pull-left">
  <AiToolEditor
    @tools={{this.allTools}}
    @model={{this.model}}
    @presets={{this.presets}}
  />
 </section>
@@ -1 +0,0 @@
 <AiUsage @model={{this.model}} />
@@ -1,151 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module Admin
    class AiLlmsController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      def index
        llms = LlmModel.all.order(:display_name)
        render json: {
                 ai_llms:
                   ActiveModel::ArraySerializer.new(
                     llms,
                     each_serializer: LlmModelSerializer,
                     root: false,
                     scope: {
                       llm_usage: DiscourseAi::Configuration::LlmEnumerator.global_usage,
                     },
                   ).as_json,
                 meta: {
                   provider_params: LlmModel.provider_params,
                   presets: DiscourseAi::Completions::Llm.presets,
                   providers: DiscourseAi::Completions::Llm.provider_names,
                   tokenizers:
                     DiscourseAi::Completions::Llm.tokenizer_names.map { |tn|
                       { id: tn, name: tn.split("::").last }
                     },
                 },
               }
      end
      def show
        llm_model = LlmModel.find(params[:id])
        render json: LlmModelSerializer.new(llm_model)
      end
      def create
        llm_model = LlmModel.new(ai_llm_params)
        if llm_model.save
          llm_model.toggle_companion_user
          render json: LlmModelSerializer.new(llm_model), status: :created
        else
          render_json_error llm_model
        end
      end
      def update
        llm_model = LlmModel.find(params[:id])
        if llm_model.seeded?
          return render_json_error(I18n.t("discourse_ai.llm.cannot_edit_builtin"), status: 403)
        end
        if llm_model.update(ai_llm_params(updating: llm_model))
          llm_model.toggle_companion_user
          render json: LlmModelSerializer.new(llm_model)
        else
          render_json_error llm_model
        end
      end
      def destroy
        llm_model = LlmModel.find(params[:id])
        if llm_model.seeded?
          return render_json_error(I18n.t("discourse_ai.llm.cannot_delete_builtin"), status: 403)
        end
        in_use_by = DiscourseAi::Configuration::LlmValidator.new.modules_using(llm_model)
        if !in_use_by.empty?
          return(
            render_json_error(
              I18n.t(
                "discourse_ai.llm.delete_failed",
                settings: in_use_by.join(", "),
                count: in_use_by.length,
              ),
              status: 409,
            )
          )
        end
        # Clean up companion users
        llm_model.enabled_chat_bot = false
        llm_model.toggle_companion_user
        if llm_model.destroy
          head :no_content
        else
          render_json_error llm_model
        end
      end
      def test
        RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!
        llm_model = LlmModel.new(ai_llm_params)
        DiscourseAi::Configuration::LlmValidator.new.run_test(llm_model)
        render json: { success: true }
      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
        render json: { success: false, error: e.message }
      end
      private
      def ai_llm_params(updating: nil)
        return {} if params[:ai_llm].blank?
        permitted =
          params.require(:ai_llm).permit(
            :display_name,
            :name,
            :provider,
            :tokenizer,
            :max_prompt_tokens,
            :api_key,
            :enabled_chat_bot,
            :vision_enabled,
          )
        provider = updating ? updating.provider : permitted[:provider]
        permit_url = provider != LlmModel::BEDROCK_PROVIDER_NAME
        new_url = params.dig(:ai_llm, :url)
        permitted[:url] = new_url if permit_url && new_url
        extra_field_names = LlmModel.provider_params.dig(provider&.to_sym)
        if extra_field_names.present?
          received_prov_params =
            params.dig(:ai_llm, :provider_params)&.slice(*extra_field_names.keys)
          if received_prov_params.present?
            received_prov_params.each do |pname, value|
              if extra_field_names[pname.to_sym] == :checkbox
                received_prov_params[pname] = ActiveModel::Type::Boolean.new.cast(value)
              end
            end
            permitted[:provider_params] = received_prov_params.permit!
          end
        end
        permitted
      end
    end
  end
 end
@@ -5,7 +5,8 @@ module DiscourseAi
    class AiPersonasController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME
-      before_action :find_ai_persona, only: %i[show update destroy create_user]
+      before_action :find_ai_persona,
                    only: %i[show update destroy create_user indexing_status_check]
      def index
        ai_personas =
@@ -18,19 +19,11 @@ module DiscourseAi
          DiscourseAi::AiBot::Personas::Persona.all_available_tools.map do |tool|
            AiToolSerializer.new(tool, root: false)
          end
        AiTool
          .where(enabled: true)
          .each do |tool|
            tools << {
              id: "custom-#{tool.id}",
              name: I18n.t("discourse_ai.tools.custom_name", name: tool.name.capitalize),
            }
          end
        llms =
          DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
            { id: hash[:value], name: hash[:name] }
          end
-        render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
+        render json: { ai_personas: ai_personas, meta: { commands: tools, llms: llms } }
      end
      def show
@@ -40,12 +33,9 @@ module DiscourseAi
      def create
        ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
        if ai_persona.save
-          RagDocumentFragment.link_target_and_uploads(ai_persona, attached_upload_ids)
+          RagDocumentFragment.link_persona_and_uploads(ai_persona, attached_upload_ids)
-          render json: {
+          render json: { ai_persona: ai_persona }, status: :created
                   ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
                 },
                 status: :created
        else
          render_json_error ai_persona
        end
@@ -58,9 +48,9 @@ module DiscourseAi
      def update
        if @ai_persona.update(ai_persona_params.except(:rag_uploads))
-          RagDocumentFragment.update_target_uploads(@ai_persona, attached_upload_ids)
+          RagDocumentFragment.update_persona_uploads(@ai_persona, attached_upload_ids)
-          render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
+          render json: @ai_persona
        else
          render_json_error @ai_persona
        end
@@ -74,93 +64,39 @@ module DiscourseAi
        end
      end
-      def stream_reply
+      def upload_file
-        persona =
+        file = params[:file] || params[:files].first
          AiPersona.find_by(name: params[:persona_name]) ||
            AiPersona.find_by(id: params[:persona_id])
        return render_json_error(I18n.t("discourse_ai.errors.persona_not_found")) if persona.nil?
-        return render_json_error(I18n.t("discourse_ai.errors.persona_disabled")) if !persona.enabled
+        if !SiteSetting.ai_embeddings_enabled?
-
+          raise Discourse::InvalidAccess.new("Embeddings not enabled")
        if persona.default_llm.blank?
          return render_json_error(I18n.t("discourse_ai.errors.no_default_llm"))
        end
-        if params[:query].blank?
+        validate_extension!(file.original_filename)
-          return render_json_error(I18n.t("discourse_ai.errors.no_query_specified"))
+        validate_file_size!(file.tempfile.size)
        end
-        if !persona.user_id
+        hijack do
-          return render_json_error(I18n.t("discourse_ai.errors.no_user_for_persona"))
+          upload =
-        end
+            UploadCreator.new(
              file.tempfile,
              file.original_filename,
              type: "discourse_ai_rag_upload",
              skip_validations: true,
            ).create_for(current_user.id)
-        if !params[:username] && !params[:user_unique_id]
+          if upload.persisted?
-          return render_json_error(I18n.t("discourse_ai.errors.no_user_specified"))
+            render json: UploadSerializer.new(upload)
-        end
+          else
-
+            render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
        user = nil
        if params[:username]
          user = User.find_by_username(params[:username])
          return render_json_error(I18n.t("discourse_ai.errors.user_not_found")) if user.nil?
        elsif params[:user_unique_id]
          user = stage_user
        end
        raise Discourse::NotFound if user.nil?
        topic_id = params[:topic_id].to_i
        topic = nil
        if topic_id > 0
          topic = Topic.find(topic_id)
          if topic.topic_allowed_users.where(user_id: user.id).empty?
            return render_json_error(I18n.t("discourse_ai.errors.user_not_allowed"))
          end
        end
      end
-        hijack = request.env["rack.hijack"]
+      def indexing_status_check
-        io = hijack.call
+        render json: RagDocumentFragment.indexing_status(@ai_persona, @ai_persona.uploads)
        DiscourseAi::AiBot::ResponseHttpStreamer.queue_streamed_reply(
          io: io,
          persona: persona,
          user: user,
          topic: topic,
          query: params[:query].to_s,
          custom_instructions: params[:custom_instructions].to_s,
          current_user: current_user,
        )
      end
      private
      AI_STREAM_CONVERSATION_UNIQUE_ID = "ai-stream-conversation-unique-id"
      def stage_user
        unique_id = params[:user_unique_id].to_s
        field = UserCustomField.find_by(name: AI_STREAM_CONVERSATION_UNIQUE_ID, value: unique_id)
        if field
          field.user
        else
          preferred_username = params[:preferred_username]
          username = UserNameSuggester.suggest(preferred_username || unique_id)
          user =
            User.new(
              username: username,
              email: "#{SecureRandom.hex}@invalid.com",
              staged: true,
              active: false,
            )
          user.custom_fields[AI_STREAM_CONVERSATION_UNIQUE_ID] = unique_id
          user.save!
          user
        end
      end
      def find_ai_persona
        @ai_persona = AiPersona.find(params[:id])
      end
@@ -181,6 +117,7 @@ module DiscourseAi
            :temperature,
            :default_llm,
            :user_id,
            :mentionable,
            :max_context_posts,
            :vision_enabled,
            :vision_max_pixels,
@@ -188,33 +125,54 @@ module DiscourseAi
            :rag_chunk_overlap_tokens,
            :rag_conversation_chunks,
            :question_consolidator_llm,
            :allow_chat_channel_mentions,
            :allow_chat_direct_messages,
            :allow_topic_mentions,
            :allow_personal_messages,
            :tool_details,
            :forced_tool_count,
            :force_default_llm,
            allowed_group_ids: [],
            rag_uploads: [:id],
          )
-        if tools = params.dig(:ai_persona, :tools)
+        if commands = params.dig(:ai_persona, :commands)
-          permitted[:tools] = permit_tools(tools)
+          permitted[:commands] = permit_commands(commands)
        end
        permitted
      end
-      def permit_tools(tools)
+      def permit_commands(commands)
-        return [] if !tools.is_a?(Array)
+        return [] if !commands.is_a?(Array)
-        tools.filter_map do |tool, options, force_tool|
+        commands.filter_map do |command, options|
-          break nil if !tool.is_a?(String)
+          break nil if !command.is_a?(String)
          options&.permit! if options && options.is_a?(ActionController::Parameters)
-          # this is simpler from a storage perspective, 1 way to store tools
+          if options
-          [tool, options, !!force_tool]
+            [command, options]
          else
            command
          end
        end
      end
      def validate_extension!(filename)
        extension = File.extname(filename)[1..-1] || ""
        authorized_extensions = %w[txt md]
        if !authorized_extensions.include?(extension)
          raise Discourse::InvalidParameters.new(
                  I18n.t(
                    "upload.unauthorized",
                    authorized_extensions: authorized_extensions.join(" "),
                  ),
                )
        end
      end
      def validate_file_size!(filesize)
        max_size_bytes = 20.megabytes
        if filesize > max_size_bytes
          raise Discourse::InvalidParameters.new(
                  I18n.t(
                    "upload.attachments.too_large_humanized",
                    max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
                  ),
                )
        end
      end
    end
@@ -1,96 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module Admin
    class AiToolsController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      before_action :find_ai_tool, only: %i[test show update destroy]
      def index
        ai_tools = AiTool.all
        render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
      end
      def show
        render_serialized(@ai_tool, AiCustomToolSerializer)
      end
      def create
        ai_tool = AiTool.new(ai_tool_params)
        ai_tool.created_by_id = current_user.id
        if ai_tool.save
          RagDocumentFragment.link_target_and_uploads(ai_tool, attached_upload_ids)
          render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
        else
          render_json_error ai_tool
        end
      end
      def update
        if @ai_tool.update(ai_tool_params)
          RagDocumentFragment.update_target_uploads(@ai_tool, attached_upload_ids)
          render_serialized(@ai_tool, AiCustomToolSerializer)
        else
          render_json_error @ai_tool
        end
      end
      def destroy
        if @ai_tool.destroy
          head :no_content
        else
          render_json_error @ai_tool
        end
      end
      def test
        @ai_tool.assign_attributes(ai_tool_params) if params[:ai_tool]
        parameters = params[:parameters].to_unsafe_h
        # we need an llm so we have a tokenizer
        # but will do without if none is available
        llm = LlmModel.first&.to_llm
        runner = @ai_tool.runner(parameters, llm: llm, bot_user: current_user, context: {})
        result = runner.invoke
        if result.is_a?(Hash) && result[:error]
          render_json_error result[:error]
        else
          render json: { output: result }
        end
      rescue ActiveRecord::RecordNotFound => e
        render_json_error e.message, status: 400
      rescue => e
        render_json_error "Error executing the tool: #{e.message}", status: 400
      end
      private
      def attached_upload_ids
        params[:ai_tool][:rag_uploads].to_a.map { |h| h[:id] }
      end
      def find_ai_tool
        @ai_tool = AiTool.find(params[:id].to_i)
      end
      def ai_tool_params
        params
          .require(:ai_tool)
          .permit(
            :name,
            :description,
            :script,
            :summary,
            :rag_chunk_tokens,
            :rag_chunk_overlap_tokens,
            rag_uploads: [:id],
            parameters: [:name, :type, :description, :required, enum: []],
          )
          .except(:rag_uploads)
      end
    end
  end
 end
@@ -1,27 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module Admin
    class AiUsageController < ::Admin::AdminController
      requires_plugin "discourse-ai"
      def show
        render json: AiUsageSerializer.new(create_report, root: false)
      end
      private
      def create_report
        report =
          DiscourseAi::Completions::Report.new(
            start_date: params[:start_date]&.to_date || 30.days.ago,
            end_date: params[:end_date]&.to_date || Time.current,
          )
        report = report.filter_by_feature(params[:feature]) if params[:feature].present?
        report = report.filter_by_model(params[:model]) if params[:model].present?
        report
      end
    end
  end
 end
@@ -1,75 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module Admin
    class RagDocumentFragmentsController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      def indexing_status_check
        if params[:target_type] == "AiPersona"
          @target = AiPersona.find(params[:target_id])
        elsif params[:target_type] == "AiTool"
          @target = AiTool.find(params[:target_id])
        else
          raise Discourse::InvalidParameters.new("Invalid target type")
        end
        render json: RagDocumentFragment.indexing_status(@target, @target.uploads)
      end
      def upload_file
        file = params[:file] || params[:files].first
        if !SiteSetting.ai_embeddings_enabled?
          raise Discourse::InvalidAccess.new("Embeddings not enabled")
        end
        validate_extension!(file.original_filename)
        validate_file_size!(file.tempfile.size)
        hijack do
          upload =
            UploadCreator.new(
              file.tempfile,
              file.original_filename,
              type: "discourse_ai_rag_upload",
              skip_validations: true,
            ).create_for(current_user.id)
          if upload.persisted?
            render json: UploadSerializer.new(upload)
          else
            render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
          end
        end
      end
      private
      def validate_extension!(filename)
        extension = File.extname(filename)[1..-1] || ""
        authorized_extensions = %w[txt md]
        if !authorized_extensions.include?(extension)
          raise Discourse::InvalidParameters.new(
                  I18n.t(
                    "upload.unauthorized",
                    authorized_extensions: authorized_extensions.join(" "),
                  ),
                )
        end
      end
      def validate_file_size!(filesize)
        max_size_bytes = 20.megabytes
        if filesize > max_size_bytes
          raise Discourse::InvalidParameters.new(
                  I18n.t(
                    "upload.attachments.too_large_humanized",
                    max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
                  ),
                )
        end
      end
    end
  end
 end
@@ -1,87 +0,0 @@
 # frozen_string_literal: true
 module DiscourseAi
  module AiBot
    class ArtifactsController < ApplicationController
      requires_plugin DiscourseAi::PLUGIN_NAME
      before_action :require_site_settings!
      skip_before_action :preload_json, :check_xhr, only: %i[show]
      def show
        artifact = AiArtifact.find(params[:id])
        post = Post.find_by(id: artifact.post_id)
        if artifact.metadata&.dig("public")
          # no guardian needed
        else
          raise Discourse::NotFound if !post&.topic&.private_message?
          raise Discourse::NotFound if !guardian.can_see?(post)
        end
        # Prepare the inner (untrusted) HTML document
        untrusted_html = <<~HTML
          <!DOCTYPE html>
          <html>
            <head>
              <meta charset="UTF-8">
              <title>#{ERB::Util.html_escape(artifact.name)}</title>
              <style>
                #{artifact.css}
              </style>
            </head>
            <body>
              #{artifact.html}
              <script>
                #{artifact.js}
              </script>
            </body>
          </html>
        HTML
        # Prepare the outer (trusted) HTML document
        trusted_html = <<~HTML
          <!DOCTYPE html>
          <html>
            <head>
              <meta charset="UTF-8">
              <title>#{ERB::Util.html_escape(artifact.name)}</title>
              <style>
                html, body, iframe {
                  margin: 0;
                  padding: 0;
                  width: 100%;
                  height: 100%;
                  border: 0;
                  overflow: hidden;
                }
                iframe {
                  overflow: auto;
                }
              </style>
            </head>
            <body>
              <iframe sandbox="allow-scripts allow-forms" height="100%" width="100%" srcdoc="#{ERB::Util.html_escape(untrusted_html)}" frameborder="0"></iframe>
            </body>
          </html>
        HTML
        response.headers.delete("X-Frame-Options")
        response.headers["Content-Security-Policy"] = "script-src 'unsafe-inline';"
        response.headers["X-Robots-Tag"] = "noindex"
        # Render the content
        render html: trusted_html.html_safe, layout: false, content_type: "text/html"
      end
      private
      def require_site_settings!
        if !SiteSetting.discourse_ai_enabled ||
             !SiteSetting.ai_artifact_security.in?(%w[lax strict])
          raise Discourse::NotFound
        end
      end
    end
  end
 end
@@ -6,14 +6,6 @@ module DiscourseAi
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      requires_login
      def show_debug_info_by_id
        log = AiApiAuditLog.find(params[:id])
        raise Discourse::NotFound if !log.topic
        guardian.ensure_can_debug_ai_bot_conversation!(log.topic)
        render json: AiApiAuditLogSerializer.new(log, root: false), status: 200
      end
      def show_debug_info
        post = Post.find(params[:post_id])
        guardian.ensure_can_debug_ai_bot_conversation!(post)
@@ -26,7 +18,7 @@ module DiscourseAi
        debug_info = AiApiAuditLog.where(post: posts).order(created_at: :desc).first
-        render json: AiApiAuditLogSerializer.new(debug_info, root: false), status: 200
+        render json: debug_info, status: 200
      end
      def stop_streaming_response
@@ -39,10 +31,12 @@ module DiscourseAi
      end
      def show_bot_username
-        bot_user = DiscourseAi::AiBot::EntryPoint.find_user_from_model(params[:username])
+        bot_user_id = DiscourseAi::AiBot::EntryPoint.map_bot_model_to_user_id(params[:username])
-        raise Discourse::InvalidParameters.new(:username) if !bot_user
+        raise Discourse::InvalidParameters.new(:username) if !bot_user_id
-        render json: { bot_username: bot_user.username_lower }, status: 200
+        bot_username_lower = User.find(bot_user_id).username_lower
        render json: { bot_username: bot_username_lower }, status: 200
      end
    end
  end
@@ -7,8 +7,7 @@ module DiscourseAi
      requires_login only: %i[create update destroy]
      before_action :require_site_settings!
-      skip_before_action :preload_json, :check_xhr, only: %i[show asset]
+      skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required, only: %i[show]
      skip_before_action :verify_authenticity_token, only: ["asset"]
      def create
        ensure_allowed_create!
@@ -20,8 +19,7 @@ module DiscourseAi
        if shared_conversation.persisted?
          render json: success_json.merge(share_key: shared_conversation.share_key)
        else
-          render json:
+          render json: failed_json.merge(error: I18n.t("discourse_ai.share_ai.failed_to_share")),
                   failed_json.merge(error: I18n.t("discourse_ai.share_ai.errors.failed_to_share")),
                 status: :unprocessable_entity
        end
      end
@@ -32,9 +30,7 @@ module DiscourseAi
        SharedAiConversation.destroy_conversation(@shared_conversation)
        render json:
-                 success_json.merge(
+                 success_json.merge(message: I18n.t("discourse_ai.share_ai.conversation_deleted"))
                   message: I18n.t("discourse_ai.share_ai.errors.conversation_deleted"),
                 )
      end
      def show
@@ -51,30 +47,6 @@ module DiscourseAi
        end
      end
      def asset
        no_cookies
        name = params[:name]
        path, content_type =
          if name == "share"
            %w[share.css text/css]
          elsif name == "highlight"
            %w[highlight.min.js application/javascript]
          else
            raise Discourse::NotFound
          end
        content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
        # note, path contains a ":version" which automatically busts the cache
        # based on file content, so this is safe
        response.headers["Last-Modified"] = 10.years.ago.httpdate
        response.headers["Content-Length"] = content.bytesize.to_s
        immutable_for 1.year
        render plain: content, disposition: :nil, content_type: content_type
      end
      def preview
        ensure_allowed_preview!
        data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
@@ -6,24 +6,12 @@ module DiscourseAi
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      requires_login
      before_action :ensure_can_request_suggestions
-      before_action :rate_limiter_performed!
+      before_action :rate_limiter_performed!, except: %i[prompts]
      include SecureUploadEndpointHelpers
      RATE_LIMITS = {
        "default" => {
          amount: 6,
          interval: 3.minutes,
        },
        "caption_image" => {
          amount: 20,
          interval: 1.minute,
        },
      }.freeze
      def suggest
        input = get_text_param!
        force_default_locale = params[:force_default_locale] || false
        prompt = CompletionPrompt.find_by(id: params[:mode])
@@ -35,7 +23,7 @@ module DiscourseAi
          prompt.custom_instruction = params[:custom_prompt]
        end
-        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+        suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
        hijack do
          render json:
@@ -43,7 +31,6 @@ module DiscourseAi
                     prompt,
                     input,
                     current_user,
                     force_default_locale,
                   ),
                 status: 200
        end
@@ -100,26 +87,18 @@ module DiscourseAi
        end
      end
-      def stream_suggestion
+      def explain
        post_id = get_post_param!
-        text = get_text_param!
+        term_to_explain = get_text_param!
        post = Post.includes(:topic).find_by(id: post_id)
        prompt = CompletionPrompt.find_by(id: params[:mode])
        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
        raise Discourse::InvalidParameters.new(:post_id) unless post
        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
          raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
        end
        Jobs.enqueue(
          :stream_post_helper,
          post_id: post.id,
          user_id: current_user.id,
-          text: text,
+          term_to_explain: term_to_explain,
          prompt: prompt.name,
          custom_prompt: params[:custom_prompt],
        )
        render json: { success: true }, status: 200
@@ -130,26 +109,18 @@ module DiscourseAi
      def caption_image
        image_url = params[:image_url]
        image_url_type = params[:image_url_type]
        raise Discourse::InvalidParameters.new(:image_url) if !image_url
        raise Discourse::InvalidParameters.new(:image_url) if !image_url_type
        if image_url_type == "short_path"
          image = Upload.find_by(sha1: Upload.sha1_from_short_path(image_url))
        elsif image_url_type == "short_url"
          image = Upload.find_by(sha1: Upload.sha1_from_short_url(image_url))
        else
          image = upload_from_full_url(image_url)
        end
        image = upload_from_full_url(image_url)
        raise Discourse::NotFound if image.blank?
-
+        final_image_url = get_caption_url(image, image_url)
        check_secure_upload_permission(image) if image.secure?
        user = current_user
        hijack do
-          caption = DiscourseAi::AiHelper::Assistant.new.generate_image_caption(image, user)
+          caption =
            DiscourseAi::AiHelper::Assistant.new.generate_image_caption(
              final_image_url,
              current_user,
            )
          render json: {
                   caption:
                     "#{caption} (#{I18n.t("discourse_ai.ai_helper.image_caption.attribution")})",
@@ -172,23 +143,22 @@ module DiscourseAi
      end
      def rate_limiter_performed!
-        action_rate_limit = RATE_LIMITS[action_name] || RATE_LIMITS["default"]
+        RateLimiter.new(current_user, "ai_assistant", 6, 3.minutes).performed!
        RateLimiter.new(
          current_user,
          "ai_assistant",
          action_rate_limit[:amount],
          action_rate_limit[:interval],
        ).performed!
      end
      def ensure_can_request_suggestions
-        allowed_groups =
+        if !current_user.in_any_groups?(SiteSetting.ai_helper_allowed_groups_map)
-          (
+          raise Discourse::InvalidAccess
-            SiteSetting.composer_ai_helper_allowed_groups_map |
+        end
-              SiteSetting.post_ai_helper_allowed_groups_map
+      end
          )
-        raise Discourse::InvalidAccess if !current_user.in_any_groups?(allowed_groups)
+      def get_caption_url(image_upload, image_url)
        if image_upload.secure?
          check_secure_upload_permission(image_upload)
          return Discourse.store.url_for(image_upload)
        end
        UrlHelper.absolute(image_url)
      end
    end
  end
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
David Taylor	aee2081356	Fix lib directory	2024-05-02 19:19:09 +01:00
David Taylor	e6bfec5bdb	DEV: Update engine config to match skeleton - adds `lib/` autoloading (and removes the plugin.rb workaround) - adds eager loading of scheduled jobs in development (otherwise they don't work at all)	2024-05-02 18:53:22 +01:00
		`@@ -0,0 +1 @@`
							`module.exports = require("@discourse/lint-configs/eslint");`
		`@@ -1,2 +0,0 @@`
			`engine-strict = true`
			`auto-install-peers = false`
		`@@ -1 +0,0 @@`
			`<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />`