DEV: Increase chunk size before splitting content for translation

Previously staff and bots would get scanned if TL was low
Additionally if somehow spam scanner user was blocked
(deactivated, silenced, banned) it would stop the feature from working

This adds an override that ensures unconditionally the user is setup correctly prior to scanning

.discourse-compatibility

@@ -1,3 +1,4 @@
+< 3.5.0.beta6-dev: 3e74eea1e5e3143888d67a8d8a11206df214dc24
 < 3.5.0.beta3-dev: 09a68414804a1447f52e5d60691ba59742cda9ec
 < 3.5.0.beta2-dev: de8624416a15b3d8e7ad350b083cc1420451ccec
 < 3.5.0.beta1-dev: bdef136080074a993e7c4f5ca562edc31a8ba756

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-features-edit.js

@@ -12,7 +12,7 @@ export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRo
 
     const { site_settings } = await ajax("/admin/config/site_settings.json", {
       data: {
-        filter_area: `ai-features/${currentFeature.ref}`,
+        filter_area: `ai-features/${currentFeature.module_name}`,
         plugin: "discourse-ai",
         category: "discourse_ai",
       },

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/index.gjs

@@ -1,156 +0,0 @@
-import Component from "@glimmer/component";
-import { service } from "@ember/service";
-import RouteTemplate from "ember-route-template";
-import { gt } from "truth-helpers";
-import DBreadcrumbsItem from "discourse/components/d-breadcrumbs-item";
-import DButton from "discourse/components/d-button";
-import DPageSubheader from "discourse/components/d-page-subheader";
-import { i18n } from "discourse-i18n";
-
-export default RouteTemplate(
-  class extends Component {
-    @service adminPluginNavManager;
-
-    get tableHeaders() {
-      const prefix = "discourse_ai.features.list.header";
-      return [
-        i18n(`${prefix}.name`),
-        i18n(`${prefix}.persona`),
-        i18n(`${prefix}.groups`),
-        "",
-      ];
-    }
-
-    get configuredFeatures() {
-      return this.args.model.filter(
-        (feature) => feature.enable_setting.value === true
-      );
-    }
-
-    get unconfiguredFeatures() {
-      return this.args.model.filter(
-        (feature) => feature.enable_setting.value === false
-      );
-    }
-
-    <template>
-      <DBreadcrumbsItem
-        @path="/admin/plugins/{{this.adminPluginNavManager.currentPlugin.name}}/ai-features"
-        @label={{i18n "discourse_ai.features.short_title"}}
-      />
-      <section class="ai-feature-list admin-detail">
-        <DPageSubheader
-          @titleLabel={{i18n "discourse_ai.features.short_title"}}
-          @descriptionLabel={{i18n "discourse_ai.features.description"}}
-          @learnMoreUrl="todo"
-        />
-
-        {{#if (gt this.configuredFeatures.length 0)}}
-          <div class="ai-feature-list__configured-features">
-            <h3>{{i18n "discourse_ai.features.list.configured_features"}}</h3>
-
-            <table class="d-admin-table">
-              <thead>
-                <tr>
-                  {{#each this.tableHeaders as |header|}}
-                    <th>{{header}}</th>
-                  {{/each}}
-                </tr>
-              </thead>
-
-              <tbody>
-                {{#each this.configuredFeatures as |feature|}}
-                  <tr
-                    class="ai-feature-list__row d-admin-row__content"
-                    data-feature-name={{feature.name}}
-                  >
-                    <td class="d-admin-row__overview ai-feature-list__row-item">
-                      <span class="ai-feature-list__row-item-name">
-                        <strong>
-                          {{feature.name}}
-                        </strong>
-                      </span>
-                      <span class="ai-feature-list__row-item-description">
-                        {{feature.description}}
-                      </span>
-                    </td>
-                    <td
-                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__persona"
-                    >
-                      <DButton
-                        class="btn-flat btn-small ai-feature-list__row-item-persona"
-                        @translatedLabel={{feature.persona.name}}
-                        @route="adminPlugins.show.discourse-ai-personas.edit"
-                        @routeModels={{feature.persona.id}}
-                      />
-                    </td>
-                    <td
-                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__groups"
-                    >
-                      {{#if (gt feature.persona.allowed_groups.length 0)}}
-                        <ul class="ai-feature-list__row-item-groups">
-                          {{#each feature.persona.allowed_groups as |group|}}
-                            <li>{{group.name}}</li>
-                          {{/each}}
-                        </ul>
-                      {{/if}}
-                    </td>
-                    <td class="d-admin-row_controls">
-                      <DButton
-                        class="btn-small edit"
-                        @label="discourse_ai.features.list.edit"
-                        @route="adminPlugins.show.discourse-ai-features.edit"
-                        @routeModels={{feature.id}}
-                      />
-                    </td>
-                  </tr>
-                {{/each}}
-              </tbody>
-            </table>
-          </div>
-        {{/if}}
-
-        {{#if (gt this.unconfiguredFeatures.length 0)}}
-          <div class="ai-feature-list__unconfigured-features">
-            <h3>{{i18n "discourse_ai.features.list.unconfigured_features"}}</h3>
-
-            <table class="d-admin-table">
-              <thead>
-                <tr>
-                  <th>{{i18n "discourse_ai.features.list.header.name"}}</th>
-                  <th></th>
-                </tr>
-              </thead>
-
-              <tbody>
-                {{#each this.unconfiguredFeatures as |feature|}}
-                  <tr class="ai-feature-list__row d-admin-row__content">
-                    <td class="d-admin-row__overview ai-feature-list__row-item">
-                      <span class="ai-feature-list__row-item-name">
-                        <strong>
-                          {{feature.name}}
-                        </strong>
-                      </span>
-                      <span class="ai-feature-list__row-item-description">
-                        {{feature.description}}
-                      </span>
-                    </td>
-
-                    <td class="d-admin-row_controls">
-                      <DButton
-                        class="btn-small"
-                        @label="discourse_ai.features.list.set_up"
-                        @route="adminPlugins.show.discourse-ai-features.edit"
-                        @routeModels={{feature.id}}
-                      />
-                    </td>
-                  </tr>
-                {{/each}}
-              </tbody>
-            </table>
-          </div>
-        {{/if}}
-      </section>
-    </template>
-  }
-);

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/index.hbs

@@ -0,0 +1 @@
+<AiFeatures @features={{this.model}} />

app/controllers/discourse_ai/admin/ai_embeddings_controller.rb

@@ -40,6 +40,7 @@ module DiscourseAi
         embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
 
         if embedding_def.save
+          log_ai_embedding_creation(embedding_def)
           render json: AiEmbeddingDefinitionSerializer.new(embedding_def), status: :created
         else
           render_json_error embedding_def
@@ -55,7 +56,10 @@ module DiscourseAi
           )
         end
 
+        initial_attributes = embedding_def.attributes.dup
+
         if embedding_def.update(ai_embeddings_params.except(:dimensions))
+          log_ai_embedding_update(embedding_def, initial_attributes)
           render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
         else
           render_json_error embedding_def
@@ -75,7 +79,16 @@ module DiscourseAi
           return render_json_error(I18n.t("discourse_ai.embeddings.delete_failed"), status: 409)
         end
 
+        embedding_details = {
+          embedding_id: embedding_def.id,
+          display_name: embedding_def.display_name,
+          provider: embedding_def.provider,
+          dimensions: embedding_def.dimensions,
+          subject: embedding_def.display_name,
+        }
+
         if embedding_def.destroy
+          log_ai_embedding_deletion(embedding_details)
           head :no_content
         else
           render_json_error embedding_def
@@ -128,6 +141,60 @@ module DiscourseAi
 
         permitted
       end
+
+      def ai_embeddings_logger_fields
+        {
+          display_name: {
+          },
+          provider: {
+          },
+          dimensions: {
+          },
+          url: {
+          },
+          tokenizer_class: {
+          },
+          max_sequence_length: {
+          },
+          embed_prompt: {
+            type: :large_text,
+          },
+          search_prompt: {
+            type: :large_text,
+          },
+          matryoshka_dimensions: {
+          },
+          api_key: {
+            type: :sensitive,
+          },
+          # JSON fields should be tracked as simple changes
+          json_fields: [:provider_params],
+        }
+      end
+
+      def log_ai_embedding_creation(embedding_def)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
+        logger.log_creation("embedding", embedding_def, ai_embeddings_logger_fields, entity_details)
+      end
+
+      def log_ai_embedding_update(embedding_def, initial_attributes)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
+
+        logger.log_update(
+          "embedding",
+          embedding_def,
+          initial_attributes,
+          ai_embeddings_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_ai_embedding_deletion(embedding_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        logger.log_deletion("embedding", embedding_details)
+      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_features_controller.rb

@@ -6,24 +6,40 @@ module DiscourseAi
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
       def index
-        render json: serialize_features(DiscourseAi::Features.features)
+        render json: serialize_modules(DiscourseAi::Configuration::Module.all)
       end
 
       def edit
         raise Discourse::InvalidParameters.new(:id) if params[:id].blank?
-        render json: serialize_feature(DiscourseAi::Features.find_feature_by_id(params[:id].to_i))
+
+        a_module = DiscourseAi::Configuration::Module.find_by(id: params[:id].to_i)
+
+        render json: serialize_module(a_module)
       end
 
       private
 
-      def serialize_features(features)
-        features.map { |feature| feature.merge(persona: serialize_persona(feature[:persona])) }
+      def serialize_modules(modules)
+        modules.map { |a_module| serialize_module(a_module) }
+      end
+
+      def serialize_module(a_module)
+        return nil if a_module.blank?
+
+        {
+          id: a_module.id,
+          module_name: a_module.name,
+          module_enabled: a_module.enabled?,
+          features: a_module.features.map { |f| serialize_feature(f) },
+        }
       end
 
       def serialize_feature(feature)
-        return nil if feature.blank?
-
-        feature.merge(persona: serialize_persona(feature[:persona]))
+        {
+          name: feature.name,
+          persona: serialize_persona(persona_id_obj_hash[feature.persona_id]),
+          enabled: feature.enabled?,
+        }
       end
 
       def serialize_persona(persona)
@@ -31,6 +47,18 @@ module DiscourseAi
 
         serialize_data(persona, AiFeaturesPersonaSerializer, root: false)
       end
+
+      private
+
+      def persona_id_obj_hash
+        @persona_id_obj_hash ||=
+          begin
+            setting_names = DiscourseAi::Configuration::Feature.all_persona_setting_names
+            ids = setting_names.map { |sn| SiteSetting.public_send(sn) }
+
+            AiPersona.where(id: ids).index_by(&:id)
+          end
+      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_llms_controller.rb

@@ -47,6 +47,7 @@ module DiscourseAi
 
         if llm_model.save
           llm_model.toggle_companion_user
+          log_llm_model_creation(llm_model)
           render json: LlmModelSerializer.new(llm_model), status: :created
         else
           render_json_error llm_model
@@ -56,6 +57,10 @@ module DiscourseAi
       def update
         llm_model = LlmModel.find(params[:id])
 
+        # Capture initial state for logging
+        initial_attributes = llm_model.attributes.dup
+        initial_quotas = llm_model.llm_quotas.map(&:attributes)
+
         if params[:ai_llm].key?(:llm_quotas)
           if quota_params
             existing_quota_group_ids = llm_model.llm_quotas.pluck(:group_id)
@@ -81,6 +86,7 @@ module DiscourseAi
 
         if llm_model.update(ai_llm_params(updating: llm_model))
           llm_model.toggle_companion_user
+          log_llm_model_update(llm_model, initial_attributes, initial_quotas)
           render json: LlmModelSerializer.new(llm_model)
         else
           render_json_error llm_model
@@ -109,11 +115,20 @@ module DiscourseAi
           )
         end
 
+        # Capture model details for logging before destruction
+        model_details = {
+          model_id: llm_model.id,
+          display_name: llm_model.display_name,
+          name: llm_model.name,
+          provider: llm_model.provider,
+        }
+
         # Clean up companion users
         llm_model.enabled_chat_bot = false
         llm_model.toggle_companion_user
 
         if llm_model.destroy
+          log_llm_model_deletion(model_details)
           head :no_content
         else
           render_json_error llm_model
@@ -190,6 +205,89 @@ module DiscourseAi
 
         permitted
       end
+
+      def ai_llm_logger_fields
+        {
+          display_name: {
+          },
+          name: {
+          },
+          provider: {
+          },
+          tokenizer: {
+          },
+          url: {
+          },
+          max_prompt_tokens: {
+          },
+          max_output_tokens: {
+          },
+          enabled_chat_bot: {
+          },
+          vision_enabled: {
+          },
+          api_key: {
+            type: :sensitive,
+          },
+          input_cost: {
+          },
+          output_cost: {
+          },
+          # JSON fields should be tracked as simple changes
+          json_fields: [:provider_params],
+        }
+      end
+
+      def log_llm_model_creation(llm_model)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
+
+        # Add quota information as a special case
+        if llm_model.llm_quotas.any?
+          entity_details[:quotas] = llm_model
+            .llm_quotas
+            .map do |quota|
+              "Group #{quota.group_id}: #{quota.max_tokens} tokens, #{quota.max_usages} usages, #{quota.duration_seconds}s"
+            end
+            .join("; ")
+        end
+
+        logger.log_creation("llm_model", llm_model, ai_llm_logger_fields, entity_details)
+      end
+
+      def log_llm_model_update(llm_model, initial_attributes, initial_quotas)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
+
+        # Track quota changes separately as they're a special case
+        current_quotas = llm_model.llm_quotas.reload.map(&:attributes)
+        if initial_quotas != current_quotas
+          initial_quota_summary =
+            initial_quotas
+              .map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
+              .join("; ")
+          current_quota_summary =
+            current_quotas
+              .map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
+              .join("; ")
+          entity_details[:quotas_changed] = true
+          entity_details[:quotas] = "#{initial_quota_summary} → #{current_quota_summary}"
+        end
+
+        logger.log_update(
+          "llm_model",
+          llm_model,
+          initial_attributes,
+          ai_llm_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_llm_model_deletion(model_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        model_details[:subject] = model_details[:display_name]
+        logger.log_deletion("llm_model", model_details)
+      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -9,15 +9,16 @@ module DiscourseAi
 
       def index
         ai_personas =
-          AiPersona.ordered.map do |persona|
-            # we use a special serializer here cause names and descriptions are
-            # localized for system personas
-            LocalizedAiPersonaSerializer.new(persona, root: false)
-          end
+          AiPersona
+            .ordered
+            .includes(:user, :uploads)
+            .map { |persona| LocalizedAiPersonaSerializer.new(persona, root: false) }
+
         tools =
           DiscourseAi::Personas::Persona.all_available_tools.map do |tool|
             AiToolSerializer.new(tool, root: false)
           end
+
         AiTool
           .where(enabled: true)
           .each do |tool|
@@ -31,10 +32,12 @@ module DiscourseAi
                 ),
             }
           end
+
         llms =
           DiscourseAi::Configuration::LlmEnumerator.values_for_serialization(
             allowed_seeded_llm_ids: SiteSetting.ai_bot_allowed_seeded_models_map,
           )
+
         render json: {
                  ai_personas: ai_personas,
                  meta: {
@@ -58,6 +61,7 @@ module DiscourseAi
         ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
         if ai_persona.save
           RagDocumentFragment.link_target_and_uploads(ai_persona, attached_upload_ids)
+          log_ai_persona_creation(ai_persona)
 
           render json: {
                    ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
@@ -74,8 +78,11 @@ module DiscourseAi
       end
 
       def update
+        initial_attributes = @ai_persona.attributes.dup
+
         if @ai_persona.update(ai_persona_params.except(:rag_uploads))
           RagDocumentFragment.update_target_uploads(@ai_persona, attached_upload_ids)
+          log_ai_persona_update(@ai_persona, initial_attributes)
 
           render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
         else
@@ -84,7 +91,14 @@ module DiscourseAi
       end
 
       def destroy
+        persona_details = {
+          persona_id: @ai_persona.id,
+          name: @ai_persona.name,
+          description: @ai_persona.description,
+        }
+
         if @ai_persona.destroy
+          log_ai_persona_deletion(persona_details)
           head :no_content
         else
           render_json_error @ai_persona
@@ -261,6 +275,92 @@ module DiscourseAi
 
         examples.map { |example_arr| example_arr.take(2).map(&:to_s) }
       end
+
+      def ai_persona_logger_fields
+        {
+          name: {
+          },
+          description: {
+          },
+          enabled: {
+          },
+          priority: {
+          },
+          system_prompt: {
+            type: :large_text,
+          },
+          default_llm_id: {
+          },
+          temperature: {
+          },
+          top_p: {
+          },
+          user_id: {
+          },
+          max_context_posts: {
+          },
+          vision_enabled: {
+          },
+          vision_max_pixels: {
+          },
+          rag_chunk_tokens: {
+          },
+          rag_chunk_overlap_tokens: {
+          },
+          rag_conversation_chunks: {
+          },
+          rag_llm_model_id: {
+          },
+          question_consolidator_llm_id: {
+          },
+          allow_chat_channel_mentions: {
+          },
+          allow_chat_direct_messages: {
+          },
+          allow_topic_mentions: {
+          },
+          allow_personal_messages: {
+          },
+          tool_details: {
+            type: :large_text,
+          },
+          forced_tool_count: {
+          },
+          force_default_llm: {
+          },
+          # JSON fields
+          json_fields: %i[tools response_format examples allowed_group_ids],
+        }
+      end
+
+      def log_ai_persona_creation(ai_persona)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
+        entity_details[:tools_count] = (ai_persona.tools || []).size
+
+        logger.log_creation("persona", ai_persona, ai_persona_logger_fields, entity_details)
+      end
+
+      def log_ai_persona_update(ai_persona, initial_attributes)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
+        entity_details[:tools_count] = ai_persona.tools.size if ai_persona.tools.present?
+
+        logger.log_update(
+          "persona",
+          ai_persona,
+          initial_attributes,
+          ai_persona_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_ai_persona_deletion(persona_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        persona_details[:subject] = persona_details[:name]
+
+        logger.log_deletion("persona", persona_details)
+      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -10,6 +10,10 @@ module DiscourseAi
       end
 
       def update
+        initial_settings = AiModerationSetting.spam
+        initial_custom_instructions = initial_settings&.data&.dig("custom_instructions")
+        initial_llm_model_id = initial_settings&.llm_model_id
+
         updated_params = {}
         if allowed_params.key?(:llm_model_id)
           llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
@@ -36,6 +40,8 @@ module DiscourseAi
           else
             AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
           end
+
+          log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, allowed_params)
         end
 
         is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
@@ -113,6 +119,30 @@ module DiscourseAi
 
       private
 
+      def log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, params)
+        changes_to_log = {}
+
+        if params.key?(:llm_model_id) && initial_llm_model_id.to_s != params[:llm_model_id].to_s
+          old_model_name =
+            LlmModel.find_by(id: initial_llm_model_id)&.display_name || initial_llm_model_id
+          new_model_name =
+            LlmModel.find_by(id: params[:llm_model_id])&.display_name || params[:llm_model_id]
+
+          changes_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"
+        end
+
+        if params.key?(:custom_instructions) &&
+             initial_custom_instructions != params[:custom_instructions]
+          changes_to_log[:custom_instructions] = params[:custom_instructions]
+        end
+
+        if changes_to_log.present?
+          changes_to_log[:subject] = I18n.t("discourse_ai.spam_detection.logging_subject")
+          logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+          logger.log_custom("update_ai_spam_settings", changes_to_log)
+        end
+      end
+
       def allowed_params
         params.permit(:is_enabled, :llm_model_id, :custom_instructions)
       end

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -25,6 +25,7 @@ module DiscourseAi
 
         if ai_tool.save
           RagDocumentFragment.link_target_and_uploads(ai_tool, attached_upload_ids)
+          log_ai_tool_creation(ai_tool)
           render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
         else
           render_json_error ai_tool
@@ -32,8 +33,11 @@ module DiscourseAi
       end
 
       def update
+        initial_attributes = @ai_tool.attributes.dup
+
         if @ai_tool.update(ai_tool_params)
           RagDocumentFragment.update_target_uploads(@ai_tool, attached_upload_ids)
+          log_ai_tool_update(@ai_tool, initial_attributes)
           render_serialized(@ai_tool, AiCustomToolSerializer)
         else
           render_json_error @ai_tool
@@ -41,7 +45,15 @@ module DiscourseAi
       end
 
       def destroy
+        tool_logger_details = {
+          tool_id: @ai_tool.id,
+          name: @ai_tool.name,
+          tool_name: @ai_tool.tool_name,
+          subject: @ai_tool.name,
+        }
+
         if @ai_tool.destroy
+          log_ai_tool_deletion(tool_logger_details)
           head :no_content
         else
           render_json_error @ai_tool
@@ -96,6 +108,60 @@ module DiscourseAi
           )
           .except(:rag_uploads)
       end
+
+      def ai_tool_logger_fields
+        {
+          name: {
+          },
+          tool_name: {
+          },
+          description: {
+          },
+          summary: {
+          },
+          enabled: {
+          },
+          rag_chunk_tokens: {
+          },
+          rag_chunk_overlap_tokens: {
+          },
+          rag_llm_model_id: {
+          },
+          script: {
+            type: :large_text,
+          },
+          parameters: {
+            type: :large_text,
+          },
+        }
+      end
+
+      def log_ai_tool_creation(ai_tool)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+
+        entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
+        entity_details[:parameter_count] = ai_tool.parameters.size if ai_tool.parameters.present?
+
+        logger.log_creation("tool", ai_tool, ai_tool_logger_fields, entity_details)
+      end
+
+      def log_ai_tool_update(ai_tool, initial_attributes)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
+
+        logger.log_update(
+          "tool",
+          ai_tool,
+          initial_attributes,
+          ai_tool_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_ai_tool_deletion(tool_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        logger.log_deletion("tool", tool_details)
+      end
     end
   end
 end

app/controllers/discourse_ai/ai_bot/artifact_key_values_controller.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module AiBot
+    class ArtifactKeyValuesController < ::ApplicationController
+      requires_plugin DiscourseAi::PLUGIN_NAME
+      before_action :ensure_logged_in, only: %i[set destroy]
+      before_action :find_artifact
+
+      PER_PAGE_MAX = 100
+
+      def index
+        page = index_params[:page].to_i
+        page = 1 if page < 1
+        per_page = index_params[:per_page].to_i
+        per_page = PER_PAGE_MAX if per_page < 1 || per_page > PER_PAGE_MAX
+
+        query = build_index_query
+
+        total_count = query.count
+        key_values =
+          query
+            .includes(:user)
+            .order(:user_id, :key, :created_at)
+            .offset((page - 1) * per_page)
+            .limit(per_page + 1)
+
+        has_more = key_values.length > per_page
+        key_values = key_values.first(per_page) if has_more
+
+        render json: {
+                 key_values:
+                   ActiveModel::ArraySerializer.new(
+                     key_values,
+                     each_serializer: AiArtifactKeyValueSerializer,
+                     keys_only: params[:keys_only] == "true",
+                   ).as_json,
+                 has_more: has_more,
+                 total_count: total_count,
+                 users:
+                   key_values
+                     .map { |kv| kv.user }
+                     .uniq
+                     .map { |u| BasicUserSerializer.new(u, root: nil).as_json },
+               }
+      end
+
+      def destroy
+        if params[:key].blank?
+          render json: { error: "Key parameter is required" }, status: :bad_request
+          return
+        end
+
+        key_value = @artifact.key_values.find_by(user_id: current_user.id, key: params[:key])
+
+        if key_value.nil?
+          render json: { error: "Key not found" }, status: :not_found
+        elsif key_value.destroy
+          head :ok
+        else
+          render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
+        end
+      end
+
+      def set
+        key_value =
+          @artifact.key_values.find_or_initialize_by(
+            user: current_user,
+            key: key_value_params[:key],
+          )
+
+        key_value.assign_attributes(key_value_params.except(:key))
+
+        if key_value.save
+          render json: AiArtifactKeyValueSerializer.new(key_value).as_json
+        else
+          render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
+        end
+      end
+
+      private
+
+      def key_value_params
+        params.permit(:key, :value, :public)
+      end
+
+      def index_params
+        @index_params ||= params.permit(:page, :per_page, :key, :keys_only, :all_users)
+      end
+
+      def build_index_query
+        query = @artifact.key_values
+
+        query =
+          if current_user&.admin?
+            query
+          elsif current_user
+            query.where("user_id = ? OR public = true", current_user.id)
+          else
+            query.where(public: true)
+          end
+
+        query = query.where("key = ?", index_params[:key]) if index_params[:key].present?
+
+        if index_params[:all_users].to_s != "true"
+          if current_user
+            query = query.where(user_id: current_user.id)
+          else
+            query = query.where("1 = 0")
+          end
+        end
+
+        query
+      end
+
+      def find_artifact
+        @artifact = AiArtifact.find_by(id: params[:artifact_id])
+        raise Discourse::NotFound if !@artifact
+        raise Discourse::NotFound if [email protected]? && guardian.anonymous?
+        raise Discourse::NotFound if [email protected]? && !guardian.can_see?(@artifact.post)
+      end
+    end
+  end
+end

app/controllers/discourse_ai/ai_helper/assistant_controller.rb

@@ -25,25 +25,24 @@ module DiscourseAi
         input = get_text_param!
         force_default_locale = params[:force_default_locale] || false
 
-        prompt = CompletionPrompt.find_by(id: params[:mode])
+        raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?
 
-        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
-
-        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
           raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
-
-          prompt.custom_instruction = params[:custom_prompt]
         end
 
-        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
+          return suggest_thumbnails(input)
+        end
 
         hijack do
           render json:
                    DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
-                     prompt,
+                     params[:mode],
                      input,
                      current_user,
                      force_default_locale: force_default_locale,
+                     custom_prompt: params[:custom_prompt],
                    ),
                  status: 200
         end
@@ -60,13 +59,10 @@ module DiscourseAi
           input = get_text_param!
         end
 
-        prompt = CompletionPrompt.enabled_by_name("generate_titles")
-        raise Discourse::InvalidParameters.new(:mode) if !prompt
-
         hijack do
           render json:
                    DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
-                     prompt,
+                     DiscourseAi::AiHelper::Assistant::GENERATE_TITLES,
                      input,
                      current_user,
                    ),
@@ -115,12 +111,12 @@ module DiscourseAi
         location = params[:location]
         raise Discourse::InvalidParameters.new(:location) if !location
 
-        prompt = CompletionPrompt.find_by(id: params[:mode])
+        raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
+          return suggest_thumbnails(input)
+        end
 
-        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
-        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
-
-        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
           raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
         end
 
@@ -133,7 +129,7 @@ module DiscourseAi
             :stream_composer_helper,
             user_id: current_user.id,
             text: text,
-            prompt: prompt.name,
+            prompt: params[:mode],
             custom_prompt: params[:custom_prompt],
             force_default_locale: params[:force_default_locale] || false,
             client_id: params[:client_id],
@@ -149,7 +145,7 @@ module DiscourseAi
             post_id: post.id,
             user_id: current_user.id,
             text: text,
-            prompt: prompt.name,
+            prompt: params[:mode],
             custom_prompt: params[:custom_prompt],
             client_id: params[:client_id],
           )

app/jobs/regular/detect_translate_post.rb

@@ -12,20 +12,27 @@ module Jobs
       post = Post.find_by(id: args[:post_id])
       return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
 
+      topic = post.topic
+      return if topic.blank?
+
       if SiteSetting.ai_translation_backfill_limit_to_public_content
-        topic = post.topic
-        if topic.blank? || topic.category&.read_restricted? ||
-             topic.archetype == Archetype.private_message
+        return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
+      else
+        if topic.archetype == Archetype.private_message &&
+             !TopicAllowedGroup.exists?(topic_id: topic.id)
           return
         end
       end
 
-      begin
-        detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
-      rescue FinalDestination::SSRFDetector::LookupFailedError
-        # this job is non-critical
-        # the backfill job will handle failures
-        return
+      # the user may fill locale in manually
+      if (detected_locale = post.locale).blank?
+        begin
+          detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # this job is non-critical
+          # the backfill job will handle failures
+          return
+        end
       end
 
       locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
@@ -33,6 +40,7 @@ module Jobs
 
       locales.each do |locale|
         next if locale == detected_locale
+        next if post.post_localizations.exists?(locale:)
 
         begin
           DiscourseAi::Translation::PostLocalizer.localize(post, locale)

app/jobs/regular/detect_translate_topic.rb

@@ -15,15 +15,22 @@ module Jobs
       end
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
-        return if topic.category&.read_restricted?
+        return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
+      else
+        if topic.archetype == Archetype.private_message &&
+             !TopicAllowedGroup.exists?(topic_id: topic.id)
+          return
+        end
       end
 
-      begin
-        detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
-      rescue FinalDestination::SSRFDetector::LookupFailedError
-        # this job is non-critical
-        # the backfill job will handle failures
-        return
+      if (detected_locale = topic.locale).blank?
+        begin
+          detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # this job is non-critical
+          # the backfill job will handle failures
+          return
+        end
       end
 
       locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
@@ -31,6 +38,7 @@ module Jobs
 
       locales.each do |locale|
         next if locale == detected_locale
+        next if topic.topic_localizations.exists?(locale:)
 
         begin
           DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)

app/jobs/regular/localize_categories.rb

@@ -17,7 +17,8 @@ module Jobs
       cat_id = args[:from_category_id] || Category.order(:id).first&.id
       last_id = nil
 
-      categories = Category.where("id >= ?", cat_id).order(:id).limit(BATCH_SIZE)
+      categories =
+        Category.where("id >= ? AND locale IS NOT NULL", cat_id).order(:id).limit(BATCH_SIZE)
       return if categories.empty?
 
       categories.each do |category|
@@ -26,9 +27,13 @@ module Jobs
           next
         end
 
-        CategoryLocalization.transaction do
-          locales.each do |locale|
-            next if CategoryLocalization.exists?(category_id: category.id, locale: locale)
+        locales.each do |locale|
+          localization = category.category_localizations.find_by(locale:)
+
+          if locale == category.locale && localization
+            localization.destroy
+          else
+            next if locale == category.locale
             begin
               DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
             rescue FinalDestination::SSRFDetector::LookupFailedError

app/jobs/regular/localize_posts.rb

@@ -29,12 +29,22 @@ module Jobs
             .where.not(locale: locale)
             .where("pl.id IS NULL")
 
+        posts = posts.joins(:topic)
+
         if SiteSetting.ai_translation_backfill_limit_to_public_content
+          # exclude all PMs
+          # and only include posts from public categories
           posts =
             posts
-              .joins(:topic)
-              .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
               .where.not(topics: { archetype: Archetype.private_message })
+              .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
+        else
+          # all regular topics, and group PMs
+          posts =
+            posts.where(
+              "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
+              Archetype.private_message,
+            )
         end
 
         if SiteSetting.ai_translation_backfill_max_age_days > 0

app/jobs/regular/localize_topics.rb

@@ -29,7 +29,19 @@ module Jobs
             .where("tl.id IS NULL")
 
         if SiteSetting.ai_translation_backfill_limit_to_public_content
-          topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+          # exclude all PMs
+          # and only include posts from public categories
+          topics =
+            topics
+              .where.not(archetype: Archetype.private_message)
+              .where(category_id: Category.where(read_restricted: false).select(:id))
+        else
+          # all regular topics, and group PMs
+          topics =
+            topics.where(
+              "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
+              Archetype.private_message,
+            )
         end
 
         if SiteSetting.ai_translation_backfill_max_age_days > 0

app/jobs/regular/stream_composer_helper.rb

@@ -10,19 +10,16 @@ module Jobs
       return unless args[:text]
       return unless args[:client_id]
 
-      prompt = CompletionPrompt.enabled_by_name(args[:prompt])
-
-      if prompt.id == CompletionPrompt::CUSTOM_PROMPT
-        prompt.custom_instruction = args[:custom_prompt]
-      end
+      helper_mode = args[:prompt]
 
       DiscourseAi::AiHelper::Assistant.new.stream_prompt(
-        prompt,
+        helper_mode,
         args[:text],
         user,
         "/discourse-ai/ai-helper/stream_composer_suggestion",
         force_default_locale: args[:force_default_locale],
         client_id: args[:client_id],
+        custom_prompt: args[:custom_prompt],
       )
     end
   end

app/jobs/regular/stream_post_helper.rb

@@ -14,16 +14,12 @@ module Jobs
 
       return unless user.guardian.can_see?(post)
 
-      prompt = CompletionPrompt.enabled_by_name(args[:prompt])
+      helper_mode = args[:prompt]
 
-      if prompt.id == CompletionPrompt::CUSTOM_PROMPT
-        prompt.custom_instruction = args[:custom_prompt]
-      end
-
-      if prompt.name == "explain"
-        input = <<~TEXT
-      <term>#{args[:text]}</term>
-      <context>#{post.raw}</context>
+      if helper_mode == DiscourseAi::AiHelper::Assistant::EXPLAIN
+        input = <<~TEXT.strip
+          <term>#{args[:text]}</term>
+          <context>#{post.raw}</context>
           <topic>#{topic.title}</topic>
           #{reply_to ? "<replyTo>#{reply_to.raw}</replyTo>" : nil}
         TEXT
@@ -32,10 +28,11 @@ module Jobs
       end
 
       DiscourseAi::AiHelper::Assistant.new.stream_prompt(
-        prompt,
+        helper_mode,
         input,
         user,
         "/discourse-ai/ai-helper/stream_suggestion/#{post.id}",
+        custom_prompt: args[:custom_prompt],
       )
     end
   end

app/jobs/scheduled/categories_locale_detection_backfill.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Jobs
+  class CategoriesLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 1.hour
+    sidekiq_options retry: false
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      categories = Category.where(locale: nil)
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        categories = categories.where(read_restricted: false)
+      end
+
+      categories = categories.limit(SiteSetting.ai_translation_backfill_rate)
+      return if categories.empty?
+
+      categories.each do |category|
+        begin
+          DiscourseAi::Translation::CategoryLocaleDetector.detect_locale(category)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to detect category #{category.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseAi::Translation::VerboseLogger.log("Detected #{categories.size} category locales")
+    end
+  end
+end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -10,6 +10,7 @@ module Jobs
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
       limit = SiteSetting.ai_translation_backfill_rate
+
       return if limit == 0
 
       topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")

app/models/ai_artifact.rb

@@ -2,6 +2,7 @@
 
 class AiArtifact < ActiveRecord::Base
   has_many :versions, class_name: "AiArtifactVersion", dependent: :destroy
+  has_many :key_values, class_name: "AiArtifactKeyValue", dependent: :destroy
   belongs_to :user
   belongs_to :post
   validates :html, length: { maximum: 65_535 }
@@ -15,13 +16,25 @@ class AiArtifact < ActiveRecord::Base
     https://ajax.googleapis.com
     https://d3js.org
     https://code.jquery.com
+    https://esm.sh
   ]
 
+  def self.artifact_version_attribute(version)
+    if version
+      "data-artifact-version='#{version}'"
+    else
+      ""
+    end
+  end
+
   def self.iframe_for(id, version = nil)
     <<~HTML
       <div class='ai-artifact'>
         <iframe src='#{url(id, version)}' frameborder="0" height="100%" width="100%"></iframe>
-        <a href='#{url(id, version)}' target='_blank'>#{I18n.t("discourse_ai.ai_artifact.link")}</a>
+        <div class='ai-artifact-controls'>
+          <a href='#{url(id, version)}' class='link-artifact' target='_blank'>#{I18n.t("discourse_ai.ai_artifact.link")}</a>
+          <a href class='copy-embed' data-artifact-id="#{id}" #{artifact_version_attribute(version)} data-url="#{url(id, version)}">#{I18n.t("discourse_ai.ai_artifact.copy_embed")}</a>
+        </div>
       </div>
     HTML
   end
@@ -37,7 +50,11 @@ class AiArtifact < ActiveRecord::Base
 
   def self.share_publicly(id:, post:)
     artifact = AiArtifact.find_by(id: id)
-    artifact.update!(metadata: { public: true }) if artifact&.post&.topic&.id == post.topic.id
+    if artifact&.post&.topic&.id == post.topic.id
+      artifact.metadata ||= {}
+      artifact.metadata[:public] = true
+      artifact.save!
+    end
   end
 
   def self.unshare_publicly(id:)

app/models/ai_artifact_key_value.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class AiArtifactKeyValue < ActiveRecord::Base
+  belongs_to :ai_artifact
+  belongs_to :user
+
+  validates :key, presence: true, length: { maximum: 50 }
+  validates :value,
+            presence: true,
+            length: {
+              maximum: ->(_) { SiteSetting.ai_artifact_kv_value_max_length },
+            }
+  attribute :public, :boolean, default: false
+  validates :ai_artifact, presence: true
+  validates :user, presence: true
+  validates :key, uniqueness: { scope: %i[ai_artifact_id user_id] }
+
+  validate :validate_max_keys_per_user_per_artifact
+
+  private
+
+  def validate_max_keys_per_user_per_artifact
+    return unless ai_artifact_id && user_id
+
+    max_keys = SiteSetting.ai_artifact_max_keys_per_user_per_artifact
+    existing_count = self.class.where(ai_artifact_id: ai_artifact_id, user_id: user_id).count
+
+    # Don't count the current record if it's being updated
+    existing_count -= 1 if persisted?
+
+    if existing_count >= max_keys
+      errors.add(
+        :base,
+        I18n.t("discourse_ai.ai_artifact.errors.max_keys_exceeded", count: max_keys),
+      )
+    end
+  end
+end
+
+# == Schema Information
+#
+# Table name: ai_artifact_key_values
+#
+#  id             :bigint           not null, primary key
+#  ai_artifact_id :bigint           not null
+#  user_id        :integer          not null
+#  key            :string(50)       not null
+#  value          :string(20000)    not null
+#  public         :boolean          default(FALSE), not null
+#  created_at     :datetime         not null
+#  updated_at     :datetime         not null
+#
+# Indexes
+#
+#  index_ai_artifact_kv_unique  (ai_artifact_id,user_id,key) UNIQUE
+#

app/models/ai_persona.rb

@@ -316,6 +316,10 @@ class AiPersona < ActiveRecord::Base
     end
   end
 
+  def features
+    DiscourseAi::Configuration::Feature.find_features_using(persona_id: id)
+  end
+
   private
 
   def chat_preconditions

app/models/completion_prompt.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
 class CompletionPrompt < ActiveRecord::Base
+  #
+  # DEPRECATED.
+  # TODO(roman): Remove after 06-17-25
+  #
+
   TRANSLATE = -301
   GENERATE_TITLES = -307
   PROOFREAD = -303

app/models/llm_model.rb

@@ -52,6 +52,7 @@ class LlmModel < ActiveRecord::Base
         disable_temperature: :checkbox,
         disable_top_p: :checkbox,
         disable_streaming: :checkbox,
+        enable_responses_api: :checkbox,
         reasoning_effort: {
           type: :enum,
           values: %w[default low medium high],

app/models/shared_ai_conversation.rb

@@ -37,7 +37,9 @@ class SharedAiConversation < ActiveRecord::Base
 
     maybe_topic = conversation.target
     if maybe_topic.is_a?(Topic)
-      AiArtifact.where(post: maybe_topic.posts).update_all(metadata: { public: false })
+      AiArtifact.where(post: maybe_topic.posts).update_all(
+        "metadata = jsonb_set(COALESCE(metadata, '{}'), '{public}', 'false')",
+      )
     end
 
     ::Jobs.enqueue(
@@ -176,7 +178,7 @@ class SharedAiConversation < ActiveRecord::Base
 
   def self.cook_artifacts(post)
     html = post.cooked
-    return html if !%w[lax strict].include?(SiteSetting.ai_artifact_security)
+    return html if !%w[lax hybrid strict].include?(SiteSetting.ai_artifact_security)
 
     doc = Nokogiri::HTML5.fragment(html)
     doc

app/serializers/ai_artifact_key_value_serializer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AiArtifactKeyValueSerializer < ApplicationSerializer
+  attributes :id, :key, :value, :public, :user_id, :created_at, :updated_at
+
+  def include_value?
+    !options[:keys_only]
+  end
+end

app/serializers/ai_features_persona_serializer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class AiFeaturesPersonaSerializer < ApplicationSerializer
-  attributes :id, :name, :system_prompt, :allowed_groups, :enabled
+  attributes :id, :name, :allowed_groups
 
   def allowed_groups
     Group