DEV: Deprecate the plugin

FEATURE: Use Personas in Automation's llm_report script (#1499 )
FIX: Process succesfully generated embeddings even if some failed (#1500 )
2025-07-14 18:14:14 +02:00 · 2025-07-14 12:47:21 -03:00 · 2025-07-10 17:51:01 -03:00 · 2025-07-10 17:06:39 +10:00 · 2025-07-09 22:21:51 +08:00 · 2025-07-09 06:39:36 -07:00
403 changed files with 14333 additions and 3376246 deletions
@@ -1,3 +1,5 @@
+< 3.5.0.beta7-dev: cd14b0c0bee0cf63c59b64b6f7213e31a37f11a7
+< 3.5.0.beta6-dev: 3e74eea1e5e3143888d67a8d8a11206df214dc24
 < 3.5.0.beta3-dev: 09a68414804a1447f52e5d60691ba59742cda9ec
 < 3.5.0.beta2-dev: de8624416a15b3d8e7ad350b083cc1420451ccec
 < 3.5.0.beta1-dev: bdef136080074a993e7c4f5ca562edc31a8ba756
@@ -6,5 +6,4 @@ evals/log
 evals/cases
 config/eval-llms.local.yml
 # this gets rid of search results from ag, ripgrep, etc
-tokenizers/
 public/ai-share/highlight.min.js
@@ -34,7 +34,7 @@ GEM
      racc
    prettier_print (1.2.1)
    racc (1.8.1)
-    rack (3.1.14)
+    rack (3.1.16)
    rainbow (3.1.1)
    regexp_parser (2.10.0)
    rubocop (1.74.0)
@@ -1,5 +1,8 @@
 # **Discourse AI** Plugin

+> [!IMPORTANT]
+> This plugin has now been bundled into Discourse core. See: https://meta.discourse.org/t/bundling-more-popular-plugins-with-discourse-core/373574
+
 **Plugin Summary**

 For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco
@@ -1,3 +1,4 @@
+import { action } from "@ember/object";
 import { ajax } from "discourse/lib/ajax";
 import DiscourseRoute from "discourse/routes/discourse";
 import SiteSetting from "admin/models/site-setting";
@@ -12,7 +13,7 @@ export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRo

    const { site_settings } = await ajax("/admin/config/site_settings.json", {
      data: {
-        filter_area: `ai-features/${currentFeature.ref}`,
+        filter_area: `ai-features/${currentFeature.module_name}`,
        plugin: "discourse-ai",
        category: "discourse_ai",
      },
@@ -24,4 +25,11 @@ export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRo

    return currentFeature;
  }
+
+  @action
+  willTransition() {
+    // site settings may amend if a feature is enabled or disabled, so refresh the model
+    // even on back button
+    this.router.refresh("adminPlugins.show.discourse-ai-features");
+  }
 }
@@ -1,156 +0,0 @@
-import Component from "@glimmer/component";
-import { service } from "@ember/service";
-import RouteTemplate from "ember-route-template";
-import { gt } from "truth-helpers";
-import DBreadcrumbsItem from "discourse/components/d-breadcrumbs-item";
-import DButton from "discourse/components/d-button";
-import DPageSubheader from "discourse/components/d-page-subheader";
-import { i18n } from "discourse-i18n";
-
-export default RouteTemplate(
-  class extends Component {
-    @service adminPluginNavManager;
-
-    get tableHeaders() {
-      const prefix = "discourse_ai.features.list.header";
-      return [
-        i18n(`${prefix}.name`),
-        i18n(`${prefix}.persona`),
-        i18n(`${prefix}.groups`),
-        "",
-      ];
-    }
-
-    get configuredFeatures() {
-      return this.args.model.filter(
-        (feature) => feature.enable_setting.value === true
-      );
-    }
-
-    get unconfiguredFeatures() {
-      return this.args.model.filter(
-        (feature) => feature.enable_setting.value === false
-      );
-    }
-
-    <template>
-      <DBreadcrumbsItem
-        @path="/admin/plugins/{{this.adminPluginNavManager.currentPlugin.name}}/ai-features"
-        @label={{i18n "discourse_ai.features.short_title"}}
-      />
-      <section class="ai-feature-list admin-detail">
-        <DPageSubheader
-          @titleLabel={{i18n "discourse_ai.features.short_title"}}
-          @descriptionLabel={{i18n "discourse_ai.features.description"}}
-          @learnMoreUrl="todo"
-        />
-
-        {{#if (gt this.configuredFeatures.length 0)}}
-          <div class="ai-feature-list__configured-features">
-            <h3>{{i18n "discourse_ai.features.list.configured_features"}}</h3>
-
-            <table class="d-admin-table">
-              <thead>
-                <tr>
-                  {{#each this.tableHeaders as |header|}}
-                    <th>{{header}}</th>
-                  {{/each}}
-                </tr>
-              </thead>
-
-              <tbody>
-                {{#each this.configuredFeatures as |feature|}}
-                  <tr
-                    class="ai-feature-list__row d-admin-row__content"
-                    data-feature-name={{feature.name}}
-                  >
-                    <td class="d-admin-row__overview ai-feature-list__row-item">
-                      <span class="ai-feature-list__row-item-name">
-                        <strong>
-                          {{feature.name}}
-                        </strong>
-                      </span>
-                      <span class="ai-feature-list__row-item-description">
-                        {{feature.description}}
-                      </span>
-                    </td>
-                    <td
-                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__persona"
-                    >
-                      <DButton
-                        class="btn-flat btn-small ai-feature-list__row-item-persona"
-                        @translatedLabel={{feature.persona.name}}
-                        @route="adminPlugins.show.discourse-ai-personas.edit"
-                        @routeModels={{feature.persona.id}}
-                      />
-                    </td>
-                    <td
-                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__groups"
-                    >
-                      {{#if (gt feature.persona.allowed_groups.length 0)}}
-                        <ul class="ai-feature-list__row-item-groups">
-                          {{#each feature.persona.allowed_groups as |group|}}
-                            <li>{{group.name}}</li>
-                          {{/each}}
-                        </ul>
-                      {{/if}}
-                    </td>
-                    <td class="d-admin-row_controls">
-                      <DButton
-                        class="btn-small edit"
-                        @label="discourse_ai.features.list.edit"
-                        @route="adminPlugins.show.discourse-ai-features.edit"
-                        @routeModels={{feature.id}}
-                      />
-                    </td>
-                  </tr>
-                {{/each}}
-              </tbody>
-            </table>
-          </div>
-        {{/if}}
-
-        {{#if (gt this.unconfiguredFeatures.length 0)}}
-          <div class="ai-feature-list__unconfigured-features">
-            <h3>{{i18n "discourse_ai.features.list.unconfigured_features"}}</h3>
-
-            <table class="d-admin-table">
-              <thead>
-                <tr>
-                  <th>{{i18n "discourse_ai.features.list.header.name"}}</th>
-                  <th></th>
-                </tr>
-              </thead>
-
-              <tbody>
-                {{#each this.unconfiguredFeatures as |feature|}}
-                  <tr class="ai-feature-list__row d-admin-row__content">
-                    <td class="d-admin-row__overview ai-feature-list__row-item">
-                      <span class="ai-feature-list__row-item-name">
-                        <strong>
-                          {{feature.name}}
-                        </strong>
-                      </span>
-                      <span class="ai-feature-list__row-item-description">
-                        {{feature.description}}
-                      </span>
-                    </td>
-
-                    <td class="d-admin-row_controls">
-                      <DButton
-                        class="btn-small"
-                        @label="discourse_ai.features.list.set_up"
-                        @route="adminPlugins.show.discourse-ai-features.edit"
-                        @routeModels={{feature.id}}
-                      />
-                    </td>
-                  </tr>
-                {{/each}}
-              </tbody>
-            </table>
-          </div>
-        {{/if}}
-      </section>
-    </template>
-  }
-);
@@ -0,0 +1 @@
+<AiFeatures @features={{this.model}} />
@@ -40,6 +40,7 @@ module DiscourseAi
        embedding_def = EmbeddingDefinition.new(ai_embeddings_params)

        if embedding_def.save
+          log_ai_embedding_creation(embedding_def)
          render json: AiEmbeddingDefinitionSerializer.new(embedding_def), status: :created
        else
          render_json_error embedding_def
@@ -55,7 +56,10 @@ module DiscourseAi
          )
        end

+        initial_attributes = embedding_def.attributes.dup
+
        if embedding_def.update(ai_embeddings_params.except(:dimensions))
+          log_ai_embedding_update(embedding_def, initial_attributes)
          render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
        else
          render_json_error embedding_def
@@ -75,7 +79,16 @@ module DiscourseAi
          return render_json_error(I18n.t("discourse_ai.embeddings.delete_failed"), status: 409)
        end

+        embedding_details = {
+          embedding_id: embedding_def.id,
+          display_name: embedding_def.display_name,
+          provider: embedding_def.provider,
+          dimensions: embedding_def.dimensions,
+          subject: embedding_def.display_name,
+        }
+
        if embedding_def.destroy
+          log_ai_embedding_deletion(embedding_details)
          head :no_content
        else
          render_json_error embedding_def
@@ -128,6 +141,60 @@ module DiscourseAi

        permitted
      end
+
+      def ai_embeddings_logger_fields
+        {
+          display_name: {
+          },
+          provider: {
+          },
+          dimensions: {
+          },
+          url: {
+          },
+          tokenizer_class: {
+          },
+          max_sequence_length: {
+          },
+          embed_prompt: {
+            type: :large_text,
+          },
+          search_prompt: {
+            type: :large_text,
+          },
+          matryoshka_dimensions: {
+          },
+          api_key: {
+            type: :sensitive,
+          },
+          # JSON fields should be tracked as simple changes
+          json_fields: [:provider_params],
+        }
+      end
+
+      def log_ai_embedding_creation(embedding_def)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
+        logger.log_creation("embedding", embedding_def, ai_embeddings_logger_fields, entity_details)
+      end
+
+      def log_ai_embedding_update(embedding_def, initial_attributes)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
+
+        logger.log_update(
+          "embedding",
+          embedding_def,
+          initial_attributes,
+          ai_embeddings_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_ai_embedding_deletion(embedding_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        logger.log_deletion("embedding", embedding_details)
+      end
    end
  end
 end
@@ -6,24 +6,44 @@ module DiscourseAi
      requires_plugin ::DiscourseAi::PLUGIN_NAME

      def index
-        render json: serialize_features(DiscourseAi::Features.features)
+        render json: serialize_modules(DiscourseAi::Configuration::Module.all)
      end

      def edit
        raise Discourse::InvalidParameters.new(:id) if params[:id].blank?
-        render json: serialize_feature(DiscourseAi::Features.find_feature_by_id(params[:id].to_i))
+
+        a_module = DiscourseAi::Configuration::Module.find_by(id: params[:id].to_i)
+
+        render json: serialize_module(a_module)
      end

      private

-      def serialize_features(features)
-        features.map { |feature| feature.merge(persona: serialize_persona(feature[:persona])) }
+      def serialize_modules(modules)
+        modules.map { |a_module| serialize_module(a_module) }
+      end
+
+      def serialize_module(a_module)
+        return nil if a_module.blank?
+
+        {
+          id: a_module.id,
+          module_name: a_module.name,
+          module_enabled: a_module.enabled?,
+          features: a_module.features.map { |f| serialize_feature(f) },
+        }
      end

      def serialize_feature(feature)
-        return nil if feature.blank?
-
-        feature.merge(persona: serialize_persona(feature[:persona]))
+        {
+          name: feature.name,
+          personas: feature.persona_ids.map { |id| serialize_persona(persona_id_obj_hash[id]) },
+          llm_models:
+            feature.llm_models.map do |llm_model|
+              { id: llm_model.id, name: llm_model.display_name }
+            end,
+          enabled: feature.enabled?,
+        }
      end

      def serialize_persona(persona)
@@ -31,6 +51,16 @@ module DiscourseAi

        serialize_data(persona, AiFeaturesPersonaSerializer, root: false)
      end
+
+      private
+
+      def persona_id_obj_hash
+        @persona_id_obj_hash ||=
+          begin
+            ids = DiscourseAi::Configuration::Feature.all.map(&:persona_ids).flatten.uniq
+            AiPersona.where(id: ids).index_by(&:id)
+          end
+      end
    end
  end
 end
@@ -47,6 +47,7 @@ module DiscourseAi

        if llm_model.save
          llm_model.toggle_companion_user
+          log_llm_model_creation(llm_model)
          render json: LlmModelSerializer.new(llm_model), status: :created
        else
          render_json_error llm_model
@@ -56,6 +57,10 @@ module DiscourseAi
      def update
        llm_model = LlmModel.find(params[:id])

+        # Capture initial state for logging
+        initial_attributes = llm_model.attributes.dup
+        initial_quotas = llm_model.llm_quotas.map(&:attributes)
+
        if params[:ai_llm].key?(:llm_quotas)
          if quota_params
            existing_quota_group_ids = llm_model.llm_quotas.pluck(:group_id)
@@ -81,6 +86,7 @@ module DiscourseAi

        if llm_model.update(ai_llm_params(updating: llm_model))
          llm_model.toggle_companion_user
+          log_llm_model_update(llm_model, initial_attributes, initial_quotas)
          render json: LlmModelSerializer.new(llm_model)
        else
          render_json_error llm_model
@@ -109,11 +115,20 @@ module DiscourseAi
          )
        end

+        # Capture model details for logging before destruction
+        model_details = {
+          model_id: llm_model.id,
+          display_name: llm_model.display_name,
+          name: llm_model.name,
+          provider: llm_model.provider,
+        }
+
        # Clean up companion users
        llm_model.enabled_chat_bot = false
        llm_model.toggle_companion_user

        if llm_model.destroy
+          log_llm_model_deletion(model_details)
          head :no_content
        else
          render_json_error llm_model
@@ -190,6 +205,89 @@ module DiscourseAi

        permitted
      end
+
+      def ai_llm_logger_fields
+        {
+          display_name: {
+          },
+          name: {
+          },
+          provider: {
+          },
+          tokenizer: {
+          },
+          url: {
+          },
+          max_prompt_tokens: {
+          },
+          max_output_tokens: {
+          },
+          enabled_chat_bot: {
+          },
+          vision_enabled: {
+          },
+          api_key: {
+            type: :sensitive,
+          },
+          input_cost: {
+          },
+          output_cost: {
+          },
+          # JSON fields should be tracked as simple changes
+          json_fields: [:provider_params],
+        }
+      end
+
+      def log_llm_model_creation(llm_model)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
+
+        # Add quota information as a special case
+        if llm_model.llm_quotas.any?
+          entity_details[:quotas] = llm_model
+            .llm_quotas
+            .map do |quota|
+              "Group #{quota.group_id}: #{quota.max_tokens} tokens, #{quota.max_usages} usages, #{quota.duration_seconds}s"
+            end
+            .join("; ")
+        end
+
+        logger.log_creation("llm_model", llm_model, ai_llm_logger_fields, entity_details)
+      end
+
+      def log_llm_model_update(llm_model, initial_attributes, initial_quotas)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
+
+        # Track quota changes separately as they're a special case
+        current_quotas = llm_model.llm_quotas.reload.map(&:attributes)
+        if initial_quotas != current_quotas
+          initial_quota_summary =
+            initial_quotas
+              .map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
+              .join("; ")
+          current_quota_summary =
+            current_quotas
+              .map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
+              .join("; ")
+          entity_details[:quotas_changed] = true
+          entity_details[:quotas] = "#{initial_quota_summary} → #{current_quota_summary}"
+        end
+
+        logger.log_update(
+          "llm_model",
+          llm_model,
+          initial_attributes,
+          ai_llm_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_llm_model_deletion(model_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        model_details[:subject] = model_details[:display_name]
+        logger.log_deletion("llm_model", model_details)
+      end
    end
  end
 end
@@ -5,19 +5,20 @@ module DiscourseAi
    class AiPersonasController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME

-      before_action :find_ai_persona, only: %i[edit update destroy create_user]
+      before_action :find_ai_persona, only: %i[edit update destroy create_user export]

      def index
        ai_personas =
-          AiPersona.ordered.map do |persona|
-            # we use a special serializer here cause names and descriptions are
-            # localized for system personas
-            LocalizedAiPersonaSerializer.new(persona, root: false)
-          end
+          AiPersona
+            .ordered
+            .includes(:user, :uploads)
+            .map { |persona| LocalizedAiPersonaSerializer.new(persona, root: false) }
+
        tools =
          DiscourseAi::Personas::Persona.all_available_tools.map do |tool|
            AiToolSerializer.new(tool, root: false)
          end
+
        AiTool
          .where(enabled: true)
          .each do |tool|
@@ -31,10 +32,12 @@ module DiscourseAi
                ),
            }
          end
+
        llms =
          DiscourseAi::Configuration::LlmEnumerator.values_for_serialization(
            allowed_seeded_llm_ids: SiteSetting.ai_bot_allowed_seeded_models_map,
          )
+
        render json: {
                 ai_personas: ai_personas,
                 meta: {
@@ -58,6 +61,7 @@ module DiscourseAi
        ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
        if ai_persona.save
          RagDocumentFragment.link_target_and_uploads(ai_persona, attached_upload_ids)
+          log_ai_persona_creation(ai_persona)

          render json: {
                   ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
@@ -74,8 +78,11 @@ module DiscourseAi
      end

      def update
+        initial_attributes = @ai_persona.attributes.dup
+
        if @ai_persona.update(ai_persona_params.except(:rag_uploads))
          RagDocumentFragment.update_target_uploads(@ai_persona, attached_upload_ids)
+          log_ai_persona_update(@ai_persona, initial_attributes)

          render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
        else
@@ -84,13 +91,57 @@ module DiscourseAi
      end

      def destroy
+        persona_details = {
+          persona_id: @ai_persona.id,
+          name: @ai_persona.name,
+          description: @ai_persona.description,
+        }
+
        if @ai_persona.destroy
+          log_ai_persona_deletion(persona_details)
          head :no_content
        else
          render_json_error @ai_persona
        end
      end

+      def export
+        persona = AiPersona.find(params[:id])
+        exporter = DiscourseAi::PersonaExporter.new(persona: persona)
+
+        response.headers[
+          "Content-Disposition"
+        ] = "attachment; filename=\"#{persona.name.parameterize}.json\""
+
+        render json: exporter.export
+      end
+
+      def import
+        name = params.dig(:persona, :name)
+        existing_persona = AiPersona.find_by(name: name)
+        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
+
+        begin
+          importer = DiscourseAi::PersonaImporter.new(json: params.to_unsafe_h)
+
+          if existing_persona && force_update
+            initial_attributes = existing_persona.attributes.dup
+            persona = importer.import!(overwrite: true)
+            log_ai_persona_update(persona, initial_attributes)
+            render json: LocalizedAiPersonaSerializer.new(persona, root: false)
+          else
+            persona = importer.import!
+            log_ai_persona_creation(persona)
+            render json: LocalizedAiPersonaSerializer.new(persona, root: false), status: :created
+          end
+        rescue DiscourseAi::PersonaImporter::ImportError => e
+          render_json_error e.message, status: :unprocessable_entity
+        rescue StandardError => e
+          Rails.logger.error("AI Persona import failed: #{e.message}")
+          render_json_error "Import failed: #{e.message}", status: :unprocessable_entity
+        end
+      end
+
      def stream_reply
        persona =
          AiPersona.find_by(name: params[:persona_name]) ||
@@ -261,6 +312,92 @@ module DiscourseAi

        examples.map { |example_arr| example_arr.take(2).map(&:to_s) }
      end
+
+      def ai_persona_logger_fields
+        {
+          name: {
+          },
+          description: {
+          },
+          enabled: {
+          },
+          priority: {
+          },
+          system_prompt: {
+            type: :large_text,
+          },
+          default_llm_id: {
+          },
+          temperature: {
+          },
+          top_p: {
+          },
+          user_id: {
+          },
+          max_context_posts: {
+          },
+          vision_enabled: {
+          },
+          vision_max_pixels: {
+          },
+          rag_chunk_tokens: {
+          },
+          rag_chunk_overlap_tokens: {
+          },
+          rag_conversation_chunks: {
+          },
+          rag_llm_model_id: {
+          },
+          question_consolidator_llm_id: {
+          },
+          allow_chat_channel_mentions: {
+          },
+          allow_chat_direct_messages: {
+          },
+          allow_topic_mentions: {
+          },
+          allow_personal_messages: {
+          },
+          tool_details: {
+            type: :large_text,
+          },
+          forced_tool_count: {
+          },
+          force_default_llm: {
+          },
+          # JSON fields
+          json_fields: %i[tools response_format examples allowed_group_ids],
+        }
+      end
+
+      def log_ai_persona_creation(ai_persona)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
+        entity_details[:tools_count] = (ai_persona.tools || []).size
+
+        logger.log_creation("persona", ai_persona, ai_persona_logger_fields, entity_details)
+      end
+
+      def log_ai_persona_update(ai_persona, initial_attributes)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
+        entity_details[:tools_count] = ai_persona.tools.size if ai_persona.tools.present?
+
+        logger.log_update(
+          "persona",
+          ai_persona,
+          initial_attributes,
+          ai_persona_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_ai_persona_deletion(persona_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        persona_details[:subject] = persona_details[:name]
+
+        logger.log_deletion("persona", persona_details)
+      end
    end
  end
 end
@@ -10,6 +10,17 @@ module DiscourseAi
      end

      def update
+        initial_settings = AiModerationSetting.spam
+
+        initial_data = {
+          custom_instructions: initial_settings&.data&.dig("custom_instructions"),
+          llm_model_id: initial_settings&.llm_model_id,
+          ai_persona_id: initial_settings&.ai_persona_id,
+        }
+
+        initial_custom_instructions = initial_settings&.data&.dig("custom_instructions")
+        initial_llm_model_id = initial_settings&.llm_model_id
+
        updated_params = {}
        if allowed_params.key?(:llm_model_id)
          llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
@@ -25,6 +36,22 @@ module DiscourseAi
            )
          end
        end
+
+        if allowed_params.key?(:ai_persona_id)
+          updated_params[:ai_persona_id] = allowed_params[:ai_persona_id]
+          persona = AiPersona.find_by(id: allowed_params[:ai_persona_id])
+          if persona.nil? ||
+               persona.response_format.to_a.none? { |rf|
+                 rf["key"] == "spam" && rf["type"] == "boolean"
+               }
+            return(
+              render_json_error(
+                I18n.t("discourse_ai.llm.configuration.invalid_persona_response_format"),
+                status: 422,
+              )
+            )
+          end
+        end
        updated_params[:data] = {
          custom_instructions: allowed_params[:custom_instructions],
        } if allowed_params.key?(:custom_instructions)
@@ -36,6 +63,8 @@ module DiscourseAi
          else
            AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
          end
+
+          log_ai_spam_update(initial_data, allowed_params)
        end

        is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
@@ -113,8 +142,44 @@ module DiscourseAi

      private

+      def log_ai_spam_update(initial_data, params)
+        changes_to_log = {}
+
+        initial_llm_model_id = initial_data[:llm_model_id]
+        if params.key?(:llm_model_id) && initial_llm_model_id.to_s != params[:llm_model_id].to_s
+          old_model_name =
+            LlmModel.find_by(id: initial_llm_model_id)&.display_name || initial_llm_model_id
+          new_model_name =
+            LlmModel.find_by(id: params[:llm_model_id])&.display_name || params[:llm_model_id]
+
+          changes_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"
+        end
+
+        initial_custom_instructions = initial_data[:custom_instructions]
+        if params.key?(:custom_instructions) &&
+             initial_custom_instructions != params[:custom_instructions]
+          changes_to_log[:custom_instructions] = params[:custom_instructions]
+        end
+
+        initial_ai_persona_id = initial_data[:ai_persona_id]
+        if params.key?(:ai_persona_id) && initial_ai_persona_id.to_s != params[:ai_persona_id].to_s
+          old_persona_name =
+            AiPersona.find_by(id: initial_ai_persona_id)&.name || initial_ai_persona_id
+          new_persona_name =
+            AiPersona.find_by(id: params[:ai_persona_id])&.name || params[:ai_persona_id]
+
+          changes_to_log[:ai_persona_id] = "#{old_persona_name} → #{new_persona_name}"
+        end
+
+        if changes_to_log.present?
+          changes_to_log[:subject] = I18n.t("discourse_ai.spam_detection.logging_subject")
+          logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+          logger.log_custom("update_ai_spam_settings", changes_to_log)
+        end
+      end
+
      def allowed_params
-        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
+        params.permit(:is_enabled, :llm_model_id, :custom_instructions, :ai_persona_id)
      end

      def spam_config
@@ -5,7 +5,7 @@ module DiscourseAi
    class AiToolsController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME

-      before_action :find_ai_tool, only: %i[test edit update destroy]
+      before_action :find_ai_tool, only: %i[test edit update destroy export]

      def index
        ai_tools = AiTool.all
@@ -25,15 +25,58 @@ module DiscourseAi

        if ai_tool.save
          RagDocumentFragment.link_target_and_uploads(ai_tool, attached_upload_ids)
+          log_ai_tool_creation(ai_tool)
          render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
        else
          render_json_error ai_tool
        end
      end

+      def export
+        response.headers[
+          "Content-Disposition"
+        ] = "attachment; filename=\"#{@ai_tool.tool_name}.json\""
+        render_serialized(@ai_tool, AiCustomToolSerializer)
+      end
+
+      def import
+        existing_tool = AiTool.find_by(tool_name: ai_tool_params[:tool_name])
+        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
+
+        if existing_tool && !force_update
+          return(
+            render_json_error "Tool with tool_name '#{ai_tool_params[:tool_name]}' already exists. Use force=true to overwrite.",
+                              status: :conflict
+          )
+        end
+
+        if existing_tool && force_update
+          initial_attributes = existing_tool.attributes.dup
+          if existing_tool.update(ai_tool_params)
+            log_ai_tool_update(existing_tool, initial_attributes)
+            render_serialized(existing_tool, AiCustomToolSerializer)
+          else
+            render_json_error existing_tool
+          end
+        else
+          ai_tool = AiTool.new(ai_tool_params)
+          ai_tool.created_by_id = current_user.id
+
+          if ai_tool.save
+            log_ai_tool_creation(ai_tool)
+            render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
+          else
+            render_json_error ai_tool
+          end
+        end
+      end
+
      def update
+        initial_attributes = @ai_tool.attributes.dup
+
        if @ai_tool.update(ai_tool_params)
          RagDocumentFragment.update_target_uploads(@ai_tool, attached_upload_ids)
+          log_ai_tool_update(@ai_tool, initial_attributes)
          render_serialized(@ai_tool, AiCustomToolSerializer)
        else
          render_json_error @ai_tool
@@ -41,7 +84,15 @@ module DiscourseAi
      end

      def destroy
+        tool_logger_details = {
+          tool_id: @ai_tool.id,
+          name: @ai_tool.name,
+          tool_name: @ai_tool.tool_name,
+          subject: @ai_tool.name,
+        }
+
        if @ai_tool.destroy
+          log_ai_tool_deletion(tool_logger_details)
          head :no_content
        else
          render_json_error @ai_tool
@@ -96,6 +147,60 @@ module DiscourseAi
          )
          .except(:rag_uploads)
      end
+
+      def ai_tool_logger_fields
+        {
+          name: {
+          },
+          tool_name: {
+          },
+          description: {
+          },
+          summary: {
+          },
+          enabled: {
+          },
+          rag_chunk_tokens: {
+          },
+          rag_chunk_overlap_tokens: {
+          },
+          rag_llm_model_id: {
+          },
+          script: {
+            type: :large_text,
+          },
+          parameters: {
+            type: :large_text,
+          },
+        }
+      end
+
+      def log_ai_tool_creation(ai_tool)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+
+        entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
+        entity_details[:parameter_count] = ai_tool.parameters.size if ai_tool.parameters.present?
+
+        logger.log_creation("tool", ai_tool, ai_tool_logger_fields, entity_details)
+      end
+
+      def log_ai_tool_update(ai_tool, initial_attributes)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
+
+        logger.log_update(
+          "tool",
+          ai_tool,
+          initial_attributes,
+          ai_tool_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_ai_tool_deletion(tool_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        logger.log_deletion("tool", tool_details)
+      end
    end
  end
 end
@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module AiBot
+    class ArtifactKeyValuesController < ::ApplicationController
+      requires_plugin DiscourseAi::PLUGIN_NAME
+      before_action :ensure_logged_in, only: %i[set destroy]
+      before_action :find_artifact
+
+      PER_PAGE_MAX = 100
+
+      def index
+        page = index_params[:page].to_i
+        page = 1 if page < 1
+        per_page = index_params[:per_page].to_i
+        per_page = PER_PAGE_MAX if per_page < 1 || per_page > PER_PAGE_MAX
+
+        query = build_index_query
+
+        total_count = query.count
+        key_values =
+          query
+            .includes(:user)
+            .order(:user_id, :key, :created_at)
+            .offset((page - 1) * per_page)
+            .limit(per_page + 1)
+
+        has_more = key_values.length > per_page
+        key_values = key_values.first(per_page) if has_more
+
+        render json: {
+                 key_values:
+                   ActiveModel::ArraySerializer.new(
+                     key_values,
+                     each_serializer: AiArtifactKeyValueSerializer,
+                     keys_only: params[:keys_only] == "true",
+                   ).as_json,
+                 has_more: has_more,
+                 total_count: total_count,
+                 users:
+                   key_values
+                     .map { |kv| kv.user }
+                     .uniq
+                     .map { |u| BasicUserSerializer.new(u, root: nil).as_json },
+               }
+      end
+
+      def destroy
+        if params[:key].blank?
+          render json: { error: "Key parameter is required" }, status: :bad_request
+          return
+        end
+
+        key_value = @artifact.key_values.find_by(user_id: current_user.id, key: params[:key])
+
+        if key_value.nil?
+          render json: { error: "Key not found" }, status: :not_found
+        elsif key_value.destroy
+          head :ok
+        else
+          render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
+        end
+      end
+
+      def set
+        key_value =
+          @artifact.key_values.find_or_initialize_by(
+            user: current_user,
+            key: key_value_params[:key],
+          )
+
+        key_value.assign_attributes(key_value_params.except(:key))
+
+        if key_value.save
+          render json: AiArtifactKeyValueSerializer.new(key_value).as_json
+        else
+          render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
+        end
+      end
+
+      private
+
+      def key_value_params
+        params.permit(:key, :value, :public)
+      end
+
+      def index_params
+        @index_params ||= params.permit(:page, :per_page, :key, :keys_only, :all_users)
+      end
+
+      def build_index_query
+        query = @artifact.key_values
+
+        query =
+          if current_user&.admin?
+            query
+          elsif current_user
+            query.where("user_id = ? OR public = true", current_user.id)
+          else
+            query.where(public: true)
+          end
+
+        query = query.where("key = ?", index_params[:key]) if index_params[:key].present?
+
+        if index_params[:all_users].to_s != "true"
+          if current_user
+            query = query.where(user_id: current_user.id)
+          else
+            query = query.where("1 = 0")
+          end
+        end
+
+        query
+      end
+
+      def find_artifact
+        @artifact = AiArtifact.find_by(id: params[:artifact_id])
+        raise Discourse::NotFound if !@artifact
+        raise Discourse::NotFound if [email protected]? && guardian.anonymous?
+        raise Discourse::NotFound if [email protected]? && !guardian.can_see?(@artifact.post)
+      end
+    end
+  end
+end
@@ -25,25 +25,24 @@ module DiscourseAi
        input = get_text_param!
        force_default_locale = params[:force_default_locale] || false

-        prompt = CompletionPrompt.find_by(id: params[:mode])
+        raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?

-        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
-
-        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
          raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
-
-          prompt.custom_instruction = params[:custom_prompt]
        end

-        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
+          return suggest_thumbnails(input)
+        end

        hijack do
          render json:
                   DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
-                     prompt,
+                     params[:mode],
                     input,
                     current_user,
                     force_default_locale: force_default_locale,
+                     custom_prompt: params[:custom_prompt],
                   ),
                 status: 200
        end
@@ -60,13 +59,10 @@ module DiscourseAi
          input = get_text_param!
        end

-        prompt = CompletionPrompt.enabled_by_name("generate_titles")
-        raise Discourse::InvalidParameters.new(:mode) if !prompt
-
        hijack do
          render json:
                   DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
-                     prompt,
+                     DiscourseAi::AiHelper::Assistant::GENERATE_TITLES,
                     input,
                     current_user,
                   ),
@@ -115,12 +111,12 @@ module DiscourseAi
        location = params[:location]
        raise Discourse::InvalidParameters.new(:location) if !location

-        prompt = CompletionPrompt.find_by(id: params[:mode])
+        raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
+          return suggest_thumbnails(input)
+        end

-        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
-        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
-
-        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
          raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
        end

@@ -128,15 +124,19 @@ module DiscourseAi
        # otherwise we may end up streaming the data to the wrong client
        raise Discourse::InvalidParameters.new(:client_id) if params[:client_id].blank?

+        channel_id = next_channel_id
+        progress_channel = "discourse_ai_helper/stream_suggestions/#{channel_id}"
+
        if location == "composer"
          Jobs.enqueue(
            :stream_composer_helper,
            user_id: current_user.id,
            text: text,
-            prompt: prompt.name,
+            prompt: params[:mode],
            custom_prompt: params[:custom_prompt],
            force_default_locale: params[:force_default_locale] || false,
            client_id: params[:client_id],
+            progress_channel:,
          )
        else
          post_id = get_post_param!
@@ -149,13 +149,14 @@ module DiscourseAi
            post_id: post.id,
            user_id: current_user.id,
            text: text,
-            prompt: prompt.name,
+            prompt: params[:mode],
            custom_prompt: params[:custom_prompt],
            client_id: params[:client_id],
+            progress_channel:,
          )
        end

-        render json: { success: true }, status: 200
+        render json: { success: true, progress_channel: }, status: 200
      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                          status: 502
@@ -196,6 +197,18 @@ module DiscourseAi

      private

+      CHANNEL_ID_KEY = "discourse_ai_helper_next_channel_id"
+
+      def next_channel_id
+        Discourse
+          .redis
+          .pipelined do |pipeline|
+            pipeline.incr(CHANNEL_ID_KEY)
+            pipeline.expire(CHANNEL_ID_KEY, 1.day)
+          end
+          .first
+      end
+
      def get_text_param!
        params[:text].tap { |t| raise Discourse::InvalidParameters.new(:text) if t.blank? }
      end
@@ -2,37 +2,47 @@

 module Jobs
  class DetectTranslatePost < ::Jobs::Base
+    cluster_concurrency 1
    sidekiq_options retry: false

    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.enabled?
      return if args[:post_id].blank?

      post = Post.find_by(id: args[:post_id])
      return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0

+      topic = post.topic
+      return if topic.blank?
+
      if SiteSetting.ai_translation_backfill_limit_to_public_content
-        topic = post.topic
-        if topic.blank? || topic.category&.read_restricted? ||
-             topic.archetype == Archetype.private_message
+        return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
+      else
+        if topic.archetype == Archetype.private_message &&
+             !TopicAllowedGroup.exists?(topic_id: topic.id)
          return
        end
      end

-      begin
-        detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
-      rescue FinalDestination::SSRFDetector::LookupFailedError
-        # this job is non-critical
-        # the backfill job will handle failures
-        return
+      # the user may fill locale in manually
+      if (detected_locale = post.locale).blank?
+        begin
+          detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # this job is non-critical
+          # the backfill job will handle failures
+          return
+        end
      end

-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if detected_locale.blank?
+      locales = SiteSetting.content_localization_supported_locales.split("|")
      return if locales.blank?

      locales.each do |locale|
-        next if locale == detected_locale
+        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        regionless_locale = locale.split("_").first
+        next if post.post_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?

        begin
          DiscourseAi::Translation::PostLocalizer.localize(post, locale)
@@ -40,7 +50,7 @@ module Jobs
          # do nothing, there are too many sporadic lookup failures
        rescue => e
          DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+            "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
          )
        end
      end
@@ -2,11 +2,11 @@

 module Jobs
  class DetectTranslateTopic < ::Jobs::Base
+    cluster_concurrency 1
    sidekiq_options retry: false

    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.enabled?
      return if args[:topic_id].blank?

      topic = Topic.find_by(id: args[:topic_id])
@@ -15,22 +15,32 @@ module Jobs
      end

      if SiteSetting.ai_translation_backfill_limit_to_public_content
-        return if topic.category&.read_restricted?
+        return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
+      else
+        if topic.archetype == Archetype.private_message &&
+             !TopicAllowedGroup.exists?(topic_id: topic.id)
+          return
+        end
      end

-      begin
-        detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
-      rescue FinalDestination::SSRFDetector::LookupFailedError
-        # this job is non-critical
-        # the backfill job will handle failures
-        return
+      if (detected_locale = topic.locale).blank?
+        begin
+          detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # this job is non-critical
+          # the backfill job will handle failures
+          return
+        end
      end

-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if detected_locale.blank?
+      locales = SiteSetting.content_localization_supported_locales.split("|")
      return if locales.blank?

      locales.each do |locale|
-        next if locale == detected_locale
+        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        regionless_locale = locale.split("_").first
+        next if topic.topic_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?

        begin
          DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
@@ -38,7 +48,7 @@ module Jobs
          # do nothing, there are too many sporadic lookup failures
        rescue => e
          DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
          )
        end
      end
@@ -5,46 +5,48 @@ module Jobs
    cluster_concurrency 1
    sidekiq_options retry: false

-    BATCH_SIZE = 50
-
    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.enabled?

-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
-      return if locales.blank?
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.nil?
+      return if limit <= 0
+      locales = SiteSetting.content_localization_supported_locales.split("|")

-      cat_id = args[:from_category_id] || Category.order(:id).first&.id
-      last_id = nil
-
-      categories = Category.where("id >= ?", cat_id).order(:id).limit(BATCH_SIZE)
+      categories = Category.where("locale IS NOT NULL")
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        categories = categories.where(read_restricted: false)
+      end
+      categories = categories.order(:id).limit(limit)
      return if categories.empty?

-      categories.each do |category|
-        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
-          last_id = category.id
-          next
-        end
+      remaining_limit = limit

-        CategoryLocalization.transaction do
-          locales.each do |locale|
-            next if CategoryLocalization.exists?(category_id: category.id, locale: locale)
-            begin
-              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
-            rescue FinalDestination::SSRFDetector::LookupFailedError
-              # do nothing, there are too many sporadic lookup failures
-            rescue => e
-              DiscourseAi::Translation::VerboseLogger.log(
-                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
-              )
-            end
+      categories.each do |category|
+        break if remaining_limit <= 0
+
+        existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)
+        missing_locales = locales - existing_locales - [category.locale]
+        missing_locales.each do |locale|
+          break if remaining_limit <= 0
+          next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, category.locale)
+
+          begin
+            DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate category #{category.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+            )
+          ensure
+            remaining_limit -= 1
          end
        end
-        last_id = category.id
-      end

-      if categories.size == BATCH_SIZE
-        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
+        if existing_locales.include?(category.locale)
+          CategoryLocalization.find_by(category_id: category.id, locale: category.locale).destroy
+        end
      end
    end
  end
@@ -5,43 +5,46 @@ module Jobs
    cluster_concurrency 1
    sidekiq_options retry: false

-    BATCH_SIZE = 50
-
    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0

-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
-      return if locales.blank?
-
-      limit = args[:limit] || BATCH_SIZE
+      return if !DiscourseAi::Translation.backfill_enabled?

+      locales = SiteSetting.content_localization_supported_locales.split("|")
      locales.each do |locale|
+        base_locale = locale.split("_").first
        posts =
          Post
            .joins(
-              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale LIKE '#{base_locale}%'",
+            )
+            .where(
+              "posts.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
            )
            .where(deleted_at: nil)
            .where("posts.user_id > 0")
            .where.not(raw: [nil, ""])
            .where.not(locale: nil)
-            .where.not(locale: locale)
+            .where("posts.locale NOT LIKE '#{base_locale}%'")
            .where("pl.id IS NULL")

+        posts = posts.joins(:topic)
+
        if SiteSetting.ai_translation_backfill_limit_to_public_content
+          # exclude all PMs
+          # and only include posts from public categories
          posts =
            posts
-              .joins(:topic)
-              .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
              .where.not(topics: { archetype: Archetype.private_message })
-        end
-
-        if SiteSetting.ai_translation_backfill_max_age_days > 0
+              .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
+        else
+          # all regular topics, and group PMs
          posts =
            posts.where(
-              "posts.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+              "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
+              Archetype.private_message,
            )
        end

@@ -56,7 +59,7 @@ module Jobs
            # do nothing, there are too many sporadic lookup failures
          rescue => e
            DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+              "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
            )
          end
        end
@@ -5,38 +5,43 @@ module Jobs
    cluster_concurrency 1
    sidekiq_options retry: false

-    BATCH_SIZE = 50
-
    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0

-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
-      return if locales.blank?
-
-      limit = args[:limit] || BATCH_SIZE
+      return if !DiscourseAi::Translation.backfill_enabled?

+      locales = SiteSetting.content_localization_supported_locales.split("|")
      locales.each do |locale|
+        base_locale = locale.split("_").first
        topics =
          Topic
            .joins(
-              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale LIKE '#{base_locale}%'",
+            )
+            .where(
+              "topics.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
            )
            .where(deleted_at: nil)
            .where("topics.user_id > 0")
            .where.not(locale: nil)
-            .where.not(locale: locale)
+            .where("topics.locale NOT LIKE '#{base_locale}%'")
            .where("tl.id IS NULL")

        if SiteSetting.ai_translation_backfill_limit_to_public_content
-          topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
-        end
-
-        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          # exclude all PMs
+          # and only include posts from public categories
+          topics =
+            topics
+              .where.not(archetype: Archetype.private_message)
+              .where(category_id: Category.where(read_restricted: false).select(:id))
+        else
+          # all regular topics, and group PMs
          topics =
            topics.where(
-              "topics.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+              "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
+              Archetype.private_message,
            )
        end

@@ -51,7 +56,7 @@ module Jobs
            # do nothing, there are too many sporadic lookup failures
          rescue => e
            DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
            )
          end
        end
@@ -9,20 +9,18 @@ module Jobs
      return unless user = User.find_by(id: args[:user_id])
      return unless args[:text]
      return unless args[:client_id]
+      return unless args[:progress_channel]

-      prompt = CompletionPrompt.enabled_by_name(args[:prompt])
-
-      if prompt.id == CompletionPrompt::CUSTOM_PROMPT
-        prompt.custom_instruction = args[:custom_prompt]
-      end
+      helper_mode = args[:prompt]

      DiscourseAi::AiHelper::Assistant.new.stream_prompt(
-        prompt,
+        helper_mode,
        args[:text],
        user,
-        "/discourse-ai/ai-helper/stream_composer_suggestion",
+        args[:progress_channel],
        force_default_locale: args[:force_default_locale],
        client_id: args[:client_id],
+        custom_prompt: args[:custom_prompt],
      )
    end
  end
@@ -8,22 +8,20 @@ module Jobs
      return unless post = Post.includes(:topic).find_by(id: args[:post_id])
      return unless user = User.find_by(id: args[:user_id])
      return unless args[:text]
+      return unless args[:progress_channel]
+      return unless args[:client_id]

      topic = post.topic
      reply_to = post.reply_to_post

      return unless user.guardian.can_see?(post)

-      prompt = CompletionPrompt.enabled_by_name(args[:prompt])
+      helper_mode = args[:prompt]

-      if prompt.id == CompletionPrompt::CUSTOM_PROMPT
-        prompt.custom_instruction = args[:custom_prompt]
-      end
-
-      if prompt.name == "explain"
-        input = <<~TEXT
-      <term>#{args[:text]}</term>
-      <context>#{post.raw}</context>
+      if helper_mode == DiscourseAi::AiHelper::Assistant::EXPLAIN
+        input = <<~TEXT.strip
+          <term>#{args[:text]}</term>
+          <context>#{post.raw}</context>
          <topic>#{topic.title}</topic>
          #{reply_to ? "<replyTo>#{reply_to.raw}</replyTo>" : nil}
        TEXT
@@ -32,10 +30,12 @@ module Jobs
      end

      DiscourseAi::AiHelper::Assistant.new.stream_prompt(
-        prompt,
+        helper_mode,
        input,
        user,
-        "/discourse-ai/ai-helper/stream_suggestion/#{post.id}",
+        args[:progress_channel],
+        custom_prompt: args[:custom_prompt],
+        client_id: args[:client_id],
      )
    end
  end
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Jobs
+  class CategoriesLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 1.hour
+    sidekiq_options retry: false
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !DiscourseAi::Translation.backfill_enabled?
+
+      categories = Category.where(locale: nil)
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        categories = categories.where(read_restricted: false)
+      end
+
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
+      categories = categories.limit(limit)
+      return if categories.empty?
+
+      categories.each do |category|
+        begin
+          DiscourseAi::Translation::CategoryLocaleDetector.detect_locale(category)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to detect category #{category.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+          )
+        end
+      end
+
+      DiscourseAi::Translation::VerboseLogger.log("Detected #{categories.size} category locales")
+    end
+  end
+end
@@ -2,15 +2,14 @@

 module Jobs
  class CategoryLocalizationBackfill < ::Jobs::Scheduled
-    every 12.hours
+    every 1.hour
    cluster_concurrency 1

    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if !DiscourseAi::Translation.backfill_enabled?
+      limit = SiteSetting.ai_translation_backfill_hourly_rate

-      Jobs.enqueue(:localize_categories)
+      Jobs.enqueue(:localize_categories, limit:)
    end
  end
 end
@@ -18,105 +18,115 @@ module Jobs
        )
      end

-      rebaked = 0
+      production_vector = DiscourseAi::Embeddings::Vector.instance

-      vector = DiscourseAi::Embeddings::Vector.instance
-      vector_def = vector.vdef
-      table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
-
-      topics =
-        Topic
-          .joins(
-            "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
+      if SiteSetting.ai_embeddings_backfill_model.present? &&
+           SiteSetting.ai_embeddings_backfill_model != SiteSetting.ai_embeddings_selected_model
+        backfill_vector =
+          DiscourseAi::Embeddings::Vector.new(
+            EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_backfill_model),
          )
-          .where(archetype: Archetype.default)
-          .where(deleted_at: nil)
-          .order("topics.bumped_at DESC")
+      end

-      rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
+      topic_work_list = []
+      topic_work_list << production_vector
+      topic_work_list << backfill_vector if backfill_vector

-      return if rebaked >= limit
+      topic_work_list.each do |vector|
+        rebaked = 0
+        table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
+        vector_def = vector.vdef

-      # Then, we'll try to backfill embeddings for topics that have outdated
-      # embeddings, be it model or strategy version
-      relation = topics.where(<<~SQL).limit(limit - rebaked)
-          #{table_name}.model_version < #{vector_def.version}
-          OR
-          #{table_name}.strategy_version < #{vector_def.strategy_version}
-        SQL
+        topics =
+          Topic
+            .joins(
+              "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
+            )
+            .where(archetype: Archetype.default)
+            .where(deleted_at: nil)
+            .order("topics.bumped_at DESC")

-      rebaked += populate_topic_embeddings(vector, relation, force: true)
+        rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))

-      return if rebaked >= limit
+        next if rebaked >= limit

-      # Finally, we'll try to backfill embeddings for topics that have outdated
-      # embeddings due to edits or new replies. Here we only do 10% of the limit
-      relation =
-        topics
-          .where("#{table_name}.updated_at < ?", 6.hours.ago)
-          .where("#{table_name}.updated_at < topics.updated_at")
+        # Then, we'll try to backfill embeddings for topics that have outdated
+        # embeddings, be it model or strategy version
+        relation = topics.where(<<~SQL).limit(limit - rebaked)
+            #{table_name}.model_version < #{vector_def.version}
+            OR
+            #{table_name}.strategy_version < #{vector_def.strategy_version}
+          SQL
+
+        rebaked += populate_topic_embeddings(vector, relation, force: true)
+
+        next if rebaked >= limit
+
+        # Finally, we'll try to backfill embeddings for topics that have outdated
+        # embeddings due to edits or new replies. Here we only do 10% of the limit
+        relation =
+          topics
+            .where("#{table_name}.updated_at < ?", 6.hours.ago)
+            .where("#{table_name}.updated_at < topics.updated_at")
+            .limit((limit - rebaked) / 10)
+
+        populate_topic_embeddings(vector, relation, force: true)
+
+        next unless SiteSetting.ai_embeddings_per_post_enabled
+
+        # Now for posts
+        table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
+        posts_batch_size = 1000
+
+        posts =
+          Post
+            .joins(
+              "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
+            )
+            .where(deleted_at: nil)
+            .where(post_type: Post.types[:regular])
+
+        # First, we'll try to backfill embeddings for posts that have none
+        posts
+          .where("#{table_name}.post_id IS NULL")
+          .limit(limit - rebaked)
+          .pluck(:id)
+          .each_slice(posts_batch_size) do |batch|
+            vector.gen_bulk_reprensentations(Post.where(id: batch))
+            rebaked += batch.length
+          end
+
+        next if rebaked >= limit
+
+        # Then, we'll try to backfill embeddings for posts that have outdated
+        # embeddings, be it model or strategy version
+        posts
+          .where(<<~SQL)
+            #{table_name}.model_version < #{vector_def.version}
+            OR
+            #{table_name}.strategy_version < #{vector_def.strategy_version}
+          SQL
+          .limit(limit - rebaked)
+          .pluck(:id)
+          .each_slice(posts_batch_size) do |batch|
+            vector.gen_bulk_reprensentations(Post.where(id: batch))
+            rebaked += batch.length
+          end
+
+        next if rebaked >= limit
+
+        # Finally, we'll try to backfill embeddings for posts that have outdated
+        # embeddings due to edits. Here we only do 10% of the limit
+        posts
+          .where("#{table_name}.updated_at < ?", 7.days.ago)
+          .order("random()")
          .limit((limit - rebaked) / 10)
-
-      populate_topic_embeddings(vector, relation, force: true)
-
-      return if rebaked >= limit
-
-      return unless SiteSetting.ai_embeddings_per_post_enabled
-
-      # Now for posts
-      table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
-      posts_batch_size = 1000
-
-      posts =
-        Post
-          .joins(
-            "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
-          )
-          .where(deleted_at: nil)
-          .where(post_type: Post.types[:regular])
-
-      # First, we'll try to backfill embeddings for posts that have none
-      posts
-        .where("#{table_name}.post_id IS NULL")
-        .limit(limit - rebaked)
-        .pluck(:id)
-        .each_slice(posts_batch_size) do |batch|
-          vector.gen_bulk_reprensentations(Post.where(id: batch))
-          rebaked += batch.length
-        end
-
-      return if rebaked >= limit
-
-      # Then, we'll try to backfill embeddings for posts that have outdated
-      # embeddings, be it model or strategy version
-      posts
-        .where(<<~SQL)
-          #{table_name}.model_version < #{vector_def.version}
-          OR
-          #{table_name}.strategy_version < #{vector_def.strategy_version}
-        SQL
-        .limit(limit - rebaked)
-        .pluck(:id)
-        .each_slice(posts_batch_size) do |batch|
-          vector.gen_bulk_reprensentations(Post.where(id: batch))
-          rebaked += batch.length
-        end
-
-      return if rebaked >= limit
-
-      # Finally, we'll try to backfill embeddings for posts that have outdated
-      # embeddings due to edits. Here we only do 10% of the limit
-      posts
-        .where("#{table_name}.updated_at < ?", 7.days.ago)
-        .order("random()")
-        .limit((limit - rebaked) / 10)
-        .pluck(:id)
-        .each_slice(posts_batch_size) do |batch|
-          vector.gen_bulk_reprensentations(Post.where(id: batch))
-          rebaked += batch.length
-        end
-
-      rebaked
+          .pluck(:id)
+          .each_slice(posts_batch_size) do |batch|
+            vector.gen_bulk_reprensentations(Post.where(id: batch))
+            rebaked += batch.length
+          end
+      end
    end

    private
@@ -6,13 +6,12 @@ module Jobs
    cluster_concurrency 1

    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.backfill_enabled?

-      return if SiteSetting.experimental_content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0

-      Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
+      Jobs.enqueue(:localize_posts, limit:)
    end
  end
 end
@@ -7,35 +7,33 @@ module Jobs
    cluster_concurrency 1

    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      return if !DiscourseAi::Translation.backfill_enabled?
+
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals

      posts =
        Post
          .where(locale: nil)
          .where(deleted_at: nil)
          .where("posts.user_id > 0")
+          .where("posts.created_at > ?", SiteSetting.ai_translation_backfill_max_age_days.days.ago)
          .where.not(raw: [nil, ""])

      if SiteSetting.ai_translation_backfill_limit_to_public_content
-        public_categories = Category.where(read_restricted: false).pluck(:id)
        posts =
          posts
            .joins(:topic)
-            .where(topics: { category_id: public_categories })
-            .where(topics: { archetype: "regular" })
-      end
-
-      if SiteSetting.ai_translation_backfill_max_age_days > 0
+            .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
+            .where("archetype != ?", Archetype.private_message)
+      else
        posts =
-          posts.where(
-            "posts.created_at > ?",
-            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+          posts.joins(:topic).where(
+            "topics.archetype != ? OR EXISTS (SELECT 1 FROM topic_allowed_groups WHERE topic_id = topics.id)",
+            Archetype.private_message,
          )
      end

-      posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
+      posts = posts.order(updated_at: :desc).limit(limit)
      return if posts.empty?

      posts.each do |post|
@@ -45,7 +43,7 @@ module Jobs
          # do nothing, there are too many sporadic lookup failures
        rescue => e
          DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect post #{post.id}'s locale: #{e.message}",
+            "Failed to detect post #{post.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
          )
        end
      end
@@ -36,7 +36,12 @@ module ::Jobs
      existing_summary = strategy.existing_summary

      if existing_summary.blank? || existing_summary.outdated
-        strategy.summarize(user)
+        begin
+          strategy.summarize(user)
+        rescue => e
+          Rails.logger.error("Error summarizing topic #{topic.id}: #{e.class.name} - #{e.message}")
+          raise e
+        end
      else
        # Hiding or deleting a post, and creating a small action alters the Topic#highest_post_number.
        # We use this as a quick way to select potential backfill candidates without relying on original_content_sha.
@@ -6,13 +6,10 @@ module Jobs
    cluster_concurrency 1

    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.backfill_enabled?

-      return if SiteSetting.experimental_content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
-
-      Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      Jobs.enqueue(:localize_topics, limit:)
    end
  end
 end
--- a/Show More
+++ b/Show More