FEATURE: bring back hidden settings

The hidden settings have general use and may be already set.

Helper one is particularly important cause it sets a default for a large
number of settings

Additionally hiding access means people can not remove LLMs without console
access

.gitignore

@@ -6,5 +6,4 @@ evals/log
 evals/cases
 config/eval-llms.local.yml
 # this gets rid of search results from ag, ripgrep, etc
-tokenizers/
 public/ai-share/highlight.min.js

Gemfile.lock

@@ -34,7 +34,7 @@ GEM
       racc
     prettier_print (1.2.1)
     racc (1.8.1)
-    rack (3.1.14)
+    rack (3.1.16)
     rainbow (3.1.1)
     regexp_parser (2.10.0)
     rubocop (1.74.0)

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-features-edit.js

@@ -1,3 +1,4 @@
+import { action } from "@ember/object";
 import { ajax } from "discourse/lib/ajax";
 import DiscourseRoute from "discourse/routes/discourse";
 import SiteSetting from "admin/models/site-setting";
@@ -24,4 +25,11 @@ export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRo
 
     return currentFeature;
   }
+
+  @action
+  willTransition() {
+    // site settings may amend if a feature is enabled or disabled, so refresh the model
+    // even on back button
+    this.router.refresh("adminPlugins.show.discourse-ai-features");
+  }
 }

app/controllers/discourse_ai/admin/ai_features_controller.rb

@@ -37,7 +37,11 @@ module DiscourseAi
       def serialize_feature(feature)
         {
           name: feature.name,
-          persona: serialize_persona(persona_id_obj_hash[feature.persona_id]),
+          personas: feature.persona_ids.map { |id| serialize_persona(persona_id_obj_hash[id]) },
+          llm_models:
+            feature.llm_models.map do |llm_model|
+              { id: llm_model.id, name: llm_model.display_name }
+            end,
           enabled: feature.enabled?,
         }
       end
@@ -53,9 +57,7 @@ module DiscourseAi
       def persona_id_obj_hash
         @persona_id_obj_hash ||=
           begin
-            setting_names = DiscourseAi::Configuration::Feature.all_persona_setting_names
-            ids = setting_names.map { |sn| SiteSetting.public_send(sn) }
-
+            ids = DiscourseAi::Configuration::Feature.all.map(&:persona_ids).flatten.uniq
             AiPersona.where(id: ids).index_by(&:id)
           end
       end

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -5,7 +5,7 @@ module DiscourseAi
     class AiPersonasController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_persona, only: %i[edit update destroy create_user]
+      before_action :find_ai_persona, only: %i[edit update destroy create_user export]
 
       def index
         ai_personas =
@@ -105,6 +105,43 @@ module DiscourseAi
         end
       end
 
+      def export
+        persona = AiPersona.find(params[:id])
+        exporter = DiscourseAi::PersonaExporter.new(persona: persona)
+
+        response.headers[
+          "Content-Disposition"
+        ] = "attachment; filename=\"#{persona.name.parameterize}.json\""
+
+        render json: exporter.export
+      end
+
+      def import
+        name = params.dig(:persona, :name)
+        existing_persona = AiPersona.find_by(name: name)
+        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
+
+        begin
+          importer = DiscourseAi::PersonaImporter.new(json: params.to_unsafe_h)
+
+          if existing_persona && force_update
+            initial_attributes = existing_persona.attributes.dup
+            persona = importer.import!(overwrite: true)
+            log_ai_persona_update(persona, initial_attributes)
+            render json: LocalizedAiPersonaSerializer.new(persona, root: false)
+          else
+            persona = importer.import!
+            log_ai_persona_creation(persona)
+            render json: LocalizedAiPersonaSerializer.new(persona, root: false), status: :created
+          end
+        rescue DiscourseAi::PersonaImporter::ImportError => e
+          render_json_error e.message, status: :unprocessable_entity
+        rescue StandardError => e
+          Rails.logger.error("AI Persona import failed: #{e.message}")
+          render_json_error "Import failed: #{e.message}", status: :unprocessable_entity
+        end
+      end
+
       def stream_reply
         persona =
           AiPersona.find_by(name: params[:persona_name]) ||

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -11,6 +11,13 @@ module DiscourseAi
 
       def update
         initial_settings = AiModerationSetting.spam
+
+        initial_data = {
+          custom_instructions: initial_settings&.data&.dig("custom_instructions"),
+          llm_model_id: initial_settings&.llm_model_id,
+          ai_persona_id: initial_settings&.ai_persona_id,
+        }
+
         initial_custom_instructions = initial_settings&.data&.dig("custom_instructions")
         initial_llm_model_id = initial_settings&.llm_model_id
 
@@ -29,6 +36,22 @@ module DiscourseAi
             )
           end
         end
+
+        if allowed_params.key?(:ai_persona_id)
+          updated_params[:ai_persona_id] = allowed_params[:ai_persona_id]
+          persona = AiPersona.find_by(id: allowed_params[:ai_persona_id])
+          if persona.nil? ||
+               persona.response_format.to_a.none? { |rf|
+                 rf["key"] == "spam" && rf["type"] == "boolean"
+               }
+            return(
+              render_json_error(
+                I18n.t("discourse_ai.llm.configuration.invalid_persona_response_format"),
+                status: 422,
+              )
+            )
+          end
+        end
         updated_params[:data] = {
           custom_instructions: allowed_params[:custom_instructions],
         } if allowed_params.key?(:custom_instructions)
@@ -41,7 +64,7 @@ module DiscourseAi
             AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
           end
 
-          log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, allowed_params)
+          log_ai_spam_update(initial_data, allowed_params)
         end
 
         is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
@@ -119,9 +142,10 @@ module DiscourseAi
 
       private
 
-      def log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, params)
+      def log_ai_spam_update(initial_data, params)
         changes_to_log = {}
 
+        initial_llm_model_id = initial_data[:llm_model_id]
         if params.key?(:llm_model_id) && initial_llm_model_id.to_s != params[:llm_model_id].to_s
           old_model_name =
             LlmModel.find_by(id: initial_llm_model_id)&.display_name || initial_llm_model_id
@@ -131,11 +155,22 @@ module DiscourseAi
           changes_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"
         end
 
+        initial_custom_instructions = initial_data[:custom_instructions]
         if params.key?(:custom_instructions) &&
              initial_custom_instructions != params[:custom_instructions]
           changes_to_log[:custom_instructions] = params[:custom_instructions]
         end
 
+        initial_ai_persona_id = initial_data[:ai_persona_id]
+        if params.key?(:ai_persona_id) && initial_ai_persona_id.to_s != params[:ai_persona_id].to_s
+          old_persona_name =
+            AiPersona.find_by(id: initial_ai_persona_id)&.name || initial_ai_persona_id
+          new_persona_name =
+            AiPersona.find_by(id: params[:ai_persona_id])&.name || params[:ai_persona_id]
+
+          changes_to_log[:ai_persona_id] = "#{old_persona_name} → #{new_persona_name}"
+        end
+
         if changes_to_log.present?
           changes_to_log[:subject] = I18n.t("discourse_ai.spam_detection.logging_subject")
           logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
@@ -144,7 +179,7 @@ module DiscourseAi
       end
 
       def allowed_params
-        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
+        params.permit(:is_enabled, :llm_model_id, :custom_instructions, :ai_persona_id)
       end
 
       def spam_config

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -5,7 +5,7 @@ module DiscourseAi
     class AiToolsController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_tool, only: %i[test edit update destroy]
+      before_action :find_ai_tool, only: %i[test edit update destroy export]
 
       def index
         ai_tools = AiTool.all
@@ -32,6 +32,45 @@ module DiscourseAi
         end
       end
 
+      def export
+        response.headers[
+          "Content-Disposition"
+        ] = "attachment; filename=\"#{@ai_tool.tool_name}.json\""
+        render_serialized(@ai_tool, AiCustomToolSerializer)
+      end
+
+      def import
+        existing_tool = AiTool.find_by(tool_name: ai_tool_params[:tool_name])
+        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
+
+        if existing_tool && !force_update
+          return(
+            render_json_error "Tool with tool_name '#{ai_tool_params[:tool_name]}' already exists. Use force=true to overwrite.",
+                              status: :conflict
+          )
+        end
+
+        if existing_tool && force_update
+          initial_attributes = existing_tool.attributes.dup
+          if existing_tool.update(ai_tool_params)
+            log_ai_tool_update(existing_tool, initial_attributes)
+            render_serialized(existing_tool, AiCustomToolSerializer)
+          else
+            render_json_error existing_tool
+          end
+        else
+          ai_tool = AiTool.new(ai_tool_params)
+          ai_tool.created_by_id = current_user.id
+
+          if ai_tool.save
+            log_ai_tool_creation(ai_tool)
+            render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
+          else
+            render_json_error ai_tool
+          end
+        end
+      end
+
       def update
         initial_attributes = @ai_tool.attributes.dup
 

app/controllers/discourse_ai/ai_helper/assistant_controller.rb

@@ -124,6 +124,9 @@ module DiscourseAi
         # otherwise we may end up streaming the data to the wrong client
         raise Discourse::InvalidParameters.new(:client_id) if params[:client_id].blank?
 
+        channel_id = next_channel_id
+        progress_channel = "discourse_ai_helper/stream_suggestions/#{channel_id}"
+
         if location == "composer"
           Jobs.enqueue(
             :stream_composer_helper,
@@ -133,6 +136,7 @@ module DiscourseAi
             custom_prompt: params[:custom_prompt],
             force_default_locale: params[:force_default_locale] || false,
             client_id: params[:client_id],
+            progress_channel:,
           )
         else
           post_id = get_post_param!
@@ -148,10 +152,11 @@ module DiscourseAi
             prompt: params[:mode],
             custom_prompt: params[:custom_prompt],
             client_id: params[:client_id],
+            progress_channel:,
           )
         end
 
-        render json: { success: true }, status: 200
+        render json: { success: true, progress_channel: }, status: 200
       rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
         render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                           status: 502
@@ -192,6 +197,18 @@ module DiscourseAi
 
       private
 
+      CHANNEL_ID_KEY = "discourse_ai_helper_next_channel_id"
+
+      def next_channel_id
+        Discourse
+          .redis
+          .pipelined do |pipeline|
+            pipeline.incr(CHANNEL_ID_KEY)
+            pipeline.expire(CHANNEL_ID_KEY, 1.day)
+          end
+          .first
+      end
+
       def get_text_param!
         params[:text].tap { |t| raise Discourse::InvalidParameters.new(:text) if t.blank? }
       end

app/jobs/regular/detect_translate_post.rb

@@ -49,7 +49,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+            "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/jobs/regular/detect_translate_topic.rb

@@ -47,7 +47,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/jobs/regular/localize_categories.rb

@@ -5,51 +5,51 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.nil?
+      return if limit <= 0
+
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
-      cat_id = args[:from_category_id] || Category.order(:id).first&.id
-      last_id = nil
+      categories = Category.where("locale IS NOT NULL")
 
-      categories =
-        Category.where("id >= ? AND locale IS NOT NULL", cat_id).order(:id).limit(BATCH_SIZE)
-      return if categories.empty?
-
-      categories.each do |category|
-        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
-          last_id = category.id
-          next
-        end
-
-        locales.each do |locale|
-          localization = category.category_localizations.find_by(locale:)
-
-          if locale == category.locale && localization
-            localization.destroy
-          else
-            next if locale == category.locale
-            begin
-              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
-            rescue FinalDestination::SSRFDetector::LookupFailedError
-              # do nothing, there are too many sporadic lookup failures
-            rescue => e
-              DiscourseAi::Translation::VerboseLogger.log(
-                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
-              )
-            end
-          end
-        end
-        last_id = category.id
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        categories = categories.where(read_restricted: false)
       end
 
-      if categories.size == BATCH_SIZE
-        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
+      categories = categories.order(:id).limit(limit)
+      return if categories.empty?
+
+      remaining_limit = limit
+
+      categories.each do |category|
+        break if remaining_limit <= 0
+
+        existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)
+        missing_locales = locales - existing_locales - [category.locale]
+        missing_locales.each do |locale|
+          break if remaining_limit <= 0
+
+          begin
+            DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate category #{category.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+            )
+          ensure
+            remaining_limit -= 1
+          end
+        end
+
+        if existing_locales.include?(category.locale)
+          CategoryLocalization.find_by(category_id: category.id, locale: category.locale).destroy
+        end
       end
     end
   end

app/jobs/regular/localize_posts.rb

@@ -5,17 +5,16 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
 
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
-      limit = args[:limit] || BATCH_SIZE
-
       locales.each do |locale|
         posts =
           Post
@@ -66,7 +65,7 @@ module Jobs
             # do nothing, there are too many sporadic lookup failures
           rescue => e
             DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+              "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
             )
           end
         end

app/jobs/regular/localize_topics.rb

@@ -5,17 +5,16 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
 
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
-      limit = args[:limit] || BATCH_SIZE
-
       locales.each do |locale|
         topics =
           Topic
@@ -63,7 +62,7 @@ module Jobs
             # do nothing, there are too many sporadic lookup failures
           rescue => e
             DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
             )
           end
         end

app/jobs/regular/stream_composer_helper.rb

@@ -9,6 +9,7 @@ module Jobs
       return unless user = User.find_by(id: args[:user_id])
       return unless args[:text]
       return unless args[:client_id]
+      return unless args[:progress_channel]
 
       helper_mode = args[:prompt]
 
@@ -16,7 +17,7 @@ module Jobs
         helper_mode,
         args[:text],
         user,
-        "/discourse-ai/ai-helper/stream_composer_suggestion",
+        args[:progress_channel],
         force_default_locale: args[:force_default_locale],
         client_id: args[:client_id],
         custom_prompt: args[:custom_prompt],

app/jobs/regular/stream_post_helper.rb

@@ -8,6 +8,8 @@ module Jobs
       return unless post = Post.includes(:topic).find_by(id: args[:post_id])
       return unless user = User.find_by(id: args[:user_id])
       return unless args[:text]
+      return unless args[:progress_channel]
+      return unless args[:client_id]
 
       topic = post.topic
       reply_to = post.reply_to_post
@@ -31,8 +33,9 @@ module Jobs
         helper_mode,
         input,
         user,
-        "/discourse-ai/ai-helper/stream_suggestion/#{post.id}",
+        args[:progress_channel],
         custom_prompt: args[:custom_prompt],
+        client_id: args[:client_id],
       )
     end
   end

app/jobs/scheduled/categories_locale_detection_backfill.rb

@@ -9,7 +9,8 @@ module Jobs
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
+      return if limit == 0
 
       categories = Category.where(locale: nil)
 
@@ -17,7 +18,7 @@ module Jobs
         categories = categories.where(read_restricted: false)
       end
 
-      categories = categories.limit(SiteSetting.ai_translation_backfill_rate)
+      categories = categories.limit(limit)
       return if categories.empty?
 
       categories.each do |category|
@@ -26,7 +27,7 @@ module Jobs
         rescue FinalDestination::SSRFDetector::LookupFailedError
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect category #{category.id}'s locale: #{e.message}",
+            "Failed to detect category #{category.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/jobs/scheduled/category_localization_backfill.rb

@@ -2,15 +2,17 @@
 
 module Jobs
   class CategoryLocalizationBackfill < ::Jobs::Scheduled
-    every 12.hours
+    every 1.hour
     cluster_concurrency 1
 
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
       return if SiteSetting.content_localization_supported_locales.blank?
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
+      return if limit == 0
 
-      Jobs.enqueue(:localize_categories)
+      Jobs.enqueue(:localize_categories, limit:)
     end
   end
 end

app/jobs/scheduled/embeddings_backfill.rb

@@ -18,105 +18,115 @@ module Jobs
         )
       end
 
-      rebaked = 0
+      production_vector = DiscourseAi::Embeddings::Vector.instance
 
-      vector = DiscourseAi::Embeddings::Vector.instance
-      vector_def = vector.vdef
-      table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
-
-      topics =
-        Topic
-          .joins(
-            "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
+      if SiteSetting.ai_embeddings_backfill_model.present? &&
+           SiteSetting.ai_embeddings_backfill_model != SiteSetting.ai_embeddings_selected_model
+        backfill_vector =
+          DiscourseAi::Embeddings::Vector.new(
+            EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_backfill_model),
           )
-          .where(archetype: Archetype.default)
-          .where(deleted_at: nil)
-          .order("topics.bumped_at DESC")
+      end
 
-      rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
+      topic_work_list = []
+      topic_work_list << production_vector
+      topic_work_list << backfill_vector if backfill_vector
 
-      return if rebaked >= limit
+      topic_work_list.each do |vector|
+        rebaked = 0
+        table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
+        vector_def = vector.vdef
 
-      # Then, we'll try to backfill embeddings for topics that have outdated
-      # embeddings, be it model or strategy version
-      relation = topics.where(<<~SQL).limit(limit - rebaked)
-          #{table_name}.model_version < #{vector_def.version}
-          OR
-          #{table_name}.strategy_version < #{vector_def.strategy_version}
-        SQL
+        topics =
+          Topic
+            .joins(
+              "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
+            )
+            .where(archetype: Archetype.default)
+            .where(deleted_at: nil)
+            .order("topics.bumped_at DESC")
 
-      rebaked += populate_topic_embeddings(vector, relation, force: true)
+        rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
 
-      return if rebaked >= limit
+        next if rebaked >= limit
 
-      # Finally, we'll try to backfill embeddings for topics that have outdated
-      # embeddings due to edits or new replies. Here we only do 10% of the limit
-      relation =
-        topics
-          .where("#{table_name}.updated_at < ?", 6.hours.ago)
-          .where("#{table_name}.updated_at < topics.updated_at")
+        # Then, we'll try to backfill embeddings for topics that have outdated
+        # embeddings, be it model or strategy version
+        relation = topics.where(<<~SQL).limit(limit - rebaked)
+            #{table_name}.model_version < #{vector_def.version}
+            OR
+            #{table_name}.strategy_version < #{vector_def.strategy_version}
+          SQL
+
+        rebaked += populate_topic_embeddings(vector, relation, force: true)
+
+        next if rebaked >= limit
+
+        # Finally, we'll try to backfill embeddings for topics that have outdated
+        # embeddings due to edits or new replies. Here we only do 10% of the limit
+        relation =
+          topics
+            .where("#{table_name}.updated_at < ?", 6.hours.ago)
+            .where("#{table_name}.updated_at < topics.updated_at")
+            .limit((limit - rebaked) / 10)
+
+        populate_topic_embeddings(vector, relation, force: true)
+
+        next unless SiteSetting.ai_embeddings_per_post_enabled
+
+        # Now for posts
+        table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
+        posts_batch_size = 1000
+
+        posts =
+          Post
+            .joins(
+              "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
+            )
+            .where(deleted_at: nil)
+            .where(post_type: Post.types[:regular])
+
+        # First, we'll try to backfill embeddings for posts that have none
+        posts
+          .where("#{table_name}.post_id IS NULL")
+          .limit(limit - rebaked)
+          .pluck(:id)
+          .each_slice(posts_batch_size) do |batch|
+            vector.gen_bulk_reprensentations(Post.where(id: batch))
+            rebaked += batch.length
+          end
+
+        next if rebaked >= limit
+
+        # Then, we'll try to backfill embeddings for posts that have outdated
+        # embeddings, be it model or strategy version
+        posts
+          .where(<<~SQL)
+            #{table_name}.model_version < #{vector_def.version}
+            OR
+            #{table_name}.strategy_version < #{vector_def.strategy_version}
+          SQL
+          .limit(limit - rebaked)
+          .pluck(:id)
+          .each_slice(posts_batch_size) do |batch|
+            vector.gen_bulk_reprensentations(Post.where(id: batch))
+            rebaked += batch.length
+          end
+
+        next if rebaked >= limit
+
+        # Finally, we'll try to backfill embeddings for posts that have outdated
+        # embeddings due to edits. Here we only do 10% of the limit
+        posts
+          .where("#{table_name}.updated_at < ?", 7.days.ago)
+          .order("random()")
           .limit((limit - rebaked) / 10)
-
-      populate_topic_embeddings(vector, relation, force: true)
-
-      return if rebaked >= limit
-
-      return unless SiteSetting.ai_embeddings_per_post_enabled
-
-      # Now for posts
-      table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
-      posts_batch_size = 1000
-
-      posts =
-        Post
-          .joins(
-            "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
-          )
-          .where(deleted_at: nil)
-          .where(post_type: Post.types[:regular])
-
-      # First, we'll try to backfill embeddings for posts that have none
-      posts
-        .where("#{table_name}.post_id IS NULL")
-        .limit(limit - rebaked)
-        .pluck(:id)
-        .each_slice(posts_batch_size) do |batch|
-          vector.gen_bulk_reprensentations(Post.where(id: batch))
-          rebaked += batch.length
-        end
-
-      return if rebaked >= limit
-
-      # Then, we'll try to backfill embeddings for posts that have outdated
-      # embeddings, be it model or strategy version
-      posts
-        .where(<<~SQL)
-          #{table_name}.model_version < #{vector_def.version}
-          OR
-          #{table_name}.strategy_version < #{vector_def.strategy_version}
-        SQL
-        .limit(limit - rebaked)
-        .pluck(:id)
-        .each_slice(posts_batch_size) do |batch|
-          vector.gen_bulk_reprensentations(Post.where(id: batch))
-          rebaked += batch.length
-        end
-
-      return if rebaked >= limit
-
-      # Finally, we'll try to backfill embeddings for posts that have outdated
-      # embeddings due to edits. Here we only do 10% of the limit
-      posts
-        .where("#{table_name}.updated_at < ?", 7.days.ago)
-        .order("random()")
-        .limit((limit - rebaked) / 10)
-        .pluck(:id)
-        .each_slice(posts_batch_size) do |batch|
-          vector.gen_bulk_reprensentations(Post.where(id: batch))
-          rebaked += batch.length
-        end
-
-      rebaked
+          .pluck(:id)
+          .each_slice(posts_batch_size) do |batch|
+            vector.gen_bulk_reprensentations(Post.where(id: batch))
+            rebaked += batch.length
+          end
+      end
     end
 
     private

app/jobs/scheduled/post_localization_backfill.rb

@@ -10,9 +10,10 @@ module Jobs
       return if !SiteSetting.ai_translation_enabled
 
       return if SiteSetting.content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
-      Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
+      Jobs.enqueue(:localize_posts, limit:)
     end
   end
 end

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -9,7 +9,8 @@ module Jobs
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
       posts =
         Post
@@ -19,12 +20,17 @@ module Jobs
           .where.not(raw: [nil, ""])
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
-        public_categories = Category.where(read_restricted: false).pluck(:id)
         posts =
           posts
             .joins(:topic)
-            .where(topics: { category_id: public_categories })
-            .where(topics: { archetype: "regular" })
+            .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
+            .where("archetype != ?", Archetype.private_message)
+      else
+        posts =
+          posts.joins(:topic).where(
+            "topics.archetype != ? OR EXISTS (SELECT 1 FROM topic_allowed_groups WHERE topic_id = topics.id)",
+            Archetype.private_message,
+          )
       end
 
       if SiteSetting.ai_translation_backfill_max_age_days > 0
@@ -35,7 +41,7 @@ module Jobs
           )
       end
 
-      posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
+      posts = posts.order(updated_at: :desc).limit(limit)
       return if posts.empty?
 
       posts.each do |post|
@@ -45,7 +51,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect post #{post.id}'s locale: #{e.message}",
+            "Failed to detect post #{post.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/jobs/scheduled/topic_localization_backfill.rb

@@ -10,9 +10,10 @@ module Jobs
       return if !SiteSetting.ai_translation_enabled
 
       return if SiteSetting.content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
-      Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
+      Jobs.enqueue(:localize_topics, limit:)
     end
   end
 end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -9,14 +9,23 @@ module Jobs
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-      limit = SiteSetting.ai_translation_backfill_rate
-
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
       return if limit == 0
 
       topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
-        topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+        topics =
+          topics.where(category_id: Category.where(read_restricted: false).select(:id)).where(
+            "archetype != ?",
+            Archetype.private_message,
+          )
+      else
+        topics =
+          topics.where(
+            "archetype != ? OR EXISTS (SELECT 1 FROM topic_allowed_groups WHERE topic_id = topics.id)",
+            Archetype.private_message,
+          )
       end
 
       if SiteSetting.ai_translation_backfill_max_age_days > 0
@@ -37,7 +46,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect topic #{topic.id}'s locale: #{e.message}",
+            "Failed to detect topic #{topic.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/models/ai_moderation_setting.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 class AiModerationSetting < ActiveRecord::Base
   belongs_to :llm_model
+  belongs_to :ai_persona
 
   validates :llm_model_id, presence: true
   validates :setting_type, presence: true
@@ -19,12 +20,13 @@ end
 #
 # Table name: ai_moderation_settings
 #
-#  id           :bigint           not null, primary key
-#  setting_type :enum             not null
-#  data         :jsonb
-#  llm_model_id :bigint           not null
-#  created_at   :datetime         not null
-#  updated_at   :datetime         not null
+#  id            :bigint           not null, primary key
+#  setting_type  :enum             not null
+#  data          :jsonb
+#  llm_model_id  :bigint           not null
+#  created_at    :datetime         not null
+#  updated_at    :datetime         not null
+#  ai_persona_id :bigint           default(-31), not null
 #
 # Indexes
 #

app/models/embedding_definition.rb

@@ -23,7 +23,8 @@ class EmbeddingDefinition < ActiveRecord::Base
         DiscourseAi::Tokenizer::GeminiTokenizer,
         DiscourseAi::Tokenizer::MultilingualE5LargeTokenizer,
         DiscourseAi::Tokenizer::OpenAiTokenizer,
-        DiscourseAi::Tokenizer::MixtralTokenizer,
+        DiscourseAi::Tokenizer::OpenAiCl100kTokenizer,
+        DiscourseAi::Tokenizer::MistralTokenizer,
         DiscourseAi::Tokenizer::QwenTokenizer,
       ].map(&:name)
     end

app/models/llm_model.rb

@@ -65,6 +65,8 @@ class LlmModel < ActiveRecord::Base
       google: {
         disable_native_tools: :checkbox,
         enable_thinking: :checkbox,
+        disable_temperature: :checkbox,
+        disable_top_p: :checkbox,
         thinking_tokens: :number,
       },
       azure: {

app/serializers/ai_spam_serializer.rb

@@ -8,7 +8,9 @@ class AiSpamSerializer < ApplicationSerializer
              :stats,
              :flagging_username,
              :spam_score_type,
-             :spam_scanning_user
+             :spam_scanning_user,
+             :ai_persona_id,
+             :available_personas
 
   def is_enabled
     object[:enabled]
@@ -18,6 +20,11 @@ class AiSpamSerializer < ApplicationSerializer
     settings&.llm_model&.id
   end
 
+  def ai_persona_id
+    settings&.ai_persona&.id ||
+      DiscourseAi::Personas::Persona.system_personas[DiscourseAi::Personas::SpamDetector]
+  end
+
   def custom_instructions
     settings&.custom_instructions
   end
@@ -28,6 +35,12 @@ class AiSpamSerializer < ApplicationSerializer
       .map { |hash| { id: hash[:value], name: hash[:name] } }
   end
 
+  def available_personas
+    DiscourseAi::Configuration::PersonaEnumerator.values.map do |h|
+      { id: h[:value], name: h[:name] }
+    end
+  end
+
   def flagging_username
     object[:flagging_username]
   end

app/services/discourse_ai/persona_exporter.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  class PersonaExporter
+    def initialize(persona:)
+      raise ArgumentError, "Invalid persona provided" if !persona.is_a?(AiPersona)
+      @persona = persona
+    end
+
+    def export
+      serialized_custom_tools = serialize_tools(@persona)
+      serialize_persona(@persona, serialized_custom_tools)
+    end
+
+    private
+
+    def serialize_tools(ai_persona)
+      custom_tool_ids =
+        (ai_persona.tools || []).filter_map do |tool_config|
+          # A tool config is an array like: ["custom-ID", {options}, force_flag]
+          if tool_config.is_a?(Array) && tool_config[0].to_s.start_with?("custom-")
+            tool_config[0].split("-", 2).last.to_i
+          end
+        end
+
+      return [] if custom_tool_ids.empty?
+
+      tools = AiTool.where(id: custom_tool_ids)
+      tools.map do |tool|
+        {
+          identifier: tool.tool_name, # Use tool_name for portability
+          name: tool.name,
+          description: tool.description,
+          tool_name: tool.tool_name,
+          parameters: tool.parameters,
+          summary: tool.summary,
+          script: tool.script,
+        }
+      end
+    end
+
+    def serialize_persona(ai_persona, serialized_custom_tools)
+      export_data = {
+        meta: {
+          version: "1.0",
+          exported_at: Time.zone.now.iso8601,
+        },
+        persona: {
+          name: ai_persona.name,
+          description: ai_persona.description,
+          system_prompt: ai_persona.system_prompt,
+          examples: ai_persona.examples,
+          temperature: ai_persona.temperature,
+          top_p: ai_persona.top_p,
+          response_format: ai_persona.response_format,
+          tools: transform_tools_for_export(ai_persona.tools, serialized_custom_tools),
+        },
+        custom_tools: serialized_custom_tools,
+      }
+
+      JSON.pretty_generate(export_data)
+    end
+
+    def transform_tools_for_export(tools_config, _serialized_custom_tools)
+      return [] if tools_config.blank?
+
+      tools_config.map do |tool_config|
+        unless tool_config.is_a?(Array) && tool_config[0].to_s.start_with?("custom-")
+          next tool_config
+        end
+
+        tool_id = tool_config[0].split("-", 2).last.to_i
+        tool = AiTool.find_by(id: tool_id)
+        next tool_config unless tool
+
+        ["custom-#{tool.tool_name}", tool_config[1], tool_config[2]]
+      end
+    end
+  end
+end

app/services/discourse_ai/persona_importer.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  class PersonaImporter
+    class ImportError < StandardError
+      attr_reader :conflicts
+
+      def initialize(message, conflicts = {})
+        super(message)
+        @conflicts = conflicts
+      end
+    end
+
+    def initialize(json:)
+      @data =
+        case json
+        when String
+          JSON.parse(json)
+        when Hash
+          json
+        else
+          raise ArgumentError, "Invalid JSON payload"
+        end
+
+      validate_payload!
+    end
+
+    def import!(overwrite: false)
+      ActiveRecord::Base.transaction do
+        check_conflicts! unless overwrite
+
+        tool_name_to_id = import_custom_tools(@data["custom_tools"] || [], overwrite: overwrite)
+        persona_data = @data["persona"]
+
+        existing_persona = AiPersona.find_by(name: persona_data["name"])
+
+        attrs = {
+          description: persona_data["description"],
+          system_prompt: persona_data["system_prompt"],
+          examples: persona_data["examples"],
+          temperature: persona_data["temperature"],
+          top_p: persona_data["top_p"],
+          response_format: persona_data["response_format"],
+          tools: transform_tools_for_import(persona_data["tools"], tool_name_to_id),
+        }
+
+        if existing_persona && overwrite
+          existing_persona.update!(**attrs)
+          existing_persona
+        else
+          attrs[:name] = persona_data["name"]
+          AiPersona.create!(**attrs)
+        end
+      end
+    end
+
+    private
+
+    def validate_payload!
+      unless @data.is_a?(Hash) && @data["persona"].is_a?(Hash)
+        raise ArgumentError, "Invalid persona export data"
+      end
+    end
+
+    def check_conflicts!
+      conflicts = {}
+
+      persona_name = @data["persona"]["name"]
+      conflicts[:persona] = persona_name if AiPersona.exists?(name: persona_name)
+
+      if @data["custom_tools"].present?
+        existing_tools = []
+        @data["custom_tools"].each do |tool_data|
+          tool_name = tool_data["tool_name"] || tool_data["identifier"]
+          existing_tools << tool_name if AiTool.exists?(tool_name: tool_name)
+        end
+        conflicts[:custom_tools] = existing_tools if existing_tools.any?
+      end
+
+      if conflicts.any?
+        message = build_conflict_message(conflicts)
+        raise ImportError.new(message, conflicts)
+      end
+    end
+
+    def build_conflict_message(conflicts)
+      messages = []
+
+      if conflicts[:persona]
+        messages << I18n.t("discourse_ai.errors.persona_already_exists", name: conflicts[:persona])
+      end
+
+      if conflicts[:custom_tools] && conflicts[:custom_tools].any?
+        tools_list = conflicts[:custom_tools].join(", ")
+        error =
+          I18n.t(
+            "discourse_ai.errors.custom_tool_exists",
+            names: tools_list,
+            count: conflicts[:custom_tools].size,
+          )
+        messages << error
+      end
+
+      messages.join("\n")
+    end
+
+    def import_custom_tools(custom_tools, overwrite:)
+      return {} if custom_tools.blank?
+
+      custom_tools.each_with_object({}) do |tool_data, map|
+        tool_name = tool_data["tool_name"] || tool_data["identifier"]
+
+        if overwrite
+          tool = AiTool.find_or_initialize_by(tool_name: tool_name)
+        else
+          tool = AiTool.new(tool_name: tool_name)
+        end
+
+        tool.tap do |t|
+          t.name = tool_data["name"]
+          t.description = tool_data["description"]
+          t.parameters = tool_data["parameters"]
+          t.script = tool_data["script"]
+          t.summary = tool_data["summary"]
+          t.created_by ||= Discourse.system_user
+          t.save!
+        end
+
+        map[tool.tool_name] = tool.id
+      end
+    end
+
+    def transform_tools_for_import(tools_config, tool_name_to_id)
+      return [] if tools_config.blank?
+
+      tools_config.map do |tool_config|
+        if tool_config.is_a?(Array) && tool_config[0].to_s.start_with?("custom-")
+          tool_name = tool_config[0].split("-", 2).last
+          tool_id = tool_name_to_id[tool_name] || AiTool.find_by(tool_name: tool_name)&.id
+          raise ArgumentError, "Custom tool '#{tool_name}' not found" unless tool_id
+
+          ["custom-#{tool_id}", tool_config[1], tool_config[2]]
+        else
+          tool_config
+        end
+      end
+    end
+  end
+end

assets/javascripts/discourse/components/ai-bot-header-icon.gjs

@@ -20,7 +20,7 @@ export default class AiBotHeaderIcon extends Component {
 
   get bots() {
     const availableBots = this.currentUser.ai_enabled_chat_bots
-      .filter((bot) => !bot.is_persosna)
+      .filter((bot) => !bot.is_persona || bot.has_default_llm)
       .filter(Boolean);
 
     return availableBots ? availableBots.map((bot) => bot.model_name) : [];