Update db/migrate/20250716005855_copy_ai_image_caption_model_to_persona_default.rb

Since translations only require a single key back, there is little point in using structured output. This PR also includes some prompt updates dealing with quotes, details, and code.

Related: #1502

This does mean reverting discourse/discourse-translator#257, but we can see how it goes.

.discourse-compatibility

@@ -1,3 +1,4 @@
+< 3.5.0.beta7-dev: cd14b0c0bee0cf63c59b64b6f7213e31a37f11a7
 < 3.5.0.beta6-dev: 3e74eea1e5e3143888d67a8d8a11206df214dc24
 < 3.5.0.beta3-dev: 09a68414804a1447f52e5d60691ba59742cda9ec
 < 3.5.0.beta2-dev: de8624416a15b3d8e7ad350b083cc1420451ccec

.gitignore

@@ -6,5 +6,4 @@ evals/log
 evals/cases
 config/eval-llms.local.yml
 # this gets rid of search results from ag, ripgrep, etc
-tokenizers/
 public/ai-share/highlight.min.js

Gemfile.lock

@@ -34,7 +34,7 @@ GEM
       racc
     prettier_print (1.2.1)
     racc (1.8.1)
-    rack (3.1.14)
+    rack (3.1.16)
     rainbow (3.1.1)
     regexp_parser (2.10.0)
     rubocop (1.74.0)

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-features-edit.js

@@ -1,3 +1,4 @@
+import { action } from "@ember/object";
 import { ajax } from "discourse/lib/ajax";
 import DiscourseRoute from "discourse/routes/discourse";
 import SiteSetting from "admin/models/site-setting";
@@ -24,4 +25,11 @@ export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRo
 
     return currentFeature;
   }
+
+  @action
+  willTransition() {
+    // site settings may amend if a feature is enabled or disabled, so refresh the model
+    // even on back button
+    this.router.refresh("adminPlugins.show.discourse-ai-features");
+  }
 }

app/controllers/discourse_ai/admin/ai_features_controller.rb

@@ -37,7 +37,11 @@ module DiscourseAi
       def serialize_feature(feature)
         {
           name: feature.name,
-          persona: serialize_persona(persona_id_obj_hash[feature.persona_id]),
+          personas: feature.persona_ids.map { |id| serialize_persona(persona_id_obj_hash[id]) },
+          llm_models:
+            feature.llm_models.map do |llm_model|
+              { id: llm_model.id, name: llm_model.display_name }
+            end,
           enabled: feature.enabled?,
         }
       end
@@ -53,9 +57,7 @@ module DiscourseAi
       def persona_id_obj_hash
         @persona_id_obj_hash ||=
           begin
-            setting_names = DiscourseAi::Configuration::Feature.all_persona_setting_names
-            ids = setting_names.map { |sn| SiteSetting.public_send(sn) }
-
+            ids = DiscourseAi::Configuration::Feature.all.map(&:persona_ids).flatten.uniq
             AiPersona.where(id: ids).index_by(&:id)
           end
       end

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -5,7 +5,7 @@ module DiscourseAi
     class AiPersonasController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_persona, only: %i[edit update destroy create_user]
+      before_action :find_ai_persona, only: %i[edit update destroy create_user export]
 
       def index
         ai_personas =
@@ -105,6 +105,43 @@ module DiscourseAi
         end
       end
 
+      def export
+        persona = AiPersona.find(params[:id])
+        exporter = DiscourseAi::PersonaExporter.new(persona: persona)
+
+        response.headers[
+          "Content-Disposition"
+        ] = "attachment; filename=\"#{persona.name.parameterize}.json\""
+
+        render json: exporter.export
+      end
+
+      def import
+        name = params.dig(:persona, :name)
+        existing_persona = AiPersona.find_by(name: name)
+        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
+
+        begin
+          importer = DiscourseAi::PersonaImporter.new(json: params.to_unsafe_h)
+
+          if existing_persona && force_update
+            initial_attributes = existing_persona.attributes.dup
+            persona = importer.import!(overwrite: true)
+            log_ai_persona_update(persona, initial_attributes)
+            render json: LocalizedAiPersonaSerializer.new(persona, root: false)
+          else
+            persona = importer.import!
+            log_ai_persona_creation(persona)
+            render json: LocalizedAiPersonaSerializer.new(persona, root: false), status: :created
+          end
+        rescue DiscourseAi::PersonaImporter::ImportError => e
+          render_json_error e.message, status: :unprocessable_entity
+        rescue StandardError => e
+          Rails.logger.error("AI Persona import failed: #{e.message}")
+          render_json_error "Import failed: #{e.message}", status: :unprocessable_entity
+        end
+      end
+
       def stream_reply
         persona =
           AiPersona.find_by(name: params[:persona_name]) ||

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -11,6 +11,13 @@ module DiscourseAi
 
       def update
         initial_settings = AiModerationSetting.spam
+
+        initial_data = {
+          custom_instructions: initial_settings&.data&.dig("custom_instructions"),
+          llm_model_id: initial_settings&.llm_model_id,
+          ai_persona_id: initial_settings&.ai_persona_id,
+        }
+
         initial_custom_instructions = initial_settings&.data&.dig("custom_instructions")
         initial_llm_model_id = initial_settings&.llm_model_id
 
@@ -29,6 +36,22 @@ module DiscourseAi
             )
           end
         end
+
+        if allowed_params.key?(:ai_persona_id)
+          updated_params[:ai_persona_id] = allowed_params[:ai_persona_id]
+          persona = AiPersona.find_by(id: allowed_params[:ai_persona_id])
+          if persona.nil? ||
+               persona.response_format.to_a.none? { |rf|
+                 rf["key"] == "spam" && rf["type"] == "boolean"
+               }
+            return(
+              render_json_error(
+                I18n.t("discourse_ai.llm.configuration.invalid_persona_response_format"),
+                status: 422,
+              )
+            )
+          end
+        end
         updated_params[:data] = {
           custom_instructions: allowed_params[:custom_instructions],
         } if allowed_params.key?(:custom_instructions)
@@ -41,7 +64,7 @@ module DiscourseAi
             AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
           end
 
-          log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, allowed_params)
+          log_ai_spam_update(initial_data, allowed_params)
         end
 
         is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
@@ -119,9 +142,10 @@ module DiscourseAi
 
       private
 
-      def log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, params)
+      def log_ai_spam_update(initial_data, params)
         changes_to_log = {}
 
+        initial_llm_model_id = initial_data[:llm_model_id]
         if params.key?(:llm_model_id) && initial_llm_model_id.to_s != params[:llm_model_id].to_s
           old_model_name =
             LlmModel.find_by(id: initial_llm_model_id)&.display_name || initial_llm_model_id
@@ -131,11 +155,22 @@ module DiscourseAi
           changes_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"
         end
 
+        initial_custom_instructions = initial_data[:custom_instructions]
         if params.key?(:custom_instructions) &&
              initial_custom_instructions != params[:custom_instructions]
           changes_to_log[:custom_instructions] = params[:custom_instructions]
         end
 
+        initial_ai_persona_id = initial_data[:ai_persona_id]
+        if params.key?(:ai_persona_id) && initial_ai_persona_id.to_s != params[:ai_persona_id].to_s
+          old_persona_name =
+            AiPersona.find_by(id: initial_ai_persona_id)&.name || initial_ai_persona_id
+          new_persona_name =
+            AiPersona.find_by(id: params[:ai_persona_id])&.name || params[:ai_persona_id]
+
+          changes_to_log[:ai_persona_id] = "#{old_persona_name} → #{new_persona_name}"
+        end
+
         if changes_to_log.present?
           changes_to_log[:subject] = I18n.t("discourse_ai.spam_detection.logging_subject")
           logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
@@ -144,7 +179,7 @@ module DiscourseAi
       end
 
       def allowed_params
-        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
+        params.permit(:is_enabled, :llm_model_id, :custom_instructions, :ai_persona_id)
       end
 
       def spam_config

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -5,7 +5,7 @@ module DiscourseAi
     class AiToolsController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_tool, only: %i[test edit update destroy]
+      before_action :find_ai_tool, only: %i[test edit update destroy export]
 
       def index
         ai_tools = AiTool.all
@@ -32,6 +32,45 @@ module DiscourseAi
         end
       end
 
+      def export
+        response.headers[
+          "Content-Disposition"
+        ] = "attachment; filename=\"#{@ai_tool.tool_name}.json\""
+        render_serialized(@ai_tool, AiCustomToolSerializer)
+      end
+
+      def import
+        existing_tool = AiTool.find_by(tool_name: ai_tool_params[:tool_name])
+        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
+
+        if existing_tool && !force_update
+          return(
+            render_json_error "Tool with tool_name '#{ai_tool_params[:tool_name]}' already exists. Use force=true to overwrite.",
+                              status: :conflict
+          )
+        end
+
+        if existing_tool && force_update
+          initial_attributes = existing_tool.attributes.dup
+          if existing_tool.update(ai_tool_params)
+            log_ai_tool_update(existing_tool, initial_attributes)
+            render_serialized(existing_tool, AiCustomToolSerializer)
+          else
+            render_json_error existing_tool
+          end
+        else
+          ai_tool = AiTool.new(ai_tool_params)
+          ai_tool.created_by_id = current_user.id
+
+          if ai_tool.save
+            log_ai_tool_creation(ai_tool)
+            render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
+          else
+            render_json_error ai_tool
+          end
+        end
+      end
+
       def update
         initial_attributes = @ai_tool.attributes.dup
 

app/controllers/discourse_ai/ai_helper/assistant_controller.rb

@@ -124,6 +124,9 @@ module DiscourseAi
         # otherwise we may end up streaming the data to the wrong client
         raise Discourse::InvalidParameters.new(:client_id) if params[:client_id].blank?
 
+        channel_id = next_channel_id
+        progress_channel = "discourse_ai_helper/stream_suggestions/#{channel_id}"
+
         if location == "composer"
           Jobs.enqueue(
             :stream_composer_helper,
@@ -133,6 +136,7 @@ module DiscourseAi
             custom_prompt: params[:custom_prompt],
             force_default_locale: params[:force_default_locale] || false,
             client_id: params[:client_id],
+            progress_channel:,
           )
         else
           post_id = get_post_param!
@@ -148,10 +152,11 @@ module DiscourseAi
             prompt: params[:mode],
             custom_prompt: params[:custom_prompt],
             client_id: params[:client_id],
+            progress_channel:,
           )
         end
 
-        render json: { success: true }, status: 200
+        render json: { success: true, progress_channel: }, status: 200
       rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
         render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                           status: 502
@@ -192,6 +197,18 @@ module DiscourseAi
 
       private
 
+      CHANNEL_ID_KEY = "discourse_ai_helper_next_channel_id"
+
+      def next_channel_id
+        Discourse
+          .redis
+          .pipelined do |pipeline|
+            pipeline.incr(CHANNEL_ID_KEY)
+            pipeline.expire(CHANNEL_ID_KEY, 1.day)
+          end
+          .first
+      end
+
       def get_text_param!
         params[:text].tap { |t| raise Discourse::InvalidParameters.new(:text) if t.blank? }
       end

app/jobs/regular/detect_translate_post.rb

@@ -2,11 +2,11 @@
 
 module Jobs
   class DetectTranslatePost < ::Jobs::Base
+    cluster_concurrency 1
     sidekiq_options retry: false
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.enabled?
       return if args[:post_id].blank?
 
       post = Post.find_by(id: args[:post_id])
@@ -35,12 +35,14 @@ module Jobs
         end
       end
 
-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if detected_locale.blank?
+      locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
       locales.each do |locale|
-        next if locale == detected_locale
-        next if post.post_localizations.exists?(locale:)
+        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        regionless_locale = locale.split("_").first
+        next if post.post_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
 
         begin
           DiscourseAi::Translation::PostLocalizer.localize(post, locale)
@@ -48,7 +50,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+            "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/jobs/regular/detect_translate_topic.rb

@@ -2,11 +2,11 @@
 
 module Jobs
   class DetectTranslateTopic < ::Jobs::Base
+    cluster_concurrency 1
     sidekiq_options retry: false
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.enabled?
       return if args[:topic_id].blank?
 
       topic = Topic.find_by(id: args[:topic_id])
@@ -33,12 +33,14 @@ module Jobs
         end
       end
 
-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if detected_locale.blank?
+      locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
       locales.each do |locale|
-        next if locale == detected_locale
-        next if topic.topic_localizations.exists?(locale:)
+        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        regionless_locale = locale.split("_").first
+        next if topic.topic_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
 
         begin
           DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
@@ -46,7 +48,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/jobs/regular/localize_categories.rb

@@ -5,51 +5,48 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.enabled?
 
-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
-      return if locales.blank?
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.nil?
+      return if limit <= 0
+      locales = SiteSetting.content_localization_supported_locales.split("|")
 
-      cat_id = args[:from_category_id] || Category.order(:id).first&.id
-      last_id = nil
-
-      categories =
-        Category.where("id >= ? AND locale IS NOT NULL", cat_id).order(:id).limit(BATCH_SIZE)
+      categories = Category.where("locale IS NOT NULL")
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        categories = categories.where(read_restricted: false)
+      end
+      categories = categories.order(:id).limit(limit)
       return if categories.empty?
 
+      remaining_limit = limit
+
       categories.each do |category|
-        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
-          last_id = category.id
-          next
-        end
+        break if remaining_limit <= 0
 
-        locales.each do |locale|
-          localization = category.category_localizations.find_by(locale:)
+        existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)
+        missing_locales = locales - existing_locales - [category.locale]
+        missing_locales.each do |locale|
+          break if remaining_limit <= 0
+          next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, category.locale)
 
-          if locale == category.locale && localization
-            localization.destroy
-          else
-            next if locale == category.locale
-            begin
-              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
-            rescue FinalDestination::SSRFDetector::LookupFailedError
-              # do nothing, there are too many sporadic lookup failures
-            rescue => e
-              DiscourseAi::Translation::VerboseLogger.log(
-                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
-              )
-            end
+          begin
+            DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate category #{category.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+            )
+          ensure
+            remaining_limit -= 1
           end
         end
-        last_id = category.id
-      end
 
-      if categories.size == BATCH_SIZE
-        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
+        if existing_locales.include?(category.locale)
+          CategoryLocalization.find_by(category_id: category.id, locale: category.locale).destroy
+        end
       end
     end
   end

app/jobs/regular/localize_posts.rb

@@ -5,28 +5,29 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
 
-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
-      return if locales.blank?
-
-      limit = args[:limit] || BATCH_SIZE
+      return if !DiscourseAi::Translation.backfill_enabled?
 
+      locales = SiteSetting.content_localization_supported_locales.split("|")
       locales.each do |locale|
+        base_locale = locale.split("_").first
         posts =
           Post
             .joins(
-              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale LIKE '#{base_locale}%'",
+            )
+            .where(
+              "posts.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
             )
             .where(deleted_at: nil)
             .where("posts.user_id > 0")
             .where.not(raw: [nil, ""])
             .where.not(locale: nil)
-            .where.not(locale: locale)
+            .where("posts.locale NOT LIKE '#{base_locale}%'")
             .where("pl.id IS NULL")
 
         posts = posts.joins(:topic)
@@ -47,14 +48,6 @@ module Jobs
             )
         end
 
-        if SiteSetting.ai_translation_backfill_max_age_days > 0
-          posts =
-            posts.where(
-              "posts.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
-            )
-        end
-
         posts = posts.order(updated_at: :desc).limit(limit)
 
         next if posts.empty?
@@ -66,7 +59,7 @@ module Jobs
             # do nothing, there are too many sporadic lookup failures
           rescue => e
             DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+              "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
             )
           end
         end

app/jobs/regular/localize_topics.rb

@@ -5,27 +5,28 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
 
-      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
-      return if locales.blank?
-
-      limit = args[:limit] || BATCH_SIZE
+      return if !DiscourseAi::Translation.backfill_enabled?
 
+      locales = SiteSetting.content_localization_supported_locales.split("|")
       locales.each do |locale|
+        base_locale = locale.split("_").first
         topics =
           Topic
             .joins(
-              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale LIKE '#{base_locale}%'",
+            )
+            .where(
+              "topics.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
             )
             .where(deleted_at: nil)
             .where("topics.user_id > 0")
             .where.not(locale: nil)
-            .where.not(locale: locale)
+            .where("topics.locale NOT LIKE '#{base_locale}%'")
             .where("tl.id IS NULL")
 
         if SiteSetting.ai_translation_backfill_limit_to_public_content
@@ -44,14 +45,6 @@ module Jobs
             )
         end
 
-        if SiteSetting.ai_translation_backfill_max_age_days > 0
-          topics =
-            topics.where(
-              "topics.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
-            )
-        end
-
         topics = topics.order(updated_at: :desc).limit(limit)
 
         next if topics.empty?
@@ -63,7 +56,7 @@ module Jobs
             # do nothing, there are too many sporadic lookup failures
           rescue => e
             DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
             )
           end
         end

app/jobs/regular/stream_composer_helper.rb

@@ -9,6 +9,7 @@ module Jobs
       return unless user = User.find_by(id: args[:user_id])
       return unless args[:text]
       return unless args[:client_id]
+      return unless args[:progress_channel]
 
       helper_mode = args[:prompt]
 
@@ -16,7 +17,7 @@ module Jobs
         helper_mode,
         args[:text],
         user,
-        "/discourse-ai/ai-helper/stream_composer_suggestion",
+        args[:progress_channel],
         force_default_locale: args[:force_default_locale],
         client_id: args[:client_id],
         custom_prompt: args[:custom_prompt],

app/jobs/regular/stream_post_helper.rb

@@ -8,6 +8,8 @@ module Jobs
       return unless post = Post.includes(:topic).find_by(id: args[:post_id])
       return unless user = User.find_by(id: args[:user_id])
       return unless args[:text]
+      return unless args[:progress_channel]
+      return unless args[:client_id]
 
       topic = post.topic
       reply_to = post.reply_to_post
@@ -31,8 +33,9 @@ module Jobs
         helper_mode,
         input,
         user,
-        "/discourse-ai/ai-helper/stream_suggestion/#{post.id}",
+        args[:progress_channel],
         custom_prompt: args[:custom_prompt],
+        client_id: args[:client_id],
       )
     end
   end

app/jobs/scheduled/categories_locale_detection_backfill.rb

@@ -7,9 +7,7 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      return if !DiscourseAi::Translation.backfill_enabled?
 
       categories = Category.where(locale: nil)
 
@@ -17,7 +15,8 @@ module Jobs
         categories = categories.where(read_restricted: false)
       end
 
-      categories = categories.limit(SiteSetting.ai_translation_backfill_rate)
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
+      categories = categories.limit(limit)
       return if categories.empty?
 
       categories.each do |category|
@@ -26,7 +25,7 @@ module Jobs
         rescue FinalDestination::SSRFDetector::LookupFailedError
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect category #{category.id}'s locale: #{e.message}",
+            "Failed to detect category #{category.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/jobs/scheduled/category_localization_backfill.rb

@@ -2,15 +2,14 @@
 
 module Jobs
   class CategoryLocalizationBackfill < ::Jobs::Scheduled
-    every 12.hours
+    every 1.hour
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if !DiscourseAi::Translation.backfill_enabled?
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
 
-      Jobs.enqueue(:localize_categories)
+      Jobs.enqueue(:localize_categories, limit:)
     end
   end
 end

app/jobs/scheduled/embeddings_backfill.rb

@@ -18,105 +18,115 @@ module Jobs
         )
       end
 
-      rebaked = 0
+      production_vector = DiscourseAi::Embeddings::Vector.instance
 
-      vector = DiscourseAi::Embeddings::Vector.instance
-      vector_def = vector.vdef
-      table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
-
-      topics =
-        Topic
-          .joins(
-            "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
+      if SiteSetting.ai_embeddings_backfill_model.present? &&
+           SiteSetting.ai_embeddings_backfill_model != SiteSetting.ai_embeddings_selected_model
+        backfill_vector =
+          DiscourseAi::Embeddings::Vector.new(
+            EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_backfill_model),
           )
-          .where(archetype: Archetype.default)
-          .where(deleted_at: nil)
-          .order("topics.bumped_at DESC")
+      end
 
-      rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
+      topic_work_list = []
+      topic_work_list << production_vector
+      topic_work_list << backfill_vector if backfill_vector
 
-      return if rebaked >= limit
+      topic_work_list.each do |vector|
+        rebaked = 0
+        table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
+        vector_def = vector.vdef
 
-      # Then, we'll try to backfill embeddings for topics that have outdated
-      # embeddings, be it model or strategy version
-      relation = topics.where(<<~SQL).limit(limit - rebaked)
-          #{table_name}.model_version < #{vector_def.version}
-          OR
-          #{table_name}.strategy_version < #{vector_def.strategy_version}
-        SQL
+        topics =
+          Topic
+            .joins(
+              "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
+            )
+            .where(archetype: Archetype.default)
+            .where(deleted_at: nil)
+            .order("topics.bumped_at DESC")
 
-      rebaked += populate_topic_embeddings(vector, relation, force: true)
+        rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
 
-      return if rebaked >= limit
+        next if rebaked >= limit
 
-      # Finally, we'll try to backfill embeddings for topics that have outdated
-      # embeddings due to edits or new replies. Here we only do 10% of the limit
-      relation =
-        topics
-          .where("#{table_name}.updated_at < ?", 6.hours.ago)
-          .where("#{table_name}.updated_at < topics.updated_at")
+        # Then, we'll try to backfill embeddings for topics that have outdated
+        # embeddings, be it model or strategy version
+        relation = topics.where(<<~SQL).limit(limit - rebaked)
+            #{table_name}.model_version < #{vector_def.version}
+            OR
+            #{table_name}.strategy_version < #{vector_def.strategy_version}
+          SQL
+
+        rebaked += populate_topic_embeddings(vector, relation, force: true)
+
+        next if rebaked >= limit
+
+        # Finally, we'll try to backfill embeddings for topics that have outdated
+        # embeddings due to edits or new replies. Here we only do 10% of the limit
+        relation =
+          topics
+            .where("#{table_name}.updated_at < ?", 6.hours.ago)
+            .where("#{table_name}.updated_at < topics.updated_at")
+            .limit((limit - rebaked) / 10)
+
+        populate_topic_embeddings(vector, relation, force: true)
+
+        next unless SiteSetting.ai_embeddings_per_post_enabled
+
+        # Now for posts
+        table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
+        posts_batch_size = 1000
+
+        posts =
+          Post
+            .joins(
+              "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
+            )
+            .where(deleted_at: nil)
+            .where(post_type: Post.types[:regular])
+
+        # First, we'll try to backfill embeddings for posts that have none
+        posts
+          .where("#{table_name}.post_id IS NULL")
+          .limit(limit - rebaked)
+          .pluck(:id)
+          .each_slice(posts_batch_size) do |batch|
+            vector.gen_bulk_reprensentations(Post.where(id: batch))
+            rebaked += batch.length
+          end
+
+        next if rebaked >= limit
+
+        # Then, we'll try to backfill embeddings for posts that have outdated
+        # embeddings, be it model or strategy version
+        posts
+          .where(<<~SQL)
+            #{table_name}.model_version < #{vector_def.version}
+            OR
+            #{table_name}.strategy_version < #{vector_def.strategy_version}
+          SQL
+          .limit(limit - rebaked)
+          .pluck(:id)
+          .each_slice(posts_batch_size) do |batch|
+            vector.gen_bulk_reprensentations(Post.where(id: batch))
+            rebaked += batch.length
+          end
+
+        next if rebaked >= limit
+
+        # Finally, we'll try to backfill embeddings for posts that have outdated
+        # embeddings due to edits. Here we only do 10% of the limit
+        posts
+          .where("#{table_name}.updated_at < ?", 7.days.ago)
+          .order("random()")
           .limit((limit - rebaked) / 10)
-
-      populate_topic_embeddings(vector, relation, force: true)
-
-      return if rebaked >= limit
-
-      return unless SiteSetting.ai_embeddings_per_post_enabled
-
-      # Now for posts
-      table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
-      posts_batch_size = 1000
-
-      posts =
-        Post
-          .joins(
-            "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
-          )
-          .where(deleted_at: nil)
-          .where(post_type: Post.types[:regular])
-
-      # First, we'll try to backfill embeddings for posts that have none
-      posts
-        .where("#{table_name}.post_id IS NULL")
-        .limit(limit - rebaked)
-        .pluck(:id)
-        .each_slice(posts_batch_size) do |batch|
-          vector.gen_bulk_reprensentations(Post.where(id: batch))
-          rebaked += batch.length
-        end
-
-      return if rebaked >= limit
-
-      # Then, we'll try to backfill embeddings for posts that have outdated
-      # embeddings, be it model or strategy version
-      posts
-        .where(<<~SQL)
-          #{table_name}.model_version < #{vector_def.version}
-          OR
-          #{table_name}.strategy_version < #{vector_def.strategy_version}
-        SQL
-        .limit(limit - rebaked)
-        .pluck(:id)
-        .each_slice(posts_batch_size) do |batch|
-          vector.gen_bulk_reprensentations(Post.where(id: batch))
-          rebaked += batch.length
-        end
-
-      return if rebaked >= limit
-
-      # Finally, we'll try to backfill embeddings for posts that have outdated
-      # embeddings due to edits. Here we only do 10% of the limit
-      posts
-        .where("#{table_name}.updated_at < ?", 7.days.ago)
-        .order("random()")
-        .limit((limit - rebaked) / 10)
-        .pluck(:id)
-        .each_slice(posts_batch_size) do |batch|
-          vector.gen_bulk_reprensentations(Post.where(id: batch))
-          rebaked += batch.length
-        end
-
-      rebaked
+          .pluck(:id)
+          .each_slice(posts_batch_size) do |batch|
+            vector.gen_bulk_reprensentations(Post.where(id: batch))
+            rebaked += batch.length
+          end
+      end
     end
 
     private

app/jobs/scheduled/post_localization_backfill.rb

@@ -6,13 +6,12 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.backfill_enabled?
 
-      return if SiteSetting.experimental_content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
-      Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
+      Jobs.enqueue(:localize_posts, limit:)
     end
   end
 end

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -7,35 +7,33 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      return if !DiscourseAi::Translation.backfill_enabled?
+
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
 
       posts =
         Post
           .where(locale: nil)
           .where(deleted_at: nil)
           .where("posts.user_id > 0")
+          .where("posts.created_at > ?", SiteSetting.ai_translation_backfill_max_age_days.days.ago)
           .where.not(raw: [nil, ""])
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
-        public_categories = Category.where(read_restricted: false).pluck(:id)
         posts =
           posts
             .joins(:topic)
-            .where(topics: { category_id: public_categories })
-            .where(topics: { archetype: "regular" })
-      end
-
-      if SiteSetting.ai_translation_backfill_max_age_days > 0
+            .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
+            .where("archetype != ?", Archetype.private_message)
+      else
         posts =
-          posts.where(
-            "posts.created_at > ?",
-            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+          posts.joins(:topic).where(
+            "topics.archetype != ? OR EXISTS (SELECT 1 FROM topic_allowed_groups WHERE topic_id = topics.id)",
+            Archetype.private_message,
           )
       end
 
-      posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
+      posts = posts.order(updated_at: :desc).limit(limit)
       return if posts.empty?
 
       posts.each do |post|
@@ -45,7 +43,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect post #{post.id}'s locale: #{e.message}",
+            "Failed to detect post #{post.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/jobs/scheduled/summaries_backfill.rb

@@ -36,7 +36,12 @@ module ::Jobs
       existing_summary = strategy.existing_summary
 
       if existing_summary.blank? || existing_summary.outdated
-        strategy.summarize(user)
+        begin
+          strategy.summarize(user)
+        rescue => e
+          Rails.logger.error("Error summarizing topic #{topic.id}: #{e.class.name} - #{e.message}")
+          raise e
+        end
       else
         # Hiding or deleting a post, and creating a small action alters the Topic#highest_post_number.
         # We use this as a quick way to select potential backfill candidates without relying on original_content_sha.

app/jobs/scheduled/topic_localization_backfill.rb

@@ -6,13 +6,10 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
+      return if !DiscourseAi::Translation.backfill_enabled?
 
-      return if SiteSetting.experimental_content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
-
-      Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      Jobs.enqueue(:localize_topics, limit:)
     end
   end
 end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -7,23 +7,27 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      limit = SiteSetting.ai_translation_backfill_rate
+      return if !DiscourseAi::Translation.backfill_enabled?
 
-      return if limit == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
 
-      topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")
+      topics =
+        Topic
+          .where(locale: nil, deleted_at: nil)
+          .where("topics.user_id > 0")
+          .where("topics.created_at > ?", SiteSetting.ai_translation_backfill_max_age_days.days.ago)
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
-        topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
-      end
-
-      if SiteSetting.ai_translation_backfill_max_age_days > 0
+        topics =
+          topics.where(category_id: Category.where(read_restricted: false).select(:id)).where(
+            "archetype != ?",
+            Archetype.private_message,
+          )
+      else
         topics =
           topics.where(
-            "topics.created_at > ?",
-            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            "archetype != ? OR EXISTS (SELECT 1 FROM topic_allowed_groups WHERE topic_id = topics.id)",
+            Archetype.private_message,
           )
       end
 
@@ -37,7 +41,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect topic #{topic.id}'s locale: #{e.message}",
+            "Failed to detect topic #{topic.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
           )
         end
       end

app/models/ai_moderation_setting.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 class AiModerationSetting < ActiveRecord::Base
   belongs_to :llm_model
+  belongs_to :ai_persona
 
   validates :llm_model_id, presence: true
   validates :setting_type, presence: true
@@ -19,12 +20,13 @@ end
 #
 # Table name: ai_moderation_settings
 #
-#  id           :bigint           not null, primary key
-#  setting_type :enum             not null
-#  data         :jsonb
-#  llm_model_id :bigint           not null
-#  created_at   :datetime         not null
-#  updated_at   :datetime         not null
+#  id            :bigint           not null, primary key
+#  setting_type  :enum             not null
+#  data          :jsonb
+#  llm_model_id  :bigint           not null
+#  created_at    :datetime         not null
+#  updated_at    :datetime         not null
+#  ai_persona_id :bigint           default(-31), not null
 #
 # Indexes
 #

app/models/ai_persona.rb

@@ -12,7 +12,6 @@ class AiPersona < ActiveRecord::Base
   validates :system_prompt, presence: true, length: { maximum: 10_000_000 }
   validate :system_persona_unchangeable, on: :update, if: :system
   validate :chat_preconditions
-  validate :allowed_seeded_model, if: :default_llm_id
   validate :well_formated_examples
   validates :max_context_posts, numericality: { greater_than: 0 }, allow_nil: true
   # leaves some room for growth but sets a maximum to avoid memory issues
@@ -365,17 +364,6 @@ class AiPersona < ActiveRecord::Base
     end
   end
 
-  def allowed_seeded_model
-    return if default_llm_id.blank?
-
-    return if default_llm.nil?
-    return if !default_llm.seeded?
-
-    return if SiteSetting.ai_bot_allowed_seeded_models_map.include?(default_llm.id.to_s)
-
-    errors.add(:default_llm, I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"))
-  end
-
   def well_formated_examples
     return if examples.blank?
 

app/models/embedding_definition.rb

@@ -23,7 +23,9 @@ class EmbeddingDefinition < ActiveRecord::Base
         DiscourseAi::Tokenizer::GeminiTokenizer,
         DiscourseAi::Tokenizer::MultilingualE5LargeTokenizer,
         DiscourseAi::Tokenizer::OpenAiTokenizer,
-        DiscourseAi::Tokenizer::MixtralTokenizer,
+        DiscourseAi::Tokenizer::OpenAiCl100kTokenizer,
+        DiscourseAi::Tokenizer::MistralTokenizer,
+        DiscourseAi::Tokenizer::QwenTokenizer,
       ].map(&:name)
     end
 
@@ -82,7 +84,7 @@ class EmbeddingDefinition < ActiveRecord::Base
               dimensions: 2000,
               max_sequence_length: 8191,
               pg_function: "<=>",
-              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiCl100kTokenizer",
               url: "https://api.openai.com/v1/embeddings",
               provider: OPEN_AI,
               matryoshka_dimensions: true,
@@ -96,7 +98,7 @@ class EmbeddingDefinition < ActiveRecord::Base
               dimensions: 1536,
               max_sequence_length: 8191,
               pg_function: "<=>",
-              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiCl100kTokenizer",
               url: "https://api.openai.com/v1/embeddings",
               provider: OPEN_AI,
               matryoshka_dimensions: true,
@@ -110,7 +112,7 @@ class EmbeddingDefinition < ActiveRecord::Base
               dimensions: 1536,
               max_sequence_length: 8191,
               pg_function: "<=>",
-              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiTokenizer",
+              tokenizer_class: "DiscourseAi::Tokenizer::OpenAiCl100kTokenizer",
               url: "https://api.openai.com/v1/embeddings",
               provider: OPEN_AI,
               provider_params: {

app/models/llm_model.rb

@@ -65,6 +65,8 @@ class LlmModel < ActiveRecord::Base
       google: {
         disable_native_tools: :checkbox,
         enable_thinking: :checkbox,
+        disable_temperature: :checkbox,
+        disable_top_p: :checkbox,
         thinking_tokens: :number,
       },
       azure: {
@@ -97,7 +99,7 @@ class LlmModel < ActiveRecord::Base
   end
 
   def identifier
-    "custom:#{id}"
+    "#{id}"
   end
 
   def toggle_companion_user

app/serializers/ai_spam_serializer.rb

@@ -8,7 +8,9 @@ class AiSpamSerializer < ApplicationSerializer
              :stats,
              :flagging_username,
              :spam_score_type,
-             :spam_scanning_user
+             :spam_scanning_user,
+             :ai_persona_id,
+             :available_personas
 
   def is_enabled
     object[:enabled]
@@ -18,6 +20,11 @@ class AiSpamSerializer < ApplicationSerializer
     settings&.llm_model&.id
   end
 
+  def ai_persona_id
+    settings&.ai_persona&.id ||
+      DiscourseAi::Personas::Persona.system_personas[DiscourseAi::Personas::SpamDetector]
+  end
+
   def custom_instructions
     settings&.custom_instructions
   end
@@ -28,6 +35,12 @@ class AiSpamSerializer < ApplicationSerializer
       .map { |hash| { id: hash[:value], name: hash[:name] } }
   end
 
+  def available_personas
+    DiscourseAi::Configuration::PersonaEnumerator.values.map do |h|
+      { id: h[:value], name: h[:name] }
+    end
+  end
+
   def flagging_username
     object[:flagging_username]
   end

app/services/discourse_ai/persona_exporter.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  class PersonaExporter
+    def initialize(persona:)
+      raise ArgumentError, "Invalid persona provided" if !persona.is_a?(AiPersona)
+      @persona = persona
+    end
+
+    def export
+      serialized_custom_tools = serialize_tools(@persona)
+      serialize_persona(@persona, serialized_custom_tools)
+    end
+
+    private
+
+    def serialize_tools(ai_persona)
+      custom_tool_ids =
+        (ai_persona.tools || []).filter_map do |tool_config|
+          # A tool config is an array like: ["custom-ID", {options}, force_flag]
+          if tool_config.is_a?(Array) && tool_config[0].to_s.start_with?("custom-")
+            tool_config[0].split("-", 2).last.to_i
+          end
+        end
+
+      return [] if custom_tool_ids.empty?
+
+      tools = AiTool.where(id: custom_tool_ids)
+      tools.map do |tool|
+        {
+          identifier: tool.tool_name, # Use tool_name for portability
+          name: tool.name,
+          description: tool.description,
+          tool_name: tool.tool_name,
+          parameters: tool.parameters,
+          summary: tool.summary,
+          script: tool.script,
+        }
+      end
+    end
+
+    def serialize_persona(ai_persona, serialized_custom_tools)
+      export_data = {
+        meta: {
+          version: "1.0",
+          exported_at: Time.zone.now.iso8601,
+        },
+        persona: {
+          name: ai_persona.name,
+          description: ai_persona.description,
+          system_prompt: ai_persona.system_prompt,
+          examples: ai_persona.examples,
+          temperature: ai_persona.temperature,
+          top_p: ai_persona.top_p,
+          response_format: ai_persona.response_format,
+          tools: transform_tools_for_export(ai_persona.tools, serialized_custom_tools),
+        },
+        custom_tools: serialized_custom_tools,
+      }
+
+      JSON.pretty_generate(export_data)
+    end
+
+    def transform_tools_for_export(tools_config, _serialized_custom_tools)
+      return [] if tools_config.blank?
+
+      tools_config.map do |tool_config|
+        unless tool_config.is_a?(Array) && tool_config[0].to_s.start_with?("custom-")
+          next tool_config
+        end
+
+        tool_id = tool_config[0].split("-", 2).last.to_i
+        tool = AiTool.find_by(id: tool_id)
+        next tool_config unless tool
+
+        ["custom-#{tool.tool_name}", tool_config[1], tool_config[2]]
+      end
+    end
+  end
+end