DEV: Increase chunk size before splitting content for translation

.discourse-compatibility

@@ -1,4 +1,3 @@
-< 3.5.0.beta7-dev: cd14b0c0bee0cf63c59b64b6f7213e31a37f11a7
 < 3.5.0.beta6-dev: 3e74eea1e5e3143888d67a8d8a11206df214dc24
 < 3.5.0.beta3-dev: 09a68414804a1447f52e5d60691ba59742cda9ec
 < 3.5.0.beta2-dev: de8624416a15b3d8e7ad350b083cc1420451ccec

.gitignore

@@ -6,4 +6,5 @@ evals/log
 evals/cases
 config/eval-llms.local.yml
 # this gets rid of search results from ag, ripgrep, etc
+tokenizers/
 public/ai-share/highlight.min.js

Gemfile.lock

@@ -34,7 +34,7 @@ GEM
       racc
     prettier_print (1.2.1)
     racc (1.8.1)
-    rack (3.1.16)
+    rack (3.1.14)
     rainbow (3.1.1)
     regexp_parser (2.10.0)
     rubocop (1.74.0)

README.md

@@ -1,8 +1,5 @@
 # **Discourse AI** Plugin
 
-> [!IMPORTANT]
-> This plugin has now been bundled into Discourse core. See: https://meta.discourse.org/t/bundling-more-popular-plugins-with-discourse-core/373574
-
 **Plugin Summary**
 
 For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-features-edit.js

@@ -1,4 +1,3 @@
-import { action } from "@ember/object";
 import { ajax } from "discourse/lib/ajax";
 import DiscourseRoute from "discourse/routes/discourse";
 import SiteSetting from "admin/models/site-setting";
@@ -25,11 +24,4 @@ export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRo
 
     return currentFeature;
   }
-
-  @action
-  willTransition() {
-    // site settings may amend if a feature is enabled or disabled, so refresh the model
-    // even on back button
-    this.router.refresh("adminPlugins.show.discourse-ai-features");
-  }
 }

app/controllers/discourse_ai/admin/ai_features_controller.rb

@@ -37,11 +37,7 @@ module DiscourseAi
       def serialize_feature(feature)
         {
           name: feature.name,
-          personas: feature.persona_ids.map { |id| serialize_persona(persona_id_obj_hash[id]) },
-          llm_models:
-            feature.llm_models.map do |llm_model|
-              { id: llm_model.id, name: llm_model.display_name }
-            end,
+          persona: serialize_persona(persona_id_obj_hash[feature.persona_id]),
           enabled: feature.enabled?,
         }
       end
@@ -57,7 +53,9 @@ module DiscourseAi
       def persona_id_obj_hash
         @persona_id_obj_hash ||=
           begin
-            ids = DiscourseAi::Configuration::Feature.all.map(&:persona_ids).flatten.uniq
+            setting_names = DiscourseAi::Configuration::Feature.all_persona_setting_names
+            ids = setting_names.map { |sn| SiteSetting.public_send(sn) }
+
             AiPersona.where(id: ids).index_by(&:id)
           end
       end

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -5,7 +5,7 @@ module DiscourseAi
     class AiPersonasController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_persona, only: %i[edit update destroy create_user export]
+      before_action :find_ai_persona, only: %i[edit update destroy create_user]
 
       def index
         ai_personas =
@@ -105,43 +105,6 @@ module DiscourseAi
         end
       end
 
-      def export
-        persona = AiPersona.find(params[:id])
-        exporter = DiscourseAi::PersonaExporter.new(persona: persona)
-
-        response.headers[
-          "Content-Disposition"
-        ] = "attachment; filename=\"#{persona.name.parameterize}.json\""
-
-        render json: exporter.export
-      end
-
-      def import
-        name = params.dig(:persona, :name)
-        existing_persona = AiPersona.find_by(name: name)
-        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
-
-        begin
-          importer = DiscourseAi::PersonaImporter.new(json: params.to_unsafe_h)
-
-          if existing_persona && force_update
-            initial_attributes = existing_persona.attributes.dup
-            persona = importer.import!(overwrite: true)
-            log_ai_persona_update(persona, initial_attributes)
-            render json: LocalizedAiPersonaSerializer.new(persona, root: false)
-          else
-            persona = importer.import!
-            log_ai_persona_creation(persona)
-            render json: LocalizedAiPersonaSerializer.new(persona, root: false), status: :created
-          end
-        rescue DiscourseAi::PersonaImporter::ImportError => e
-          render_json_error e.message, status: :unprocessable_entity
-        rescue StandardError => e
-          Rails.logger.error("AI Persona import failed: #{e.message}")
-          render_json_error "Import failed: #{e.message}", status: :unprocessable_entity
-        end
-      end
-
       def stream_reply
         persona =
           AiPersona.find_by(name: params[:persona_name]) ||

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -11,13 +11,6 @@ module DiscourseAi
 
       def update
         initial_settings = AiModerationSetting.spam
-
-        initial_data = {
-          custom_instructions: initial_settings&.data&.dig("custom_instructions"),
-          llm_model_id: initial_settings&.llm_model_id,
-          ai_persona_id: initial_settings&.ai_persona_id,
-        }
-
         initial_custom_instructions = initial_settings&.data&.dig("custom_instructions")
         initial_llm_model_id = initial_settings&.llm_model_id
 
@@ -36,22 +29,6 @@ module DiscourseAi
             )
           end
         end
-
-        if allowed_params.key?(:ai_persona_id)
-          updated_params[:ai_persona_id] = allowed_params[:ai_persona_id]
-          persona = AiPersona.find_by(id: allowed_params[:ai_persona_id])
-          if persona.nil? ||
-               persona.response_format.to_a.none? { |rf|
-                 rf["key"] == "spam" && rf["type"] == "boolean"
-               }
-            return(
-              render_json_error(
-                I18n.t("discourse_ai.llm.configuration.invalid_persona_response_format"),
-                status: 422,
-              )
-            )
-          end
-        end
         updated_params[:data] = {
           custom_instructions: allowed_params[:custom_instructions],
         } if allowed_params.key?(:custom_instructions)
@@ -64,7 +41,7 @@ module DiscourseAi
             AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
           end
 
-          log_ai_spam_update(initial_data, allowed_params)
+          log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, allowed_params)
         end
 
         is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
@@ -142,10 +119,9 @@ module DiscourseAi
 
       private
 
-      def log_ai_spam_update(initial_data, params)
+      def log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, params)
         changes_to_log = {}
 
-        initial_llm_model_id = initial_data[:llm_model_id]
         if params.key?(:llm_model_id) && initial_llm_model_id.to_s != params[:llm_model_id].to_s
           old_model_name =
             LlmModel.find_by(id: initial_llm_model_id)&.display_name || initial_llm_model_id
@@ -155,22 +131,11 @@ module DiscourseAi
           changes_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"
         end
 
-        initial_custom_instructions = initial_data[:custom_instructions]
         if params.key?(:custom_instructions) &&
              initial_custom_instructions != params[:custom_instructions]
           changes_to_log[:custom_instructions] = params[:custom_instructions]
         end
 
-        initial_ai_persona_id = initial_data[:ai_persona_id]
-        if params.key?(:ai_persona_id) && initial_ai_persona_id.to_s != params[:ai_persona_id].to_s
-          old_persona_name =
-            AiPersona.find_by(id: initial_ai_persona_id)&.name || initial_ai_persona_id
-          new_persona_name =
-            AiPersona.find_by(id: params[:ai_persona_id])&.name || params[:ai_persona_id]
-
-          changes_to_log[:ai_persona_id] = "#{old_persona_name} → #{new_persona_name}"
-        end
-
         if changes_to_log.present?
           changes_to_log[:subject] = I18n.t("discourse_ai.spam_detection.logging_subject")
           logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
@@ -179,7 +144,7 @@ module DiscourseAi
       end
 
       def allowed_params
-        params.permit(:is_enabled, :llm_model_id, :custom_instructions, :ai_persona_id)
+        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
       end
 
       def spam_config

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -5,7 +5,7 @@ module DiscourseAi
     class AiToolsController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_tool, only: %i[test edit update destroy export]
+      before_action :find_ai_tool, only: %i[test edit update destroy]
 
       def index
         ai_tools = AiTool.all
@@ -32,45 +32,6 @@ module DiscourseAi
         end
       end
 
-      def export
-        response.headers[
-          "Content-Disposition"
-        ] = "attachment; filename=\"#{@ai_tool.tool_name}.json\""
-        render_serialized(@ai_tool, AiCustomToolSerializer)
-      end
-
-      def import
-        existing_tool = AiTool.find_by(tool_name: ai_tool_params[:tool_name])
-        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
-
-        if existing_tool && !force_update
-          return(
-            render_json_error "Tool with tool_name '#{ai_tool_params[:tool_name]}' already exists. Use force=true to overwrite.",
-                              status: :conflict
-          )
-        end
-
-        if existing_tool && force_update
-          initial_attributes = existing_tool.attributes.dup
-          if existing_tool.update(ai_tool_params)
-            log_ai_tool_update(existing_tool, initial_attributes)
-            render_serialized(existing_tool, AiCustomToolSerializer)
-          else
-            render_json_error existing_tool
-          end
-        else
-          ai_tool = AiTool.new(ai_tool_params)
-          ai_tool.created_by_id = current_user.id
-
-          if ai_tool.save
-            log_ai_tool_creation(ai_tool)
-            render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
-          else
-            render_json_error ai_tool
-          end
-        end
-      end
-
       def update
         initial_attributes = @ai_tool.attributes.dup
 

app/controllers/discourse_ai/ai_helper/assistant_controller.rb

@@ -124,9 +124,6 @@ module DiscourseAi
         # otherwise we may end up streaming the data to the wrong client
         raise Discourse::InvalidParameters.new(:client_id) if params[:client_id].blank?
 
-        channel_id = next_channel_id
-        progress_channel = "discourse_ai_helper/stream_suggestions/#{channel_id}"
-
         if location == "composer"
           Jobs.enqueue(
             :stream_composer_helper,
@@ -136,7 +133,6 @@ module DiscourseAi
             custom_prompt: params[:custom_prompt],
             force_default_locale: params[:force_default_locale] || false,
             client_id: params[:client_id],
-            progress_channel:,
           )
         else
           post_id = get_post_param!
@@ -152,11 +148,10 @@ module DiscourseAi
             prompt: params[:mode],
             custom_prompt: params[:custom_prompt],
             client_id: params[:client_id],
-            progress_channel:,
           )
         end
 
-        render json: { success: true, progress_channel: }, status: 200
+        render json: { success: true }, status: 200
       rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
         render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                           status: 502
@@ -197,18 +192,6 @@ module DiscourseAi
 
       private
 
-      CHANNEL_ID_KEY = "discourse_ai_helper_next_channel_id"
-
-      def next_channel_id
-        Discourse
-          .redis
-          .pipelined do |pipeline|
-            pipeline.incr(CHANNEL_ID_KEY)
-            pipeline.expire(CHANNEL_ID_KEY, 1.day)
-          end
-          .first
-      end
-
       def get_text_param!
         params[:text].tap { |t| raise Discourse::InvalidParameters.new(:text) if t.blank? }
       end

app/jobs/regular/detect_translate_post.rb

@@ -2,11 +2,11 @@
 
 module Jobs
   class DetectTranslatePost < ::Jobs::Base
-    cluster_concurrency 1
     sidekiq_options retry: false
 
     def execute(args)
-      return if !DiscourseAi::Translation.enabled?
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
       return if args[:post_id].blank?
 
       post = Post.find_by(id: args[:post_id])
@@ -35,14 +35,12 @@ module Jobs
         end
       end
 
-      return if detected_locale.blank?
-      locales = SiteSetting.content_localization_supported_locales.split("|")
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
       return if locales.blank?
 
       locales.each do |locale|
-        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
-        regionless_locale = locale.split("_").first
-        next if post.post_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
+        next if locale == detected_locale
+        next if post.post_localizations.exists?(locale:)
 
         begin
           DiscourseAi::Translation::PostLocalizer.localize(post, locale)
@@ -50,7 +48,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+            "Failed to translate post #{post.id} to #{locale}: #{e.message}",
           )
         end
       end

app/jobs/regular/detect_translate_topic.rb

@@ -2,11 +2,11 @@
 
 module Jobs
   class DetectTranslateTopic < ::Jobs::Base
-    cluster_concurrency 1
     sidekiq_options retry: false
 
     def execute(args)
-      return if !DiscourseAi::Translation.enabled?
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
       return if args[:topic_id].blank?
 
       topic = Topic.find_by(id: args[:topic_id])
@@ -33,14 +33,12 @@ module Jobs
         end
       end
 
-      return if detected_locale.blank?
-      locales = SiteSetting.content_localization_supported_locales.split("|")
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
       return if locales.blank?
 
       locales.each do |locale|
-        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
-        regionless_locale = locale.split("_").first
-        next if topic.topic_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
+        next if locale == detected_locale
+        next if topic.topic_localizations.exists?(locale:)
 
         begin
           DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
@@ -48,7 +46,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
           )
         end
       end

app/jobs/regular/localize_categories.rb

@@ -5,48 +5,51 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
+    BATCH_SIZE = 50
+
     def execute(args)
-      return if !DiscourseAi::Translation.enabled?
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
 
-      limit = args[:limit]
-      raise Discourse::InvalidParameters.new(:limit) if limit.nil?
-      return if limit <= 0
-      locales = SiteSetting.content_localization_supported_locales.split("|")
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
 
-      categories = Category.where("locale IS NOT NULL")
-      if SiteSetting.ai_translation_backfill_limit_to_public_content
-        categories = categories.where(read_restricted: false)
-      end
-      categories = categories.order(:id).limit(limit)
+      cat_id = args[:from_category_id] || Category.order(:id).first&.id
+      last_id = nil
+
+      categories =
+        Category.where("id >= ? AND locale IS NOT NULL", cat_id).order(:id).limit(BATCH_SIZE)
       return if categories.empty?
 
-      remaining_limit = limit
-
       categories.each do |category|
-        break if remaining_limit <= 0
+        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
+          last_id = category.id
+          next
+        end
 
-        existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)
-        missing_locales = locales - existing_locales - [category.locale]
-        missing_locales.each do |locale|
-          break if remaining_limit <= 0
-          next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, category.locale)
+        locales.each do |locale|
+          localization = category.category_localizations.find_by(locale:)
 
-          begin
-            DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
-          rescue FinalDestination::SSRFDetector::LookupFailedError
-            # do nothing, there are too many sporadic lookup failures
-          rescue => e
-            DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate category #{category.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
-            )
-          ensure
-            remaining_limit -= 1
+          if locale == category.locale && localization
+            localization.destroy
+          else
+            next if locale == category.locale
+            begin
+              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+            rescue FinalDestination::SSRFDetector::LookupFailedError
+              # do nothing, there are too many sporadic lookup failures
+            rescue => e
+              DiscourseAi::Translation::VerboseLogger.log(
+                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
+              )
+            end
           end
         end
+        last_id = category.id
+      end
 
-        if existing_locales.include?(category.locale)
-          CategoryLocalization.find_by(category_id: category.id, locale: category.locale).destroy
-        end
+      if categories.size == BATCH_SIZE
+        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
       end
     end
   end

app/jobs/regular/localize_posts.rb

@@ -5,29 +5,28 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
+    BATCH_SIZE = 50
+
     def execute(args)
-      limit = args[:limit]
-      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
 
-      return if !DiscourseAi::Translation.backfill_enabled?
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
 
-      locales = SiteSetting.content_localization_supported_locales.split("|")
       locales.each do |locale|
-        base_locale = locale.split("_").first
         posts =
           Post
             .joins(
-              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale LIKE '#{base_locale}%'",
-            )
-            .where(
-              "posts.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
             )
             .where(deleted_at: nil)
             .where("posts.user_id > 0")
             .where.not(raw: [nil, ""])
             .where.not(locale: nil)
-            .where("posts.locale NOT LIKE '#{base_locale}%'")
+            .where.not(locale: locale)
             .where("pl.id IS NULL")
 
         posts = posts.joins(:topic)
@@ -48,6 +47,14 @@ module Jobs
             )
         end
 
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          posts =
+            posts.where(
+              "posts.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
         posts = posts.order(updated_at: :desc).limit(limit)
 
         next if posts.empty?
@@ -59,7 +66,7 @@ module Jobs
             # do nothing, there are too many sporadic lookup failures
           rescue => e
             DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+              "Failed to translate post #{post.id} to #{locale}: #{e.message}",
             )
           end
         end

app/jobs/regular/localize_topics.rb

@@ -5,28 +5,27 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
+    BATCH_SIZE = 50
+
     def execute(args)
-      limit = args[:limit]
-      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
 
-      return if !DiscourseAi::Translation.backfill_enabled?
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
 
-      locales = SiteSetting.content_localization_supported_locales.split("|")
       locales.each do |locale|
-        base_locale = locale.split("_").first
         topics =
           Topic
             .joins(
-              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale LIKE '#{base_locale}%'",
-            )
-            .where(
-              "topics.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
             )
             .where(deleted_at: nil)
             .where("topics.user_id > 0")
             .where.not(locale: nil)
-            .where("topics.locale NOT LIKE '#{base_locale}%'")
+            .where.not(locale: locale)
             .where("tl.id IS NULL")
 
         if SiteSetting.ai_translation_backfill_limit_to_public_content
@@ -45,6 +44,14 @@ module Jobs
             )
         end
 
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          topics =
+            topics.where(
+              "topics.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
         topics = topics.order(updated_at: :desc).limit(limit)
 
         next if topics.empty?
@@ -56,7 +63,7 @@ module Jobs
             # do nothing, there are too many sporadic lookup failures
           rescue => e
             DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
             )
           end
         end

app/jobs/regular/stream_composer_helper.rb

@@ -9,7 +9,6 @@ module Jobs
       return unless user = User.find_by(id: args[:user_id])
       return unless args[:text]
       return unless args[:client_id]
-      return unless args[:progress_channel]
 
       helper_mode = args[:prompt]
 
@@ -17,7 +16,7 @@ module Jobs
         helper_mode,
         args[:text],
         user,
-        args[:progress_channel],
+        "/discourse-ai/ai-helper/stream_composer_suggestion",
         force_default_locale: args[:force_default_locale],
         client_id: args[:client_id],
         custom_prompt: args[:custom_prompt],

app/jobs/regular/stream_post_helper.rb

@@ -8,8 +8,6 @@ module Jobs
       return unless post = Post.includes(:topic).find_by(id: args[:post_id])
       return unless user = User.find_by(id: args[:user_id])
       return unless args[:text]
-      return unless args[:progress_channel]
-      return unless args[:client_id]
 
       topic = post.topic
       reply_to = post.reply_to_post
@@ -33,9 +31,8 @@ module Jobs
         helper_mode,
         input,
         user,
-        args[:progress_channel],
+        "/discourse-ai/ai-helper/stream_suggestion/#{post.id}",
         custom_prompt: args[:custom_prompt],
-        client_id: args[:client_id],
       )
     end
   end

app/jobs/scheduled/categories_locale_detection_backfill.rb

@@ -7,7 +7,9 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !DiscourseAi::Translation.backfill_enabled?
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.ai_translation_backfill_rate == 0
 
       categories = Category.where(locale: nil)
 
@@ -15,8 +17,7 @@ module Jobs
         categories = categories.where(read_restricted: false)
       end
 
-      limit = SiteSetting.ai_translation_backfill_hourly_rate
-      categories = categories.limit(limit)
+      categories = categories.limit(SiteSetting.ai_translation_backfill_rate)
       return if categories.empty?
 
       categories.each do |category|
@@ -25,7 +26,7 @@ module Jobs
         rescue FinalDestination::SSRFDetector::LookupFailedError
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect category #{category.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+            "Failed to detect category #{category.id}'s locale: #{e.message}",
           )
         end
       end

app/jobs/scheduled/category_localization_backfill.rb

@@ -2,14 +2,15 @@
 
 module Jobs
   class CategoryLocalizationBackfill < ::Jobs::Scheduled
-    every 1.hour
+    every 12.hours
     cluster_concurrency 1
 
     def execute(args)
-      return if !DiscourseAi::Translation.backfill_enabled?
-      limit = SiteSetting.ai_translation_backfill_hourly_rate
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
 
-      Jobs.enqueue(:localize_categories, limit:)
+      Jobs.enqueue(:localize_categories)
     end
   end
 end

app/jobs/scheduled/embeddings_backfill.rb

@@ -18,115 +18,105 @@ module Jobs
         )
       end
 
-      production_vector = DiscourseAi::Embeddings::Vector.instance
+      rebaked = 0
 
-      if SiteSetting.ai_embeddings_backfill_model.present? &&
-           SiteSetting.ai_embeddings_backfill_model != SiteSetting.ai_embeddings_selected_model
-        backfill_vector =
-          DiscourseAi::Embeddings::Vector.new(
-            EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_backfill_model),
+      vector = DiscourseAi::Embeddings::Vector.instance
+      vector_def = vector.vdef
+      table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
+
+      topics =
+        Topic
+          .joins(
+            "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
           )
-      end
+          .where(archetype: Archetype.default)
+          .where(deleted_at: nil)
+          .order("topics.bumped_at DESC")
 
-      topic_work_list = []
-      topic_work_list << production_vector
-      topic_work_list << backfill_vector if backfill_vector
+      rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
 
-      topic_work_list.each do |vector|
-        rebaked = 0
-        table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
-        vector_def = vector.vdef
+      return if rebaked >= limit
 
-        topics =
-          Topic
-            .joins(
-              "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
-            )
-            .where(archetype: Archetype.default)
-            .where(deleted_at: nil)
-            .order("topics.bumped_at DESC")
+      # Then, we'll try to backfill embeddings for topics that have outdated
+      # embeddings, be it model or strategy version
+      relation = topics.where(<<~SQL).limit(limit - rebaked)
+          #{table_name}.model_version < #{vector_def.version}
+          OR
+          #{table_name}.strategy_version < #{vector_def.strategy_version}
+        SQL
 
-        rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
+      rebaked += populate_topic_embeddings(vector, relation, force: true)
 
-        next if rebaked >= limit
+      return if rebaked >= limit
 
-        # Then, we'll try to backfill embeddings for topics that have outdated
-        # embeddings, be it model or strategy version
-        relation = topics.where(<<~SQL).limit(limit - rebaked)
-            #{table_name}.model_version < #{vector_def.version}
-            OR
-            #{table_name}.strategy_version < #{vector_def.strategy_version}
-          SQL
-
-        rebaked += populate_topic_embeddings(vector, relation, force: true)
-
-        next if rebaked >= limit
-
-        # Finally, we'll try to backfill embeddings for topics that have outdated
-        # embeddings due to edits or new replies. Here we only do 10% of the limit
-        relation =
-          topics
-            .where("#{table_name}.updated_at < ?", 6.hours.ago)
-            .where("#{table_name}.updated_at < topics.updated_at")
-            .limit((limit - rebaked) / 10)
-
-        populate_topic_embeddings(vector, relation, force: true)
-
-        next unless SiteSetting.ai_embeddings_per_post_enabled
-
-        # Now for posts
-        table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
-        posts_batch_size = 1000
-
-        posts =
-          Post
-            .joins(
-              "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
-            )
-            .where(deleted_at: nil)
-            .where(post_type: Post.types[:regular])
-
-        # First, we'll try to backfill embeddings for posts that have none
-        posts
-          .where("#{table_name}.post_id IS NULL")
-          .limit(limit - rebaked)
-          .pluck(:id)
-          .each_slice(posts_batch_size) do |batch|
-            vector.gen_bulk_reprensentations(Post.where(id: batch))
-            rebaked += batch.length
-          end
-
-        next if rebaked >= limit
-
-        # Then, we'll try to backfill embeddings for posts that have outdated
-        # embeddings, be it model or strategy version
-        posts
-          .where(<<~SQL)
-            #{table_name}.model_version < #{vector_def.version}
-            OR
-            #{table_name}.strategy_version < #{vector_def.strategy_version}
-          SQL
-          .limit(limit - rebaked)
-          .pluck(:id)
-          .each_slice(posts_batch_size) do |batch|
-            vector.gen_bulk_reprensentations(Post.where(id: batch))
-            rebaked += batch.length
-          end
-
-        next if rebaked >= limit
-
-        # Finally, we'll try to backfill embeddings for posts that have outdated
-        # embeddings due to edits. Here we only do 10% of the limit
-        posts
-          .where("#{table_name}.updated_at < ?", 7.days.ago)
-          .order("random()")
+      # Finally, we'll try to backfill embeddings for topics that have outdated
+      # embeddings due to edits or new replies. Here we only do 10% of the limit
+      relation =
+        topics
+          .where("#{table_name}.updated_at < ?", 6.hours.ago)
+          .where("#{table_name}.updated_at < topics.updated_at")
           .limit((limit - rebaked) / 10)
-          .pluck(:id)
-          .each_slice(posts_batch_size) do |batch|
-            vector.gen_bulk_reprensentations(Post.where(id: batch))
-            rebaked += batch.length
-          end
-      end
+
+      populate_topic_embeddings(vector, relation, force: true)
+
+      return if rebaked >= limit
+
+      return unless SiteSetting.ai_embeddings_per_post_enabled
+
+      # Now for posts
+      table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
+      posts_batch_size = 1000
+
+      posts =
+        Post
+          .joins(
+            "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
+          )
+          .where(deleted_at: nil)
+          .where(post_type: Post.types[:regular])
+
+      # First, we'll try to backfill embeddings for posts that have none
+      posts
+        .where("#{table_name}.post_id IS NULL")
+        .limit(limit - rebaked)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
+        end
+
+      return if rebaked >= limit
+
+      # Then, we'll try to backfill embeddings for posts that have outdated
+      # embeddings, be it model or strategy version
+      posts
+        .where(<<~SQL)
+          #{table_name}.model_version < #{vector_def.version}
+          OR
+          #{table_name}.strategy_version < #{vector_def.strategy_version}
+        SQL
+        .limit(limit - rebaked)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
+        end
+
+      return if rebaked >= limit
+
+      # Finally, we'll try to backfill embeddings for posts that have outdated
+      # embeddings due to edits. Here we only do 10% of the limit
+      posts
+        .where("#{table_name}.updated_at < ?", 7.days.ago)
+        .order("random()")
+        .limit((limit - rebaked) / 10)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
+        end
+
+      rebaked
     end
 
     private

app/jobs/scheduled/post_localization_backfill.rb

@@ -6,12 +6,13 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !DiscourseAi::Translation.backfill_enabled?
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
 
-      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
-      return if limit == 0
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if SiteSetting.ai_translation_backfill_rate == 0
 
-      Jobs.enqueue(:localize_posts, limit:)
+      Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
     end
   end
 end

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -7,33 +7,35 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !DiscourseAi::Translation.backfill_enabled?
-
-      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.ai_translation_backfill_rate == 0
 
       posts =
         Post
           .where(locale: nil)
           .where(deleted_at: nil)
           .where("posts.user_id > 0")
-          .where("posts.created_at > ?", SiteSetting.ai_translation_backfill_max_age_days.days.ago)
           .where.not(raw: [nil, ""])
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
+        public_categories = Category.where(read_restricted: false).pluck(:id)
         posts =
           posts
             .joins(:topic)
-            .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
-            .where("archetype != ?", Archetype.private_message)
-      else
+            .where(topics: { category_id: public_categories })
+            .where(topics: { archetype: "regular" })
+      end
+
+      if SiteSetting.ai_translation_backfill_max_age_days > 0
         posts =
-          posts.joins(:topic).where(
-            "topics.archetype != ? OR EXISTS (SELECT 1 FROM topic_allowed_groups WHERE topic_id = topics.id)",
-            Archetype.private_message,
+          posts.where(
+            "posts.created_at > ?",
+            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
           )
       end
 
-      posts = posts.order(updated_at: :desc).limit(limit)
+      posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
       return if posts.empty?
 
       posts.each do |post|
@@ -43,7 +45,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect post #{post.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+            "Failed to detect post #{post.id}'s locale: #{e.message}",
           )
         end
       end

app/jobs/scheduled/summaries_backfill.rb

@@ -36,12 +36,7 @@ module ::Jobs
       existing_summary = strategy.existing_summary
 
       if existing_summary.blank? || existing_summary.outdated
-        begin
-          strategy.summarize(user)
-        rescue => e
-          Rails.logger.error("Error summarizing topic #{topic.id}: #{e.class.name} - #{e.message}")
-          raise e
-        end
+        strategy.summarize(user)
       else
         # Hiding or deleting a post, and creating a small action alters the Topic#highest_post_number.
         # We use this as a quick way to select potential backfill candidates without relying on original_content_sha.

app/jobs/scheduled/topic_localization_backfill.rb

@@ -6,10 +6,13 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !DiscourseAi::Translation.backfill_enabled?
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
 
-      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
-      Jobs.enqueue(:localize_topics, limit:)
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
     end
   end
 end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -7,27 +7,23 @@ module Jobs
     cluster_concurrency 1
 
     def execute(args)
-      return if !DiscourseAi::Translation.backfill_enabled?
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      limit = SiteSetting.ai_translation_backfill_rate
 
-      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
-      topics =
-        Topic
-          .where(locale: nil, deleted_at: nil)
-          .where("topics.user_id > 0")
-          .where("topics.created_at > ?", SiteSetting.ai_translation_backfill_max_age_days.days.ago)
+      topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")
 
       if SiteSetting.ai_translation_backfill_limit_to_public_content
-        topics =
-          topics.where(category_id: Category.where(read_restricted: false).select(:id)).where(
-            "archetype != ?",
-            Archetype.private_message,
-          )
-      else
+        topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+      end
+
+      if SiteSetting.ai_translation_backfill_max_age_days > 0
         topics =
           topics.where(
-            "archetype != ? OR EXISTS (SELECT 1 FROM topic_allowed_groups WHERE topic_id = topics.id)",
-            Archetype.private_message,
+            "topics.created_at > ?",
+            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
           )
       end
 
@@ -41,7 +37,7 @@ module Jobs
           # do nothing, there are too many sporadic lookup failures
         rescue => e
           DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect topic #{topic.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
+            "Failed to detect topic #{topic.id}'s locale: #{e.message}",
           )
         end
       end

app/models/ai_moderation_setting.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 class AiModerationSetting < ActiveRecord::Base
   belongs_to :llm_model
-  belongs_to :ai_persona
 
   validates :llm_model_id, presence: true
   validates :setting_type, presence: true
@@ -20,13 +19,12 @@ end
 #
 # Table name: ai_moderation_settings
 #
-#  id            :bigint           not null, primary key
-#  setting_type  :enum             not null
-#  data          :jsonb
-#  llm_model_id  :bigint           not null
-#  created_at    :datetime         not null
-#  updated_at    :datetime         not null
-#  ai_persona_id :bigint           default(-31), not null
+#  id           :bigint           not null, primary key
+#  setting_type :enum             not null
+#  data         :jsonb
+#  llm_model_id :bigint           not null
+#  created_at   :datetime         not null
+#  updated_at   :datetime         not null
 #
 # Indexes
 #

app/models/embedding_definition.rb

@@ -23,9 +23,7 @@ class EmbeddingDefinition < ActiveRecord::Base
         DiscourseAi::Tokenizer::GeminiTokenizer,
         DiscourseAi::Tokenizer::MultilingualE5LargeTokenizer,
         DiscourseAi::Tokenizer::OpenAiTokenizer,
-        DiscourseAi::Tokenizer::OpenAiCl100kTokenizer,
-        DiscourseAi::Tokenizer::MistralTokenizer,
-        DiscourseAi::Tokenizer::QwenTokenizer,
+        DiscourseAi::Tokenizer::MixtralTokenizer,
       ].map(&:name)
     end
 

app/models/llm_model.rb

@@ -65,8 +65,6 @@ class LlmModel < ActiveRecord::Base
       google: {
         disable_native_tools: :checkbox,
         enable_thinking: :checkbox,
-        disable_temperature: :checkbox,
-        disable_top_p: :checkbox,
         thinking_tokens: :number,
       },
       azure: {

app/serializers/ai_spam_serializer.rb

@@ -8,9 +8,7 @@ class AiSpamSerializer < ApplicationSerializer
              :stats,
              :flagging_username,
              :spam_score_type,
-             :spam_scanning_user,
-             :ai_persona_id,
-             :available_personas
+             :spam_scanning_user
 
   def is_enabled
     object[:enabled]
@@ -20,11 +18,6 @@ class AiSpamSerializer < ApplicationSerializer
     settings&.llm_model&.id
   end
 
-  def ai_persona_id
-    settings&.ai_persona&.id ||
-      DiscourseAi::Personas::Persona.system_personas[DiscourseAi::Personas::SpamDetector]
-  end
-
   def custom_instructions
     settings&.custom_instructions
   end
@@ -35,12 +28,6 @@ class AiSpamSerializer < ApplicationSerializer
       .map { |hash| { id: hash[:value], name: hash[:name] } }
   end
 
-  def available_personas
-    DiscourseAi::Configuration::PersonaEnumerator.values.map do |h|
-      { id: h[:value], name: h[:name] }
-    end
-  end
-
   def flagging_username
     object[:flagging_username]
   end

app/services/discourse_ai/persona_exporter.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  class PersonaExporter
-    def initialize(persona:)
-      raise ArgumentError, "Invalid persona provided" if !persona.is_a?(AiPersona)
-      @persona = persona
-    end
-
-    def export
-      serialized_custom_tools = serialize_tools(@persona)
-      serialize_persona(@persona, serialized_custom_tools)
-    end
-
-    private
-
-    def serialize_tools(ai_persona)
-      custom_tool_ids =
-        (ai_persona.tools || []).filter_map do |tool_config|
-          # A tool config is an array like: ["custom-ID", {options}, force_flag]
-          if tool_config.is_a?(Array) && tool_config[0].to_s.start_with?("custom-")
-            tool_config[0].split("-", 2).last.to_i
-          end
-        end
-
-      return [] if custom_tool_ids.empty?
-
-      tools = AiTool.where(id: custom_tool_ids)
-      tools.map do |tool|
-        {
-          identifier: tool.tool_name, # Use tool_name for portability
-          name: tool.name,
-          description: tool.description,
-          tool_name: tool.tool_name,
-          parameters: tool.parameters,
-          summary: tool.summary,
-          script: tool.script,
-        }
-      end
-    end
-
-    def serialize_persona(ai_persona, serialized_custom_tools)
-      export_data = {
-        meta: {
-          version: "1.0",
-          exported_at: Time.zone.now.iso8601,
-        },
-        persona: {
-          name: ai_persona.name,
-          description: ai_persona.description,
-          system_prompt: ai_persona.system_prompt,
-          examples: ai_persona.examples,
-          temperature: ai_persona.temperature,
-          top_p: ai_persona.top_p,
-          response_format: ai_persona.response_format,
-          tools: transform_tools_for_export(ai_persona.tools, serialized_custom_tools),
-        },
-        custom_tools: serialized_custom_tools,
-      }
-
-      JSON.pretty_generate(export_data)
-    end
-
-    def transform_tools_for_export(tools_config, _serialized_custom_tools)
-      return [] if tools_config.blank?
-
-      tools_config.map do |tool_config|
-        unless tool_config.is_a?(Array) && tool_config[0].to_s.start_with?("custom-")
-          next tool_config
-        end
-
-        tool_id = tool_config[0].split("-", 2).last.to_i
-        tool = AiTool.find_by(id: tool_id)
-        next tool_config unless tool
-
-        ["custom-#{tool.tool_name}", tool_config[1], tool_config[2]]
-      end
-    end
-  end
-end