Update translations

This index helps to speed up queries that joins the `topics` table
against the `ai_topics_embeddings` table on the `topic_id` column. There
are a number of queries which filters on `ai_topics_embeddings.model_id`
so we are including that in the index as well.

.discourse-compatibility

@@ -1,3 +1,4 @@
+< 3.5.0.beta8-dev: 5d80a34589d63e381d80273828072badc18955b1
 < 3.5.0.beta7-dev: cd14b0c0bee0cf63c59b64b6f7213e31a37f11a7
 < 3.5.0.beta6-dev: 3e74eea1e5e3143888d67a8d8a11206df214dc24
 < 3.5.0.beta3-dev: 09a68414804a1447f52e5d60691ba59742cda9ec

app/jobs/regular/detect_translate_post.rb

@@ -40,7 +40,7 @@ module Jobs
       return if locales.blank?
 
       locales.each do |locale|
-        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        next if LocaleNormalizer.is_same?(locale, detected_locale)
         regionless_locale = locale.split("_").first
         next if post.post_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
 

app/jobs/regular/detect_translate_topic.rb

@@ -38,7 +38,7 @@ module Jobs
       return if locales.blank?
 
       locales.each do |locale|
-        next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, detected_locale)
+        next if LocaleNormalizer.is_same?(locale, detected_locale)
         regionless_locale = locale.split("_").first
         next if topic.topic_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
 

app/jobs/regular/localize_categories.rb

@@ -29,7 +29,7 @@ module Jobs
         missing_locales = locales - existing_locales - [category.locale]
         missing_locales.each do |locale|
           break if remaining_limit <= 0
-          next if DiscourseAi::Translation::LocaleNormalizer.is_same?(locale, category.locale)
+          next if LocaleNormalizer.is_same?(locale, category.locale)
 
           begin
             DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)

config/locales/client.de.yml

@@ -110,6 +110,9 @@ de:
             top_p:
               label: "Top P"
               description: "Top P für die LLM, erhöhen, um die Zufälligkeit zu erhöhen (leer lassen, um das Standardmodell zu verwenden)"
+            persona_id:
+              label: "Persona"
+              description: "KI Persona zur Berichtserstellung"
         llm_tool_triage:
           fields:
             model:
@@ -179,6 +182,9 @@ de:
             temperature:
               label: "Temperatur"
               description: "Temperatur, die für das LLM verwendet werden soll. Erhöhen, um die Zufälligkeit zu erhöhen (leer lassen, um das Standardmodell zu verwenden)"
+            max_output_tokens:
+              label: "Maximale Ausgabetoken"
+              description: "Wenn angegeben, wird eine Obergrenze für die maximale Anzahl von Token festgelegt, die das Modell erzeugen kann. Beachtet die Obergrenze für die Anzahl der ausgegebenen Token des LLM"
     discourse_ai:
       title: "KI"
       features:
@@ -836,6 +842,7 @@ de:
           last_7_days: "Letzte 7 Tage"
           last_30_days: "Letzte 30 Tage"
           upload_files: "Dateien hochladen"
+          uploads_in_progress: "Absenden nicht möglich, solange Uploads nicht abgeschlossen sind"
       sentiments:
         dashboard:
           title: "Stimmung"

config/locales/client.en.yml

@@ -180,6 +180,9 @@ en:
             temperature:
               label: "Temperature"
               description: "Temperature to use for the LLM. Increase to increase randomness (leave empty to use model default)"
+            max_output_tokens:
+              label: "Max output tokens"
+              description: "When specified, sets an upper bound to the maximum number of tokens the model can generate. Respects LLM's max output tokens limit"
 
     discourse_ai:
       title: "AI"

config/locales/client.he.yml

@@ -109,6 +109,9 @@ he:
             top_p:
               label: "ה־P המובילים"
               description: "ה־P המוביל לשימוש למודל השפה הגדול (LLM), הגדלה תגדיל את האקראיות (להשאיר ריק כדי להשתמש בברירת המחדל של הדגם)"
+            persona_id:
+              label: "דמות"
+              description: "דמות בינה מלאכותית לשימוש להפקת דוחות"
         llm_tool_triage:
           fields:
             model:
@@ -167,6 +170,8 @@ he:
             temperature:
               label: "טמפרטורה"
               description: "טמפרטורה לשימוש למודל השפה הגדול (LLM), הגדלה תגדיל את האקראיות (להשאיר ריק כדי להשתמש בברירת המחדל של הדגם)"
+            max_output_tokens:
+              label: "כמות אסימוני פלט מרבית"
     discourse_ai:
       title: "בינה מלאכותית"
       features:
@@ -773,6 +778,7 @@ he:
           last_7_days: "7 הימים האחרונים"
           last_30_days: "30 הימים האחרונים"
           upload_files: "העלאת קבצים"
+          uploads_in_progress: "לא ניתן להגיש תוך כדי שמתבצעות העלאות"
       sentiments:
         dashboard:
           title: "רגש"

config/locales/client.lt.yml

@@ -110,6 +110,8 @@ lt:
             top_p:
               label: "Viršutinė P"
               description: "Viršutinė P, naudojama LLM, padidinkite, kad padidėtų atsitiktinumas (palikite tuščią, jei norite naudoti numatytuosius modelio parametrus)"
+            persona_id:
+              label: "Persona"
         llm_tool_triage:
           fields:
             model:

config/locales/client.pl_PL.yml

@@ -88,6 +88,8 @@ pl_PL:
             temperature:
               label: "Temperatura"
               description: "Temperatura używana dla LLM. Zwiększ, aby zwiększyć losowość (pozostaw puste, aby użyć domyślnego modelu)."
+            persona_id:
+              label: "Osobowość"
         llm_tool_triage:
           fields:
             model:

config/locales/client.zh_CN.yml

@@ -94,6 +94,8 @@ zh_CN:
             top_p:
               label: "Top P"
               description: "用于 LLM 的 Top P 值，增加该值会提高生成内容的随机性（留空则使用模型默认值）"
+            persona_id:
+              label: "角色模型"
         llm_tool_triage:
           fields:
             model:

config/locales/server.de.yml

@@ -384,6 +384,9 @@ de:
         content_creator:
           name: "Inhaltsersteller"
           description: "Standard-Persona für die HyDE-Suche"
+        report_runner:
+          name: "Berichtsausführer"
+          description: "Standard-Persona, die im Automatisierungsskript für Berichte verwendet wird."
       topic_not_found: "Zusammenfassung nicht verfügbar, Thema nicht gefunden!"
       summarizing: "Thema zusammenfassen"
       searching: "Suche nach: „%{query}“"

config/locales/server.he.yml

@@ -302,6 +302,9 @@ he:
           name: "בקשה מותאמת אישית"
         smart_dates:
           name: "תאריכים חכמים"
+        report_runner:
+          name: "אצן דוחות"
+          description: "דמות ברירת המחדל שסקריפט אוטומציית הדוחות משתמש בה"
       topic_not_found: "תקציר לא זמין, לא נמצא נושא!"
       summarizing: "הנושא מסוכם"
       searching: "חיפוש אחר: ‚%{query}’"

config/settings.yml

@@ -527,6 +527,8 @@ discourse_ai:
     area: "ai-features/translation"
   ai_translation_backfill_max_age_days:
     default: 5
+    min: 0
+    max: 20000
     client: false
     area: "ai-features/translation"
   ai_translation_verbose_logs:

db/migrate/20250717075002_set_translation_backfill_max_age.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+class SetTranslationBackfillMaxAge < ActiveRecord::Migration[7.2]
+  def up
+    execute <<~SQL
+      UPDATE site_settings
+      SET value = '20000'
+      WHERE name = 'ai_translation_backfill_max_age_days'
+      AND value::integer > 20000;
+    SQL
+
+    execute <<~SQL
+      UPDATE site_settings
+      SET value = '0'
+      WHERE name = 'ai_translation_backfill_max_age_days'
+      AND value::integer < 0;
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20250722082515_add_index_to_ai_topics_embeddings.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddIndexToAiTopicsEmbeddings < ActiveRecord::Migration[7.2]
+  def up
+    add_index :ai_topics_embeddings, %i[topic_id model_id]
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

discourse_automation/llm_triage.rb

@@ -24,6 +24,7 @@ if defined?(DiscourseAutomation)
     field :max_post_tokens, component: :text
     field :stop_sequences, component: :text_list, required: false
     field :temperature, component: :text
+    field :max_output_tokens, component: :text
 
     # Actions
     field :category, component: :category
@@ -85,6 +86,9 @@ if defined?(DiscourseAutomation)
         temperature = temperature.to_f
       end
 
+      max_output_tokens = fields.dig("max_output_tokens", "value").to_i
+      max_output_tokens = nil if max_output_tokens <= 0
+
       max_post_tokens = nil if max_post_tokens <= 0
 
       stop_sequences = fields.dig("stop_sequences", "value")
@@ -122,6 +126,7 @@ if defined?(DiscourseAutomation)
           stop_sequences: stop_sequences,
           automation: self.automation,
           temperature: temperature,
+          max_output_tokens: max_output_tokens,
           action: context["action"],
         )
       rescue => e

lib/automation/llm_triage.rb

@@ -21,6 +21,7 @@ module DiscourseAi
         temperature: nil,
         whisper: nil,
         reply_persona_id: nil,
+        max_output_tokens: nil,
         action: nil
       )
         if category_id.blank? && tags.blank? && canned_reply.blank? && hide_topic.blank? &&
@@ -59,8 +60,8 @@ module DiscourseAi
         result =
           llm.generate(
             prompt,
+            max_tokens: max_output_tokens,
             temperature: temperature,
-            max_tokens: 700, # ~500 words
             user: Discourse.system_user,
             stop_sequences: stop_sequences,
             feature_name: "llm_triage",

lib/translation/base_translator.rb

@@ -46,12 +46,24 @@ module DiscourseAi
             topic: @topic,
             post: @post,
           )
+        max_tokens = get_max_tokens(text)
+        llm_args = { max_tokens: }
 
         result = +""
-        bot.reply(context) { |partial| result << partial }
+        bot.reply(context, llm_args:) { |partial| result << partial }
         result
       end
 
+      def get_max_tokens(text)
+        if text.length < 100
+          500
+        elsif text.length < 500
+          1000
+        else
+          text.length * 2
+        end
+      end
+
       def persona_setting
         raise NotImplementedError
       end

lib/translation/locale_normalizer.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Translation
-    class LocaleNormalizer
-      # Normalizes locale string, matching the list of I18n.locales where possible
-      # @param locale [String,Symbol] the locale to normalize
-      # @return [String] the normalized locale
-      def self.normalize_to_i18n(locale)
-        return nil if locale.blank?
-        locale = locale.to_s.gsub("-", "_")
-
-        i18n_pairs.each { |downcased, value| return value if locale.downcase == downcased }
-
-        locale
-      end
-
-      def self.is_same?(locale1, locale2)
-        return true if locale1 == locale2
-        locale1 = locale1.gsub("-", "_").downcase
-        locale2 = locale2.gsub("-", "_").downcase
-        locale1.split("_").first == locale2.split("_").first
-      end
-
-      private
-
-      def self.i18n_pairs
-        # they should look like this for the input to match against:
-        # {
-        #   "lowercased" => "actual",
-        #   "en" => "en",
-        #   "zh_cn" => "zh_CN",
-        #   "zh" => "zh_CN",
-        # }
-        @locale_map ||=
-          I18n
-            .available_locales
-            .reduce({}) do |output, sym|
-              locale = sym.to_s
-              output[locale.downcase] = locale
-              if locale.include?("_")
-                short = locale.split("_").first
-                output[short] = locale if output[short].blank?
-              end
-              output
-            end
-      end
-    end
-  end
-end

spec/lib/discourse_automation/llm_triage_spec.rb

@@ -95,6 +95,7 @@ describe DiscourseAi::Automation::LlmTriage do
     reply_user.update!(admin: true)
     add_automation_field("include_personal_messages", true, type: :boolean)
     add_automation_field("temperature", "0.2")
+    add_automation_field("max_output_tokens", "700")
     post = Fabricate(:post, topic: personal_message)
 
     prompt_options = nil
@@ -107,6 +108,7 @@ describe DiscourseAi::Automation::LlmTriage do
     end
 
     expect(prompt_options[:temperature]).to eq(0.2)
+    expect(prompt_options[:max_tokens]).to eq(700)
 
     last_post = post.topic.reload.posts.order(:post_number).last
     expect(last_post.raw).to eq(canned_reply_text)

spec/lib/translation/base_translator_spec.rb

@@ -20,6 +20,7 @@ describe DiscourseAi::Translation::BaseTranslator do
     let(:target_locale) { "de" }
     let(:llm_response) { "hur dur hur dur!" }
     fab!(:post)
+    fab!(:topic) { post.topic }
 
     it "creates the correct prompt" do
       post_translator =
@@ -36,6 +37,62 @@ describe DiscourseAi::Translation::BaseTranslator do
       end
     end
 
+    it "creates BotContext with the correct parameters and calls bot.reply with correct args" do
+      post_translator =
+        DiscourseAi::Translation::PostRawTranslator.new(text:, target_locale:, post:, topic:)
+
+      expected_content = { content: text, target_locale: target_locale }.to_json
+
+      bot_context = instance_double(DiscourseAi::Personas::BotContext)
+      allow(DiscourseAi::Personas::BotContext).to receive(:new).and_return(bot_context)
+
+      mock_bot = instance_double(DiscourseAi::Personas::Bot)
+      allow(DiscourseAi::Personas::Bot).to receive(:as).and_return(mock_bot)
+      allow(mock_bot).to receive(:reply).and_yield(llm_response)
+
+      post_translator.translate
+
+      expect(DiscourseAi::Personas::BotContext).to have_received(:new).with(
+        user: an_instance_of(User),
+        skip_tool_details: true,
+        feature_name: "translation",
+        messages: [{ type: :user, content: expected_content }],
+        topic: topic,
+        post: post,
+      )
+
+      expect(DiscourseAi::Personas::Bot).to have_received(:as)
+      expect(mock_bot).to have_received(:reply).with(bot_context, llm_args: { max_tokens: 500 })
+    end
+
+    it "sets max_tokens correctly based on text length" do
+      test_cases = [
+        ["Short text", 500], # Short text (< 100 chars)
+        ["a" * 200, 1000], # Medium text (100-500 chars)
+        ["a" * 600, 1200], # Long text (> 500 chars, 600*2=1200)
+      ]
+
+      test_cases.each do |text, expected_max_tokens|
+        translator = DiscourseAi::Translation::PostRawTranslator.new(text: text, target_locale:)
+
+        bot_context = instance_double(DiscourseAi::Personas::BotContext)
+        allow(DiscourseAi::Personas::BotContext).to receive(:new).and_return(bot_context)
+
+        mock_bot = instance_double(DiscourseAi::Personas::Bot)
+        allow(DiscourseAi::Personas::Bot).to receive(:as).and_return(mock_bot)
+        allow(mock_bot).to receive(:reply).and_yield("translated #{text[0..10]}")
+
+        translator.translate
+
+        expect(mock_bot).to have_received(:reply).with(
+          bot_context,
+          llm_args: {
+            max_tokens: expected_max_tokens,
+          },
+        )
+      end
+    end
+
     it "returns the translation from the llm's response" do
       DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do
         expect(

spec/lib/translation/locale_normalizer_spec.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-describe DiscourseAi::Translation::LocaleNormalizer do
-  describe ".normalize_to_i18n" do
-    it "matches input locales to i18n locales" do
-      expect(described_class.normalize_to_i18n("en-GB")).to eq("en_GB")
-      expect(described_class.normalize_to_i18n("en")).to eq("en")
-      expect(described_class.normalize_to_i18n("zh")).to eq("zh_CN")
-      expect(described_class.normalize_to_i18n("tr")).to eq("tr_TR")
-    end
-
-    it "converts dashes to underscores" do
-      expect(described_class.normalize_to_i18n("a-b")).to eq("a_b")
-    end
-  end
-
-  describe "#is_same?" do
-    it "returns true for the same locale" do
-      expect(described_class.is_same?("en", "en")).to be true
-    end
-
-    it "returns true for locales with different cases" do
-      expect(described_class.is_same?("en", "EN")).to be true
-    end
-
-    it "returns true for locales with different separators" do
-      expect(described_class.is_same?("en-US", "en_US")).to be true
-    end
-
-    it "returns false for different locales" do
-      expect(described_class.is_same?("en", "ja")).to be false
-    end
-
-    it "returns true for locales with the same base language" do
-      expect(described_class.is_same?("zh-CN", "zh_TW")).to be true
-    end
-
-    it "returns false for completely different locales" do
-      expect(described_class.is_same?("en", "ja")).to be false
-    end
-  end
-end