FEATURE: Automatic translation and localization of posts, topics, categories (#1376)

Related: https://github.com/discourse/discourse-translator/pull/310

This commit includes all the jobs and event hooks to localize posts, topics, and categories.

A few notes:
- `feature_name: "translation"` because the site setting is `ai-translation` and module is `Translation`
- we will switch to proper ai-feature in the near future, and can consider using the persona_user as `localization.localizer_user_id`
- keeping things flat within the module for now as we will be moving to ai-feature soon and have to rearrange
- Settings renamed/introduced are:
  - ai_translation_backfill_rate (0)
  - ai_translation_backfill_limit_to_public_content (true)
  - ai_translation_backfill_max_age_days (5)
  - ai_translation_verbose_logs (false)

lib/translation/base_translator.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class BaseTranslator
+      def initialize(text, target_language)
+        @text = text
+        @target_language = target_language
+      end
+
+      def translate
+        prompt =
+          DiscourseAi::Completions::Prompt.new(
+            prompt_template,
+            messages: [{ type: :user, content: formatted_content, id: "user" }],
+          )
+
+        structured_output =
+          DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_translation_model).generate(
+            prompt,
+            user: Discourse.system_user,
+            feature_name: "translation",
+            response_format: response_format,
+          )
+
+        structured_output&.read_buffered_property(:translation)
+      end
+
+      def formatted_content
+        { content: @text, target_language: @target_language }.to_json
+      end
+
+      def response_format
+        {
+          type: "json_schema",
+          json_schema: {
+            name: "reply",
+            schema: {
+              type: "object",
+              properties: {
+                translation: {
+                  type: "string",
+                },
+              },
+              required: ["translation"],
+              additionalProperties: false,
+            },
+            strict: true,
+          },
+        }
+      end
+
+      private
+
+      def prompt_template
+        raise NotImplementedError
+      end
+    end
+  end
+end

lib/translation/category_localizer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class CategoryLocalizer
+      def self.localize(category, target_locale = I18n.locale)
+        return if category.blank? || target_locale.blank?
+
+        target_locale_sym = target_locale.to_s.sub("-", "_").to_sym
+
+        translated_name = ShortTextTranslator.new(category.name, target_locale_sym).translate
+        # category descriptions are first paragraphs of posts
+        translated_description =
+          PostRawTranslator.new(category.description, target_locale_sym).translate
+
+        localization =
+          CategoryLocalization.find_or_initialize_by(
+            category_id: category.id,
+            locale: target_locale_sym.to_s,
+          )
+
+        localization.name = translated_name
+        localization.description = translated_description
+        localization.save!
+        localization
+      end
+    end
+  end
+end

lib/translation/content_splitter.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class ContentSplitter
+      CHUNK_SIZE = 3000
+
+      BBCODE_PATTERNS = [
+        %r{\[table.*?\].*?\[/table\]}m,
+        %r{\[quote.*?\].*?\[/quote\]}m,
+        %r{\[details.*?\].*?\[/details\]}m,
+        %r{\<details.*?\>.*?\</details\>}m,
+        %r{\[spoiler.*?\].*?\[/spoiler\]}m,
+        %r{\[code.*?\].*?\[/code\]}m,
+        /```.*?```/m,
+      ].freeze
+
+      TEXT_BOUNDARIES = [
+        /\n\s*\n\s*|\r\n\s*\r\n\s*/, # double newlines with optional spaces
+        /[.!?]\s+/, # sentence endings
+        /[,;]\s+/, # clause endings
+        /\n|\r\n/, # single newlines
+        /\s+/, # any whitespace
+      ].freeze
+
+      def self.split(content)
+        return [] if content.nil?
+        return [""] if content.empty?
+        return [content] if content.length <= CHUNK_SIZE
+
+        chunks = []
+        remaining = content.dup
+
+        while remaining.present?
+          chunk = extract_mixed_chunk(remaining)
+          break if chunk.empty?
+          chunks << chunk
+          remaining = remaining[chunk.length..-1]
+        end
+
+        chunks
+      end
+
+      private
+
+      def self.extract_mixed_chunk(text, size: CHUNK_SIZE)
+        return text if text.length <= size
+        flexible_size = size * 1.5
+
+        # try each splitting strategy in order
+        split_point =
+          [
+            -> { find_nearest_html_end_index(text, size) },
+            -> { find_nearest_bbcode_end_index(text, size) },
+            -> { find_text_boundary(text, size) },
+            -> { size },
+          ].lazy.map(&:call).compact.find { |pos| pos <= flexible_size }
+
+        text[0...split_point]
+      end
+
+      def self.find_nearest_html_end_index(text, target_pos)
+        return nil if !text.include?("<")
+
+        begin
+          doc = Nokogiri::HTML5.fragment(text)
+          current_length = 0
+
+          doc.children.each do |node|
+            html = node.to_html
+            end_pos = current_length + html.length
+            return end_pos if end_pos > target_pos
+            current_length = end_pos
+          end
+          nil
+        rescue Nokogiri::SyntaxError
+          nil
+        end
+      end
+
+      def self.find_nearest_bbcode_end_index(text, target_pos)
+        BBCODE_PATTERNS.each do |pattern|
+          text.scan(pattern) do |_|
+            match = $~
+            tag_start = match.begin(0)
+            tag_end = match.end(0)
+
+            return tag_end if tag_start <= target_pos && tag_end > target_pos
+          end
+        end
+
+        nil
+      end
+
+      def self.find_text_boundary(text, target_pos)
+        search_text = text
+
+        TEXT_BOUNDARIES.each do |pattern|
+          if pos = search_text.rindex(pattern, target_pos)
+            # Include all trailing whitespace
+            pos += 1 while pos < search_text.length && search_text[pos].match?(/\s/)
+            return pos
+          end
+        end
+        nil
+      end
+    end
+  end
+end

lib/translation/entry_point.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class EntryPoint
+      def inject_into(plugin)
+        plugin.on(:post_process_cooked) do |_, post|
+          if SiteSetting.discourse_ai_enabled && SiteSetting.ai_translation_enabled
+            Jobs.enqueue(:detect_translate_post, post_id: post.id)
+          end
+        end
+
+        plugin.on(:topic_created) do |topic|
+          if SiteSetting.discourse_ai_enabled && SiteSetting.ai_translation_enabled
+            Jobs.enqueue(:detect_translate_topic, topic_id: topic.id)
+          end
+        end
+
+        plugin.on(:post_edited) do |post, topic_changed|
+          if SiteSetting.discourse_ai_enabled && SiteSetting.ai_translation_enabled && topic_changed
+            Jobs.enqueue(:detect_translate_topic, topic_id: post.topic_id)
+          end
+        end
+      end
+    end
+  end
+end

lib/translation/language_detector.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class LanguageDetector
+      DETECTION_CHAR_LIMIT = 1000
+      PROMPT_TEXT = <<~TEXT
+      You will be given a piece of text, and your task is to detect the locale (language) of the text and return it in a specific JSON format.
+
+      To complete this task, follow these steps:
+
+      1. Carefully read and analyze the provided text.
+      2. Determine the language of the text based on its characteristics, such as vocabulary, grammar, and sentence structure.
+      3. Do not use links or programing code in the text to detect the locale
+      4. Identify the appropriate language code for the detected language.
+
+      Here is a list of common language codes for reference:
+      - English: en
+      - Spanish: es
+      - French: fr
+      - German: de
+      - Italian: it
+      - Brazilian Portuguese: pt-BR
+      - Russian: ru
+      - Simplified Chinese: zh-CN
+      - Japanese: ja
+      - Korean: ko
+
+      If the language is not in this list, use the appropriate IETF language tag code.
+
+      5. Format your response as a JSON object with a single key "locale" and the value as the language code.
+
+      Your output should be in the following format:
+      <output>
+      {"locale": "xx"}
+      </output>
+
+      Where "xx" is replaced by the appropriate language code.
+
+      Important: Base your analysis solely on the provided text. Do not use any external information or make assumptions about the text's origin or context beyond what is explicitly provided.
+    TEXT
+
+      def initialize(text)
+        @text = text
+      end
+
+      def detect
+        prompt =
+          DiscourseAi::Completions::Prompt.new(
+            PROMPT_TEXT,
+            messages: [{ type: :user, content: @text, id: "user" }],
+          )
+
+        structured_output =
+          DiscourseAi::Completions::Llm.proxy(SiteSetting.ai_translation_model).generate(
+            prompt,
+            user: Discourse.system_user,
+            feature_name: "translation",
+            response_format: response_format,
+          )
+
+        structured_output&.read_buffered_property(:locale)
+      end
+
+      def response_format
+        {
+          type: "json_schema",
+          json_schema: {
+            name: "reply",
+            schema: {
+              type: "object",
+              properties: {
+                locale: {
+                  type: "string",
+                },
+              },
+              required: ["locale"],
+              additionalProperties: false,
+            },
+            strict: true,
+          },
+        }
+      end
+    end
+  end
+end

lib/translation/locale_normalizer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class LocaleNormalizer
+      # Normalizes locale string, matching the list of I18n.locales where possible
+      # @param locale [String,Symbol] the locale to normalize
+      # @return [String] the normalized locale
+      def self.normalize_to_i18n(locale)
+        return nil if locale.blank?
+        locale = locale.to_s.gsub("-", "_")
+
+        i18n_pairs.each { |downcased, value| return value if locale.downcase == downcased }
+
+        locale
+      end
+
+      private
+
+      def self.i18n_pairs
+        # they should look like this for the input to match against:
+        # {
+        #   "lowercased" => "actual",
+        #   "en" => "en",
+        #   "zh_cn" => "zh_CN",
+        #   "zh" => "zh_CN",
+        # }
+        @locale_map ||=
+          I18n
+            .available_locales
+            .reduce({}) do |output, sym|
+              locale = sym.to_s
+              output[locale.downcase] = locale
+              if locale.include?("_")
+                short = locale.split("_").first
+                output[short] = locale if output[short].blank?
+              end
+              output
+            end
+      end
+    end
+  end
+end

lib/translation/post_locale_detector.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class PostLocaleDetector
+      def self.detect_locale(post)
+        return if post.blank?
+
+        detected_locale = LanguageDetector.new(post.raw).detect
+        locale = LocaleNormalizer.normalize_to_i18n(detected_locale)
+        post.update_column(:locale, locale)
+        locale
+      end
+    end
+  end
+end

lib/translation/post_localizer.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class PostLocalizer
+      def self.localize(post, target_locale = I18n.locale)
+        return if post.blank? || target_locale.blank? || post.locale == target_locale.to_s
+        target_locale_sym = target_locale.to_s.sub("-", "_").to_sym
+
+        translated_raw =
+          ContentSplitter
+            .split(post.raw)
+            .map { |chunk| PostRawTranslator.new(chunk, target_locale_sym).translate }
+            .join("")
+
+        localization =
+          PostLocalization.find_or_initialize_by(post_id: post.id, locale: target_locale_sym.to_s)
+
+        localization.raw = translated_raw
+        localization.cooked = PrettyText.cook(translated_raw)
+        localization.post_version = post.version
+        localization.localizer_user_id = Discourse.system_user.id
+        localization.save!
+        localization
+      end
+    end
+  end
+end

lib/translation/post_raw_translator.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class PostRawTranslator < BaseTranslator
+      PROMPT_TEMPLATE = <<~TEXT.freeze
+      You are a highly skilled translator tasked with translating content from one language to another. Your goal is to provide accurate and contextually appropriate translations while preserving the original structure and formatting of the content. Follow these instructions carefully:
+
+      Translation Instructions:
+      1. Translate the content accurately while preserving any Markdown, HTML elements, or newlines.
+      2. Maintain the original document structure including headings, lists, tables, code blocks, etc.
+      3. Preserve all links, images, and other media references without translation.
+      4. Handle code snippets appropriately:
+        - Do not translate variable names, functions, or syntax within code blocks (```).
+        - Translate comments within code blocks.
+      5. For technical terminology:
+        - Provide the accepted target language term if it exists.
+        - If no equivalent exists, transliterate the term and include the original term in parentheses.
+      6. For ambiguous terms or phrases, choose the most contextually appropriate translation.
+      7. Do not add any content besides the translation.
+      8. Ensure the translation only contains the original language and the target language.
+
+      Output your translation in the following JSON format:
+      {"translation": "Your TARGET_LANGUAGE translation here"}
+
+      Here are three examples of correct translations:
+
+      Original: {"content":"New Update for Minecraft Adds Underwater Temples", "target_language":"Spanish"}
+      Correct translation: {"translation": "Nueva actualización para Minecraft añade templos submarinos"}
+
+      Original: {"content": "# Machine Learning 101\n\nMachine Learning (ML) is a subset of Artificial Intelligence (AI) that focuses on the development of algorithms and statistical models that enable computer systems to improve their performance on a specific task through experience.\n\n## Key Concepts\n\n1. **Supervised Learning**: The algorithm learns from labeled training data.\n2. **Unsupervised Learning**: The algorithm finds patterns in unlabeled data.\n3. **Reinforcement Learning**: The algorithm learns through interaction with an environment.\n\n```python\n# Simple example of a machine learning model\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\n\n# Assuming X and y are your features and target variables\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\nmodel = LogisticRegression()\nmodel.fit(X_train, y_train)\n\n# Evaluate the model\naccuracy = model.score(X_test, y_test)\nprint(f'Model accuracy: {accuracy}')\n```\n\nFor more information, visit [Machine Learning on Wikipedia](https://en.wikipedia.org/wiki/Machine_learning).", "target_language":"French"}
+      Correct translation: {"translation": "# Machine Learning 101\n\nLe Machine Learning (ML) est un sous-ensemble de l'Intelligence Artificielle (IA) qui se concentre sur le développement d'algorithmes et de modèles statistiques permettant aux systèmes informatiques d'améliorer leurs performances sur une tâche spécifique grâce à l'expérience.\n\n## Concepts clés\n\n1. **Apprentissage supervisé** : L'algorithme apprend à partir de données d'entraînement étiquetées.\n2. **Apprentissage non supervisé** : L'algorithme trouve des motifs dans des données non étiquetées.\n3. **Apprentissage par renforcement** : L'algorithme apprend à travers l'interaction avec un environnement.\n\n```python\n# Exemple simple d'un modèle de machine learning\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\n\n# En supposant que X et y sont vos variables de caractéristiques et cibles\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\nmodel = LogisticRegression()\nmodel.fit(X_train, y_train)\n\n# Évaluer le modèle\naccuracy = model.score(X_test, y_test)\nprint(f'Model accuracy: {accuracy}')\n```\n\nPour plus d'informations, visitez [Machine Learning sur Wikipedia](https://en.wikipedia.org/wiki/Machine_learning)."}
+
+      Original: {"content": "**Heathrow fechado**: paralisação de voos deve continuar nos próximos dias, diz gestora do aeroporto de *Londres*", "target_language": "English"}
+      Correct translation: {"translation": "**Heathrow closed**: flight disruption expected to continue in coming days, says *London* airport management"}
+
+      Remember, you are being consumed via an API. Only return the translated text in the specified JSON format. Do not include any additional information or explanations in your response.
+    TEXT
+
+      private def prompt_template
+        PROMPT_TEMPLATE
+      end
+    end
+  end
+end

lib/translation/short_text_translator.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class ShortTextTranslator < BaseTranslator
+      PROMPT_TEMPLATE = <<~TEXT.freeze
+      You are a translation service specializing in translating short pieces of text or a few words.
+      These words may be things like a name, description, or title. Adhere to the following guidelines:
+
+      1. Keep proper nouns and technical terms in their original language
+      2. Keep the translated content close to the original length
+      3. Translation maintains the original meaning
+      4. Preserving any Markdown, HTML elements, links, parenthesis, or newlines
+
+      Provide your translation in the following JSON format:
+
+      <output>
+      {"translation": "target_language translation here"}
+      </output>
+
+      Here are three examples of correct translation
+
+      Original: {"content":"Japan", "target_language":"Spanish"}
+      Correct translation: {"translation": "Japón"}
+
+      Original: {"name":"Cats and Dogs", "target_language":"Chinese"}
+      Correct translation: {"translation": "猫和狗"}
+
+      Original: {"name": "Q&A", "target_language": "Portuguese"}
+      Correct translation: {"translation": "Perguntas e Respostas"}
+
+      Remember to keep proper nouns like "Minecraft" and "Toyota" in their original form. Translate the text now and provide your answer in the specified JSON format.
+    TEXT
+
+      private def prompt_template
+        PROMPT_TEMPLATE
+      end
+    end
+  end
+end

lib/translation/topic_locale_detector.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class TopicLocaleDetector
+      def self.detect_locale(topic)
+        return if topic.blank?
+
+        text = topic.title.dup
+        text << " #{topic.first_post.raw}" if topic.first_post.raw
+
+        detected_locale = LanguageDetector.new(text).detect
+        locale = LocaleNormalizer.normalize_to_i18n(detected_locale)
+        topic.update_column(:locale, locale)
+        locale
+      end
+    end
+  end
+end

lib/translation/topic_localizer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class TopicLocalizer
+      def self.localize(topic, target_locale = I18n.locale)
+        return if topic.blank? || target_locale.blank? || topic.locale == target_locale.to_s
+
+        target_locale_sym = target_locale.to_s.sub("-", "_").to_sym
+
+        translated_title = TopicTitleTranslator.new(topic.title, target_locale_sym).translate
+        translated_excerpt = ShortTextTranslator.new(topic.excerpt, target_locale_sym).translate
+
+        localization =
+          TopicLocalization.find_or_initialize_by(
+            topic_id: topic.id,
+            locale: target_locale_sym.to_s,
+          )
+
+        localization.title = translated_title
+        localization.fancy_title = Topic.fancy_title(translated_title)
+        localization.excerpt = translated_excerpt
+        localization.localizer_user_id = Discourse.system_user.id
+        localization.save!
+        localization
+      end
+    end
+  end
+end

lib/translation/topic_title_translator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class TopicTitleTranslator < BaseTranslator
+      PROMPT_TEMPLATE = <<~TEXT.freeze
+      You are a translation service specializing in translating forum post titles from English to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines:
+
+      1. Translate the given title from English to target_language asked.
+      2. Keep proper nouns and technical terms in their original language.
+      3. Attempt to keep the translated title length close to the original when possible.
+      4. Ensure the translation maintains the original meaning and tone.
+
+      To complete this task:
+
+      1. Read and understand the title carefully.
+      2. Identify any proper nouns or technical terms that should remain untranslated.
+      3. Translate the remaining words and phrases into the target_language, ensuring the meaning is preserved.
+      4. Adjust the translation if necessary to keep the length similar to the original title.
+      5. Review your translation for accuracy and naturalness in the target_language.
+
+      Provide your translation in the following JSON format:
+
+      <output>
+      {"translation": "Your target_language translation here"}
+      </output>
+
+      Here are three examples of correct translation
+
+      Original: {"title":"New Update for Minecraft Adds Underwater Temples", "target_language":"Spanish"}
+      Correct translation: {"translation": "Nueva actualización para Minecraft añade templos submarinos"}
+
+      Original: {"title":"Toyota announces revolutionary battery technology", "target_language":"French"}
+      Correct translation: {"translation": "Toyota annonce une technologie de batteries révolutionnaire"}
+
+      Original: {"title": "Heathrow fechado: paralisação de voos deve continuar nos próximos dias, diz gestora do aeroporto de Londres", "target_language": "English"}
+      Correct translation: {"translation": "Heathrow closed: flight disruption expected to continue in coming days, says London airport management"}
+
+      Remember to keep proper nouns like "Minecraft" and "Toyota" in their original form. Translate the title now and provide your answer in the specified JSON format.
+    TEXT
+
+      private def prompt_template
+        PROMPT_TEMPLATE
+      end
+    end
+  end
+end

lib/translation/verbose_logger.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Translation
+    class VerboseLogger
+      def self.log(message, opts = { level: :warn })
+        if SiteSetting.ai_translation_verbose_logs
+          Rails.logger.send(opts[:level], "DiscourseAi::Translation: #{message}")
+        end
+      end
+    end
+  end
+end