DEV: Re-introduce validator

- add sleep function for tool polling with rate limits
- Support base64 encoding for HTTP requests and uploads
-  Enhance forum researcher with cost warnings and comprehensive planning
- Add cancellation support for research operations
- Include feature_name parameter for bot analytics
- richer research support (OR queries)

app/jobs/regular/create_ai_reply.rb

@@ -15,7 +15,7 @@ module ::Jobs
 
         bot = DiscourseAi::Personas::Bot.as(bot_user, persona: persona.new)
 
-        DiscourseAi::AiBot::Playground.new(bot).reply_to(post)
+        DiscourseAi::AiBot::Playground.new(bot).reply_to(post, feature_name: "bot")
       rescue DiscourseAi::Personas::Bot::BOT_NOT_FOUND
         Rails.logger.warn(
           "Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",

app/jobs/regular/detect_translate_post.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslatePost < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if args[:post_id].blank?
+
+      post = Post.find_by(id: args[:post_id])
+      return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        topic = post.topic
+        if topic.blank? || topic.category&.read_restricted? ||
+             topic.archetype == Archetype.private_message
+          return
+        end
+      end
+
+      begin
+        detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+
+        begin
+          DiscourseAi::Translation::PostLocalizer.localize(post, locale)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+
+      MessageBus.publish("/topic/#{post.topic_id}", type: :localized, id: post.id)
+    end
+  end
+end

app/jobs/regular/detect_translate_topic.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslateTopic < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if args[:topic_id].blank?
+
+      topic = Topic.find_by(id: args[:topic_id])
+      if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0
+        return
+      end
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        return if topic.category&.read_restricted?
+      end
+
+      begin
+        detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+
+        begin
+          DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+
+      MessageBus.publish("/topic/#{topic.id}", type: :localized, id: 1)
+    end
+  end
+end

app/jobs/regular/generate_inferred_concepts.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateInferredConcepts < ::Jobs::Base
+    sidekiq_options queue: "low"
+
+    # Process items to generate new concepts
+    #
+    # @param args [Hash] Contains job arguments
+    # @option args [String] :item_type Required - Type of items to process ('topics' or 'posts')
+    # @option args [Array<Integer>] :item_ids Required - List of item IDs to process
+    # @option args [Integer] :batch_size (100) Number of items to process in each batch
+    # @option args [Boolean] :match_only (false) Only match against existing concepts without generating new ones
+    def execute(args = {})
+      return if args[:item_ids].blank? || args[:item_type].blank?
+
+      if %w[topics posts].exclude?(args[:item_type])
+        Rails.logger.error("Invalid item_type for GenerateInferredConcepts: #{args[:item_type]}")
+        return
+      end
+
+      # Process items in smaller batches to avoid memory issues
+      batch_size = args[:batch_size] || 100
+
+      # Get the list of item IDs
+      item_ids = args[:item_ids]
+      match_only = args[:match_only] || false
+
+      # Process items in batches
+      item_ids.each_slice(batch_size) do |batch_item_ids|
+        process_batch(batch_item_ids, args[:item_type], match_only)
+      end
+    end
+
+    private
+
+    def process_batch(item_ids, item_type, match_only)
+      klass = item_type.singularize.classify.constantize
+      items = klass.where(id: item_ids)
+      manager = DiscourseAi::InferredConcepts::Manager.new
+
+      items.each do |item|
+        begin
+          process_item(item, item_type, match_only, manager)
+        rescue => e
+          Rails.logger.error(
+            "Error generating concepts from #{item_type.singularize} #{item.id}: #{e.message}\n#{e.backtrace.join("\n")}",
+          )
+        end
+      end
+    end
+
+    def process_item(item, item_type, match_only, manager)
+      # Use the Manager method that handles both identifying and creating concepts
+      if match_only
+        if item_type == "topics"
+          manager.match_topic_to_concepts(item)
+        else # posts
+          manager.match_post_to_concepts(item)
+        end
+      else
+        if item_type == "topics"
+          manager.generate_concepts_from_topic(item)
+        else # posts
+          manager.generate_concepts_from_post(item)
+        end
+      end
+    end
+  end
+end

app/jobs/regular/localize_categories.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizeCategories < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      cat_id = args[:from_category_id] || Category.order(:id).first&.id
+      last_id = nil
+
+      categories = Category.where("id >= ?", cat_id).order(:id).limit(BATCH_SIZE)
+      return if categories.empty?
+
+      categories.each do |category|
+        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
+          last_id = category.id
+          next
+        end
+
+        CategoryLocalization.transaction do
+          locales.each do |locale|
+            next if CategoryLocalization.exists?(category_id: category.id, locale: locale)
+            begin
+              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+            rescue FinalDestination::SSRFDetector::LookupFailedError
+              # do nothing, there are too many sporadic lookup failures
+            rescue => e
+              DiscourseAi::Translation::VerboseLogger.log(
+                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
+              )
+            end
+          end
+        end
+        last_id = category.id
+      end
+
+      if categories.size == BATCH_SIZE
+        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
+      end
+    end
+  end
+end

app/jobs/regular/localize_posts.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizePosts < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        posts =
+          Post
+            .joins(
+              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("posts.user_id > 0")
+            .where.not(raw: [nil, ""])
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("pl.id IS NULL")
+
+        if SiteSetting.ai_translation_backfill_limit_to_public_content
+          posts =
+            posts
+              .joins(:topic)
+              .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
+              .where.not(topics: { archetype: Archetype.private_message })
+        end
+
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          posts =
+            posts.where(
+              "posts.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        posts = posts.order(updated_at: :desc).limit(limit)
+
+        next if posts.empty?
+
+        posts.each do |post|
+          begin
+            DiscourseAi::Translation::PostLocalizer.localize(post, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
+      end
+    end
+  end
+end

app/jobs/regular/localize_topics.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizeTopics < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        topics =
+          Topic
+            .joins(
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("topics.user_id > 0")
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("tl.id IS NULL")
+
+        if SiteSetting.ai_translation_backfill_limit_to_public_content
+          topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+        end
+
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          topics =
+            topics.where(
+              "topics.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        topics = topics.order(updated_at: :desc).limit(limit)
+
+        next if topics.empty?
+
+        topics.each do |topic|
+          begin
+            DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseAi::Translation::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
+      end
+    end
+  end
+end

app/jobs/scheduled/category_localization_backfill.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Jobs
+  class CategoryLocalizationBackfill < ::Jobs::Scheduled
+    every 12.hours
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+
+      Jobs.enqueue(:localize_categories)
+    end
+  end
+end

app/jobs/scheduled/generate_concepts_from_popular_items.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateConceptsFromPopularItems < ::Jobs::Scheduled
+    every 1.day
+
+    # This job runs daily and generates new concepts from popular topics and posts
+    # It selects items based on engagement metrics and generates concepts from their content
+    def execute(_args)
+      return unless SiteSetting.inferred_concepts_enabled
+
+      process_popular_topics
+      process_popular_posts
+    end
+
+    private
+
+    def process_popular_topics
+      # Find candidate topics that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_topics(
+          limit: SiteSetting.inferred_concepts_daily_topics_limit || 20,
+          min_posts: SiteSetting.inferred_concepts_min_posts || 5,
+          min_likes: SiteSetting.inferred_concepts_min_likes || 10,
+          min_views: SiteSetting.inferred_concepts_min_views || 100,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate topics - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "topics",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "topics",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+
+    def process_popular_posts
+      # Find candidate posts that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_posts(
+          limit: SiteSetting.inferred_concepts_daily_posts_limit || 30,
+          min_likes: SiteSetting.inferred_concepts_post_min_likes || 5,
+          exclude_first_posts: true,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate posts - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "posts",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match against existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "posts",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+  end
+end

app/jobs/scheduled/post_localization_backfill.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Jobs
+  class PostLocalizationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
+    end
+  end
+end

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Jobs
+  class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    sidekiq_options retry: false
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      posts =
+        Post
+          .where(locale: nil)
+          .where(deleted_at: nil)
+          .where("posts.user_id > 0")
+          .where.not(raw: [nil, ""])
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        public_categories = Category.where(read_restricted: false).pluck(:id)
+        posts =
+          posts
+            .joins(:topic)
+            .where(topics: { category_id: public_categories })
+            .where(topics: { archetype: "regular" })
+      end
+
+      if SiteSetting.ai_translation_backfill_max_age_days > 0
+        posts =
+          posts.where(
+            "posts.created_at > ?",
+            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
+      return if posts.empty?
+
+      posts.each do |post|
+        begin
+          DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to detect post #{post.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseAi::Translation::VerboseLogger.log("Detected #{posts.size} post locales")
+    end
+  end
+end

app/jobs/scheduled/topic_localization_backfill.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicLocalizationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
+    end
+  end
+end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    sidekiq_options retry: false
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      limit = SiteSetting.ai_translation_backfill_rate
+      return if limit == 0
+
+      topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+      end
+
+      if SiteSetting.ai_translation_backfill_max_age_days > 0
+        topics =
+          topics.where(
+            "topics.created_at > ?",
+            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      topics = topics.order(updated_at: :desc).limit(limit)
+      return if topics.empty?
+
+      topics.each do |topic|
+        begin
+          DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to detect topic #{topic.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseAi::Translation::VerboseLogger.log("Detected #{topics.size} topic locales")
+    end
+  end
+end

app/models/ai_tool.rb

@@ -22,6 +22,8 @@ class AiTool < ActiveRecord::Base
               message: I18n.t("discourse_ai.tools.name.characters"),
             }
 
+  validate :validate_parameters_enum
+
   def signature
     {
       name: function_call_name,
@@ -57,6 +59,30 @@ class AiTool < ActiveRecord::Base
     end
   end
 
+  def validate_parameters_enum
+    return unless parameters.is_a?(Array)
+
+    parameters.each_with_index do |param, index|
+      next if !param.is_a?(Hash) || !param.key?("enum")
+      enum_values = param["enum"]
+
+      if enum_values.empty?
+        errors.add(
+          :parameters,
+          "Parameter '#{param["name"]}' at index #{index}: enum cannot be empty",
+        )
+        next
+      end
+
+      if enum_values.uniq.length != enum_values.length
+        errors.add(
+          :parameters,
+          "Parameter '#{param["name"]}' at index #{index}: enum values must be unique",
+        )
+      end
+    end
+  end
+
   def self.preamble
     <<~JS
      /**
@@ -142,6 +168,7 @@ class AiTool < ActiveRecord::Base
      *      base_64_content (string): Base64 encoded content of the file.
      *    Returns: { id: number, url: string, short_url: string } - Details of the created upload record.
      *
+     *    upload.getUrl(shortUrl): Given a short URL, eg upload://12345, returns the full CDN friendly URL of the upload.
      * 5. chain
      *    Controls the execution flow.
      *

app/models/inferred_concept.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class InferredConcept < ActiveRecord::Base
+  has_many :inferred_concept_topics
+  has_many :topics, through: :inferred_concept_topics
+
+  has_many :inferred_concept_posts
+  has_many :posts, through: :inferred_concept_posts
+
+  validates :name, presence: true, uniqueness: true
+end
+
+# == Schema Information
+#
+# Table name: inferred_concepts
+#
+#  id         :bigint           not null, primary key
+#  name       :string           not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concepts_on_name  (name) UNIQUE
+#

app/models/inferred_concept_post.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class InferredConceptPost < ActiveRecord::Base
+  belongs_to :inferred_concept
+  belongs_to :post
+
+  validates :inferred_concept_id, presence: true
+  validates :post_id, presence: true
+  validates :inferred_concept_id, uniqueness: { scope: :post_id }
+end
+
+# == Schema Information
+#
+# Table name: inferred_concept_posts
+#
+#  inferred_concept_id :bigint
+#  post_id             :bigint
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concept_posts_on_inferred_concept_id  (inferred_concept_id)
+#  index_inferred_concept_posts_uniqueness              (post_id,inferred_concept_id) UNIQUE
+#

app/models/inferred_concept_topic.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class InferredConceptTopic < ActiveRecord::Base
+  belongs_to :inferred_concept
+  belongs_to :topic
+
+  validates :inferred_concept_id, presence: true
+  validates :topic_id, presence: true
+  validates :inferred_concept_id, uniqueness: { scope: :topic_id }
+end
+
+# == Schema Information
+#
+# Table name: inferred_concept_topics
+#
+#  inferred_concept_id :bigint
+#  topic_id            :bigint
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concept_topics_on_inferred_concept_id  (inferred_concept_id)
+#  index_inferred_concept_topics_uniqueness              (topic_id,inferred_concept_id) UNIQUE
+#

app/serializers/ai_inferred_concept_post_serializer.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class AiInferredConceptPostSerializer < ApplicationSerializer
+  attributes :id,
+             :post_number,
+             :topic_id,
+             :topic_title,
+             :username,
+             :avatar_template,
+             :created_at,
+             :updated_at,
+             :excerpt,
+             :truncated,
+             :inferred_concepts
+
+  def avatar_template
+    User.avatar_template(object.username, object.uploaded_avatar_id)
+  end
+
+  def excerpt
+    Post.excerpt(object.cooked)
+  end
+
+  def truncated
+    object.cooked.length > SiteSetting.post_excerpt_maxlength
+  end
+
+  def inferred_concepts
+    ActiveModel::ArraySerializer.new(
+      object.inferred_concepts,
+      each_serializer: InferredConceptSerializer,
+    )
+  end
+end

app/serializers/inferred_concept_serializer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class InferredConceptSerializer < ApplicationSerializer
+  attributes :id, :name, :created_at, :updated_at
+end

assets/javascripts/discourse/components/ai-tool-editor-form.gjs

@@ -30,11 +30,16 @@ export default class AiToolEditorForm extends Component {
 
   get formData() {
     const parameters = (this.args.editingModel.parameters ?? []).map(
-      (parameter) => ({
-        ...parameter,
-        isEnum: !!parameter.enum?.length,
-        enum: (parameter.enum ??= []),
-      })
+      (parameter) => {
+        const mappedParameter = {
+          ...parameter,
+        };
+        mappedParameter.isEnum = parameter.enum && parameter.enum.length > 0;
+        if (!mappedParameter.isEnum) {
+          delete mappedParameter.enum;
+        }
+        return mappedParameter;
+      }
     );
 
     return {
@@ -63,8 +68,19 @@ export default class AiToolEditorForm extends Component {
   async save(data) {
     this.isSaving = true;
 
+    // we injected a isEnum thing, we need to clean it up
+    const copiedData = JSON.parse(JSON.stringify(data));
+    if (copiedData.parameters) {
+      copiedData.parameters.forEach((parameter) => {
+        if (!parameter.isEnum) {
+          delete parameter.enum;
+        }
+        delete parameter.isEnum;
+      });
+    }
+
     try {
-      await this.args.model.save(data);
+      await this.args.model.save(copiedData);
 
       this.toasts.success({
         data: { message: i18n("discourse_ai.tools.saved") },

assets/javascripts/discourse/components/modal/ai-persona-response-format-editor.gjs

@@ -22,10 +22,20 @@ export default class AiPersonaResponseFormatEditor extends Component {
           type: "string",
         },
         type: {
+          type: "string",
+          enum: ["string", "integer", "boolean", "array"],
+        },
+        array_type: {
           type: "string",
           enum: ["string", "integer", "boolean"],
+          options: {
+            dependencies: {
+              type: "array",
+            },
+          },
         },
       },
+      required: ["key", "type"],
     },
   };
 
@@ -41,7 +51,11 @@ export default class AiPersonaResponseFormatEditor extends Component {
     const toDisplay = {};
 
     this.args.data.response_format.forEach((keyDesc) => {
-      toDisplay[keyDesc.key] = keyDesc.type;
+      if (keyDesc.type === "array") {
+        toDisplay[keyDesc.key] = `[${keyDesc.array_type}]`;
+      } else {
+        toDisplay[keyDesc.key] = keyDesc.type;
+      }
     });
 
     return prettyJSON(toDisplay);

assets/javascripts/discourse/components/modal/debug-ai-modal.gjs

@@ -144,11 +144,11 @@ export default class DebugAiModal extends Component {
             >{{i18n "discourse_ai.ai_bot.debug_ai_modal.response"}}</a></li>
         </ul>
         <div class="ai-debug-modal__tokens">
-          <span>
+          <span class="ai-debug-modal__tokens__count">
             {{i18n "discourse_ai.ai_bot.debug_ai_modal.request_tokens"}}
             {{this.info.request_tokens}}
           </span>
-          <span>
+          <span class="ai-debug-modal__tokens__count">
             {{i18n "discourse_ai.ai_bot.debug_ai_modal.response_tokens"}}
             {{this.info.response_tokens}}
           </span>

assets/javascripts/discourse/components/modal/diff-modal.gjs

@@ -5,6 +5,7 @@ import didInsert from "@ember/render-modifiers/modifiers/did-insert";
 import willDestroy from "@ember/render-modifiers/modifiers/will-destroy";
 import { service } from "@ember/service";
 import { htmlSafe } from "@ember/template";
+import { or } from "truth-helpers";
 import CookText from "discourse/components/cook-text";
 import DButton from "discourse/components/d-button";
 import DModal from "discourse/components/d-modal";
@@ -41,6 +42,10 @@ export default class ModalDiffModal extends Component {
   }
 
   get diffResult() {
+    if (this.loading) {
+      return this.escapedSelectedText;
+    }
+
     if (this.diffStreamer.diff?.length > 0) {
       return this.diffStreamer.diff;
     }
@@ -50,10 +55,22 @@ export default class ModalDiffModal extends Component {
     return this.escapedSelectedText;
   }
 
+  get smoothStreamerResult() {
+    if (this.loading) {
+      return this.escapedSelectedText;
+    }
+
+    return this.smoothStreamer.renderedText;
+  }
+
   get isStreaming() {
     // diffStreamer stops Streaming when it is finished with a chunk, looking at isDone is safe
     // it starts off not done
-    return !this.diffStreamer.isDone || this.smoothStreamer.isStreaming;
+    if (this.args.model.showResultAsDiff) {
+      return !this.diffStreamer.isDone;
+    }
+
+    return this.smoothStreamer.isStreaming;
   }
 
   get primaryBtnLabel() {
@@ -154,42 +171,37 @@ export default class ModalDiffModal extends Component {
           {{willDestroy this.cleanup}}
           class="text-preview"
         >
-          {{#if this.loading}}
-            <div class="composer-ai-helper-modal__loading">
-              {{[email protected]~}}
-            </div>
-          {{else}}
-            <div
-              class={{concatClass
-                "composer-ai-helper-modal__suggestion"
-                "streamable-content"
-                (if this.isStreaming "streaming")
-                (if @model.showResultAsDiff "inline-diff")
-                (if this.diffStreamer.isThinking "thinking")
-              }}
-            >
-              {{~#if @model.showResultAsDiff~}}
-                <span class="diff-inner">{{htmlSafe this.diffResult}}</span>
+          <div
+            class={{concatClass
+              "composer-ai-helper-modal__suggestion"
+              "streamable-content"
+              (if this.isStreaming "streaming")
+              (if @model.showResultAsDiff "inline-diff")
+              (if this.diffStreamer.isThinking "thinking")
+              (if this.loading "composer-ai-helper-modal__loading")
+            }}
+          >
+            {{~#if @model.showResultAsDiff~}}
+              <span class="diff-inner">{{htmlSafe this.diffResult}}</span>
+            {{else}}
+              {{#if (or this.loading this.smoothStreamer.isStreaming)}}
+                <CookText
+                  @rawText={{this.smoothStreamerResult}}
+                  class="cooked"
+                />
               {{else}}
-                {{#if this.smoothStreamer.isStreaming}}
+                <div class="composer-ai-helper-modal__old-value">
+                  {{~this.escapedSelectedText~}}
+                </div>
+                <div class="composer-ai-helper-modal__new-value">
                   <CookText
-                    @rawText={{this.smoothStreamer.renderedText}}
+                    @rawText={{this.smoothStreamerResult}}
                     class="cooked"
                   />
-                {{else}}
-                  <div class="composer-ai-helper-modal__old-value">
-                    {{@model.selectedText}}
-                  </div>
-                  <div class="composer-ai-helper-modal__new-value">
-                    <CookText
-                      @rawText={{this.smoothStreamer.renderedText}}
-                      class="cooked"
-                    />
-                  </div>
-                {{/if}}
+                </div>
               {{/if}}
-            </div>
-          {{/if}}
+            {{/if}}
+          </div>
         </div>
       </:body>
 

assets/javascripts/discourse/connectors/full-page-search-below-search-header/ai-full-page-discobot-discoveries.gjs

@@ -15,13 +15,12 @@ export default class AiFullPageDiscobotDiscoveries extends Component {
     );
   }
 
+  @service capabilities;
   @service discobotDiscoveries;
   @service site;
 
   get previewLength() {
-    // todo: replace with js breakpoint API
-    // https://github.com/discourse/discourse/pull/32060
-    if (this.site.mobileView || this.site.narrowDesktopView) {
+    if (!this.capabilities.viewport.md) {
       return 50;
     } else {
       return 10000;

assets/javascripts/initializers/translation.js

@@ -0,0 +1,24 @@
+import { apiInitializer } from "discourse/lib/api";
+import cookie from "discourse/lib/cookie";
+
+export default apiInitializer((api) => {
+  const settings = api.container.lookup("service:site-settings");
+
+  if (!settings.discourse_ai_enabled || !settings.ai_translation_enabled) {
+    return;
+  }
+
+  api.registerCustomPostMessageCallback(
+    "localized",
+    (topicController, data) => {
+      if (!cookie("content-localization-show-original")) {
+        const postStream = topicController.get("model.postStream");
+        postStream.triggerChangedPost(data.id, data.updated_at).then(() => {
+          topicController.appEvents.trigger("post-stream:refresh", {
+            id: data.id,
+          });
+        });
+      }
+    }
+  );
+});

assets/stylesheets/common/streaming.scss

@@ -111,7 +111,7 @@ mark.highlight {
   animation-name: mark-blink;
 }
 
-.composer-ai-helper-modal__loading {
+.composer-ai-helper-modal__loading.inline-diff {
   white-space: pre-wrap;
 }
 

assets/stylesheets/modules/ai-bot/common/bot-replies.scss

@@ -139,7 +139,7 @@ span.onebox-ai-llm-title {
   }
 }
 
-.ai-debug-modal__tokens span {
+.ai-debug-modal__tokens__count {
   display: block;
 }
 

assets/stylesheets/modules/ai-helper/common/ai-helper.scss

@@ -1,11 +1,13 @@
 @use "lib/viewport";
 
 .composer-ai-helper-modal {
-  .text-preview,
   .inline-diff {
     font-family: var(--d-font-family--monospace);
     font-variant-ligatures: none;
+  }
 
+  .text-preview,
+  .inline-diff {
     ins {
       background-color: var(--success-low);
       text-decoration: none;
@@ -55,13 +57,16 @@
   }
 
   &__old-value {
-    background-color: var(--danger-low);
+    white-space: pre-wrap;
+    border-left: 2px solid var(--danger);
+    padding-left: 1rem;
     color: var(--danger);
     margin-bottom: 1rem;
   }
 
   &__new-value {
-    background-color: var(--success-low);
+    border-left: 2px solid var(--success);
+    padding-left: 1rem;
     color: var(--success);
   }
 
@@ -77,7 +82,6 @@
 }
 
 .ai-composer-helper-menu {
-  padding: 0.25rem;
   max-width: 25rem;
   list-style: none;
 
@@ -701,7 +705,7 @@
     width: 100%;
     border-radius: 0;
     margin: 0;
-    padding: 0.5em 1rem;
+    padding: 0.7rem 1rem;
 
     &:focus,
     &:hover {

config/locales/server.en.yml

@@ -330,6 +330,15 @@ en:
         short_summarizer:
           name: "Summarizer (short form)"
           description: "Default persona used to power AI short summaries for topic lists' items"
+        concept_finder:
+          name: "Concept Finder"
+          description: "AI Bot specialized in identifying concepts and themes in content"
+        concept_matcher:
+          name: "Concept Matcher"
+          description: "AI Bot specialized in matching content against existing concepts"
+        concept_deduplicator:
+          name: "Concept Deduplicator"
+          description: "AI Bot specialized in deduplicating concepts"
       topic_not_found: "Summary unavailable, topic not found!"
       summarizing: "Summarizing topic"
       searching: "Searching for: '%{query}'"
@@ -549,6 +558,9 @@ en:
       discord_search:
         name: "Discord Search"
         description: "Adds the ability to search Discord channels"
+      inferred_concepts:
+        name: "Inferred Concepts"
+        description: "Classifies topics and posts into areas of interest / labels."
 
     errors:
       quota_exceeded: "You have exceeded the quota for this model. Please try again in %{relative_time}."