DEV: Re-introduce validator

- add sleep function for tool polling with rate limits
- Support base64 encoding for HTTP requests and uploads
-  Enhance forum researcher with cost warnings and comprehensive planning
- Add cancellation support for research operations
- Include feature_name parameter for bot analytics
- richer research support (OR queries)

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -50,7 +50,7 @@ module DiscourseAi
             )
           end
 
-          SiteSetting.ai_spam_detection_enabled = is_enabled
+          SiteSetting.set_and_log("ai_spam_detection_enabled", is_enabled, current_user)
         end
 
         render json: AiSpamSerializer.new(spam_config, root: false)

app/jobs/regular/create_ai_reply.rb

@@ -15,7 +15,7 @@ module ::Jobs
 
         bot = DiscourseAi::Personas::Bot.as(bot_user, persona: persona.new)
 
-        DiscourseAi::AiBot::Playground.new(bot).reply_to(post)
+        DiscourseAi::AiBot::Playground.new(bot).reply_to(post, feature_name: "bot")
       rescue DiscourseAi::Personas::Bot::BOT_NOT_FOUND
         Rails.logger.warn(
           "Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",

app/jobs/regular/detect_translate_post.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslatePost < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if args[:post_id].blank?
+
+      post = Post.find_by(id: args[:post_id])
+      return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        topic = post.topic
+        if topic.blank? || topic.category&.read_restricted? ||
+             topic.archetype == Archetype.private_message
+          return
+        end
+      end
+
+      begin
+        detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+
+        begin
+          DiscourseAi::Translation::PostLocalizer.localize(post, locale)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+
+      MessageBus.publish("/topic/#{post.topic_id}", type: :localized, id: post.id)
+    end
+  end
+end

app/jobs/regular/detect_translate_topic.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslateTopic < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if args[:topic_id].blank?
+
+      topic = Topic.find_by(id: args[:topic_id])
+      if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0
+        return
+      end
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        return if topic.category&.read_restricted?
+      end
+
+      begin
+        detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+
+        begin
+          DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+
+      MessageBus.publish("/topic/#{topic.id}", type: :localized, id: 1)
+    end
+  end
+end

app/jobs/regular/generate_inferred_concepts.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateInferredConcepts < ::Jobs::Base
+    sidekiq_options queue: "low"
+
+    # Process items to generate new concepts
+    #
+    # @param args [Hash] Contains job arguments
+    # @option args [String] :item_type Required - Type of items to process ('topics' or 'posts')
+    # @option args [Array<Integer>] :item_ids Required - List of item IDs to process
+    # @option args [Integer] :batch_size (100) Number of items to process in each batch
+    # @option args [Boolean] :match_only (false) Only match against existing concepts without generating new ones
+    def execute(args = {})
+      return if args[:item_ids].blank? || args[:item_type].blank?
+
+      if %w[topics posts].exclude?(args[:item_type])
+        Rails.logger.error("Invalid item_type for GenerateInferredConcepts: #{args[:item_type]}")
+        return
+      end
+
+      # Process items in smaller batches to avoid memory issues
+      batch_size = args[:batch_size] || 100
+
+      # Get the list of item IDs
+      item_ids = args[:item_ids]
+      match_only = args[:match_only] || false
+
+      # Process items in batches
+      item_ids.each_slice(batch_size) do |batch_item_ids|
+        process_batch(batch_item_ids, args[:item_type], match_only)
+      end
+    end
+
+    private
+
+    def process_batch(item_ids, item_type, match_only)
+      klass = item_type.singularize.classify.constantize
+      items = klass.where(id: item_ids)
+      manager = DiscourseAi::InferredConcepts::Manager.new
+
+      items.each do |item|
+        begin
+          process_item(item, item_type, match_only, manager)
+        rescue => e
+          Rails.logger.error(
+            "Error generating concepts from #{item_type.singularize} #{item.id}: #{e.message}\n#{e.backtrace.join("\n")}",
+          )
+        end
+      end
+    end
+
+    def process_item(item, item_type, match_only, manager)
+      # Use the Manager method that handles both identifying and creating concepts
+      if match_only
+        if item_type == "topics"
+          manager.match_topic_to_concepts(item)
+        else # posts
+          manager.match_post_to_concepts(item)
+        end
+      else
+        if item_type == "topics"
+          manager.generate_concepts_from_topic(item)
+        else # posts
+          manager.generate_concepts_from_post(item)
+        end
+      end
+    end
+  end
+end

app/jobs/regular/localize_categories.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizeCategories < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      cat_id = args[:from_category_id] || Category.order(:id).first&.id
+      last_id = nil
+
+      categories = Category.where("id >= ?", cat_id).order(:id).limit(BATCH_SIZE)
+      return if categories.empty?
+
+      categories.each do |category|
+        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
+          last_id = category.id
+          next
+        end
+
+        CategoryLocalization.transaction do
+          locales.each do |locale|
+            next if CategoryLocalization.exists?(category_id: category.id, locale: locale)
+            begin
+              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+            rescue FinalDestination::SSRFDetector::LookupFailedError
+              # do nothing, there are too many sporadic lookup failures
+            rescue => e
+              DiscourseAi::Translation::VerboseLogger.log(
+                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
+              )
+            end
+          end
+        end
+        last_id = category.id
+      end
+
+      if categories.size == BATCH_SIZE
+        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
+      end
+    end
+  end
+end

app/jobs/regular/localize_posts.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizePosts < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        posts =
+          Post
+            .joins(
+              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("posts.user_id > 0")
+            .where.not(raw: [nil, ""])
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("pl.id IS NULL")
+
+        if SiteSetting.ai_translation_backfill_limit_to_public_content
+          posts =
+            posts
+              .joins(:topic)
+              .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
+              .where.not(topics: { archetype: Archetype.private_message })
+        end
+
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          posts =
+            posts.where(
+              "posts.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        posts = posts.order(updated_at: :desc).limit(limit)
+
+        next if posts.empty?
+
+        posts.each do |post|
+          begin
+            DiscourseAi::Translation::PostLocalizer.localize(post, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
+      end
+    end
+  end
+end

app/jobs/regular/localize_topics.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizeTopics < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        topics =
+          Topic
+            .joins(
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("topics.user_id > 0")
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("tl.id IS NULL")
+
+        if SiteSetting.ai_translation_backfill_limit_to_public_content
+          topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+        end
+
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          topics =
+            topics.where(
+              "topics.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        topics = topics.order(updated_at: :desc).limit(limit)
+
+        next if topics.empty?
+
+        topics.each do |topic|
+          begin
+            DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseAi::Translation::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
+      end
+    end
+  end
+end

app/jobs/scheduled/category_localization_backfill.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Jobs
+  class CategoryLocalizationBackfill < ::Jobs::Scheduled
+    every 12.hours
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+
+      Jobs.enqueue(:localize_categories)
+    end
+  end
+end

app/jobs/scheduled/generate_concepts_from_popular_items.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateConceptsFromPopularItems < ::Jobs::Scheduled
+    every 1.day
+
+    # This job runs daily and generates new concepts from popular topics and posts
+    # It selects items based on engagement metrics and generates concepts from their content
+    def execute(_args)
+      return unless SiteSetting.inferred_concepts_enabled
+
+      process_popular_topics
+      process_popular_posts
+    end
+
+    private
+
+    def process_popular_topics
+      # Find candidate topics that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_topics(
+          limit: SiteSetting.inferred_concepts_daily_topics_limit || 20,
+          min_posts: SiteSetting.inferred_concepts_min_posts || 5,
+          min_likes: SiteSetting.inferred_concepts_min_likes || 10,
+          min_views: SiteSetting.inferred_concepts_min_views || 100,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate topics - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "topics",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "topics",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+
+    def process_popular_posts
+      # Find candidate posts that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_posts(
+          limit: SiteSetting.inferred_concepts_daily_posts_limit || 30,
+          min_likes: SiteSetting.inferred_concepts_post_min_likes || 5,
+          exclude_first_posts: true,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate posts - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "posts",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match against existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "posts",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+  end
+end

app/jobs/scheduled/post_localization_backfill.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Jobs
+  class PostLocalizationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
+    end
+  end
+end

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Jobs
+  class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    sidekiq_options retry: false
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      posts =
+        Post
+          .where(locale: nil)
+          .where(deleted_at: nil)
+          .where("posts.user_id > 0")
+          .where.not(raw: [nil, ""])
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        public_categories = Category.where(read_restricted: false).pluck(:id)
+        posts =
+          posts
+            .joins(:topic)
+            .where(topics: { category_id: public_categories })
+            .where(topics: { archetype: "regular" })
+      end
+
+      if SiteSetting.ai_translation_backfill_max_age_days > 0
+        posts =
+          posts.where(
+            "posts.created_at > ?",
+            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
+      return if posts.empty?
+
+      posts.each do |post|
+        begin
+          DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to detect post #{post.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseAi::Translation::VerboseLogger.log("Detected #{posts.size} post locales")
+    end
+  end
+end

app/jobs/scheduled/topic_localization_backfill.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicLocalizationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
+    end
+  end
+end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    sidekiq_options retry: false
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      limit = SiteSetting.ai_translation_backfill_rate
+      return if limit == 0
+
+      topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+      end
+
+      if SiteSetting.ai_translation_backfill_max_age_days > 0
+        topics =
+          topics.where(
+            "topics.created_at > ?",
+            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      topics = topics.order(updated_at: :desc).limit(limit)
+      return if topics.empty?
+
+      topics.each do |topic|
+        begin
+          DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to detect topic #{topic.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseAi::Translation::VerboseLogger.log("Detected #{topics.size} topic locales")
+    end
+  end
+end

app/models/ai_tool.rb

@@ -22,6 +22,8 @@ class AiTool < ActiveRecord::Base
               message: I18n.t("discourse_ai.tools.name.characters"),
             }
 
+  validate :validate_parameters_enum
+
   def signature
     {
       name: function_call_name,
@@ -57,6 +59,30 @@ class AiTool < ActiveRecord::Base
     end
   end
 
+  def validate_parameters_enum
+    return unless parameters.is_a?(Array)
+
+    parameters.each_with_index do |param, index|
+      next if !param.is_a?(Hash) || !param.key?("enum")
+      enum_values = param["enum"]
+
+      if enum_values.empty?
+        errors.add(
+          :parameters,
+          "Parameter '#{param["name"]}' at index #{index}: enum cannot be empty",
+        )
+        next
+      end
+
+      if enum_values.uniq.length != enum_values.length
+        errors.add(
+          :parameters,
+          "Parameter '#{param["name"]}' at index #{index}: enum values must be unique",
+        )
+      end
+    end
+  end
+
   def self.preamble
     <<~JS
      /**
@@ -142,6 +168,7 @@ class AiTool < ActiveRecord::Base
      *      base_64_content (string): Base64 encoded content of the file.
      *    Returns: { id: number, url: string, short_url: string } - Details of the created upload record.
      *
+     *    upload.getUrl(shortUrl): Given a short URL, eg upload://12345, returns the full CDN friendly URL of the upload.
      * 5. chain
      *    Controls the execution flow.
      *

app/models/inferred_concept.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class InferredConcept < ActiveRecord::Base
+  has_many :inferred_concept_topics
+  has_many :topics, through: :inferred_concept_topics
+
+  has_many :inferred_concept_posts
+  has_many :posts, through: :inferred_concept_posts
+
+  validates :name, presence: true, uniqueness: true
+end
+
+# == Schema Information
+#
+# Table name: inferred_concepts
+#
+#  id         :bigint           not null, primary key
+#  name       :string           not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concepts_on_name  (name) UNIQUE
+#

app/models/inferred_concept_post.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class InferredConceptPost < ActiveRecord::Base
+  belongs_to :inferred_concept
+  belongs_to :post
+
+  validates :inferred_concept_id, presence: true
+  validates :post_id, presence: true
+  validates :inferred_concept_id, uniqueness: { scope: :post_id }
+end
+
+# == Schema Information
+#
+# Table name: inferred_concept_posts
+#
+#  inferred_concept_id :bigint
+#  post_id             :bigint
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concept_posts_on_inferred_concept_id  (inferred_concept_id)
+#  index_inferred_concept_posts_uniqueness              (post_id,inferred_concept_id) UNIQUE
+#

app/models/inferred_concept_topic.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class InferredConceptTopic < ActiveRecord::Base
+  belongs_to :inferred_concept
+  belongs_to :topic
+
+  validates :inferred_concept_id, presence: true
+  validates :topic_id, presence: true
+  validates :inferred_concept_id, uniqueness: { scope: :topic_id }
+end
+
+# == Schema Information
+#
+# Table name: inferred_concept_topics
+#
+#  inferred_concept_id :bigint
+#  topic_id            :bigint
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concept_topics_on_inferred_concept_id  (inferred_concept_id)
+#  index_inferred_concept_topics_uniqueness              (topic_id,inferred_concept_id) UNIQUE
+#

app/serializers/ai_inferred_concept_post_serializer.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class AiInferredConceptPostSerializer < ApplicationSerializer
+  attributes :id,
+             :post_number,
+             :topic_id,
+             :topic_title,
+             :username,
+             :avatar_template,
+             :created_at,
+             :updated_at,
+             :excerpt,
+             :truncated,
+             :inferred_concepts
+
+  def avatar_template
+    User.avatar_template(object.username, object.uploaded_avatar_id)
+  end
+
+  def excerpt
+    Post.excerpt(object.cooked)
+  end
+
+  def truncated
+    object.cooked.length > SiteSetting.post_excerpt_maxlength
+  end
+
+  def inferred_concepts
+    ActiveModel::ArraySerializer.new(
+      object.inferred_concepts,
+      each_serializer: InferredConceptSerializer,
+    )
+  end
+end

app/serializers/inferred_concept_serializer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class InferredConceptSerializer < ApplicationSerializer
+  attributes :id, :name, :created_at, :updated_at
+end

assets/javascripts/discourse/components/ai-bot-sidebar-new-conversation.gjs

@@ -5,6 +5,7 @@ import DButton from "discourse/components/d-button";
 import { AI_CONVERSATIONS_PANEL } from "../services/ai-conversations-sidebar-manager";
 
 export default class AiBotSidebarNewConversation extends Component {
+  @service appEvents;
   @service router;
   @service sidebarState;
 
@@ -14,6 +15,8 @@ export default class AiBotSidebarNewConversation extends Component {
 
   @action
   routeTo() {
+    this.appEvents.trigger("discourse-ai:new-conversation-btn-clicked");
+
     if (this.router.currentRouteName !== "discourse-ai-bot-conversations") {
       this.router.transitionTo("/discourse-ai/ai-bot/conversations");
     }

assets/javascripts/discourse/components/ai-persona-example.gjs

@@ -45,7 +45,7 @@ export default class AiPersonaCollapsableExample extends Component {
               (if (eq pairIdx 0) "user" "model")
             )
           }}
-          @validation="required|length:1,100"
+          @validation="required|length:1,5000"
           @disabled={{@system}}
           as |field|
         >

assets/javascripts/discourse/components/ai-tool-editor-form.gjs

@@ -30,11 +30,16 @@ export default class AiToolEditorForm extends Component {
 
   get formData() {
     const parameters = (this.args.editingModel.parameters ?? []).map(
-      (parameter) => ({
-        ...parameter,
-        isEnum: !!parameter.enum?.length,
-        enum: (parameter.enum ??= []),
-      })
+      (parameter) => {
+        const mappedParameter = {
+          ...parameter,
+        };
+        mappedParameter.isEnum = parameter.enum && parameter.enum.length > 0;
+        if (!mappedParameter.isEnum) {
+          delete mappedParameter.enum;
+        }
+        return mappedParameter;
+      }
     );
 
     return {
@@ -63,8 +68,19 @@ export default class AiToolEditorForm extends Component {
   async save(data) {
     this.isSaving = true;
 
+    // we injected a isEnum thing, we need to clean it up
+    const copiedData = JSON.parse(JSON.stringify(data));
+    if (copiedData.parameters) {
+      copiedData.parameters.forEach((parameter) => {
+        if (!parameter.isEnum) {
+          delete parameter.enum;
+        }
+        delete parameter.isEnum;
+      });
+    }
+
     try {
-      await this.args.model.save(data);
+      await this.args.model.save(copiedData);
 
       this.toasts.success({
         data: { message: i18n("discourse_ai.tools.saved") },

assets/javascripts/discourse/components/modal/ai-persona-response-format-editor.gjs

@@ -22,10 +22,20 @@ export default class AiPersonaResponseFormatEditor extends Component {
           type: "string",
         },
         type: {
+          type: "string",
+          enum: ["string", "integer", "boolean", "array"],
+        },
+        array_type: {
           type: "string",
           enum: ["string", "integer", "boolean"],
+          options: {
+            dependencies: {
+              type: "array",
+            },
+          },
         },
       },
+      required: ["key", "type"],
     },
   };
 
@@ -41,7 +51,11 @@ export default class AiPersonaResponseFormatEditor extends Component {
     const toDisplay = {};
 
     this.args.data.response_format.forEach((keyDesc) => {
-      toDisplay[keyDesc.key] = keyDesc.type;
+      if (keyDesc.type === "array") {
+        toDisplay[keyDesc.key] = `[${keyDesc.array_type}]`;
+      } else {
+        toDisplay[keyDesc.key] = keyDesc.type;
+      }
     });
 
     return prettyJSON(toDisplay);

assets/javascripts/discourse/components/modal/debug-ai-modal.gjs

@@ -144,11 +144,11 @@ export default class DebugAiModal extends Component {
             >{{i18n "discourse_ai.ai_bot.debug_ai_modal.response"}}</a></li>
         </ul>
         <div class="ai-debug-modal__tokens">
-          <span>
+          <span class="ai-debug-modal__tokens__count">
             {{i18n "discourse_ai.ai_bot.debug_ai_modal.request_tokens"}}
             {{this.info.request_tokens}}
           </span>
-          <span>
+          <span class="ai-debug-modal__tokens__count">
             {{i18n "discourse_ai.ai_bot.debug_ai_modal.response_tokens"}}
             {{this.info.response_tokens}}
           </span>

assets/javascripts/discourse/components/modal/diff-modal.gjs

@@ -5,6 +5,7 @@ import didInsert from "@ember/render-modifiers/modifiers/did-insert";
 import willDestroy from "@ember/render-modifiers/modifiers/will-destroy";
 import { service } from "@ember/service";
 import { htmlSafe } from "@ember/template";
+import { or } from "truth-helpers";
 import CookText from "discourse/components/cook-text";
 import DButton from "discourse/components/d-button";
 import DModal from "discourse/components/d-modal";
@@ -18,14 +19,16 @@ import DiffStreamer from "../../lib/diff-streamer";
 import SmoothStreamer from "../../lib/smooth-streamer";
 import AiIndicatorWave from "../ai-indicator-wave";
 
+const CHANNEL = "/discourse-ai/ai-helper/stream_composer_suggestion";
+
 export default class ModalDiffModal extends Component {
   @service currentUser;
   @service messageBus;
 
   @tracked loading = false;
   @tracked finalResult = "";
-  @tracked selectedText = escapeExpression(this.args.model.selectedText);
-  @tracked diffStreamer = new DiffStreamer(this.selectedText);
+  @tracked escapedSelectedText = escapeExpression(this.args.model.selectedText);
+  @tracked diffStreamer = new DiffStreamer(this.args.model.selectedText);
   @tracked suggestion = "";
   @tracked
   smoothStreamer = new SmoothStreamer(
@@ -39,17 +42,35 @@ export default class ModalDiffModal extends Component {
   }
 
   get diffResult() {
+    if (this.loading) {
+      return this.escapedSelectedText;
+    }
+
     if (this.diffStreamer.diff?.length > 0) {
       return this.diffStreamer.diff;
     }
 
     // Prevents flash by showing the
     // original text when the diff is empty
-    return this.selectedText;
+    return this.escapedSelectedText;
+  }
+
+  get smoothStreamerResult() {
+    if (this.loading) {
+      return this.escapedSelectedText;
+    }
+
+    return this.smoothStreamer.renderedText;
   }
 
   get isStreaming() {
-    return this.diffStreamer.isStreaming || this.smoothStreamer.isStreaming;
+    // diffStreamer stops Streaming when it is finished with a chunk, looking at isDone is safe
+    // it starts off not done
+    if (this.args.model.showResultAsDiff) {
+      return !this.diffStreamer.isDone;
+    }
+
+    return this.smoothStreamer.isStreaming;
   }
 
   get primaryBtnLabel() {
@@ -64,25 +85,23 @@ export default class ModalDiffModal extends Component {
 
   @bind
   subscribe() {
-    const channel = "/discourse-ai/ai-helper/stream_composer_suggestion";
-    this.messageBus.subscribe(channel, this.updateResult);
+    this.messageBus.subscribe(CHANNEL, this.updateResult);
   }
 
   @bind
-  unsubscribe() {
-    const channel = "/discourse-ai/ai-helper/stream_composer_suggestion";
-    this.messageBus.subscribe(channel, this.updateResult);
+  cleanup() {
+    // stop all callbacks so it does not end up streaming pointlessly
+    this.smoothStreamer.resetStreaming();
+    this.diffStreamer.reset();
+    this.messageBus.unsubscribe(CHANNEL, this.updateResult);
   }
 
   @action
-  async updateResult(result) {
+  updateResult(result) {
     this.loading = false;
 
     if (result.done) {
       this.finalResult = result.result;
-    }
-
-    if (result.done) {
       this.loading = false;
     }
 
@@ -105,7 +124,7 @@ export default class ModalDiffModal extends Component {
         data: {
           location: "composer",
           mode: this.args.model.mode,
-          text: this.selectedText,
+          text: this.args.model.selectedText,
           custom_prompt: this.args.model.customPromptValue,
           force_default_locale: true,
           client_id: this.messageBus.clientId,
@@ -122,7 +141,7 @@ export default class ModalDiffModal extends Component {
 
     if (this.suggestion) {
       this.args.model.toolbarEvent.replaceText(
-        this.selectedText,
+        this.args.model.selectedText,
         this.suggestion
       );
     }
@@ -131,8 +150,12 @@ export default class ModalDiffModal extends Component {
       this.finalResult?.length > 0
         ? this.finalResult
         : this.diffStreamer.suggestion;
+
     if (this.args.model.showResultAsDiff && finalResult) {
-      this.args.model.toolbarEvent.replaceText(this.selectedText, finalResult);
+      this.args.model.toolbarEvent.replaceText(
+        this.args.model.selectedText,
+        finalResult
+      );
     }
   }
 
@@ -145,45 +168,40 @@ export default class ModalDiffModal extends Component {
       <:body>
         <div
           {{didInsert this.subscribe}}
-          {{willDestroy this.unsubscribe}}
+          {{willDestroy this.cleanup}}
           class="text-preview"
         >
-          {{#if this.loading}}
-            <div class="composer-ai-helper-modal__loading">
-              {{[email protected]~}}
-            </div>
-          {{else}}
-            <div
-              class={{concatClass
-                "composer-ai-helper-modal__suggestion"
-                "streamable-content"
-                (if this.isStreaming "streaming")
-                (if @model.showResultAsDiff "inline-diff")
-                (if this.diffStreamer.isThinking "thinking")
-              }}
-            >
-              {{~#if @model.showResultAsDiff~}}
-                <span class="diff-inner">{{htmlSafe this.diffResult}}</span>
+          <div
+            class={{concatClass
+              "composer-ai-helper-modal__suggestion"
+              "streamable-content"
+              (if this.isStreaming "streaming")
+              (if @model.showResultAsDiff "inline-diff")
+              (if this.diffStreamer.isThinking "thinking")
+              (if this.loading "composer-ai-helper-modal__loading")
+            }}
+          >
+            {{~#if @model.showResultAsDiff~}}
+              <span class="diff-inner">{{htmlSafe this.diffResult}}</span>
+            {{else}}
+              {{#if (or this.loading this.smoothStreamer.isStreaming)}}
+                <CookText
+                  @rawText={{this.smoothStreamerResult}}
+                  class="cooked"
+                />
               {{else}}
-                {{#if this.smoothStreamer.isStreaming}}
+                <div class="composer-ai-helper-modal__old-value">
+                  {{~this.escapedSelectedText~}}
+                </div>
+                <div class="composer-ai-helper-modal__new-value">
                   <CookText
-                    @rawText={{this.smoothStreamer.renderedText}}
+                    @rawText={{this.smoothStreamerResult}}
                     class="cooked"
                   />
-                {{else}}
-                  <div class="composer-ai-helper-modal__old-value">
-                    {{@model.selectedText}}
-                  </div>
-                  <div class="composer-ai-helper-modal__new-value">
-                    <CookText
-                      @rawText={{this.smoothStreamer.renderedText}}
-                      class="cooked"
-                    />
-                  </div>
-                {{/if}}
+                </div>
               {{/if}}
-            </div>
-          {{/if}}
+            {{/if}}
+          </div>
         </div>
       </:body>
 

assets/javascripts/discourse/connectors/full-page-search-below-search-header/ai-full-page-discobot-discoveries.gjs

@@ -15,13 +15,12 @@ export default class AiFullPageDiscobotDiscoveries extends Component {
     );
   }
 
+  @service capabilities;
   @service discobotDiscoveries;
   @service site;
 
   get previewLength() {
-    // todo: replace with js breakpoint API
-    // https://github.com/discourse/discourse/pull/32060
-    if (this.site.mobileView || this.site.narrowDesktopView) {
+    if (!this.capabilities.viewport.md) {
       return 50;
     } else {
       return 10000;

assets/javascripts/discourse/connectors/post-text-buttons/ai-post-helper-trigger.gjs

@@ -19,7 +19,6 @@ export default class AiPostHelperTrigger extends Component {
   @tracked menuState = this.MENU_STATES.triggers;
   @tracked showMainButtons = true;
   @tracked showAiButtons = true;
-  @tracked originalPostHTML = null;
   @tracked postHighlighted = false;
   @tracked currentMenu = this.menu.getByIdentifier(
     "post-text-selection-toolbar"
@@ -45,7 +44,6 @@ export default class AiPostHelperTrigger extends Component {
       return;
     }
 
-    this.originalPostHTML = postElement.innerHTML;
     this.selectedText = this.args.outletArgs.data.quoteState.buffer;
 
     const selection = window.getSelection();
@@ -121,7 +119,15 @@ export default class AiPostHelperTrigger extends Component {
       return;
     }
 
-    postElement.innerHTML = this.originalPostHTML;
+    const highlightedSpans = postElement.querySelectorAll(
+      "span.ai-helper-highlighted-selection"
+    );
+
+    highlightedSpans.forEach((span) => {
+      const textNode = document.createTextNode(span.textContent);
+      span.parentNode.replaceChild(textNode, span);
+    });
+
     this.postHighlighted = false;
   }
 

assets/javascripts/discourse/connectors/topic-map-participants-after/ai-conversation-invite.gjs

@@ -18,8 +18,8 @@ export default class AiConversationInvite extends Component {
 
   get participants() {
     const participants = [
-      ...this.header.topicInfo.details.allowed_users,
-      ...this.header.topicInfo.details.allowed_groups,
+      ...(this.header.topicInfo.details?.allowed_users ?? []),
+      ...(this.header.topicInfo.details?.allowed_groups ?? []),
     ];
     return participants;
   }