DEV: Re-introduce validator

DEV: improve tool infra, improve forum researcher prompts, improve logging (#1391 )
- add sleep function for tool polling with rate limits - Support base64 encoding for HTTP requests and uploads - Enhance forum researcher with cost warnings and comprehensive planning - Add cancellation support for research operations - Include feature_name parameter for bot analytics - richer research support (OR queries)
2025-06-03 06:30:50 -07:00 · 2025-06-03 15:17:55 +10:00 · 2025-06-02 13:57:34 -07:00 · 2025-06-02 17:56:47 -03:00 · 2025-06-02 14:29:20 -03:00 · 2025-05-30 21:47:30 +01:00
377 changed files with 8808 additions and 3172 deletions
@@ -1,19 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class AdminPluginsShowDiscourseAiAgentsEdit extends DiscourseRoute {
-  async model(params) {
-    const allAgents = this.modelFor(
-      "adminPlugins.show.discourse-ai-agents"
-    );
-    const id = parseInt(params.id, 10);
-    return allAgents.findBy("id", id);
-  }
-
-  setupController(controller, model) {
-    super.setupController(controller, model);
-    controller.set(
-      "allAgents",
-      this.modelFor("adminPlugins.show.discourse-ai-agents")
-    );
-  }
-}
@@ -1,7 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiAiAgentsRoute extends DiscourseRoute {
-  model() {
-    return this.store.findAll("ai-agent");
-  }
-}
@@ -0,0 +1,19 @@
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
+  async model(params) {
+    const allPersonas = this.modelFor(
+      "adminPlugins.show.discourse-ai-personas"
+    );
+    const id = parseInt(params.id, 10);
+    return allPersonas.findBy("id", id);
+  }
+
+  setupController(controller, model) {
+    super.setupController(controller, model);
+    controller.set(
+      "allPersonas",
+      this.modelFor("adminPlugins.show.discourse-ai-personas")
+    );
+  }
+}
@@ -1,16 +1,16 @@
 import { AUTO_GROUPS } from "discourse/lib/constants";
 import DiscourseRoute from "discourse/routes/discourse";

-export default class AdminPluginsShowDiscourseAiAgentsNew extends DiscourseRoute {
+export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
  async model() {
-    const record = this.store.createRecord("ai-agent");
+    const record = this.store.createRecord("ai-persona");
    record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
    record.set("rag_uploads", []);
    // these match the defaults on the table
    record.set("rag_chunk_tokens", 374);
    record.set("rag_chunk_overlap_tokens", 10);
    record.set("rag_conversation_chunks", 10);
-    record.set("allow_agentl_messages", true);
+    record.set("allow_personal_messages", true);
    record.set("tool_details", false);
    return record;
  }
@@ -18,8 +18,8 @@ export default class AdminPluginsShowDiscourseAiAgentsNew extends DiscourseRoute
  setupController(controller, model) {
    super.setupController(controller, model);
    controller.set(
-      "allAgents",
-      this.modelFor("adminPlugins.show.discourse-ai-agents")
+      "allPersonas",
+      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
  }
 }
@@ -0,0 +1,7 @@
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default class DiscourseAiAiPersonasRoute extends DiscourseRoute {
+  model() {
+    return this.store.findAll("ai-persona");
+  }
+}
@@ -1,4 +0,0 @@
-<AiAgentListEditor
-  @agents={{this.allAgents}}
-  @currentAgent={{this.model}}
-/>
@@ -1 +0,0 @@
-<AiAgentListEditor @agents={{this.model}} />
@@ -1,4 +0,0 @@
-<AiAgentListEditor
-  @agents={{this.allAgents}}
-  @currentAgent={{this.model}}
-/>
@@ -15,7 +15,7 @@ export default RouteTemplate(
      const prefix = "discourse_ai.features.list.header";
      return [
        i18n(`${prefix}.name`),
-        i18n(`${prefix}.agent`),
+        i18n(`${prefix}.persona`),
        i18n(`${prefix}.groups`),
        "",
      ];
@@ -75,21 +75,21 @@ export default RouteTemplate(
                      </span>
                    </td>
                    <td
-                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__agent"
+                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__persona"
                    >
                      <DButton
-                        class="btn-flat btn-small ai-feature-list__row-item-agent"
-                        @translatedLabel={{feature.agent.name}}
-                        @route="adminPlugins.show.discourse-ai-agents.edit"
-                        @routeModels={{feature.agent.id}}
+                        class="btn-flat btn-small ai-feature-list__row-item-persona"
+                        @translatedLabel={{feature.persona.name}}
+                        @route="adminPlugins.show.discourse-ai-personas.edit"
+                        @routeModels={{feature.persona.id}}
                      />
                    </td>
                    <td
                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__groups"
                    >
-                      {{#if (gt feature.agent.allowed_groups.length 0)}}
+                      {{#if (gt feature.persona.allowed_groups.length 0)}}
                        <ul class="ai-feature-list__row-item-groups">
-                          {{#each feature.agent.allowed_groups as |group|}}
+                          {{#each feature.persona.allowed_groups as |group|}}
                            <li>{{group.name}}</li>
                          {{/each}}
                        </ul>
@@ -0,0 +1,4 @@
+<AiPersonaListEditor
+  @personas={{this.allPersonas}}
+  @currentPersona={{this.model}}
+/>
@@ -0,0 +1 @@
+<AiPersonaListEditor @personas={{this.model}} />
@@ -0,0 +1,4 @@
+<AiPersonaListEditor
+  @personas={{this.allPersonas}}
+  @currentPersona={{this.model}}
+/>
@@ -1,4 +1,4 @@
-<section class="ai-agent-tool-editor__current admin-detail pull-left">
+<section class="ai-persona-tool-editor__current admin-detail pull-left">
  <AiToolEditor
    @tools={{this.allTools}}
    @model={{this.model}}
@@ -1,4 +1,4 @@
-<section class="ai-agent-tool-editor__current admin-detail pull-left">
+<section class="ai-persona-tool-editor__current admin-detail pull-left">
  <AiToolEditor
    @tools={{this.allTools}}
    @model={{this.model}}
@@ -17,19 +17,19 @@ module DiscourseAi
      private

      def serialize_features(features)
-        features.map { |feature| feature.merge(agent: serialize_agent(feature[:agent])) }
+        features.map { |feature| feature.merge(persona: serialize_persona(feature[:persona])) }
      end

      def serialize_feature(feature)
        return nil if feature.blank?

-        feature.merge(agent: serialize_agent(feature[:agent]))
+        feature.merge(persona: serialize_persona(feature[:persona]))
      end

-      def serialize_agent(agent)
-        return nil if agent.blank?
+      def serialize_persona(persona)
+        return nil if persona.blank?

-        serialize_data(agent, AiFeaturesAgentSerializer, root: false)
+        serialize_data(persona, AiFeaturesPersonaSerializer, root: false)
      end
    end
  end
@@ -2,20 +2,20 @@

 module DiscourseAi
  module Admin
-    class AiAgentsController < ::Admin::AdminController
+    class AiPersonasController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME

-      before_action :find_ai_agent, only: %i[edit update destroy create_user]
+      before_action :find_ai_persona, only: %i[edit update destroy create_user]

      def index
-        ai_agents =
-          AiAgent.ordered.map do |agent|
+        ai_personas =
+          AiPersona.ordered.map do |persona|
            # we use a special serializer here cause names and descriptions are
-            # localized for system agents
-            LocalizedAiAgentSerializer.new(agent, root: false)
+            # localized for system personas
+            LocalizedAiPersonaSerializer.new(persona, root: false)
          end
        tools =
-          DiscourseAi::Agents::Agent.all_available_tools.map do |tool|
+          DiscourseAi::Personas::Persona.all_available_tools.map do |tool|
            AiToolSerializer.new(tool, root: false)
          end
        AiTool
@@ -36,7 +36,7 @@ module DiscourseAi
            allowed_seeded_llm_ids: SiteSetting.ai_bot_allowed_seeded_models_map,
          )
        render json: {
-                 ai_agents: ai_agents,
+                 ai_personas: ai_personas,
                 meta: {
                   tools: tools,
                   llms: llms,
@@ -51,55 +51,55 @@ module DiscourseAi
      end

      def edit
-        render json: LocalizedAiAgentSerializer.new(@ai_agent)
+        render json: LocalizedAiPersonaSerializer.new(@ai_persona)
      end

      def create
-        ai_agent = AiAgent.new(ai_agent_params.except(:rag_uploads))
-        if ai_agent.save
-          RagDocumentFragment.link_target_and_uploads(ai_agent, attached_upload_ids)
+        ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
+        if ai_persona.save
+          RagDocumentFragment.link_target_and_uploads(ai_persona, attached_upload_ids)

          render json: {
-                   ai_agent: LocalizedAiAgentSerializer.new(ai_agent, root: false),
+                   ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
                 },
                 status: :created
        else
-          render_json_error ai_agent
+          render_json_error ai_persona
        end
      end

      def create_user
-        user = @ai_agent.create_user!
+        user = @ai_persona.create_user!
        render json: BasicUserSerializer.new(user, root: "user")
      end

      def update
-        if @ai_agent.update(ai_agent_params.except(:rag_uploads))
-          RagDocumentFragment.update_target_uploads(@ai_agent, attached_upload_ids)
+        if @ai_persona.update(ai_persona_params.except(:rag_uploads))
+          RagDocumentFragment.update_target_uploads(@ai_persona, attached_upload_ids)

-          render json: LocalizedAiAgentSerializer.new(@ai_agent, root: false)
+          render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
        else
-          render_json_error @ai_agent
+          render_json_error @ai_persona
        end
      end

      def destroy
-        if @ai_agent.destroy
+        if @ai_persona.destroy
          head :no_content
        else
-          render_json_error @ai_agent
+          render_json_error @ai_persona
        end
      end

      def stream_reply
-        agent =
-          AiAgent.find_by(name: params[:agent_name]) ||
-            AiAgent.find_by(id: params[:agent_id])
-        return render_json_error(I18n.t("discourse_ai.errors.agent_not_found")) if agent.nil?
+        persona =
+          AiPersona.find_by(name: params[:persona_name]) ||
+            AiPersona.find_by(id: params[:persona_id])
+        return render_json_error(I18n.t("discourse_ai.errors.persona_not_found")) if persona.nil?

-        return render_json_error(I18n.t("discourse_ai.errors.agent_disabled")) if !agent.enabled
+        return render_json_error(I18n.t("discourse_ai.errors.persona_disabled")) if !persona.enabled

-        if agent.default_llm.blank?
+        if persona.default_llm.blank?
          return render_json_error(I18n.t("discourse_ai.errors.no_default_llm"))
        end

@@ -107,8 +107,8 @@ module DiscourseAi
          return render_json_error(I18n.t("discourse_ai.errors.no_query_specified"))
        end

-        if !agent.user_id
-          return render_json_error(I18n.t("discourse_ai.errors.no_user_for_agent"))
+        if !persona.user_id
+          return render_json_error(I18n.t("discourse_ai.errors.no_user_for_persona"))
        end

        if !params[:username] && !params[:user_unique_id]
@@ -142,7 +142,7 @@ module DiscourseAi

        DiscourseAi::AiBot::ResponseHttpStreamer.queue_streamed_reply(
          io: io,
-          agent: agent,
+          persona: persona,
          user: user,
          topic: topic,
          query: params[:query].to_s,
@@ -178,17 +178,17 @@ module DiscourseAi
        end
      end

-      def find_ai_agent
-        @ai_agent = AiAgent.find(params[:id])
+      def find_ai_persona
+        @ai_persona = AiPersona.find(params[:id])
      end

      def attached_upload_ids
-        ai_agent_params[:rag_uploads].to_a.map { |h| h[:id] }
+        ai_persona_params[:rag_uploads].to_a.map { |h| h[:id] }
      end

-      def ai_agent_params
+      def ai_persona_params
        permitted =
-          params.require(:ai_agent).permit(
+          params.require(:ai_persona).permit(
            :name,
            :description,
            :enabled,
@@ -209,7 +209,7 @@ module DiscourseAi
            :allow_chat_channel_mentions,
            :allow_chat_direct_messages,
            :allow_topic_mentions,
-            :allow_agentl_messages,
+            :allow_personal_messages,
            :tool_details,
            :forced_tool_count,
            :force_default_llm,
@@ -217,15 +217,15 @@ module DiscourseAi
            rag_uploads: [:id],
          )

-        if tools = params.dig(:ai_agent, :tools)
+        if tools = params.dig(:ai_persona, :tools)
          permitted[:tools] = permit_tools(tools)
        end

-        if response_format = params.dig(:ai_agent, :response_format)
+        if response_format = params.dig(:ai_persona, :response_format)
          permitted[:response_format] = permit_response_format(response_format)
        end

-        if examples = params.dig(:ai_agent, :examples)
+        if examples = params.dig(:ai_persona, :examples)
          permitted[:examples] = permit_examples(examples)
        end

@@ -6,8 +6,8 @@ module DiscourseAi
      requires_plugin ::DiscourseAi::PLUGIN_NAME

      def indexing_status_check
-        if params[:target_type] == "AiAgent"
-          @target = AiAgent.find(params[:target_id])
+        if params[:target_type] == "AiPersona"
+          @target = AiPersona.find(params[:target_id])
        elsif params[:target_type] == "AiTool"
          @target = AiTool.find(params[:target_id])
        else
@@ -46,17 +46,17 @@ module DiscourseAi
      end

      def discover
-        ai_agent =
-          AiAgent
-            .all_agents(enabled_only: false)
-            .find { |agent| agent.id == SiteSetting.ai_bot_discover_agent.to_i }
+        ai_persona =
+          AiPersona
+            .all_personas(enabled_only: false)
+            .find { |persona| persona.id == SiteSetting.ai_bot_discover_persona.to_i }

-        if ai_agent.nil? || !current_user.in_any_groups?(ai_agent.allowed_group_ids.to_a)
+        if ai_persona.nil? || !current_user.in_any_groups?(ai_persona.allowed_group_ids.to_a)
          raise Discourse::InvalidAccess.new
        end

-        if ai_agent.default_llm_id.blank?
-          render_json_error "Discover agent is missing a default LLM model.", status: 503
+        if ai_persona.default_llm_id.blank?
+          render_json_error "Discover persona is missing a default LLM model.", status: 503
          return
        end

@@ -77,7 +77,7 @@ module DiscourseAi

        user = User.find(params[:user_id])

-        bot_user_id = AiAgent.find_by(id: SiteSetting.ai_bot_discover_agent).user_id
+        bot_user_id = AiPersona.find_by(id: SiteSetting.ai_bot_discover_persona).user_id
        bot_username = User.find_by(id: bot_user_id).username

        query = params[:query]
@@ -11,12 +11,12 @@ module ::Jobs
      message = ::Chat::Message.find_by(id: args[:message_id])
      return if message.blank?

-      agentClass =
-        DiscourseAi::Agents::Agent.find_by(id: args[:agent_id], user: message.user)
-      return if agentClass.blank?
+      personaClass =
+        DiscourseAi::Personas::Persona.find_by(id: args[:persona_id], user: message.user)
+      return if personaClass.blank?

-      user = User.find_by(id: agentClass.user_id)
-      bot = DiscourseAi::Agents::Bot.as(user, agent: agentClass.new)
+      user = User.find_by(id: personaClass.user_id)
+      bot = DiscourseAi::Personas::Bot.as(user, persona: personaClass.new)

      DiscourseAi::AiBot::Playground.new(bot).reply_to_chat_message(
        message,
@@ -7,18 +7,18 @@ module ::Jobs
    def execute(args)
      return unless bot_user = User.find_by(id: args[:bot_user_id])
      return unless post = Post.includes(:topic).find_by(id: args[:post_id])
-      agent_id = args[:agent_id]
+      persona_id = args[:persona_id]

      begin
-        agent = DiscourseAi::Agents::Agent.find_by(user: post.user, id: agent_id)
-        raise DiscourseAi::Agents::Bot::BOT_NOT_FOUND if agent.nil?
+        persona = DiscourseAi::Personas::Persona.find_by(user: post.user, id: persona_id)
+        raise DiscourseAi::Personas::Bot::BOT_NOT_FOUND if persona.nil?

-        bot = DiscourseAi::Agents::Bot.as(bot_user, agent: agent.new)
+        bot = DiscourseAi::Personas::Bot.as(bot_user, persona: persona.new)

-        DiscourseAi::AiBot::Playground.new(bot).reply_to(post)
-      rescue DiscourseAi::Agents::Bot::BOT_NOT_FOUND
+        DiscourseAi::AiBot::Playground.new(bot).reply_to(post, feature_name: "bot")
+      rescue DiscourseAi::Personas::Bot::BOT_NOT_FOUND
        Rails.logger.warn(
-          "Bot not found for post #{post.id} - perhaps agent was deleted or bot was disabled",
+          "Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",
        )
      end
    end
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslatePost < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if args[:post_id].blank?
+
+      post = Post.find_by(id: args[:post_id])
+      return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        topic = post.topic
+        if topic.blank? || topic.category&.read_restricted? ||
+             topic.archetype == Archetype.private_message
+          return
+        end
+      end
+
+      begin
+        detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+
+        begin
+          DiscourseAi::Translation::PostLocalizer.localize(post, locale)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+
+      MessageBus.publish("/topic/#{post.topic_id}", type: :localized, id: post.id)
+    end
+  end
+end
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslateTopic < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if args[:topic_id].blank?
+
+      topic = Topic.find_by(id: args[:topic_id])
+      if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0
+        return
+      end
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        return if topic.category&.read_restricted?
+      end
+
+      begin
+        detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
+      rescue FinalDestination::SSRFDetector::LookupFailedError
+        # this job is non-critical
+        # the backfill job will handle failures
+        return
+      end
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+
+        begin
+          DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+
+      MessageBus.publish("/topic/#{topic.id}", type: :localized, id: 1)
+    end
+  end
+end
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateInferredConcepts < ::Jobs::Base
+    sidekiq_options queue: "low"
+
+    # Process items to generate new concepts
+    #
+    # @param args [Hash] Contains job arguments
+    # @option args [String] :item_type Required - Type of items to process ('topics' or 'posts')
+    # @option args [Array<Integer>] :item_ids Required - List of item IDs to process
+    # @option args [Integer] :batch_size (100) Number of items to process in each batch
+    # @option args [Boolean] :match_only (false) Only match against existing concepts without generating new ones
+    def execute(args = {})
+      return if args[:item_ids].blank? || args[:item_type].blank?
+
+      if %w[topics posts].exclude?(args[:item_type])
+        Rails.logger.error("Invalid item_type for GenerateInferredConcepts: #{args[:item_type]}")
+        return
+      end
+
+      # Process items in smaller batches to avoid memory issues
+      batch_size = args[:batch_size] || 100
+
+      # Get the list of item IDs
+      item_ids = args[:item_ids]
+      match_only = args[:match_only] || false
+
+      # Process items in batches
+      item_ids.each_slice(batch_size) do |batch_item_ids|
+        process_batch(batch_item_ids, args[:item_type], match_only)
+      end
+    end
+
+    private
+
+    def process_batch(item_ids, item_type, match_only)
+      klass = item_type.singularize.classify.constantize
+      items = klass.where(id: item_ids)
+      manager = DiscourseAi::InferredConcepts::Manager.new
+
+      items.each do |item|
+        begin
+          process_item(item, item_type, match_only, manager)
+        rescue => e
+          Rails.logger.error(
+            "Error generating concepts from #{item_type.singularize} #{item.id}: #{e.message}\n#{e.backtrace.join("\n")}",
+          )
+        end
+      end
+    end
+
+    def process_item(item, item_type, match_only, manager)
+      # Use the Manager method that handles both identifying and creating concepts
+      if match_only
+        if item_type == "topics"
+          manager.match_topic_to_concepts(item)
+        else # posts
+          manager.match_post_to_concepts(item)
+        end
+      else
+        if item_type == "topics"
+          manager.generate_concepts_from_topic(item)
+        else # posts
+          manager.generate_concepts_from_post(item)
+        end
+      end
+    end
+  end
+end
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizeCategories < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      cat_id = args[:from_category_id] || Category.order(:id).first&.id
+      last_id = nil
+
+      categories = Category.where("id >= ?", cat_id).order(:id).limit(BATCH_SIZE)
+      return if categories.empty?
+
+      categories.each do |category|
+        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
+          last_id = category.id
+          next
+        end
+
+        CategoryLocalization.transaction do
+          locales.each do |locale|
+            next if CategoryLocalization.exists?(category_id: category.id, locale: locale)
+            begin
+              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+            rescue FinalDestination::SSRFDetector::LookupFailedError
+              # do nothing, there are too many sporadic lookup failures
+            rescue => e
+              DiscourseAi::Translation::VerboseLogger.log(
+                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
+              )
+            end
+          end
+        end
+        last_id = category.id
+      end
+
+      if categories.size == BATCH_SIZE
+        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
+      end
+    end
+  end
+end
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizePosts < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        posts =
+          Post
+            .joins(
+              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("posts.user_id > 0")
+            .where.not(raw: [nil, ""])
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("pl.id IS NULL")
+
+        if SiteSetting.ai_translation_backfill_limit_to_public_content
+          posts =
+            posts
+              .joins(:topic)
+              .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
+              .where.not(topics: { archetype: Archetype.private_message })
+        end
+
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          posts =
+            posts.where(
+              "posts.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        posts = posts.order(updated_at: :desc).limit(limit)
+
+        next if posts.empty?
+
+        posts.each do |post|
+          begin
+            DiscourseAi::Translation::PostLocalizer.localize(post, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
+      end
+    end
+  end
+end
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizeTopics < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        topics =
+          Topic
+            .joins(
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("topics.user_id > 0")
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("tl.id IS NULL")
+
+        if SiteSetting.ai_translation_backfill_limit_to_public_content
+          topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+        end
+
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          topics =
+            topics.where(
+              "topics.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        topics = topics.order(updated_at: :desc).limit(limit)
+
+        next if topics.empty?
+
+        topics.each do |topic|
+          begin
+            DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseAi::Translation::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
+      end
+    end
+  end
+end
@@ -9,8 +9,8 @@ module Jobs

      return unless SiteSetting.ai_discord_search_enabled

-      if SiteSetting.ai_discord_search_mode == "agent"
-        DiscourseAi::Discord::Bot::AgentReplier.new(interaction).handle_interaction!
+      if SiteSetting.ai_discord_search_mode == "persona"
+        DiscourseAi::Discord::Bot::PersonaReplier.new(interaction).handle_interaction!
      else
        DiscourseAi::Discord::Bot::Search.new(interaction).handle_interaction!
      end
@@ -8,20 +8,20 @@ module Jobs
      return if (user = User.find_by(id: args[:user_id])).nil?
      return if (query = args[:query]).blank?

-      ai_agent_klass =
-        AiAgent
-          .all_agents(enabled_only: false)
-          .find { |agent| agent.id == SiteSetting.ai_bot_discover_agent.to_i }
+      ai_persona_klass =
+        AiPersona
+          .all_personas(enabled_only: false)
+          .find { |persona| persona.id == SiteSetting.ai_bot_discover_persona.to_i }

-      if ai_agent_klass.nil? || !user.in_any_groups?(ai_agent_klass.allowed_group_ids.to_a)
+      if ai_persona_klass.nil? || !user.in_any_groups?(ai_persona_klass.allowed_group_ids.to_a)
        return
      end
-      return if (llm_model = LlmModel.find_by(id: ai_agent_klass.default_llm_id)).nil?
+      return if (llm_model = LlmModel.find_by(id: ai_persona_klass.default_llm_id)).nil?

      bot =
-        DiscourseAi::Agents::Bot.as(
+        DiscourseAi::Personas::Bot.as(
          Discourse.system_user,
-          agent: ai_agent_klass.new,
+          persona: ai_persona_klass.new,
          model: llm_model,
        )

@@ -31,7 +31,7 @@ module Jobs
      base = { query: query, model_used: llm_model.display_name }

      context =
-        DiscourseAi::Agents::BotContext.new(
+        DiscourseAi::Personas::BotContext.new(
          messages: [{ type: :user, content: query }],
          skip_tool_details: true,
        )
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Jobs
+  class CategoryLocalizationBackfill < ::Jobs::Scheduled
+    every 12.hours
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+
+      Jobs.enqueue(:localize_categories)
+    end
+  end
+end
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateConceptsFromPopularItems < ::Jobs::Scheduled
+    every 1.day
+
+    # This job runs daily and generates new concepts from popular topics and posts
+    # It selects items based on engagement metrics and generates concepts from their content
+    def execute(_args)
+      return unless SiteSetting.inferred_concepts_enabled
+
+      process_popular_topics
+      process_popular_posts
+    end
+
+    private
+
+    def process_popular_topics
+      # Find candidate topics that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_topics(
+          limit: SiteSetting.inferred_concepts_daily_topics_limit || 20,
+          min_posts: SiteSetting.inferred_concepts_min_posts || 5,
+          min_likes: SiteSetting.inferred_concepts_min_likes || 10,
+          min_views: SiteSetting.inferred_concepts_min_views || 100,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate topics - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "topics",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "topics",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+
+    def process_popular_posts
+      # Find candidate posts that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_posts(
+          limit: SiteSetting.inferred_concepts_daily_posts_limit || 30,
+          min_likes: SiteSetting.inferred_concepts_post_min_likes || 5,
+          exclude_first_posts: true,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate posts - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "posts",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match against existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "posts",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+  end
+end
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Keegan George	9c80396942	DEV: Re-introduce validator	2025-06-03 06:30:50 -07:00
Sam	4dffd0b2c5	DEV: improve tool infra, improve forum researcher prompts, improve logging (#1391 ) - add sleep function for tool polling with rate limits - Support base64 encoding for HTTP requests and uploads - Enhance forum researcher with cost warnings and comprehensive planning - Add cancellation support for research operations - Include feature_name parameter for bot analytics - richer research support (OR queries)	2025-06-03 15:17:55 +10:00
Keegan George	4c0660d6fd	DEV: Remove validator for spam (#1393 )	2025-06-02 13:57:34 -07:00
Rafael dos Santos Silva	27de71fc4f	FIX: Proper default LLM detection for inferred concepts (#1392 )	2025-06-02 17:56:47 -03:00
Rafael dos Santos Silva Roman Rizzi Keegan George	478f31de47	FEATURE: add inferred concepts system (#1330 ) * FEATURE: add inferred concepts system This commit adds a new inferred concepts system that: - Creates a model for storing concept labels that can be applied to topics - Provides AI personas for finding new concepts and matching existing ones - Adds jobs for generating concepts from popular topics - Includes a scheduled job that automatically processes engaging topics * FEATURE: Extend inferred concepts to include posts * Adds support for concepts to be inferred from and applied to posts * Replaces daily task with one that handles both topics and posts * Adds database migration for posts_inferred_concepts join table * Updates PersonaContext to include inferred concepts Co-authored-by: Roman Rizzi <[email protected]> Co-authored-by: Keegan George <[email protected]>	2025-06-02 14:29:20 -03:00
David Taylor	4ce8973e56	PERF: Optimize `.ai-debug-modal__tokens` selector (#1390 ) This is showing as the most expensive CSS selector in Discourse at the moment. Adding specific classes and dropping the general `span` selector will make this much cheaper.	2025-05-30 21:47:30 +01:00
Keegan George	34c98de864	FIX: Exporting overall sentiment fails (#1388 ) ## 🔍 Overview When exporting an Overall Sentiment report in the admin panel, the export fails with: ```ruby Job exception: no implicit conversion of Symbol into Integer ``` This was happening because we are passing a single _Hash_ to `report.data` however, exports expect `report.data` to be an _Array of Hashes_. This update fixes this issue by wrapping the data in an array.	2025-05-30 11:58:28 -07:00
Keegan George	38f7e9c2c4	UX: AI composer helper refinements (#1387 ) This update includes a variety of small refinements to the AI composer helper: - prevent height jump when going from loading text placeholder → proofreading text streaming - update padding on AI helper options list to be more suitable with typical Discourse menu design - for composer helper results that are not `showResultAsDiff` (i.e. translation): - update before/after diff design to be more subtle - results should be in normal font (as the text is cooked and not raw markdown) - fix: smooth streaming animation stuck showing dot icon even after smooth streaming is done	2025-05-30 10:35:53 -07:00
Sam	e6876aabd5	FIX: enum handling needs to be done on save as well (#1386 )	2025-05-30 17:58:16 +10:00
Sam	b5d393b4bc	FIX: custom tools incorrectly setting all fields to blank enum (#1385 ) Previous to this change, enum was set to [] which broke all non enum tools	2025-05-30 17:12:24 +10:00
Sam	77ae426d95	FEATURE: support upload.getUrl in custom tools (#1384 ) * FEATURE: support upload.getUrl in custom tools Some tools need to share images with an API. A common pattern is for APIs to expect a URL. This allows converting upload://123123 to a proper CDN friendly URL from within a custom tool * no support for secure uploads, so be explicit about it.	2025-05-30 15:47:07 +10:00
Alan Guo Xiang Tan	9f43df0302	FIX: Full page search broken (#1383 ) Follow-up to `59d6e2b467`	2025-05-30 10:07:48 +08:00
David Taylor	59d6e2b467	DEV: Replace narrowDesktopView logic with viewport API (#1372 )	2025-05-30 06:52:39 +10:00
SamandKeegan George	c06d7b07d5	FEATURE: simplify streaming implementation - rush last update (#1380 ) * FEATURE: simplify streaming implementation - rush last update Previous to this change we would simply "flash" the final update on the screen, this amends it so we quickly update the UI in about 1 second in the end with all the final update. This makes the UI feel more interactive to end users. * DEV: Updates... - Remove unnecessary comments - Add doc style comments for all methods - Organize methods (private at bottom) - Update some variable names --------- Co-authored-by: Keegan George <[email protected]>	2025-05-30 06:50:12 +10:00
Natalie Tay	373e2305d6	FEATURE: Automatic translation and localization of posts, topics, categories (#1376 ) Related: https://github.com/discourse/discourse-translator/pull/310 This commit includes all the jobs and event hooks to localize posts, topics, and categories. A few notes: - `feature_name: "translation"` because the site setting is `ai-translation` and module is `Translation` - we will switch to proper ai-feature in the near future, and can consider using the persona_user as `localization.localizer_user_id` - keeping things flat within the module for now as we will be moving to ai-feature soon and have to rearrange - Settings renamed/introduced are: - ai_translation_backfill_rate (0) - ai_translation_backfill_limit_to_public_content (true) - ai_translation_backfill_max_age_days (5) - ai_translation_verbose_logs (false)	2025-05-29 17:28:06 +08:00
				`@@ -1 +0,0 @@`
				`<AiAgentListEditor @agents={{this.model}} />`
				`@@ -0,0 +1 @@`
				`<AiPersonaListEditor @personas={{this.model}} />`