DEV: Increase chunk size before splitting content for translation

Previously staff and bots would get scanned if TL was low
Additionally if somehow spam scanner user was blocked
(deactivated, silenced, banned) it would stop the feature from working

This adds an override that ensures unconditionally the user is setup correctly prior to scanning

.discourse-compatibility

@@ -1,3 +1,4 @@
+< 3.5.0.beta6-dev: 3e74eea1e5e3143888d67a8d8a11206df214dc24
 < 3.5.0.beta3-dev: 09a68414804a1447f52e5d60691ba59742cda9ec
 < 3.5.0.beta2-dev: de8624416a15b3d8e7ad350b083cc1420451ccec
 < 3.5.0.beta1-dev: bdef136080074a993e7c4f5ca562edc31a8ba756

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-features-edit.js

@@ -12,7 +12,7 @@ export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRo
 
     const { site_settings } = await ajax("/admin/config/site_settings.json", {
       data: {
-        filter_area: `ai-features/${currentFeature.ref}`,
+        filter_area: `ai-features/${currentFeature.module_name}`,
         plugin: "discourse-ai",
         category: "discourse_ai",
       },

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/index.gjs

@@ -1,156 +0,0 @@
-import Component from "@glimmer/component";
-import { service } from "@ember/service";
-import RouteTemplate from "ember-route-template";
-import { gt } from "truth-helpers";
-import DBreadcrumbsItem from "discourse/components/d-breadcrumbs-item";
-import DButton from "discourse/components/d-button";
-import DPageSubheader from "discourse/components/d-page-subheader";
-import { i18n } from "discourse-i18n";
-
-export default RouteTemplate(
-  class extends Component {
-    @service adminPluginNavManager;
-
-    get tableHeaders() {
-      const prefix = "discourse_ai.features.list.header";
-      return [
-        i18n(`${prefix}.name`),
-        i18n(`${prefix}.persona`),
-        i18n(`${prefix}.groups`),
-        "",
-      ];
-    }
-
-    get configuredFeatures() {
-      return this.args.model.filter(
-        (feature) => feature.enable_setting.value === true
-      );
-    }
-
-    get unconfiguredFeatures() {
-      return this.args.model.filter(
-        (feature) => feature.enable_setting.value === false
-      );
-    }
-
-    <template>
-      <DBreadcrumbsItem
-        @path="/admin/plugins/{{this.adminPluginNavManager.currentPlugin.name}}/ai-features"
-        @label={{i18n "discourse_ai.features.short_title"}}
-      />
-      <section class="ai-feature-list admin-detail">
-        <DPageSubheader
-          @titleLabel={{i18n "discourse_ai.features.short_title"}}
-          @descriptionLabel={{i18n "discourse_ai.features.description"}}
-          @learnMoreUrl="todo"
-        />
-
-        {{#if (gt this.configuredFeatures.length 0)}}
-          <div class="ai-feature-list__configured-features">
-            <h3>{{i18n "discourse_ai.features.list.configured_features"}}</h3>
-
-            <table class="d-admin-table">
-              <thead>
-                <tr>
-                  {{#each this.tableHeaders as |header|}}
-                    <th>{{header}}</th>
-                  {{/each}}
-                </tr>
-              </thead>
-
-              <tbody>
-                {{#each this.configuredFeatures as |feature|}}
-                  <tr
-                    class="ai-feature-list__row d-admin-row__content"
-                    data-feature-name={{feature.name}}
-                  >
-                    <td class="d-admin-row__overview ai-feature-list__row-item">
-                      <span class="ai-feature-list__row-item-name">
-                        <strong>
-                          {{feature.name}}
-                        </strong>
-                      </span>
-                      <span class="ai-feature-list__row-item-description">
-                        {{feature.description}}
-                      </span>
-                    </td>
-                    <td
-                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__persona"
-                    >
-                      <DButton
-                        class="btn-flat btn-small ai-feature-list__row-item-persona"
-                        @translatedLabel={{feature.persona.name}}
-                        @route="adminPlugins.show.discourse-ai-personas.edit"
-                        @routeModels={{feature.persona.id}}
-                      />
-                    </td>
-                    <td
-                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__groups"
-                    >
-                      {{#if (gt feature.persona.allowed_groups.length 0)}}
-                        <ul class="ai-feature-list__row-item-groups">
-                          {{#each feature.persona.allowed_groups as |group|}}
-                            <li>{{group.name}}</li>
-                          {{/each}}
-                        </ul>
-                      {{/if}}
-                    </td>
-                    <td class="d-admin-row_controls">
-                      <DButton
-                        class="btn-small edit"
-                        @label="discourse_ai.features.list.edit"
-                        @route="adminPlugins.show.discourse-ai-features.edit"
-                        @routeModels={{feature.id}}
-                      />
-                    </td>
-                  </tr>
-                {{/each}}
-              </tbody>
-            </table>
-          </div>
-        {{/if}}
-
-        {{#if (gt this.unconfiguredFeatures.length 0)}}
-          <div class="ai-feature-list__unconfigured-features">
-            <h3>{{i18n "discourse_ai.features.list.unconfigured_features"}}</h3>
-
-            <table class="d-admin-table">
-              <thead>
-                <tr>
-                  <th>{{i18n "discourse_ai.features.list.header.name"}}</th>
-                  <th></th>
-                </tr>
-              </thead>
-
-              <tbody>
-                {{#each this.unconfiguredFeatures as |feature|}}
-                  <tr class="ai-feature-list__row d-admin-row__content">
-                    <td class="d-admin-row__overview ai-feature-list__row-item">
-                      <span class="ai-feature-list__row-item-name">
-                        <strong>
-                          {{feature.name}}
-                        </strong>
-                      </span>
-                      <span class="ai-feature-list__row-item-description">
-                        {{feature.description}}
-                      </span>
-                    </td>
-
-                    <td class="d-admin-row_controls">
-                      <DButton
-                        class="btn-small"
-                        @label="discourse_ai.features.list.set_up"
-                        @route="adminPlugins.show.discourse-ai-features.edit"
-                        @routeModels={{feature.id}}
-                      />
-                    </td>
-                  </tr>
-                {{/each}}
-              </tbody>
-            </table>
-          </div>
-        {{/if}}
-      </section>
-    </template>
-  }
-);

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/index.hbs

@@ -0,0 +1 @@
+<AiFeatures @features={{this.model}} />

app/controllers/discourse_ai/admin/ai_embeddings_controller.rb

@@ -40,6 +40,7 @@ module DiscourseAi
         embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
 
         if embedding_def.save
+          log_ai_embedding_creation(embedding_def)
           render json: AiEmbeddingDefinitionSerializer.new(embedding_def), status: :created
         else
           render_json_error embedding_def
@@ -55,7 +56,10 @@ module DiscourseAi
           )
         end
 
+        initial_attributes = embedding_def.attributes.dup
+
         if embedding_def.update(ai_embeddings_params.except(:dimensions))
+          log_ai_embedding_update(embedding_def, initial_attributes)
           render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
         else
           render_json_error embedding_def
@@ -75,7 +79,16 @@ module DiscourseAi
           return render_json_error(I18n.t("discourse_ai.embeddings.delete_failed"), status: 409)
         end
 
+        embedding_details = {
+          embedding_id: embedding_def.id,
+          display_name: embedding_def.display_name,
+          provider: embedding_def.provider,
+          dimensions: embedding_def.dimensions,
+          subject: embedding_def.display_name,
+        }
+
         if embedding_def.destroy
+          log_ai_embedding_deletion(embedding_details)
           head :no_content
         else
           render_json_error embedding_def
@@ -128,6 +141,60 @@ module DiscourseAi
 
         permitted
       end
+
+      def ai_embeddings_logger_fields
+        {
+          display_name: {
+          },
+          provider: {
+          },
+          dimensions: {
+          },
+          url: {
+          },
+          tokenizer_class: {
+          },
+          max_sequence_length: {
+          },
+          embed_prompt: {
+            type: :large_text,
+          },
+          search_prompt: {
+            type: :large_text,
+          },
+          matryoshka_dimensions: {
+          },
+          api_key: {
+            type: :sensitive,
+          },
+          # JSON fields should be tracked as simple changes
+          json_fields: [:provider_params],
+        }
+      end
+
+      def log_ai_embedding_creation(embedding_def)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
+        logger.log_creation("embedding", embedding_def, ai_embeddings_logger_fields, entity_details)
+      end
+
+      def log_ai_embedding_update(embedding_def, initial_attributes)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
+
+        logger.log_update(
+          "embedding",
+          embedding_def,
+          initial_attributes,
+          ai_embeddings_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_ai_embedding_deletion(embedding_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        logger.log_deletion("embedding", embedding_details)
+      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_features_controller.rb

@@ -6,24 +6,40 @@ module DiscourseAi
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
       def index
-        render json: serialize_features(DiscourseAi::Features.features)
+        render json: serialize_modules(DiscourseAi::Configuration::Module.all)
       end
 
       def edit
         raise Discourse::InvalidParameters.new(:id) if params[:id].blank?
-        render json: serialize_feature(DiscourseAi::Features.find_feature_by_id(params[:id].to_i))
+
+        a_module = DiscourseAi::Configuration::Module.find_by(id: params[:id].to_i)
+
+        render json: serialize_module(a_module)
       end
 
       private
 
-      def serialize_features(features)
-        features.map { |feature| feature.merge(persona: serialize_persona(feature[:persona])) }
+      def serialize_modules(modules)
+        modules.map { |a_module| serialize_module(a_module) }
+      end
+
+      def serialize_module(a_module)
+        return nil if a_module.blank?
+
+        {
+          id: a_module.id,
+          module_name: a_module.name,
+          module_enabled: a_module.enabled?,
+          features: a_module.features.map { |f| serialize_feature(f) },
+        }
       end
 
       def serialize_feature(feature)
-        return nil if feature.blank?
-
-        feature.merge(persona: serialize_persona(feature[:persona]))
+        {
+          name: feature.name,
+          persona: serialize_persona(persona_id_obj_hash[feature.persona_id]),
+          enabled: feature.enabled?,
+        }
       end
 
       def serialize_persona(persona)
@@ -31,6 +47,18 @@ module DiscourseAi
 
         serialize_data(persona, AiFeaturesPersonaSerializer, root: false)
       end
+
+      private
+
+      def persona_id_obj_hash
+        @persona_id_obj_hash ||=
+          begin
+            setting_names = DiscourseAi::Configuration::Feature.all_persona_setting_names
+            ids = setting_names.map { |sn| SiteSetting.public_send(sn) }
+
+            AiPersona.where(id: ids).index_by(&:id)
+          end
+      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_llms_controller.rb

@@ -47,6 +47,7 @@ module DiscourseAi
 
         if llm_model.save
           llm_model.toggle_companion_user
+          log_llm_model_creation(llm_model)
           render json: LlmModelSerializer.new(llm_model), status: :created
         else
           render_json_error llm_model
@@ -56,6 +57,10 @@ module DiscourseAi
       def update
         llm_model = LlmModel.find(params[:id])
 
+        # Capture initial state for logging
+        initial_attributes = llm_model.attributes.dup
+        initial_quotas = llm_model.llm_quotas.map(&:attributes)
+
         if params[:ai_llm].key?(:llm_quotas)
           if quota_params
             existing_quota_group_ids = llm_model.llm_quotas.pluck(:group_id)
@@ -81,6 +86,7 @@ module DiscourseAi
 
         if llm_model.update(ai_llm_params(updating: llm_model))
           llm_model.toggle_companion_user
+          log_llm_model_update(llm_model, initial_attributes, initial_quotas)
           render json: LlmModelSerializer.new(llm_model)
         else
           render_json_error llm_model
@@ -109,11 +115,20 @@ module DiscourseAi
           )
         end
 
+        # Capture model details for logging before destruction
+        model_details = {
+          model_id: llm_model.id,
+          display_name: llm_model.display_name,
+          name: llm_model.name,
+          provider: llm_model.provider,
+        }
+
         # Clean up companion users
         llm_model.enabled_chat_bot = false
         llm_model.toggle_companion_user
 
         if llm_model.destroy
+          log_llm_model_deletion(model_details)
           head :no_content
         else
           render_json_error llm_model
@@ -190,6 +205,89 @@ module DiscourseAi
 
         permitted
       end
+
+      def ai_llm_logger_fields
+        {
+          display_name: {
+          },
+          name: {
+          },
+          provider: {
+          },
+          tokenizer: {
+          },
+          url: {
+          },
+          max_prompt_tokens: {
+          },
+          max_output_tokens: {
+          },
+          enabled_chat_bot: {
+          },
+          vision_enabled: {
+          },
+          api_key: {
+            type: :sensitive,
+          },
+          input_cost: {
+          },
+          output_cost: {
+          },
+          # JSON fields should be tracked as simple changes
+          json_fields: [:provider_params],
+        }
+      end
+
+      def log_llm_model_creation(llm_model)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
+
+        # Add quota information as a special case
+        if llm_model.llm_quotas.any?
+          entity_details[:quotas] = llm_model
+            .llm_quotas
+            .map do |quota|
+              "Group #{quota.group_id}: #{quota.max_tokens} tokens, #{quota.max_usages} usages, #{quota.duration_seconds}s"
+            end
+            .join("; ")
+        end
+
+        logger.log_creation("llm_model", llm_model, ai_llm_logger_fields, entity_details)
+      end
+
+      def log_llm_model_update(llm_model, initial_attributes, initial_quotas)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
+
+        # Track quota changes separately as they're a special case
+        current_quotas = llm_model.llm_quotas.reload.map(&:attributes)
+        if initial_quotas != current_quotas
+          initial_quota_summary =
+            initial_quotas
+              .map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
+              .join("; ")
+          current_quota_summary =
+            current_quotas
+              .map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
+              .join("; ")
+          entity_details[:quotas_changed] = true
+          entity_details[:quotas] = "#{initial_quota_summary} → #{current_quota_summary}"
+        end
+
+        logger.log_update(
+          "llm_model",
+          llm_model,
+          initial_attributes,
+          ai_llm_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_llm_model_deletion(model_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        model_details[:subject] = model_details[:display_name]
+        logger.log_deletion("llm_model", model_details)
+      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -9,15 +9,16 @@ module DiscourseAi
 
       def index
         ai_personas =
-          AiPersona.ordered.map do |persona|
-            # we use a special serializer here cause names and descriptions are
-            # localized for system personas
-            LocalizedAiPersonaSerializer.new(persona, root: false)
-          end
+          AiPersona
+            .ordered
+            .includes(:user, :uploads)
+            .map { |persona| LocalizedAiPersonaSerializer.new(persona, root: false) }
+
         tools =
           DiscourseAi::Personas::Persona.all_available_tools.map do |tool|
             AiToolSerializer.new(tool, root: false)
           end
+
         AiTool
           .where(enabled: true)
           .each do |tool|
@@ -31,10 +32,12 @@ module DiscourseAi
                 ),
             }
           end
+
         llms =
           DiscourseAi::Configuration::LlmEnumerator.values_for_serialization(
             allowed_seeded_llm_ids: SiteSetting.ai_bot_allowed_seeded_models_map,
           )
+
         render json: {
                  ai_personas: ai_personas,
                  meta: {
@@ -58,6 +61,7 @@ module DiscourseAi
         ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
         if ai_persona.save
           RagDocumentFragment.link_target_and_uploads(ai_persona, attached_upload_ids)
+          log_ai_persona_creation(ai_persona)
 
           render json: {
                    ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
@@ -74,8 +78,11 @@ module DiscourseAi
       end
 
       def update
+        initial_attributes = @ai_persona.attributes.dup
+
         if @ai_persona.update(ai_persona_params.except(:rag_uploads))
           RagDocumentFragment.update_target_uploads(@ai_persona, attached_upload_ids)
+          log_ai_persona_update(@ai_persona, initial_attributes)
 
           render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
         else
@@ -84,7 +91,14 @@ module DiscourseAi
       end
 
       def destroy
+        persona_details = {
+          persona_id: @ai_persona.id,
+          name: @ai_persona.name,
+          description: @ai_persona.description,
+        }
+
         if @ai_persona.destroy
+          log_ai_persona_deletion(persona_details)
           head :no_content
         else
           render_json_error @ai_persona
@@ -261,6 +275,92 @@ module DiscourseAi
 
         examples.map { |example_arr| example_arr.take(2).map(&:to_s) }
       end
+
+      def ai_persona_logger_fields
+        {
+          name: {
+          },
+          description: {
+          },
+          enabled: {
+          },
+          priority: {
+          },
+          system_prompt: {
+            type: :large_text,
+          },
+          default_llm_id: {
+          },
+          temperature: {
+          },
+          top_p: {
+          },
+          user_id: {
+          },
+          max_context_posts: {
+          },
+          vision_enabled: {
+          },
+          vision_max_pixels: {
+          },
+          rag_chunk_tokens: {
+          },
+          rag_chunk_overlap_tokens: {
+          },
+          rag_conversation_chunks: {
+          },
+          rag_llm_model_id: {
+          },
+          question_consolidator_llm_id: {
+          },
+          allow_chat_channel_mentions: {
+          },
+          allow_chat_direct_messages: {
+          },
+          allow_topic_mentions: {
+          },
+          allow_personal_messages: {
+          },
+          tool_details: {
+            type: :large_text,
+          },
+          forced_tool_count: {
+          },
+          force_default_llm: {
+          },
+          # JSON fields
+          json_fields: %i[tools response_format examples allowed_group_ids],
+        }
+      end
+
+      def log_ai_persona_creation(ai_persona)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
+        entity_details[:tools_count] = (ai_persona.tools || []).size
+
+        logger.log_creation("persona", ai_persona, ai_persona_logger_fields, entity_details)
+      end
+
+      def log_ai_persona_update(ai_persona, initial_attributes)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
+        entity_details[:tools_count] = ai_persona.tools.size if ai_persona.tools.present?
+
+        logger.log_update(
+          "persona",
+          ai_persona,
+          initial_attributes,
+          ai_persona_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_ai_persona_deletion(persona_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        persona_details[:subject] = persona_details[:name]
+
+        logger.log_deletion("persona", persona_details)
+      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -10,6 +10,10 @@ module DiscourseAi
       end
 
       def update
+        initial_settings = AiModerationSetting.spam
+        initial_custom_instructions = initial_settings&.data&.dig("custom_instructions")
+        initial_llm_model_id = initial_settings&.llm_model_id
+
         updated_params = {}
         if allowed_params.key?(:llm_model_id)
           llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
@@ -36,6 +40,8 @@ module DiscourseAi
           else
             AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
           end
+
+          log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, allowed_params)
         end
 
         is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
@@ -50,7 +56,7 @@ module DiscourseAi
             )
           end
 
-          SiteSetting.ai_spam_detection_enabled = is_enabled
+          SiteSetting.set_and_log("ai_spam_detection_enabled", is_enabled, current_user)
         end
 
         render json: AiSpamSerializer.new(spam_config, root: false)
@@ -113,6 +119,30 @@ module DiscourseAi
 
       private
 
+      def log_ai_spam_update(initial_llm_model_id, initial_custom_instructions, params)
+        changes_to_log = {}
+
+        if params.key?(:llm_model_id) && initial_llm_model_id.to_s != params[:llm_model_id].to_s
+          old_model_name =
+            LlmModel.find_by(id: initial_llm_model_id)&.display_name || initial_llm_model_id
+          new_model_name =
+            LlmModel.find_by(id: params[:llm_model_id])&.display_name || params[:llm_model_id]
+
+          changes_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"
+        end
+
+        if params.key?(:custom_instructions) &&
+             initial_custom_instructions != params[:custom_instructions]
+          changes_to_log[:custom_instructions] = params[:custom_instructions]
+        end
+
+        if changes_to_log.present?
+          changes_to_log[:subject] = I18n.t("discourse_ai.spam_detection.logging_subject")
+          logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+          logger.log_custom("update_ai_spam_settings", changes_to_log)
+        end
+      end
+
       def allowed_params
         params.permit(:is_enabled, :llm_model_id, :custom_instructions)
       end

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -25,6 +25,7 @@ module DiscourseAi
 
         if ai_tool.save
           RagDocumentFragment.link_target_and_uploads(ai_tool, attached_upload_ids)
+          log_ai_tool_creation(ai_tool)
           render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
         else
           render_json_error ai_tool
@@ -32,8 +33,11 @@ module DiscourseAi
       end
 
       def update
+        initial_attributes = @ai_tool.attributes.dup
+
         if @ai_tool.update(ai_tool_params)
           RagDocumentFragment.update_target_uploads(@ai_tool, attached_upload_ids)
+          log_ai_tool_update(@ai_tool, initial_attributes)
           render_serialized(@ai_tool, AiCustomToolSerializer)
         else
           render_json_error @ai_tool
@@ -41,7 +45,15 @@ module DiscourseAi
       end
 
       def destroy
+        tool_logger_details = {
+          tool_id: @ai_tool.id,
+          name: @ai_tool.name,
+          tool_name: @ai_tool.tool_name,
+          subject: @ai_tool.name,
+        }
+
         if @ai_tool.destroy
+          log_ai_tool_deletion(tool_logger_details)
           head :no_content
         else
           render_json_error @ai_tool
@@ -96,6 +108,60 @@ module DiscourseAi
           )
           .except(:rag_uploads)
       end
+
+      def ai_tool_logger_fields
+        {
+          name: {
+          },
+          tool_name: {
+          },
+          description: {
+          },
+          summary: {
+          },
+          enabled: {
+          },
+          rag_chunk_tokens: {
+          },
+          rag_chunk_overlap_tokens: {
+          },
+          rag_llm_model_id: {
+          },
+          script: {
+            type: :large_text,
+          },
+          parameters: {
+            type: :large_text,
+          },
+        }
+      end
+
+      def log_ai_tool_creation(ai_tool)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+
+        entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
+        entity_details[:parameter_count] = ai_tool.parameters.size if ai_tool.parameters.present?
+
+        logger.log_creation("tool", ai_tool, ai_tool_logger_fields, entity_details)
+      end
+
+      def log_ai_tool_update(ai_tool, initial_attributes)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
+
+        logger.log_update(
+          "tool",
+          ai_tool,
+          initial_attributes,
+          ai_tool_logger_fields,
+          entity_details,
+        )
+      end
+
+      def log_ai_tool_deletion(tool_details)
+        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
+        logger.log_deletion("tool", tool_details)
+      end
     end
   end
 end

app/controllers/discourse_ai/ai_bot/artifact_key_values_controller.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module AiBot
+    class ArtifactKeyValuesController < ::ApplicationController
+      requires_plugin DiscourseAi::PLUGIN_NAME
+      before_action :ensure_logged_in, only: %i[set destroy]
+      before_action :find_artifact
+
+      PER_PAGE_MAX = 100
+
+      def index
+        page = index_params[:page].to_i
+        page = 1 if page < 1
+        per_page = index_params[:per_page].to_i
+        per_page = PER_PAGE_MAX if per_page < 1 || per_page > PER_PAGE_MAX
+
+        query = build_index_query
+
+        total_count = query.count
+        key_values =
+          query
+            .includes(:user)
+            .order(:user_id, :key, :created_at)
+            .offset((page - 1) * per_page)
+            .limit(per_page + 1)
+
+        has_more = key_values.length > per_page
+        key_values = key_values.first(per_page) if has_more
+
+        render json: {
+                 key_values:
+                   ActiveModel::ArraySerializer.new(
+                     key_values,
+                     each_serializer: AiArtifactKeyValueSerializer,
+                     keys_only: params[:keys_only] == "true",
+                   ).as_json,
+                 has_more: has_more,
+                 total_count: total_count,
+                 users:
+                   key_values
+                     .map { |kv| kv.user }
+                     .uniq
+                     .map { |u| BasicUserSerializer.new(u, root: nil).as_json },
+               }
+      end
+
+      def destroy
+        if params[:key].blank?
+          render json: { error: "Key parameter is required" }, status: :bad_request
+          return
+        end
+
+        key_value = @artifact.key_values.find_by(user_id: current_user.id, key: params[:key])
+
+        if key_value.nil?
+          render json: { error: "Key not found" }, status: :not_found
+        elsif key_value.destroy
+          head :ok
+        else
+          render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
+        end
+      end
+
+      def set
+        key_value =
+          @artifact.key_values.find_or_initialize_by(
+            user: current_user,
+            key: key_value_params[:key],
+          )
+
+        key_value.assign_attributes(key_value_params.except(:key))
+
+        if key_value.save
+          render json: AiArtifactKeyValueSerializer.new(key_value).as_json
+        else
+          render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
+        end
+      end
+
+      private
+
+      def key_value_params
+        params.permit(:key, :value, :public)
+      end
+
+      def index_params
+        @index_params ||= params.permit(:page, :per_page, :key, :keys_only, :all_users)
+      end
+
+      def build_index_query
+        query = @artifact.key_values
+
+        query =
+          if current_user&.admin?
+            query
+          elsif current_user
+            query.where("user_id = ? OR public = true", current_user.id)
+          else
+            query.where(public: true)
+          end
+
+        query = query.where("key = ?", index_params[:key]) if index_params[:key].present?
+
+        if index_params[:all_users].to_s != "true"
+          if current_user
+            query = query.where(user_id: current_user.id)
+          else
+            query = query.where("1 = 0")
+          end
+        end
+
+        query
+      end
+
+      def find_artifact
+        @artifact = AiArtifact.find_by(id: params[:artifact_id])
+        raise Discourse::NotFound if !@artifact
+        raise Discourse::NotFound if [email protected]? && guardian.anonymous?
+        raise Discourse::NotFound if [email protected]? && !guardian.can_see?(@artifact.post)
+      end
+    end
+  end
+end

app/controllers/discourse_ai/ai_bot/shared_ai_conversations_controller.rb

@@ -8,7 +8,7 @@ module DiscourseAi
       before_action :require_site_settings!
 
       skip_before_action :preload_json, :check_xhr, only: %i[show asset]
-      skip_before_action :verify_authenticity_token, only: ["asset"]
+      skip_before_action :redirect_to_login_if_required, :verify_authenticity_token, only: %i[asset]
 
       def create
         ensure_allowed_create!

app/controllers/discourse_ai/ai_helper/assistant_controller.rb

@@ -25,25 +25,24 @@ module DiscourseAi
         input = get_text_param!
         force_default_locale = params[:force_default_locale] || false
 
-        prompt = CompletionPrompt.find_by(id: params[:mode])
+        raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?
 
-        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
-
-        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
           raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
-
-          prompt.custom_instruction = params[:custom_prompt]
         end
 
-        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
+          return suggest_thumbnails(input)
+        end
 
         hijack do
           render json:
                    DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
-                     prompt,
+                     params[:mode],
                      input,
                      current_user,
                      force_default_locale: force_default_locale,
+                     custom_prompt: params[:custom_prompt],
                    ),
                  status: 200
         end
@@ -60,13 +59,10 @@ module DiscourseAi
           input = get_text_param!
         end
 
-        prompt = CompletionPrompt.enabled_by_name("generate_titles")
-        raise Discourse::InvalidParameters.new(:mode) if !prompt
-
         hijack do
           render json:
                    DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
-                     prompt,
+                     DiscourseAi::AiHelper::Assistant::GENERATE_TITLES,
                      input,
                      current_user,
                    ),
@@ -115,23 +111,28 @@ module DiscourseAi
         location = params[:location]
         raise Discourse::InvalidParameters.new(:location) if !location
 
-        prompt = CompletionPrompt.find_by(id: params[:mode])
+        raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
+          return suggest_thumbnails(input)
+        end
 
-        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
-        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
-
-        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+        if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
           raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
         end
 
+        # to stream we must have an appropriate client_id
+        # otherwise we may end up streaming the data to the wrong client
+        raise Discourse::InvalidParameters.new(:client_id) if params[:client_id].blank?
+
         if location == "composer"
           Jobs.enqueue(
             :stream_composer_helper,
             user_id: current_user.id,
             text: text,
-            prompt: prompt.name,
+            prompt: params[:mode],
             custom_prompt: params[:custom_prompt],
             force_default_locale: params[:force_default_locale] || false,
+            client_id: params[:client_id],
           )
         else
           post_id = get_post_param!
@@ -144,8 +145,9 @@ module DiscourseAi
             post_id: post.id,
             user_id: current_user.id,
             text: text,
-            prompt: prompt.name,
+            prompt: params[:mode],
             custom_prompt: params[:custom_prompt],
+            client_id: params[:client_id],
           )
         end
 

app/jobs/regular/create_ai_reply.rb

@@ -15,7 +15,7 @@ module ::Jobs
 
         bot = DiscourseAi::Personas::Bot.as(bot_user, persona: persona.new)
 
-        DiscourseAi::AiBot::Playground.new(bot).reply_to(post)
+        DiscourseAi::AiBot::Playground.new(bot).reply_to(post, feature_name: "bot")
       rescue DiscourseAi::Personas::Bot::BOT_NOT_FOUND
         Rails.logger.warn(
           "Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",

app/jobs/regular/detect_translate_post.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslatePost < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if args[:post_id].blank?
+
+      post = Post.find_by(id: args[:post_id])
+      return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
+
+      topic = post.topic
+      return if topic.blank?
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
+      else
+        if topic.archetype == Archetype.private_message &&
+             !TopicAllowedGroup.exists?(topic_id: topic.id)
+          return
+        end
+      end
+
+      # the user may fill locale in manually
+      if (detected_locale = post.locale).blank?
+        begin
+          detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # this job is non-critical
+          # the backfill job will handle failures
+          return
+        end
+      end
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+        next if post.post_localizations.exists?(locale:)
+
+        begin
+          DiscourseAi::Translation::PostLocalizer.localize(post, locale)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+
+      MessageBus.publish("/topic/#{post.topic_id}", type: :localized, id: post.id)
+    end
+  end
+end

app/jobs/regular/detect_translate_topic.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectTranslateTopic < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if args[:topic_id].blank?
+
+      topic = Topic.find_by(id: args[:topic_id])
+      if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0
+        return
+      end
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
+      else
+        if topic.archetype == Archetype.private_message &&
+             !TopicAllowedGroup.exists?(topic_id: topic.id)
+          return
+        end
+      end
+
+      if (detected_locale = topic.locale).blank?
+        begin
+          detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # this job is non-critical
+          # the backfill job will handle failures
+          return
+        end
+      end
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      locales.each do |locale|
+        next if locale == detected_locale
+        next if topic.topic_localizations.exists?(locale:)
+
+        begin
+          DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+          )
+        end
+      end
+
+      MessageBus.publish("/topic/#{topic.id}", type: :localized, id: 1)
+    end
+  end
+end

app/jobs/regular/generate_inferred_concepts.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateInferredConcepts < ::Jobs::Base
+    sidekiq_options queue: "low"
+
+    # Process items to generate new concepts
+    #
+    # @param args [Hash] Contains job arguments
+    # @option args [String] :item_type Required - Type of items to process ('topics' or 'posts')
+    # @option args [Array<Integer>] :item_ids Required - List of item IDs to process
+    # @option args [Integer] :batch_size (100) Number of items to process in each batch
+    # @option args [Boolean] :match_only (false) Only match against existing concepts without generating new ones
+    def execute(args = {})
+      return if args[:item_ids].blank? || args[:item_type].blank?
+
+      if %w[topics posts].exclude?(args[:item_type])
+        Rails.logger.error("Invalid item_type for GenerateInferredConcepts: #{args[:item_type]}")
+        return
+      end
+
+      # Process items in smaller batches to avoid memory issues
+      batch_size = args[:batch_size] || 100
+
+      # Get the list of item IDs
+      item_ids = args[:item_ids]
+      match_only = args[:match_only] || false
+
+      # Process items in batches
+      item_ids.each_slice(batch_size) do |batch_item_ids|
+        process_batch(batch_item_ids, args[:item_type], match_only)
+      end
+    end
+
+    private
+
+    def process_batch(item_ids, item_type, match_only)
+      klass = item_type.singularize.classify.constantize
+      items = klass.where(id: item_ids)
+      manager = DiscourseAi::InferredConcepts::Manager.new
+
+      items.each do |item|
+        begin
+          process_item(item, item_type, match_only, manager)
+        rescue => e
+          Rails.logger.error(
+            "Error generating concepts from #{item_type.singularize} #{item.id}: #{e.message}\n#{e.backtrace.join("\n")}",
+          )
+        end
+      end
+    end
+
+    def process_item(item, item_type, match_only, manager)
+      # Use the Manager method that handles both identifying and creating concepts
+      if match_only
+        if item_type == "topics"
+          manager.match_topic_to_concepts(item)
+        else # posts
+          manager.match_post_to_concepts(item)
+        end
+      else
+        if item_type == "topics"
+          manager.generate_concepts_from_topic(item)
+        else # posts
+          manager.generate_concepts_from_post(item)
+        end
+      end
+    end
+  end
+end

app/jobs/regular/localize_categories.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizeCategories < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      cat_id = args[:from_category_id] || Category.order(:id).first&.id
+      last_id = nil
+
+      categories =
+        Category.where("id >= ? AND locale IS NOT NULL", cat_id).order(:id).limit(BATCH_SIZE)
+      return if categories.empty?
+
+      categories.each do |category|
+        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
+          last_id = category.id
+          next
+        end
+
+        locales.each do |locale|
+          localization = category.category_localizations.find_by(locale:)
+
+          if locale == category.locale && localization
+            localization.destroy
+          else
+            next if locale == category.locale
+            begin
+              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+            rescue FinalDestination::SSRFDetector::LookupFailedError
+              # do nothing, there are too many sporadic lookup failures
+            rescue => e
+              DiscourseAi::Translation::VerboseLogger.log(
+                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
+              )
+            end
+          end
+        end
+        last_id = category.id
+      end
+
+      if categories.size == BATCH_SIZE
+        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
+      end
+    end
+  end
+end

app/jobs/regular/localize_posts.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizePosts < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        posts =
+          Post
+            .joins(
+              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("posts.user_id > 0")
+            .where.not(raw: [nil, ""])
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("pl.id IS NULL")
+
+        posts = posts.joins(:topic)
+
+        if SiteSetting.ai_translation_backfill_limit_to_public_content
+          # exclude all PMs
+          # and only include posts from public categories
+          posts =
+            posts
+              .where.not(topics: { archetype: Archetype.private_message })
+              .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
+        else
+          # all regular topics, and group PMs
+          posts =
+            posts.where(
+              "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
+              Archetype.private_message,
+            )
+        end
+
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          posts =
+            posts.where(
+              "posts.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        posts = posts.order(updated_at: :desc).limit(limit)
+
+        next if posts.empty?
+
+        posts.each do |post|
+          begin
+            DiscourseAi::Translation::PostLocalizer.localize(post, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate post #{post.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
+      end
+    end
+  end
+end

app/jobs/regular/localize_topics.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Jobs
+  class LocalizeTopics < ::Jobs::Base
+    cluster_concurrency 1
+    sidekiq_options retry: false
+
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      locales = SiteSetting.experimental_content_localization_supported_locales.split("|")
+      return if locales.blank?
+
+      limit = args[:limit] || BATCH_SIZE
+
+      locales.each do |locale|
+        topics =
+          Topic
+            .joins(
+              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
+            )
+            .where(deleted_at: nil)
+            .where("topics.user_id > 0")
+            .where.not(locale: nil)
+            .where.not(locale: locale)
+            .where("tl.id IS NULL")
+
+        if SiteSetting.ai_translation_backfill_limit_to_public_content
+          # exclude all PMs
+          # and only include posts from public categories
+          topics =
+            topics
+              .where.not(archetype: Archetype.private_message)
+              .where(category_id: Category.where(read_restricted: false).select(:id))
+        else
+          # all regular topics, and group PMs
+          topics =
+            topics.where(
+              "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
+              Archetype.private_message,
+            )
+        end
+
+        if SiteSetting.ai_translation_backfill_max_age_days > 0
+          topics =
+            topics.where(
+              "topics.created_at > ?",
+              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+            )
+        end
+
+        topics = topics.order(updated_at: :desc).limit(limit)
+
+        next if topics.empty?
+
+        topics.each do |topic|
+          begin
+            DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+
+        DiscourseAi::Translation::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
+      end
+    end
+  end
+end

app/jobs/regular/stream_composer_helper.rb

@@ -8,19 +8,18 @@ module Jobs
       return unless args[:prompt]
       return unless user = User.find_by(id: args[:user_id])
       return unless args[:text]
+      return unless args[:client_id]
 
-      prompt = CompletionPrompt.enabled_by_name(args[:prompt])
-
-      if prompt.id == CompletionPrompt::CUSTOM_PROMPT
-        prompt.custom_instruction = args[:custom_prompt]
-      end
+      helper_mode = args[:prompt]
 
       DiscourseAi::AiHelper::Assistant.new.stream_prompt(
-        prompt,
+        helper_mode,
         args[:text],
         user,
         "/discourse-ai/ai-helper/stream_composer_suggestion",
         force_default_locale: args[:force_default_locale],
+        client_id: args[:client_id],
+        custom_prompt: args[:custom_prompt],
       )
     end
   end

app/jobs/regular/stream_post_helper.rb

@@ -14,16 +14,12 @@ module Jobs
 
       return unless user.guardian.can_see?(post)
 
-      prompt = CompletionPrompt.enabled_by_name(args[:prompt])
+      helper_mode = args[:prompt]
 
-      if prompt.id == CompletionPrompt::CUSTOM_PROMPT
-        prompt.custom_instruction = args[:custom_prompt]
-      end
-
-      if prompt.name == "explain"
-        input = <<~TEXT
-      <term>#{args[:text]}</term>
-      <context>#{post.raw}</context>
+      if helper_mode == DiscourseAi::AiHelper::Assistant::EXPLAIN
+        input = <<~TEXT.strip
+          <term>#{args[:text]}</term>
+          <context>#{post.raw}</context>
           <topic>#{topic.title}</topic>
           #{reply_to ? "<replyTo>#{reply_to.raw}</replyTo>" : nil}
         TEXT
@@ -32,10 +28,11 @@ module Jobs
       end
 
       DiscourseAi::AiHelper::Assistant.new.stream_prompt(
-        prompt,
+        helper_mode,
         input,
         user,
         "/discourse-ai/ai-helper/stream_suggestion/#{post.id}",
+        custom_prompt: args[:custom_prompt],
       )
     end
   end

app/jobs/scheduled/categories_locale_detection_backfill.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Jobs
+  class CategoriesLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 1.hour
+    sidekiq_options retry: false
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      categories = Category.where(locale: nil)
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        categories = categories.where(read_restricted: false)
+      end
+
+      categories = categories.limit(SiteSetting.ai_translation_backfill_rate)
+      return if categories.empty?
+
+      categories.each do |category|
+        begin
+          DiscourseAi::Translation::CategoryLocaleDetector.detect_locale(category)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to detect category #{category.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseAi::Translation::VerboseLogger.log("Detected #{categories.size} category locales")
+    end
+  end
+end

app/jobs/scheduled/category_localization_backfill.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Jobs
+  class CategoryLocalizationBackfill < ::Jobs::Scheduled
+    every 12.hours
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+
+      Jobs.enqueue(:localize_categories)
+    end
+  end
+end

app/jobs/scheduled/generate_concepts_from_popular_items.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateConceptsFromPopularItems < ::Jobs::Scheduled
+    every 1.day
+
+    # This job runs daily and generates new concepts from popular topics and posts
+    # It selects items based on engagement metrics and generates concepts from their content
+    def execute(_args)
+      return unless SiteSetting.inferred_concepts_enabled
+
+      process_popular_topics
+      process_popular_posts
+    end
+
+    private
+
+    def process_popular_topics
+      # Find candidate topics that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_topics(
+          limit: SiteSetting.inferred_concepts_daily_topics_limit || 20,
+          min_posts: SiteSetting.inferred_concepts_min_posts || 5,
+          min_likes: SiteSetting.inferred_concepts_min_likes || 10,
+          min_views: SiteSetting.inferred_concepts_min_views || 100,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate topics - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "topics",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "topics",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+
+    def process_popular_posts
+      # Find candidate posts that are popular and don't have concepts yet
+      manager = DiscourseAi::InferredConcepts::Manager.new
+      candidates =
+        manager.find_candidate_posts(
+          limit: SiteSetting.inferred_concepts_daily_posts_limit || 30,
+          min_likes: SiteSetting.inferred_concepts_post_min_likes || 5,
+          exclude_first_posts: true,
+          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
+        )
+
+      return if candidates.blank?
+
+      # Process candidate posts - first generate concepts, then match
+      Jobs.enqueue(
+        :generate_inferred_concepts,
+        item_type: "posts",
+        item_ids: candidates.map(&:id),
+        batch_size: 10,
+      )
+
+      if SiteSetting.inferred_concepts_background_match
+        # Schedule a follow-up job to match against existing concepts
+        Jobs.enqueue_in(
+          1.hour,
+          :generate_inferred_concepts,
+          item_type: "posts",
+          item_ids: candidates.map(&:id),
+          batch_size: 10,
+          match_only: true,
+        )
+      end
+    end
+  end
+end

app/jobs/scheduled/post_localization_backfill.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Jobs
+  class PostLocalizationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
+    end
+  end
+end

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Jobs
+  class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    sidekiq_options retry: false
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      posts =
+        Post
+          .where(locale: nil)
+          .where(deleted_at: nil)
+          .where("posts.user_id > 0")
+          .where.not(raw: [nil, ""])
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        public_categories = Category.where(read_restricted: false).pluck(:id)
+        posts =
+          posts
+            .joins(:topic)
+            .where(topics: { category_id: public_categories })
+            .where(topics: { archetype: "regular" })
+      end
+
+      if SiteSetting.ai_translation_backfill_max_age_days > 0
+        posts =
+          posts.where(
+            "posts.created_at > ?",
+            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
+      return if posts.empty?
+
+      posts.each do |post|
+        begin
+          DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to detect post #{post.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseAi::Translation::VerboseLogger.log("Detected #{posts.size} post locales")
+    end
+  end
+end

app/jobs/scheduled/topic_localization_backfill.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicLocalizationBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+
+      return if SiteSetting.experimental_content_localization_supported_locales.blank?
+      return if SiteSetting.ai_translation_backfill_rate == 0
+
+      Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
+    end
+  end
+end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    sidekiq_options retry: false
+    cluster_concurrency 1
+
+    def execute(args)
+      return if !SiteSetting.discourse_ai_enabled
+      return if !SiteSetting.ai_translation_enabled
+      limit = SiteSetting.ai_translation_backfill_rate
+
+      return if limit == 0
+
+      topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")
+
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
+        topics = topics.where(category_id: Category.where(read_restricted: false).select(:id))
+      end
+
+      if SiteSetting.ai_translation_backfill_max_age_days > 0
+        topics =
+          topics.where(
+            "topics.created_at > ?",
+            SiteSetting.ai_translation_backfill_max_age_days.days.ago,
+          )
+      end
+
+      topics = topics.order(updated_at: :desc).limit(limit)
+      return if topics.empty?
+
+      topics.each do |topic|
+        begin
+          DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
+        rescue FinalDestination::SSRFDetector::LookupFailedError
+          # do nothing, there are too many sporadic lookup failures
+        rescue => e
+          DiscourseAi::Translation::VerboseLogger.log(
+            "Failed to detect topic #{topic.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseAi::Translation::VerboseLogger.log("Detected #{topics.size} topic locales")
+    end
+  end
+end