DEV: output raw response in dev mode

Makes it easier to debug issues, for example if rate limits are hit.

.discourse-compatibility

@@ -1,5 +1,3 @@
-< 3.5.0.beta7-dev: cd14b0c0bee0cf63c59b64b6f7213e31a37f11a7
-< 3.5.0.beta6-dev: 3e74eea1e5e3143888d67a8d8a11206df214dc24
 < 3.5.0.beta3-dev: 09a68414804a1447f52e5d60691ba59742cda9ec
 < 3.5.0.beta2-dev: de8624416a15b3d8e7ad350b083cc1420451ccec
 < 3.5.0.beta1-dev: bdef136080074a993e7c4f5ca562edc31a8ba756

.gitignore

@@ -6,4 +6,5 @@ evals/log
 evals/cases
 config/eval-llms.local.yml
 # this gets rid of search results from ag, ripgrep, etc
+tokenizers/
 public/ai-share/highlight.min.js

Gemfile.lock

@@ -34,7 +34,7 @@ GEM
       racc
     prettier_print (1.2.1)
     racc (1.8.1)
-    rack (3.1.16)
+    rack (3.1.14)
     rainbow (3.1.1)
     regexp_parser (2.10.0)
     rubocop (1.74.0)

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-features-edit.js

@@ -1,4 +1,3 @@
-import { action } from "@ember/object";
 import { ajax } from "discourse/lib/ajax";
 import DiscourseRoute from "discourse/routes/discourse";
 import SiteSetting from "admin/models/site-setting";
@@ -13,7 +12,7 @@ export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRo
 
     const { site_settings } = await ajax("/admin/config/site_settings.json", {
       data: {
-        filter_area: `ai-features/${currentFeature.module_name}`,
+        filter_area: `ai-features/${currentFeature.ref}`,
         plugin: "discourse-ai",
         category: "discourse_ai",
       },
@@ -25,11 +24,4 @@ export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRo
 
     return currentFeature;
   }
-
-  @action
-  willTransition() {
-    // site settings may amend if a feature is enabled or disabled, so refresh the model
-    // even on back button
-    this.router.refresh("adminPlugins.show.discourse-ai-features");
-  }
 }

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/index.gjs

@@ -0,0 +1,156 @@
+import Component from "@glimmer/component";
+import { service } from "@ember/service";
+import RouteTemplate from "ember-route-template";
+import { gt } from "truth-helpers";
+import DBreadcrumbsItem from "discourse/components/d-breadcrumbs-item";
+import DButton from "discourse/components/d-button";
+import DPageSubheader from "discourse/components/d-page-subheader";
+import { i18n } from "discourse-i18n";
+
+export default RouteTemplate(
+  class extends Component {
+    @service adminPluginNavManager;
+
+    get tableHeaders() {
+      const prefix = "discourse_ai.features.list.header";
+      return [
+        i18n(`${prefix}.name`),
+        i18n(`${prefix}.persona`),
+        i18n(`${prefix}.groups`),
+        "",
+      ];
+    }
+
+    get configuredFeatures() {
+      return this.args.model.filter(
+        (feature) => feature.enable_setting.value === true
+      );
+    }
+
+    get unconfiguredFeatures() {
+      return this.args.model.filter(
+        (feature) => feature.enable_setting.value === false
+      );
+    }
+
+    <template>
+      <DBreadcrumbsItem
+        @path="/admin/plugins/{{this.adminPluginNavManager.currentPlugin.name}}/ai-features"
+        @label={{i18n "discourse_ai.features.short_title"}}
+      />
+      <section class="ai-feature-list admin-detail">
+        <DPageSubheader
+          @titleLabel={{i18n "discourse_ai.features.short_title"}}
+          @descriptionLabel={{i18n "discourse_ai.features.description"}}
+          @learnMoreUrl="todo"
+        />
+
+        {{#if (gt this.configuredFeatures.length 0)}}
+          <div class="ai-feature-list__configured-features">
+            <h3>{{i18n "discourse_ai.features.list.configured_features"}}</h3>
+
+            <table class="d-admin-table">
+              <thead>
+                <tr>
+                  {{#each this.tableHeaders as |header|}}
+                    <th>{{header}}</th>
+                  {{/each}}
+                </tr>
+              </thead>
+
+              <tbody>
+                {{#each this.configuredFeatures as |feature|}}
+                  <tr
+                    class="ai-feature-list__row d-admin-row__content"
+                    data-feature-name={{feature.name}}
+                  >
+                    <td class="d-admin-row__overview ai-feature-list__row-item">
+                      <span class="ai-feature-list__row-item-name">
+                        <strong>
+                          {{feature.name}}
+                        </strong>
+                      </span>
+                      <span class="ai-feature-list__row-item-description">
+                        {{feature.description}}
+                      </span>
+                    </td>
+                    <td
+                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__persona"
+                    >
+                      <DButton
+                        class="btn-flat btn-small ai-feature-list__row-item-persona"
+                        @translatedLabel={{feature.persona.name}}
+                        @route="adminPlugins.show.discourse-ai-personas.edit"
+                        @routeModels={{feature.persona.id}}
+                      />
+                    </td>
+                    <td
+                      class="d-admin-row__detail ai-feature-list__row-item ai-feature-list__groups"
+                    >
+                      {{#if (gt feature.persona.allowed_groups.length 0)}}
+                        <ul class="ai-feature-list__row-item-groups">
+                          {{#each feature.persona.allowed_groups as |group|}}
+                            <li>{{group.name}}</li>
+                          {{/each}}
+                        </ul>
+                      {{/if}}
+                    </td>
+                    <td class="d-admin-row_controls">
+                      <DButton
+                        class="btn-small edit"
+                        @label="discourse_ai.features.list.edit"
+                        @route="adminPlugins.show.discourse-ai-features.edit"
+                        @routeModels={{feature.id}}
+                      />
+                    </td>
+                  </tr>
+                {{/each}}
+              </tbody>
+            </table>
+          </div>
+        {{/if}}
+
+        {{#if (gt this.unconfiguredFeatures.length 0)}}
+          <div class="ai-feature-list__unconfigured-features">
+            <h3>{{i18n "discourse_ai.features.list.unconfigured_features"}}</h3>
+
+            <table class="d-admin-table">
+              <thead>
+                <tr>
+                  <th>{{i18n "discourse_ai.features.list.header.name"}}</th>
+                  <th></th>
+                </tr>
+              </thead>
+
+              <tbody>
+                {{#each this.unconfiguredFeatures as |feature|}}
+                  <tr class="ai-feature-list__row d-admin-row__content">
+                    <td class="d-admin-row__overview ai-feature-list__row-item">
+                      <span class="ai-feature-list__row-item-name">
+                        <strong>
+                          {{feature.name}}
+                        </strong>
+                      </span>
+                      <span class="ai-feature-list__row-item-description">
+                        {{feature.description}}
+                      </span>
+                    </td>
+
+                    <td class="d-admin-row_controls">
+                      <DButton
+                        class="btn-small"
+                        @label="discourse_ai.features.list.set_up"
+                        @route="adminPlugins.show.discourse-ai-features.edit"
+                        @routeModels={{feature.id}}
+                      />
+                    </td>
+                  </tr>
+                {{/each}}
+              </tbody>
+            </table>
+          </div>
+        {{/if}}
+      </section>
+    </template>
+  }
+);

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/index.hbs

@@ -1 +0,0 @@
-<AiFeatures @features={{this.model}} />

app/controllers/discourse_ai/admin/ai_embeddings_controller.rb

@@ -40,7 +40,6 @@ module DiscourseAi
         embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
 
         if embedding_def.save
-          log_ai_embedding_creation(embedding_def)
           render json: AiEmbeddingDefinitionSerializer.new(embedding_def), status: :created
         else
           render_json_error embedding_def
@@ -56,10 +55,7 @@ module DiscourseAi
           )
         end
 
-        initial_attributes = embedding_def.attributes.dup
-
         if embedding_def.update(ai_embeddings_params.except(:dimensions))
-          log_ai_embedding_update(embedding_def, initial_attributes)
           render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
         else
           render_json_error embedding_def
@@ -79,16 +75,7 @@ module DiscourseAi
           return render_json_error(I18n.t("discourse_ai.embeddings.delete_failed"), status: 409)
         end
 
-        embedding_details = {
-          embedding_id: embedding_def.id,
-          display_name: embedding_def.display_name,
-          provider: embedding_def.provider,
-          dimensions: embedding_def.dimensions,
-          subject: embedding_def.display_name,
-        }
-
         if embedding_def.destroy
-          log_ai_embedding_deletion(embedding_details)
           head :no_content
         else
           render_json_error embedding_def
@@ -141,60 +128,6 @@ module DiscourseAi
 
         permitted
       end
-
-      def ai_embeddings_logger_fields
-        {
-          display_name: {
-          },
-          provider: {
-          },
-          dimensions: {
-          },
-          url: {
-          },
-          tokenizer_class: {
-          },
-          max_sequence_length: {
-          },
-          embed_prompt: {
-            type: :large_text,
-          },
-          search_prompt: {
-            type: :large_text,
-          },
-          matryoshka_dimensions: {
-          },
-          api_key: {
-            type: :sensitive,
-          },
-          # JSON fields should be tracked as simple changes
-          json_fields: [:provider_params],
-        }
-      end
-
-      def log_ai_embedding_creation(embedding_def)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
-        logger.log_creation("embedding", embedding_def, ai_embeddings_logger_fields, entity_details)
-      end
-
-      def log_ai_embedding_update(embedding_def, initial_attributes)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
-
-        logger.log_update(
-          "embedding",
-          embedding_def,
-          initial_attributes,
-          ai_embeddings_logger_fields,
-          entity_details,
-        )
-      end
-
-      def log_ai_embedding_deletion(embedding_details)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        logger.log_deletion("embedding", embedding_details)
-      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_features_controller.rb

@@ -6,44 +6,24 @@ module DiscourseAi
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
       def index
-        render json: serialize_modules(DiscourseAi::Configuration::Module.all)
+        render json: serialize_features(DiscourseAi::Features.features)
       end
 
       def edit
         raise Discourse::InvalidParameters.new(:id) if params[:id].blank?
-
+        render json: serialize_feature(DiscourseAi::Features.find_feature_by_id(params[:id].to_i))
-        a_module = DiscourseAi::Configuration::Module.find_by(id: params[:id].to_i)
-
-        render json: serialize_module(a_module)
       end
 
       private
 
-      def serialize_modules(modules)
+      def serialize_features(features)
-        modules.map { |a_module| serialize_module(a_module) }
+        features.map { |feature| feature.merge(persona: serialize_persona(feature[:persona])) }
-      end
-
-      def serialize_module(a_module)
-        return nil if a_module.blank?
-
-        {
-          id: a_module.id,
-          module_name: a_module.name,
-          module_enabled: a_module.enabled?,
-          features: a_module.features.map { |f| serialize_feature(f) },
-        }
       end
 
       def serialize_feature(feature)
-        {
+        return nil if feature.blank?
-          name: feature.name,
+
-          personas: feature.persona_ids.map { |id| serialize_persona(persona_id_obj_hash[id]) },
+        feature.merge(persona: serialize_persona(feature[:persona]))
-          llm_models:
-            feature.llm_models.map do |llm_model|
-              { id: llm_model.id, name: llm_model.display_name }
-            end,
-          enabled: feature.enabled?,
-        }
       end
 
       def serialize_persona(persona)
@@ -51,16 +31,6 @@ module DiscourseAi
 
         serialize_data(persona, AiFeaturesPersonaSerializer, root: false)
       end
-
-      private
-
-      def persona_id_obj_hash
-        @persona_id_obj_hash ||=
-          begin
-            ids = DiscourseAi::Configuration::Feature.all.map(&:persona_ids).flatten.uniq
-            AiPersona.where(id: ids).index_by(&:id)
-          end
-      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_llms_controller.rb

@@ -47,7 +47,6 @@ module DiscourseAi
 
         if llm_model.save
           llm_model.toggle_companion_user
-          log_llm_model_creation(llm_model)
           render json: LlmModelSerializer.new(llm_model), status: :created
         else
           render_json_error llm_model
@@ -57,10 +56,6 @@ module DiscourseAi
       def update
         llm_model = LlmModel.find(params[:id])
 
-        # Capture initial state for logging
-        initial_attributes = llm_model.attributes.dup
-        initial_quotas = llm_model.llm_quotas.map(&:attributes)
-
         if params[:ai_llm].key?(:llm_quotas)
           if quota_params
             existing_quota_group_ids = llm_model.llm_quotas.pluck(:group_id)
@@ -86,7 +81,6 @@ module DiscourseAi
 
         if llm_model.update(ai_llm_params(updating: llm_model))
           llm_model.toggle_companion_user
-          log_llm_model_update(llm_model, initial_attributes, initial_quotas)
           render json: LlmModelSerializer.new(llm_model)
         else
           render_json_error llm_model
@@ -115,20 +109,11 @@ module DiscourseAi
           )
         end
 
-        # Capture model details for logging before destruction
-        model_details = {
-          model_id: llm_model.id,
-          display_name: llm_model.display_name,
-          name: llm_model.name,
-          provider: llm_model.provider,
-        }
-
         # Clean up companion users
         llm_model.enabled_chat_bot = false
         llm_model.toggle_companion_user
 
         if llm_model.destroy
-          log_llm_model_deletion(model_details)
           head :no_content
         else
           render_json_error llm_model
@@ -205,89 +190,6 @@ module DiscourseAi
 
         permitted
       end
-
-      def ai_llm_logger_fields
-        {
-          display_name: {
-          },
-          name: {
-          },
-          provider: {
-          },
-          tokenizer: {
-          },
-          url: {
-          },
-          max_prompt_tokens: {
-          },
-          max_output_tokens: {
-          },
-          enabled_chat_bot: {
-          },
-          vision_enabled: {
-          },
-          api_key: {
-            type: :sensitive,
-          },
-          input_cost: {
-          },
-          output_cost: {
-          },
-          # JSON fields should be tracked as simple changes
-          json_fields: [:provider_params],
-        }
-      end
-
-      def log_llm_model_creation(llm_model)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
-
-        # Add quota information as a special case
-        if llm_model.llm_quotas.any?
-          entity_details[:quotas] = llm_model
-            .llm_quotas
-            .map do |quota|
-              "Group #{quota.group_id}: #{quota.max_tokens} tokens, #{quota.max_usages} usages, #{quota.duration_seconds}s"
-            end
-            .join("; ")
-        end
-
-        logger.log_creation("llm_model", llm_model, ai_llm_logger_fields, entity_details)
-      end
-
-      def log_llm_model_update(llm_model, initial_attributes, initial_quotas)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
-
-        # Track quota changes separately as they're a special case
-        current_quotas = llm_model.llm_quotas.reload.map(&:attributes)
-        if initial_quotas != current_quotas
-          initial_quota_summary =
-            initial_quotas
-              .map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
-              .join("; ")
-          current_quota_summary =
-            current_quotas
-              .map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
-              .join("; ")
-          entity_details[:quotas_changed] = true
-          entity_details[:quotas] = "#{initial_quota_summary} → #{current_quota_summary}"
-        end
-
-        logger.log_update(
-          "llm_model",
-          llm_model,
-          initial_attributes,
-          ai_llm_logger_fields,
-          entity_details,
-        )
-      end
-
-      def log_llm_model_deletion(model_details)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        model_details[:subject] = model_details[:display_name]
-        logger.log_deletion("llm_model", model_details)
-      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -5,20 +5,19 @@ module DiscourseAi
     class AiPersonasController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_persona, only: %i[edit update destroy create_user export]
+      before_action :find_ai_persona, only: %i[edit update destroy create_user]
 
       def index
         ai_personas =
-          AiPersona
+          AiPersona.ordered.map do |persona|
-            .ordered
+            # we use a special serializer here cause names and descriptions are
-            .includes(:user, :uploads)
+            # localized for system personas
-            .map { |persona| LocalizedAiPersonaSerializer.new(persona, root: false) }
+            LocalizedAiPersonaSerializer.new(persona, root: false)
-
+          end
         tools =
           DiscourseAi::Personas::Persona.all_available_tools.map do |tool|
             AiToolSerializer.new(tool, root: false)
           end
-
         AiTool
           .where(enabled: true)
           .each do |tool|
@@ -32,12 +31,10 @@ module DiscourseAi
                 ),
             }
           end
-
         llms =
           DiscourseAi::Configuration::LlmEnumerator.values_for_serialization(
             allowed_seeded_llm_ids: SiteSetting.ai_bot_allowed_seeded_models_map,
           )
-
         render json: {
                  ai_personas: ai_personas,
                  meta: {
@@ -61,7 +58,6 @@ module DiscourseAi
         ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
         if ai_persona.save
           RagDocumentFragment.link_target_and_uploads(ai_persona, attached_upload_ids)
-          log_ai_persona_creation(ai_persona)
 
           render json: {
                    ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
@@ -78,11 +74,8 @@ module DiscourseAi
       end
 
       def update
-        initial_attributes = @ai_persona.attributes.dup
-
         if @ai_persona.update(ai_persona_params.except(:rag_uploads))
           RagDocumentFragment.update_target_uploads(@ai_persona, attached_upload_ids)
-          log_ai_persona_update(@ai_persona, initial_attributes)
 
           render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
         else
@@ -91,57 +84,13 @@ module DiscourseAi
       end
 
       def destroy
-        persona_details = {
-          persona_id: @ai_persona.id,
-          name: @ai_persona.name,
-          description: @ai_persona.description,
-        }
-
         if @ai_persona.destroy
-          log_ai_persona_deletion(persona_details)
           head :no_content
         else
           render_json_error @ai_persona
         end
       end
 
-      def export
-        persona = AiPersona.find(params[:id])
-        exporter = DiscourseAi::PersonaExporter.new(persona: persona)
-
-        response.headers[
-          "Content-Disposition"
-        ] = "attachment; filename=\"#{persona.name.parameterize}.json\""
-
-        render json: exporter.export
-      end
-
-      def import
-        name = params.dig(:persona, :name)
-        existing_persona = AiPersona.find_by(name: name)
-        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
-
-        begin
-          importer = DiscourseAi::PersonaImporter.new(json: params.to_unsafe_h)
-
-          if existing_persona && force_update
-            initial_attributes = existing_persona.attributes.dup
-            persona = importer.import!(overwrite: true)
-            log_ai_persona_update(persona, initial_attributes)
-            render json: LocalizedAiPersonaSerializer.new(persona, root: false)
-          else
-            persona = importer.import!
-            log_ai_persona_creation(persona)
-            render json: LocalizedAiPersonaSerializer.new(persona, root: false), status: :created
-          end
-        rescue DiscourseAi::PersonaImporter::ImportError => e
-          render_json_error e.message, status: :unprocessable_entity
-        rescue StandardError => e
-          Rails.logger.error("AI Persona import failed: #{e.message}")
-          render_json_error "Import failed: #{e.message}", status: :unprocessable_entity
-        end
-      end
-
       def stream_reply
         persona =
           AiPersona.find_by(name: params[:persona_name]) ||
@@ -312,92 +261,6 @@ module DiscourseAi
 
         examples.map { |example_arr| example_arr.take(2).map(&:to_s) }
       end
-
-      def ai_persona_logger_fields
-        {
-          name: {
-          },
-          description: {
-          },
-          enabled: {
-          },
-          priority: {
-          },
-          system_prompt: {
-            type: :large_text,
-          },
-          default_llm_id: {
-          },
-          temperature: {
-          },
-          top_p: {
-          },
-          user_id: {
-          },
-          max_context_posts: {
-          },
-          vision_enabled: {
-          },
-          vision_max_pixels: {
-          },
-          rag_chunk_tokens: {
-          },
-          rag_chunk_overlap_tokens: {
-          },
-          rag_conversation_chunks: {
-          },
-          rag_llm_model_id: {
-          },
-          question_consolidator_llm_id: {
-          },
-          allow_chat_channel_mentions: {
-          },
-          allow_chat_direct_messages: {
-          },
-          allow_topic_mentions: {
-          },
-          allow_personal_messages: {
-          },
-          tool_details: {
-            type: :large_text,
-          },
-          forced_tool_count: {
-          },
-          force_default_llm: {
-          },
-          # JSON fields
-          json_fields: %i[tools response_format examples allowed_group_ids],
-        }
-      end
-
-      def log_ai_persona_creation(ai_persona)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
-        entity_details[:tools_count] = (ai_persona.tools || []).size
-
-        logger.log_creation("persona", ai_persona, ai_persona_logger_fields, entity_details)
-      end
-
-      def log_ai_persona_update(ai_persona, initial_attributes)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
-        entity_details[:tools_count] = ai_persona.tools.size if ai_persona.tools.present?
-
-        logger.log_update(
-          "persona",
-          ai_persona,
-          initial_attributes,
-          ai_persona_logger_fields,
-          entity_details,
-        )
-      end
-
-      def log_ai_persona_deletion(persona_details)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        persona_details[:subject] = persona_details[:name]
-
-        logger.log_deletion("persona", persona_details)
-      end
     end
   end
 end

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -10,17 +10,6 @@ module DiscourseAi
       end
 
       def update
-        initial_settings = AiModerationSetting.spam
-
-        initial_data = {
-          custom_instructions: initial_settings&.data&.dig("custom_instructions"),
-          llm_model_id: initial_settings&.llm_model_id,
-          ai_persona_id: initial_settings&.ai_persona_id,
-        }
-
-        initial_custom_instructions = initial_settings&.data&.dig("custom_instructions")
-        initial_llm_model_id = initial_settings&.llm_model_id
-
         updated_params = {}
         if allowed_params.key?(:llm_model_id)
           llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
@@ -36,22 +25,6 @@ module DiscourseAi
             )
           end
         end
-
-        if allowed_params.key?(:ai_persona_id)
-          updated_params[:ai_persona_id] = allowed_params[:ai_persona_id]
-          persona = AiPersona.find_by(id: allowed_params[:ai_persona_id])
-          if persona.nil? ||
-               persona.response_format.to_a.none? { |rf|
-                 rf["key"] == "spam" && rf["type"] == "boolean"
-               }
-            return(
-              render_json_error(
-                I18n.t("discourse_ai.llm.configuration.invalid_persona_response_format"),
-                status: 422,
-              )
-            )
-          end
-        end
         updated_params[:data] = {
           custom_instructions: allowed_params[:custom_instructions],
         } if allowed_params.key?(:custom_instructions)
@@ -63,8 +36,6 @@ module DiscourseAi
           else
             AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
           end
-
-          log_ai_spam_update(initial_data, allowed_params)
         end
 
         is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
@@ -79,7 +50,7 @@ module DiscourseAi
             )
           end
 
-          SiteSetting.set_and_log("ai_spam_detection_enabled", is_enabled, current_user)
+          SiteSetting.ai_spam_detection_enabled = is_enabled
         end
 
         render json: AiSpamSerializer.new(spam_config, root: false)
@@ -142,44 +113,8 @@ module DiscourseAi
 
       private
 
-      def log_ai_spam_update(initial_data, params)
-        changes_to_log = {}
-
-        initial_llm_model_id = initial_data[:llm_model_id]
-        if params.key?(:llm_model_id) && initial_llm_model_id.to_s != params[:llm_model_id].to_s
-          old_model_name =
-            LlmModel.find_by(id: initial_llm_model_id)&.display_name || initial_llm_model_id
-          new_model_name =
-            LlmModel.find_by(id: params[:llm_model_id])&.display_name || params[:llm_model_id]
-
-          changes_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"
-        end
-
-        initial_custom_instructions = initial_data[:custom_instructions]
-        if params.key?(:custom_instructions) &&
-             initial_custom_instructions != params[:custom_instructions]
-          changes_to_log[:custom_instructions] = params[:custom_instructions]
-        end
-
-        initial_ai_persona_id = initial_data[:ai_persona_id]
-        if params.key?(:ai_persona_id) && initial_ai_persona_id.to_s != params[:ai_persona_id].to_s
-          old_persona_name =
-            AiPersona.find_by(id: initial_ai_persona_id)&.name || initial_ai_persona_id
-          new_persona_name =
-            AiPersona.find_by(id: params[:ai_persona_id])&.name || params[:ai_persona_id]
-
-          changes_to_log[:ai_persona_id] = "#{old_persona_name} → #{new_persona_name}"
-        end
-
-        if changes_to_log.present?
-          changes_to_log[:subject] = I18n.t("discourse_ai.spam_detection.logging_subject")
-          logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-          logger.log_custom("update_ai_spam_settings", changes_to_log)
-        end
-      end
-
       def allowed_params
-        params.permit(:is_enabled, :llm_model_id, :custom_instructions, :ai_persona_id)
+        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
       end
 
       def spam_config

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -5,7 +5,7 @@ module DiscourseAi
     class AiToolsController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_tool, only: %i[test edit update destroy export]
+      before_action :find_ai_tool, only: %i[test edit update destroy]
 
       def index
         ai_tools = AiTool.all
@@ -25,58 +25,15 @@ module DiscourseAi
 
         if ai_tool.save
           RagDocumentFragment.link_target_and_uploads(ai_tool, attached_upload_ids)
-          log_ai_tool_creation(ai_tool)
           render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
         else
           render_json_error ai_tool
         end
       end
 
-      def export
-        response.headers[
-          "Content-Disposition"
-        ] = "attachment; filename=\"#{@ai_tool.tool_name}.json\""
-        render_serialized(@ai_tool, AiCustomToolSerializer)
-      end
-
-      def import
-        existing_tool = AiTool.find_by(tool_name: ai_tool_params[:tool_name])
-        force_update = params[:force].present? && params[:force].to_s.downcase == "true"
-
-        if existing_tool && !force_update
-          return(
-            render_json_error "Tool with tool_name '#{ai_tool_params[:tool_name]}' already exists. Use force=true to overwrite.",
-                              status: :conflict
-          )
-        end
-
-        if existing_tool && force_update
-          initial_attributes = existing_tool.attributes.dup
-          if existing_tool.update(ai_tool_params)
-            log_ai_tool_update(existing_tool, initial_attributes)
-            render_serialized(existing_tool, AiCustomToolSerializer)
-          else
-            render_json_error existing_tool
-          end
-        else
-          ai_tool = AiTool.new(ai_tool_params)
-          ai_tool.created_by_id = current_user.id
-
-          if ai_tool.save
-            log_ai_tool_creation(ai_tool)
-            render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
-          else
-            render_json_error ai_tool
-          end
-        end
-      end
-
       def update
-        initial_attributes = @ai_tool.attributes.dup
-
         if @ai_tool.update(ai_tool_params)
           RagDocumentFragment.update_target_uploads(@ai_tool, attached_upload_ids)
-          log_ai_tool_update(@ai_tool, initial_attributes)
           render_serialized(@ai_tool, AiCustomToolSerializer)
         else
           render_json_error @ai_tool
@@ -84,15 +41,7 @@ module DiscourseAi
       end
 
       def destroy
-        tool_logger_details = {
-          tool_id: @ai_tool.id,
-          name: @ai_tool.name,
-          tool_name: @ai_tool.tool_name,
-          subject: @ai_tool.name,
-        }
-
         if @ai_tool.destroy
-          log_ai_tool_deletion(tool_logger_details)
           head :no_content
         else
           render_json_error @ai_tool
@@ -147,60 +96,6 @@ module DiscourseAi
           )
           .except(:rag_uploads)
       end
-
-      def ai_tool_logger_fields
-        {
-          name: {
-          },
-          tool_name: {
-          },
-          description: {
-          },
-          summary: {
-          },
-          enabled: {
-          },
-          rag_chunk_tokens: {
-          },
-          rag_chunk_overlap_tokens: {
-          },
-          rag_llm_model_id: {
-          },
-          script: {
-            type: :large_text,
-          },
-          parameters: {
-            type: :large_text,
-          },
-        }
-      end
-
-      def log_ai_tool_creation(ai_tool)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-
-        entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
-        entity_details[:parameter_count] = ai_tool.parameters.size if ai_tool.parameters.present?
-
-        logger.log_creation("tool", ai_tool, ai_tool_logger_fields, entity_details)
-      end
-
-      def log_ai_tool_update(ai_tool, initial_attributes)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
-
-        logger.log_update(
-          "tool",
-          ai_tool,
-          initial_attributes,
-          ai_tool_logger_fields,
-          entity_details,
-        )
-      end
-
-      def log_ai_tool_deletion(tool_details)
-        logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
-        logger.log_deletion("tool", tool_details)
-      end
     end
   end
 end

app/controllers/discourse_ai/ai_bot/artifact_key_values_controller.rb

@@ -1,124 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module AiBot
-    class ArtifactKeyValuesController < ::ApplicationController
-      requires_plugin DiscourseAi::PLUGIN_NAME
-      before_action :ensure_logged_in, only: %i[set destroy]
-      before_action :find_artifact
-
-      PER_PAGE_MAX = 100
-
-      def index
-        page = index_params[:page].to_i
-        page = 1 if page < 1
-        per_page = index_params[:per_page].to_i
-        per_page = PER_PAGE_MAX if per_page < 1 || per_page > PER_PAGE_MAX
-
-        query = build_index_query
-
-        total_count = query.count
-        key_values =
-          query
-            .includes(:user)
-            .order(:user_id, :key, :created_at)
-            .offset((page - 1) * per_page)
-            .limit(per_page + 1)
-
-        has_more = key_values.length > per_page
-        key_values = key_values.first(per_page) if has_more
-
-        render json: {
-                 key_values:
-                   ActiveModel::ArraySerializer.new(
-                     key_values,
-                     each_serializer: AiArtifactKeyValueSerializer,
-                     keys_only: params[:keys_only] == "true",
-                   ).as_json,
-                 has_more: has_more,
-                 total_count: total_count,
-                 users:
-                   key_values
-                     .map { |kv| kv.user }
-                     .uniq
-                     .map { |u| BasicUserSerializer.new(u, root: nil).as_json },
-               }
-      end
-
-      def destroy
-        if params[:key].blank?
-          render json: { error: "Key parameter is required" }, status: :bad_request
-          return
-        end
-
-        key_value = @artifact.key_values.find_by(user_id: current_user.id, key: params[:key])
-
-        if key_value.nil?
-          render json: { error: "Key not found" }, status: :not_found
-        elsif key_value.destroy
-          head :ok
-        else
-          render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
-        end
-      end
-
-      def set
-        key_value =
-          @artifact.key_values.find_or_initialize_by(
-            user: current_user,
-            key: key_value_params[:key],
-          )
-
-        key_value.assign_attributes(key_value_params.except(:key))
-
-        if key_value.save
-          render json: AiArtifactKeyValueSerializer.new(key_value).as_json
-        else
-          render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
-        end
-      end
-
-      private
-
-      def key_value_params
-        params.permit(:key, :value, :public)
-      end
-
-      def index_params
-        @index_params ||= params.permit(:page, :per_page, :key, :keys_only, :all_users)
-      end
-
-      def build_index_query
-        query = @artifact.key_values
-
-        query =
-          if current_user&.admin?
-            query
-          elsif current_user
-            query.where("user_id = ? OR public = true", current_user.id)
-          else
-            query.where(public: true)
-          end
-
-        query = query.where("key = ?", index_params[:key]) if index_params[:key].present?
-
-        if index_params[:all_users].to_s != "true"
-          if current_user
-            query = query.where(user_id: current_user.id)
-          else
-            query = query.where("1 = 0")
-          end
-        end
-
-        query
-      end
-
-      def find_artifact
-        @artifact = AiArtifact.find_by(id: params[:artifact_id])
-        raise Discourse::NotFound if !@artifact
-        raise Discourse::NotFound if [email protected]? && guardian.anonymous?
-        raise Discourse::NotFound if [email protected]? && !guardian.can_see?(@artifact.post)
-      end
-    end
-  end
-end

app/controllers/discourse_ai/ai_bot/shared_ai_conversations_controller.rb

@@ -8,7 +8,7 @@ module DiscourseAi
       before_action :require_site_settings!
 
       skip_before_action :preload_json, :check_xhr, only: %i[show asset]
-      skip_before_action :redirect_to_login_if_required, :verify_authenticity_token, only: %i[asset]
+      skip_before_action :verify_authenticity_token, only: ["asset"]
 
       def create
         ensure_allowed_create!

app/controllers/discourse_ai/ai_helper/assistant_controller.rb

@@ -25,24 +25,25 @@ module DiscourseAi
         input = get_text_param!
         force_default_locale = params[:force_default_locale] || false
 
-        raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?
+        prompt = CompletionPrompt.find_by(id: params[:mode])
 
-        if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
+        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
+
+        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
           raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
+
+          prompt.custom_instruction = params[:custom_prompt]
         end
 
-        if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
+        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
-          return suggest_thumbnails(input)
-        end
 
         hijack do
           render json:
                    DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
-                     params[:mode],
+                     prompt,
                      input,
                      current_user,
                      force_default_locale: force_default_locale,
-                     custom_prompt: params[:custom_prompt],
                    ),
                  status: 200
         end
@@ -59,10 +60,13 @@ module DiscourseAi
           input = get_text_param!
         end
 
+        prompt = CompletionPrompt.enabled_by_name("generate_titles")
+        raise Discourse::InvalidParameters.new(:mode) if !prompt
+
         hijack do
           render json:
                    DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
-                     DiscourseAi::AiHelper::Assistant::GENERATE_TITLES,
+                     prompt,
                      input,
                      current_user,
                    ),
@@ -111,32 +115,23 @@ module DiscourseAi
         location = params[:location]
         raise Discourse::InvalidParameters.new(:location) if !location
 
-        raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?
+        prompt = CompletionPrompt.find_by(id: params[:mode])
-        if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
-          return suggest_thumbnails(input)
-        end
 
-        if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
+        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
+        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+
+        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
           raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
         end
 
-        # to stream we must have an appropriate client_id
-        # otherwise we may end up streaming the data to the wrong client
-        raise Discourse::InvalidParameters.new(:client_id) if params[:client_id].blank?
-
-        channel_id = next_channel_id
-        progress_channel = "discourse_ai_helper/stream_suggestions/#{channel_id}"
-
         if location == "composer"
           Jobs.enqueue(
             :stream_composer_helper,
             user_id: current_user.id,
             text: text,
-            prompt: params[:mode],
+            prompt: prompt.name,
             custom_prompt: params[:custom_prompt],
             force_default_locale: params[:force_default_locale] || false,
-            client_id: params[:client_id],
-            progress_channel:,
           )
         else
           post_id = get_post_param!
@@ -149,14 +144,12 @@ module DiscourseAi
             post_id: post.id,
             user_id: current_user.id,
             text: text,
-            prompt: params[:mode],
+            prompt: prompt.name,
             custom_prompt: params[:custom_prompt],
-            client_id: params[:client_id],
-            progress_channel:,
           )
         end
 
-        render json: { success: true, progress_channel: }, status: 200
+        render json: { success: true }, status: 200
       rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
         render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                           status: 502
@@ -197,18 +190,6 @@ module DiscourseAi
 
       private
 
-      CHANNEL_ID_KEY = "discourse_ai_helper_next_channel_id"
-
-      def next_channel_id
-        Discourse
-          .redis
-          .pipelined do |pipeline|
-            pipeline.incr(CHANNEL_ID_KEY)
-            pipeline.expire(CHANNEL_ID_KEY, 1.day)
-          end
-          .first
-      end
-
       def get_text_param!
         params[:text].tap { |t| raise Discourse::InvalidParameters.new(:text) if t.blank? }
       end

app/jobs/regular/create_ai_reply.rb

@@ -15,7 +15,7 @@ module ::Jobs
 
         bot = DiscourseAi::Personas::Bot.as(bot_user, persona: persona.new)
 
-        DiscourseAi::AiBot::Playground.new(bot).reply_to(post, feature_name: "bot")
+        DiscourseAi::AiBot::Playground.new(bot).reply_to(post)
       rescue DiscourseAi::Personas::Bot::BOT_NOT_FOUND
         Rails.logger.warn(
           "Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",

app/jobs/regular/detect_translate_post.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class DetectTranslatePost < ::Jobs::Base
-    cluster_concurrency 1
-    sidekiq_options retry: false
-
-    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      return if args[:post_id].blank?
-
-      post = Post.find_by(id: args[:post_id])
-      return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
-
-      topic = post.topic
-      return if topic.blank?
-
-      if SiteSetting.ai_translation_backfill_limit_to_public_content
-        return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
-      else
-        if topic.archetype == Archetype.private_message &&
-             !TopicAllowedGroup.exists?(topic_id: topic.id)
-          return
-        end
-      end
-
-      # the user may fill locale in manually
-      if (detected_locale = post.locale).blank?
-        begin
-          detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
-        rescue FinalDestination::SSRFDetector::LookupFailedError
-          # this job is non-critical
-          # the backfill job will handle failures
-          return
-        end
-      end
-
-      locales = SiteSetting.content_localization_supported_locales.split("|")
-      return if locales.blank?
-
-      locales.each do |locale|
-        next if locale == detected_locale
-        next if post.post_localizations.exists?(locale:)
-
-        begin
-          DiscourseAi::Translation::PostLocalizer.localize(post, locale)
-        rescue FinalDestination::SSRFDetector::LookupFailedError
-          # do nothing, there are too many sporadic lookup failures
-        rescue => e
-          DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
-          )
-        end
-      end
-
-      MessageBus.publish("/topic/#{post.topic_id}", type: :localized, id: post.id)
-    end
-  end
-end

app/jobs/regular/detect_translate_topic.rb

@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class DetectTranslateTopic < ::Jobs::Base
-    cluster_concurrency 1
-    sidekiq_options retry: false
-
-    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      return if args[:topic_id].blank?
-
-      topic = Topic.find_by(id: args[:topic_id])
-      if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0
-        return
-      end
-
-      if SiteSetting.ai_translation_backfill_limit_to_public_content
-        return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
-      else
-        if topic.archetype == Archetype.private_message &&
-             !TopicAllowedGroup.exists?(topic_id: topic.id)
-          return
-        end
-      end
-
-      if (detected_locale = topic.locale).blank?
-        begin
-          detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
-        rescue FinalDestination::SSRFDetector::LookupFailedError
-          # this job is non-critical
-          # the backfill job will handle failures
-          return
-        end
-      end
-
-      locales = SiteSetting.content_localization_supported_locales.split("|")
-      return if locales.blank?
-
-      locales.each do |locale|
-        next if locale == detected_locale
-        next if topic.topic_localizations.exists?(locale:)
-
-        begin
-          DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
-        rescue FinalDestination::SSRFDetector::LookupFailedError
-          # do nothing, there are too many sporadic lookup failures
-        rescue => e
-          DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
-          )
-        end
-      end
-
-      MessageBus.publish("/topic/#{topic.id}", type: :localized, id: 1)
-    end
-  end
-end

app/jobs/regular/generate_inferred_concepts.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class GenerateInferredConcepts < ::Jobs::Base
-    sidekiq_options queue: "low"
-
-    # Process items to generate new concepts
-    #
-    # @param args [Hash] Contains job arguments
-    # @option args [String] :item_type Required - Type of items to process ('topics' or 'posts')
-    # @option args [Array<Integer>] :item_ids Required - List of item IDs to process
-    # @option args [Integer] :batch_size (100) Number of items to process in each batch
-    # @option args [Boolean] :match_only (false) Only match against existing concepts without generating new ones
-    def execute(args = {})
-      return if args[:item_ids].blank? || args[:item_type].blank?
-
-      if %w[topics posts].exclude?(args[:item_type])
-        Rails.logger.error("Invalid item_type for GenerateInferredConcepts: #{args[:item_type]}")
-        return
-      end
-
-      # Process items in smaller batches to avoid memory issues
-      batch_size = args[:batch_size] || 100
-
-      # Get the list of item IDs
-      item_ids = args[:item_ids]
-      match_only = args[:match_only] || false
-
-      # Process items in batches
-      item_ids.each_slice(batch_size) do |batch_item_ids|
-        process_batch(batch_item_ids, args[:item_type], match_only)
-      end
-    end
-
-    private
-
-    def process_batch(item_ids, item_type, match_only)
-      klass = item_type.singularize.classify.constantize
-      items = klass.where(id: item_ids)
-      manager = DiscourseAi::InferredConcepts::Manager.new
-
-      items.each do |item|
-        begin
-          process_item(item, item_type, match_only, manager)
-        rescue => e
-          Rails.logger.error(
-            "Error generating concepts from #{item_type.singularize} #{item.id}: #{e.message}\n#{e.backtrace.join("\n")}",
-          )
-        end
-      end
-    end
-
-    def process_item(item, item_type, match_only, manager)
-      # Use the Manager method that handles both identifying and creating concepts
-      if match_only
-        if item_type == "topics"
-          manager.match_topic_to_concepts(item)
-        else # posts
-          manager.match_post_to_concepts(item)
-        end
-      else
-        if item_type == "topics"
-          manager.generate_concepts_from_topic(item)
-        else # posts
-          manager.generate_concepts_from_post(item)
-        end
-      end
-    end
-  end
-end

app/jobs/regular/localize_categories.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class LocalizeCategories < ::Jobs::Base
-    cluster_concurrency 1
-    sidekiq_options retry: false
-
-    def execute(args)
-      limit = args[:limit]
-      raise Discourse::InvalidParameters.new(:limit) if limit.nil?
-      return if limit <= 0
-
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      locales = SiteSetting.content_localization_supported_locales.split("|")
-      return if locales.blank?
-
-      categories = Category.where("locale IS NOT NULL")
-
-      if SiteSetting.ai_translation_backfill_limit_to_public_content
-        categories = categories.where(read_restricted: false)
-      end
-
-      categories = categories.order(:id).limit(limit)
-      return if categories.empty?
-
-      remaining_limit = limit
-
-      categories.each do |category|
-        break if remaining_limit <= 0
-
-        existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)
-        missing_locales = locales - existing_locales - [category.locale]
-        missing_locales.each do |locale|
-          break if remaining_limit <= 0
-
-          begin
-            DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
-          rescue FinalDestination::SSRFDetector::LookupFailedError
-            # do nothing, there are too many sporadic lookup failures
-          rescue => e
-            DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate category #{category.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
-            )
-          ensure
-            remaining_limit -= 1
-          end
-        end
-
-        if existing_locales.include?(category.locale)
-          CategoryLocalization.find_by(category_id: category.id, locale: category.locale).destroy
-        end
-      end
-    end
-  end
-end

app/jobs/regular/localize_posts.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class LocalizePosts < ::Jobs::Base
-    cluster_concurrency 1
-    sidekiq_options retry: false
-
-    def execute(args)
-      limit = args[:limit]
-      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
-
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-
-      locales = SiteSetting.content_localization_supported_locales.split("|")
-      return if locales.blank?
-
-      locales.each do |locale|
-        posts =
-          Post
-            .joins(
-              "LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
-            )
-            .where(deleted_at: nil)
-            .where("posts.user_id > 0")
-            .where.not(raw: [nil, ""])
-            .where.not(locale: nil)
-            .where.not(locale: locale)
-            .where("pl.id IS NULL")
-
-        posts = posts.joins(:topic)
-
-        if SiteSetting.ai_translation_backfill_limit_to_public_content
-          # exclude all PMs
-          # and only include posts from public categories
-          posts =
-            posts
-              .where.not(topics: { archetype: Archetype.private_message })
-              .where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
-        else
-          # all regular topics, and group PMs
-          posts =
-            posts.where(
-              "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
-              Archetype.private_message,
-            )
-        end
-
-        if SiteSetting.ai_translation_backfill_max_age_days > 0
-          posts =
-            posts.where(
-              "posts.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
-            )
-        end
-
-        posts = posts.order(updated_at: :desc).limit(limit)
-
-        next if posts.empty?
-
-        posts.each do |post|
-          begin
-            DiscourseAi::Translation::PostLocalizer.localize(post, locale)
-          rescue FinalDestination::SSRFDetector::LookupFailedError
-            # do nothing, there are too many sporadic lookup failures
-          rescue => e
-            DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
-            )
-          end
-        end
-
-        DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
-      end
-    end
-  end
-end

app/jobs/regular/localize_topics.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class LocalizeTopics < ::Jobs::Base
-    cluster_concurrency 1
-    sidekiq_options retry: false
-
-    def execute(args)
-      limit = args[:limit]
-      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
-
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-
-      locales = SiteSetting.content_localization_supported_locales.split("|")
-      return if locales.blank?
-
-      locales.each do |locale|
-        topics =
-          Topic
-            .joins(
-              "LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale = #{ActiveRecord::Base.connection.quote(locale)}",
-            )
-            .where(deleted_at: nil)
-            .where("topics.user_id > 0")
-            .where.not(locale: nil)
-            .where.not(locale: locale)
-            .where("tl.id IS NULL")
-
-        if SiteSetting.ai_translation_backfill_limit_to_public_content
-          # exclude all PMs
-          # and only include posts from public categories
-          topics =
-            topics
-              .where.not(archetype: Archetype.private_message)
-              .where(category_id: Category.where(read_restricted: false).select(:id))
-        else
-          # all regular topics, and group PMs
-          topics =
-            topics.where(
-              "topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
-              Archetype.private_message,
-            )
-        end
-
-        if SiteSetting.ai_translation_backfill_max_age_days > 0
-          topics =
-            topics.where(
-              "topics.created_at > ?",
-              SiteSetting.ai_translation_backfill_max_age_days.days.ago,
-            )
-        end
-
-        topics = topics.order(updated_at: :desc).limit(limit)
-
-        next if topics.empty?
-
-        topics.each do |topic|
-          begin
-            DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
-          rescue FinalDestination::SSRFDetector::LookupFailedError
-            # do nothing, there are too many sporadic lookup failures
-          rescue => e
-            DiscourseAi::Translation::VerboseLogger.log(
-              "Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
-            )
-          end
-        end
-
-        DiscourseAi::Translation::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
-      end
-    end
-  end
-end

app/jobs/regular/stream_composer_helper.rb

@@ -8,19 +8,19 @@ module Jobs
       return unless args[:prompt]
       return unless user = User.find_by(id: args[:user_id])
       return unless args[:text]
-      return unless args[:client_id]
-      return unless args[:progress_channel]
 
-      helper_mode = args[:prompt]
+      prompt = CompletionPrompt.enabled_by_name(args[:prompt])
+
+      if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+        prompt.custom_instruction = args[:custom_prompt]
+      end
 
       DiscourseAi::AiHelper::Assistant.new.stream_prompt(
-        helper_mode,
+        prompt,
         args[:text],
         user,
-        args[:progress_channel],
+        "/discourse-ai/ai-helper/stream_composer_suggestion",
         force_default_locale: args[:force_default_locale],
-        client_id: args[:client_id],
-        custom_prompt: args[:custom_prompt],
       )
     end
   end

app/jobs/regular/stream_post_helper.rb

@@ -8,20 +8,22 @@ module Jobs
       return unless post = Post.includes(:topic).find_by(id: args[:post_id])
       return unless user = User.find_by(id: args[:user_id])
       return unless args[:text]
-      return unless args[:progress_channel]
-      return unless args[:client_id]
 
       topic = post.topic
       reply_to = post.reply_to_post
 
       return unless user.guardian.can_see?(post)
 
-      helper_mode = args[:prompt]
+      prompt = CompletionPrompt.enabled_by_name(args[:prompt])
 
-      if helper_mode == DiscourseAi::AiHelper::Assistant::EXPLAIN
+      if prompt.id == CompletionPrompt::CUSTOM_PROMPT
-        input = <<~TEXT.strip
+        prompt.custom_instruction = args[:custom_prompt]
-          <term>#{args[:text]}</term>
+      end
-          <context>#{post.raw}</context>
+
+      if prompt.name == "explain"
+        input = <<~TEXT
+      <term>#{args[:text]}</term>
+      <context>#{post.raw}</context>
           <topic>#{topic.title}</topic>
           #{reply_to ? "<replyTo>#{reply_to.raw}</replyTo>" : nil}
         TEXT
@@ -30,12 +32,10 @@ module Jobs
       end
 
       DiscourseAi::AiHelper::Assistant.new.stream_prompt(
-        helper_mode,
+        prompt,
         input,
         user,
-        args[:progress_channel],
+        "/discourse-ai/ai-helper/stream_suggestion/#{post.id}",
-        custom_prompt: args[:custom_prompt],
-        client_id: args[:client_id],
       )
     end
   end

app/jobs/scheduled/categories_locale_detection_backfill.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class CategoriesLocaleDetectionBackfill < ::Jobs::Scheduled
-    every 1.hour
-    sidekiq_options retry: false
-    cluster_concurrency 1
-
-    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      limit = SiteSetting.ai_translation_backfill_hourly_rate
-      return if limit == 0
-
-      categories = Category.where(locale: nil)
-
-      if SiteSetting.ai_translation_backfill_limit_to_public_content
-        categories = categories.where(read_restricted: false)
-      end
-
-      categories = categories.limit(limit)
-      return if categories.empty?
-
-      categories.each do |category|
-        begin
-          DiscourseAi::Translation::CategoryLocaleDetector.detect_locale(category)
-        rescue FinalDestination::SSRFDetector::LookupFailedError
-        rescue => e
-          DiscourseAi::Translation::VerboseLogger.log(
-            "Failed to detect category #{category.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
-          )
-        end
-      end
-
-      DiscourseAi::Translation::VerboseLogger.log("Detected #{categories.size} category locales")
-    end
-  end
-end

app/jobs/scheduled/category_localization_backfill.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class CategoryLocalizationBackfill < ::Jobs::Scheduled
-    every 1.hour
-    cluster_concurrency 1
-
-    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.content_localization_supported_locales.blank?
-      limit = SiteSetting.ai_translation_backfill_hourly_rate
-      return if limit == 0
-
-      Jobs.enqueue(:localize_categories, limit:)
-    end
-  end
-end

app/jobs/scheduled/embeddings_backfill.rb

@@ -18,115 +18,105 @@ module Jobs
         )
       end
 
-      production_vector = DiscourseAi::Embeddings::Vector.instance
+      rebaked = 0
 
-      if SiteSetting.ai_embeddings_backfill_model.present? &&
+      vector = DiscourseAi::Embeddings::Vector.instance
-           SiteSetting.ai_embeddings_backfill_model != SiteSetting.ai_embeddings_selected_model
+      vector_def = vector.vdef
-        backfill_vector =
+      table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
-          DiscourseAi::Embeddings::Vector.new(
+
-            EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_backfill_model),
+      topics =
+        Topic
+          .joins(
+            "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
           )
-      end
+          .where(archetype: Archetype.default)
+          .where(deleted_at: nil)
+          .order("topics.bumped_at DESC")
 
-      topic_work_list = []
+      rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
-      topic_work_list << production_vector
-      topic_work_list << backfill_vector if backfill_vector
 
-      topic_work_list.each do |vector|
+      return if rebaked >= limit
-        rebaked = 0
-        table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
-        vector_def = vector.vdef
 
-        topics =
+      # Then, we'll try to backfill embeddings for topics that have outdated
-          Topic
+      # embeddings, be it model or strategy version
-            .joins(
+      relation = topics.where(<<~SQL).limit(limit - rebaked)
-              "LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
+          #{table_name}.model_version < #{vector_def.version}
-            )
+          OR
-            .where(archetype: Archetype.default)
+          #{table_name}.strategy_version < #{vector_def.strategy_version}
-            .where(deleted_at: nil)
+        SQL
-            .order("topics.bumped_at DESC")
 
-        rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
+      rebaked += populate_topic_embeddings(vector, relation, force: true)
 
-        next if rebaked >= limit
+      return if rebaked >= limit
 
-        # Then, we'll try to backfill embeddings for topics that have outdated
+      # Finally, we'll try to backfill embeddings for topics that have outdated
-        # embeddings, be it model or strategy version
+      # embeddings due to edits or new replies. Here we only do 10% of the limit
-        relation = topics.where(<<~SQL).limit(limit - rebaked)
+      relation =
-            #{table_name}.model_version < #{vector_def.version}
+        topics
-            OR
+          .where("#{table_name}.updated_at < ?", 6.hours.ago)
-            #{table_name}.strategy_version < #{vector_def.strategy_version}
+          .where("#{table_name}.updated_at < topics.updated_at")
-          SQL
-
-        rebaked += populate_topic_embeddings(vector, relation, force: true)
-
-        next if rebaked >= limit
-
-        # Finally, we'll try to backfill embeddings for topics that have outdated
-        # embeddings due to edits or new replies. Here we only do 10% of the limit
-        relation =
-          topics
-            .where("#{table_name}.updated_at < ?", 6.hours.ago)
-            .where("#{table_name}.updated_at < topics.updated_at")
-            .limit((limit - rebaked) / 10)
-
-        populate_topic_embeddings(vector, relation, force: true)
-
-        next unless SiteSetting.ai_embeddings_per_post_enabled
-
-        # Now for posts
-        table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
-        posts_batch_size = 1000
-
-        posts =
-          Post
-            .joins(
-              "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
-            )
-            .where(deleted_at: nil)
-            .where(post_type: Post.types[:regular])
-
-        # First, we'll try to backfill embeddings for posts that have none
-        posts
-          .where("#{table_name}.post_id IS NULL")
-          .limit(limit - rebaked)
-          .pluck(:id)
-          .each_slice(posts_batch_size) do |batch|
-            vector.gen_bulk_reprensentations(Post.where(id: batch))
-            rebaked += batch.length
-          end
-
-        next if rebaked >= limit
-
-        # Then, we'll try to backfill embeddings for posts that have outdated
-        # embeddings, be it model or strategy version
-        posts
-          .where(<<~SQL)
-            #{table_name}.model_version < #{vector_def.version}
-            OR
-            #{table_name}.strategy_version < #{vector_def.strategy_version}
-          SQL
-          .limit(limit - rebaked)
-          .pluck(:id)
-          .each_slice(posts_batch_size) do |batch|
-            vector.gen_bulk_reprensentations(Post.where(id: batch))
-            rebaked += batch.length
-          end
-
-        next if rebaked >= limit
-
-        # Finally, we'll try to backfill embeddings for posts that have outdated
-        # embeddings due to edits. Here we only do 10% of the limit
-        posts
-          .where("#{table_name}.updated_at < ?", 7.days.ago)
-          .order("random()")
           .limit((limit - rebaked) / 10)
-          .pluck(:id)
+
-          .each_slice(posts_batch_size) do |batch|
+      populate_topic_embeddings(vector, relation, force: true)
-            vector.gen_bulk_reprensentations(Post.where(id: batch))
+
-            rebaked += batch.length
+      return if rebaked >= limit
-          end
+
-      end
+      return unless SiteSetting.ai_embeddings_per_post_enabled
+
+      # Now for posts
+      table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
+      posts_batch_size = 1000
+
+      posts =
+        Post
+          .joins(
+            "LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
+          )
+          .where(deleted_at: nil)
+          .where(post_type: Post.types[:regular])
+
+      # First, we'll try to backfill embeddings for posts that have none
+      posts
+        .where("#{table_name}.post_id IS NULL")
+        .limit(limit - rebaked)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
+        end
+
+      return if rebaked >= limit
+
+      # Then, we'll try to backfill embeddings for posts that have outdated
+      # embeddings, be it model or strategy version
+      posts
+        .where(<<~SQL)
+          #{table_name}.model_version < #{vector_def.version}
+          OR
+          #{table_name}.strategy_version < #{vector_def.strategy_version}
+        SQL
+        .limit(limit - rebaked)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
+        end
+
+      return if rebaked >= limit
+
+      # Finally, we'll try to backfill embeddings for posts that have outdated
+      # embeddings due to edits. Here we only do 10% of the limit
+      posts
+        .where("#{table_name}.updated_at < ?", 7.days.ago)
+        .order("random()")
+        .limit((limit - rebaked) / 10)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
+        end
+
+      rebaked
     end
 
     private

app/jobs/scheduled/generate_concepts_from_popular_items.rb

@@ -1,87 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class GenerateConceptsFromPopularItems < ::Jobs::Scheduled
-    every 1.day
-
-    # This job runs daily and generates new concepts from popular topics and posts
-    # It selects items based on engagement metrics and generates concepts from their content
-    def execute(_args)
-      return unless SiteSetting.inferred_concepts_enabled
-
-      process_popular_topics
-      process_popular_posts
-    end
-
-    private
-
-    def process_popular_topics
-      # Find candidate topics that are popular and don't have concepts yet
-      manager = DiscourseAi::InferredConcepts::Manager.new
-      candidates =
-        manager.find_candidate_topics(
-          limit: SiteSetting.inferred_concepts_daily_topics_limit || 20,
-          min_posts: SiteSetting.inferred_concepts_min_posts || 5,
-          min_likes: SiteSetting.inferred_concepts_min_likes || 10,
-          min_views: SiteSetting.inferred_concepts_min_views || 100,
-          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
-        )
-
-      return if candidates.blank?
-
-      # Process candidate topics - first generate concepts, then match
-      Jobs.enqueue(
-        :generate_inferred_concepts,
-        item_type: "topics",
-        item_ids: candidates.map(&:id),
-        batch_size: 10,
-      )
-
-      if SiteSetting.inferred_concepts_background_match
-        # Schedule a follow-up job to match existing concepts
-        Jobs.enqueue_in(
-          1.hour,
-          :generate_inferred_concepts,
-          item_type: "topics",
-          item_ids: candidates.map(&:id),
-          batch_size: 10,
-          match_only: true,
-        )
-      end
-    end
-
-    def process_popular_posts
-      # Find candidate posts that are popular and don't have concepts yet
-      manager = DiscourseAi::InferredConcepts::Manager.new
-      candidates =
-        manager.find_candidate_posts(
-          limit: SiteSetting.inferred_concepts_daily_posts_limit || 30,
-          min_likes: SiteSetting.inferred_concepts_post_min_likes || 5,
-          exclude_first_posts: true,
-          created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
-        )
-
-      return if candidates.blank?
-
-      # Process candidate posts - first generate concepts, then match
-      Jobs.enqueue(
-        :generate_inferred_concepts,
-        item_type: "posts",
-        item_ids: candidates.map(&:id),
-        batch_size: 10,
-      )
-
-      if SiteSetting.inferred_concepts_background_match
-        # Schedule a follow-up job to match against existing concepts
-        Jobs.enqueue_in(
-          1.hour,
-          :generate_inferred_concepts,
-          item_type: "posts",
-          item_ids: candidates.map(&:id),
-          batch_size: 10,
-          match_only: true,
-        )
-      end
-    end
-  end
-end

app/jobs/scheduled/post_localization_backfill.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class PostLocalizationBackfill < ::Jobs::Scheduled
-    every 5.minutes
-    cluster_concurrency 1
-
-    def execute(args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_translation_enabled
-
-      return if SiteSetting.content_localization_supported_locales.blank?
-      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
-      return if limit == 0
-
-      Jobs.enqueue(:localize_posts, limit:)
-    end
-  end
-end