wip experiment with /filter route

Previously was AdminPageSubheader until https://github.com/discourse/discourse/pull/30146

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-edit.js

@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default DiscourseRoute.extend({
+export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
   async model(params) {
     const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
     const id = parseInt(params.id, 10);
     const record = allLlms.findBy("id", id);
     record.provider_params = record.provider_params || {};
     return record;
-  },
+  }
 
   setupController(controller, model) {
-    this._super(controller, model);
+    super.setupController(controller, model);
     controller.set(
       "allLlms",
       this.modelFor("adminPlugins.show.discourse-ai-llms")
     );
-  },
-});
+  }
+}

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-new.js

@@ -1,18 +1,18 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default DiscourseRoute.extend({
-  queryParams: {
+export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
+  queryParams = {
     llmTemplate: { refreshModel: true },
-  },
+  };
 
   async model() {
     const record = this.store.createRecord("ai-llm");
     record.provider_params = {};
     return record;
-  },
+  }
 
   setupController(controller, model) {
-    this._super(controller, model);
+    super.setupController(controller, model);
     controller.set(
       "allLlms",
       this.modelFor("adminPlugins.show.discourse-ai-llms")
@@ -21,5 +21,5 @@ export default DiscourseRoute.extend({
       "llmTemplate",
       this.paramsFor(this.routeName).llmTemplate || null
     );
-  },
-});
+  }
+}

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-personas-edit.js

@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default DiscourseRoute.extend({
+export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
   async model(params) {
     const allPersonas = this.modelFor(
       "adminPlugins.show.discourse-ai-personas"
     );
     const id = parseInt(params.id, 10);
     return allPersonas.findBy("id", id);
-  },
+  }
 
   setupController(controller, model) {
-    this._super(controller, model);
+    super.setupController(controller, model);
     controller.set(
       "allPersonas",
       this.modelFor("adminPlugins.show.discourse-ai-personas")
     );
-  },
-});
+  }
+}

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-personas-new.js

@@ -1,7 +1,7 @@
 import { AUTO_GROUPS } from "discourse/lib/constants";
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default DiscourseRoute.extend({
+export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
   async model() {
     const record = this.store.createRecord("ai-persona");
     record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
@@ -13,13 +13,13 @@ export default DiscourseRoute.extend({
     record.set("allow_personal_messages", true);
     record.set("tool_details", false);
     return record;
-  },
+  }
 
   setupController(controller, model) {
-    this._super(controller, model);
+    super.setupController(controller, model);
     controller.set(
       "allPersonas",
       this.modelFor("adminPlugins.show.discourse-ai-personas")
     );
-  },
-});
+  }
+}

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-spam.js

@@ -0,0 +1,11 @@
+import { service } from "@ember/service";
+import { ajax } from "discourse/lib/ajax";
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default class DiscourseAiSpamRoute extends DiscourseRoute {
+  @service store;
+
+  model() {
+    return ajax("/admin/plugins/discourse-ai/ai-spam.json");
+  }
+}

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-tools-edit.js

@@ -1,6 +1,6 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
+export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
   async model(params) {
     const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
     const id = parseInt(params.id, 10);

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-usage.js

@@ -0,0 +1,11 @@
+import { service } from "@ember/service";
+import { ajax } from "discourse/lib/ajax";
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default class DiscourseAiUsageRoute extends DiscourseRoute {
+  @service store;
+
+  model() {
+    return ajax("/admin/plugins/discourse-ai/ai-usage.json");
+  }
+}

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-spam.hbs

@@ -0,0 +1 @@
+<AiSpam @model={{this.model}} />

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-usage.hbs

@@ -0,0 +1 @@
+<AiUsage @model={{this.model}} />

app/controllers/discourse_ai/admin/ai_llms_controller.rb

@@ -30,7 +30,10 @@ module DiscourseAi
                }
       end
 
-      def show
+      def new
+      end
+
+      def edit
         llm_model = LlmModel.find(params[:id])
         render json: LlmModelSerializer.new(llm_model)
       end

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -5,7 +5,7 @@ module DiscourseAi
     class AiPersonasController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_persona, only: %i[show update destroy create_user]
+      before_action :find_ai_persona, only: %i[edit update destroy create_user]
 
       def index
         ai_personas =
@@ -33,7 +33,10 @@ module DiscourseAi
         render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
       end
 
-      def show
+      def new
+      end
+
+      def edit
         render json: LocalizedAiPersonaSerializer.new(@ai_persona)
       end
 

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class AiSpamController < ::Admin::AdminController
+      requires_plugin "discourse-ai"
+
+      def show
+        render json: AiSpamSerializer.new(spam_config, root: false)
+      end
+
+      def update
+        updated_params = {}
+        if allowed_params.key?(:llm_model_id)
+          llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
+          if llm_model_id.to_i < 0 &&
+               !SiteSetting.ai_spam_detection_model_allowed_seeded_models_map.include?(
+                 "custom:#{llm_model_id}",
+               )
+            return(
+              render_json_error(
+                I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"),
+                status: 422,
+              )
+            )
+          end
+        end
+        updated_params[:data] = {
+          custom_instructions: allowed_params[:custom_instructions],
+        } if allowed_params.key?(:custom_instructions)
+
+        if updated_params.present?
+          # not using upsert cause we will not get the correct validation errors
+          if AiModerationSetting.spam
+            AiModerationSetting.spam.update!(updated_params)
+          else
+            AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
+          end
+        end
+
+        is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
+
+        if allowed_params.key?(:is_enabled)
+          if is_enabled && !AiModerationSetting.spam&.llm_model_id
+            return(
+              render_json_error(
+                I18n.t("discourse_ai.llm.configuration.must_select_model"),
+                status: 422,
+              )
+            )
+          end
+
+          SiteSetting.ai_spam_detection_enabled = is_enabled
+        end
+
+        render json: AiSpamSerializer.new(spam_config, root: false)
+      end
+
+      def test
+        url = params[:post_url].to_s
+        post = nil
+
+        if url.match?(/^\d+$/)
+          post_id = url.to_i
+          post = Post.find_by(id: post_id)
+        end
+
+        route = UrlHelper.rails_route_from_url(url) if !post
+
+        if route
+          if route[:controller] == "topics"
+            post_number = route[:post_number] || 1
+            post = Post.with_deleted.find_by(post_number: post_number, topic_id: route[:topic_id])
+          end
+        end
+
+        raise Discourse::NotFound if !post
+
+        result =
+          DiscourseAi::AiModeration::SpamScanner.test_post(
+            post,
+            custom_instructions: params[:custom_instructions],
+            llm_id: params[:llm_id],
+          )
+
+        render json: result
+      end
+
+      private
+
+      def allowed_params
+        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
+      end
+
+      def spam_config
+        spam_config = {
+          enabled: SiteSetting.ai_spam_detection_enabled,
+          settings: AiModerationSetting.spam,
+        }
+
+        spam_config[:stats] = DiscourseAi::AiModeration::SpamReport.generate(min_date: 1.week.ago)
+
+        if spam_config[:stats].scanned_count > 0
+          spam_config[
+            :flagging_username
+          ] = DiscourseAi::AiModeration::SpamScanner.flagging_user&.username
+        end
+        spam_config
+      end
+    end
+  end
+end

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -5,14 +5,17 @@ module DiscourseAi
     class AiToolsController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_tool, only: %i[test show update destroy]
+      before_action :find_ai_tool, only: %i[test edit update destroy]
 
       def index
         ai_tools = AiTool.all
         render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
       end
 
-      def show
+      def new
+      end
+
+      def edit
         render_serialized(@ai_tool, AiCustomToolSerializer)
       end
 

app/controllers/discourse_ai/admin/ai_usage_controller.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class AiUsageController < ::Admin::AdminController
+      requires_plugin "discourse-ai"
+
+      def show
+      end
+
+      def report
+        render json: AiUsageSerializer.new(create_report, root: false)
+      end
+
+      private
+
+      def create_report
+        report =
+          DiscourseAi::Completions::Report.new(
+            start_date: params[:start_date]&.to_date || 30.days.ago,
+            end_date: params[:end_date]&.to_date || Time.current,
+          )
+
+        report = report.filter_by_feature(params[:feature]) if params[:feature].present?
+        report = report.filter_by_model(params[:model]) if params[:model].present?
+        report
+      end
+    end
+  end
+end

app/controllers/discourse_ai/ai_bot/artifacts_controller.rb

@@ -19,22 +19,33 @@ module DiscourseAi
           raise Discourse::NotFound if !guardian.can_see?(post)
         end
 
+        name = artifact.name
+
+        if params[:version].present?
+          artifact = artifact.versions.find_by(version_number: params[:version])
+          raise Discourse::NotFound if !artifact
+        end
+
+        js = artifact.js || ""
+        if !js.match?(%r{\A\s*<script.*</script>}mi)
+          mod = ""
+          mod = " type=\"module\"" if js.match?(/\A\s*import.*/)
+          js = "<script#{mod}>\n#{js}\n</script>"
+        end
         # Prepare the inner (untrusted) HTML document
         untrusted_html = <<~HTML
           <!DOCTYPE html>
           <html>
             <head>
               <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(artifact.name)}</title>
+              <title>#{ERB::Util.html_escape(name)}</title>
               <style>
                 #{artifact.css}
               </style>
             </head>
             <body>
               #{artifact.html}
-              <script>
-                #{artifact.js}
-              </script>
+              #{js}
             </body>
           </html>
         HTML
@@ -45,7 +56,7 @@ module DiscourseAi
           <html>
             <head>
               <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(artifact.name)}</title>
+              <title>#{ERB::Util.html_escape(name)}</title>
               <style>
                 html, body, iframe {
                   margin: 0;
@@ -67,7 +78,9 @@ module DiscourseAi
         HTML
 
         response.headers.delete("X-Frame-Options")
-        response.headers["Content-Security-Policy"] = "script-src 'unsafe-inline';"
+        response.headers[
+          "Content-Security-Policy"
+        ] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://unpkg.com https://cdnjs.cloudflare.com https://ajax.googleapis.com https://cdn.jsdelivr.net;"
         response.headers["X-Robots-Tag"] = "noindex"
 
         # Render the content

app/jobs/regular/ai_spam_scan.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Jobs
+  class AiSpamScan < ::Jobs::Base
+    def execute(args)
+      return if !args[:post_id]
+      post = Post.find_by(id: args[:post_id])
+      return if !post
+
+      DiscourseAi::AiModeration::SpamScanner.perform_scan(post)
+    end
+  end
+end

app/jobs/regular/digest_rag_upload.rb

@@ -18,9 +18,7 @@ module ::Jobs
       target = target_type.constantize.find_by(id: target_id)
       return if !target
 
-      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+      vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
 
       tokenizer = vector_rep.tokenizer
       chunk_tokens = target.rag_chunk_tokens

app/jobs/regular/evaluate_post_uploads.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class EvaluatePostUploads < ::Jobs::Base
-    def execute(args)
-      return unless SiteSetting.ai_nsfw_detection_enabled
-      return if (post_id = args[:post_id]).blank?
-
-      post = Post.includes(:uploads).find_by_id(post_id)
-      return if post.nil? || post.uploads.empty?
-
-      return if post.uploads.none? { |u| FileHelper.is_supported_image?(u.url) }
-
-      DiscourseAi::PostClassificator.new(DiscourseAi::Nsfw::Classification.new).classify!(post)
-    end
-  end
-end

app/jobs/regular/fast_track_topic_gist.rb

@@ -14,7 +14,7 @@ module ::Jobs
 
       summarizer = DiscourseAi::Summarization.topic_gist(topic)
       gist = summarizer.existing_summary
-      return if gist.present? && !gist.outdated
+      return if gist.present? && (!gist.outdated || gist.created_at >= 5.minutes.ago)
 
       summarizer.force_summarize(Discourse.system_user)
     end

app/jobs/regular/generate_embeddings.rb

@@ -16,11 +16,7 @@ module Jobs
       return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
       return if post.raw.blank?
 
-      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
-
-      vector_rep.generate_representation_from(target)
+      DiscourseAi::Embeddings::Vector.instance.generate_representation_from(target)
     end
   end
 end

app/jobs/regular/generate_rag_embeddings.rb

@@ -8,13 +8,11 @@ module ::Jobs
     def execute(args)
       return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
 
-      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+      vector = DiscourseAi::Embeddings::Vector.instance
 
       # generate_representation_from checks compares the digest value to make sure
       # the embedding is only generated once per fragment unless something changes.
-      fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
+      fragments.map { |fragment| vector.generate_representation_from(fragment) }
 
       last_fragment = fragments.last
       target = last_fragment.target

app/jobs/regular/post_sentiment_analysis.rb

@@ -9,9 +9,7 @@ module ::Jobs
       post = Post.find_by(id: post_id, post_type: Post.types[:regular])
       return if post&.raw.blank?
 
-      DiscourseAi::PostClassificator.new(
-        DiscourseAi::Sentiment::SentimentClassification.new,
-      ).classify!(post)
+      DiscourseAi::Sentiment::PostClassification.new.classify!(post)
     end
   end
 end

app/jobs/regular/toxicity_classify_chat_message.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module ::Jobs
-  class ToxicityClassifyChatMessage < ::Jobs::Base
-    def execute(args)
-      return unless SiteSetting.ai_toxicity_enabled
-
-      return if (chat_message_id = args[:chat_message_id]).blank?
-
-      chat_message = ::Chat::Message.find_by(id: chat_message_id)
-      return if chat_message&.message.blank?
-
-      DiscourseAi::ChatMessageClassificator.new(
-        DiscourseAi::Toxicity::ToxicityClassification.new,
-      ).classify!(chat_message)
-    end
-  end
-end

app/jobs/regular/toxicity_classify_post.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module ::Jobs
-  class ToxicityClassifyPost < ::Jobs::Base
-    def execute(args)
-      return unless SiteSetting.ai_toxicity_enabled
-
-      post_id = args[:post_id]
-      return if post_id.blank?
-
-      post = Post.find_by(id: post_id, post_type: Post.types[:regular])
-      return if post&.raw.blank?
-
-      DiscourseAi::PostClassificator.new(
-        DiscourseAi::Toxicity::ToxicityClassification.new,
-      ).classify!(post)
-    end
-  end
-end

app/jobs/scheduled/embeddings_backfill.rb

@@ -20,10 +20,9 @@ module Jobs
 
       rebaked = 0
 
-      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
-      table_name = vector_rep.topic_table_name
+      vector = DiscourseAi::Embeddings::Vector.instance
+      vector_def = vector.vdef
+      table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
 
       topics =
         Topic
@@ -31,21 +30,20 @@ module Jobs
           .where(archetype: Archetype.default)
           .where(deleted_at: nil)
           .order("topics.bumped_at DESC")
-          .limit(limit - rebaked)
 
-      rebaked += populate_topic_embeddings(vector_rep, topics)
+      rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
 
       return if rebaked >= limit
 
       # Then, we'll try to backfill embeddings for topics that have outdated
       # embeddings, be it model or strategy version
-      relation = topics.where(<<~SQL)
-          #{table_name}.model_version < #{vector_rep.version}
+      relation = topics.where(<<~SQL).limit(limit - rebaked)
+          #{table_name}.model_version < #{vector_def.version}
           OR
-          #{table_name}.strategy_version < #{strategy.version}
+          #{table_name}.strategy_version < #{vector_def.strategy_version}
         SQL
 
-      rebaked += populate_topic_embeddings(vector_rep, relation)
+      rebaked += populate_topic_embeddings(vector, relation)
 
       return if rebaked >= limit
 
@@ -57,28 +55,30 @@ module Jobs
           .where("#{table_name}.updated_at < topics.updated_at")
           .limit((limit - rebaked) / 10)
 
-      populate_topic_embeddings(vector_rep, relation, force: true)
+      populate_topic_embeddings(vector, relation, force: true)
 
       return if rebaked >= limit
 
       return unless SiteSetting.ai_embeddings_per_post_enabled
 
       # Now for posts
-      table_name = vector_rep.post_table_name
+      table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
+      posts_batch_size = 1000
 
       posts =
         Post
           .joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
           .where(deleted_at: nil)
           .where(post_type: Post.types[:regular])
-          .limit(limit - rebaked)
 
       # First, we'll try to backfill embeddings for posts that have none
       posts
         .where("#{table_name}.post_id IS NULL")
-        .find_in_batches do |batch|
-          vector_rep.gen_bulk_reprensentations(batch)
-          rebaked += batch.size
+        .limit(limit - rebaked)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
         end
 
       return if rebaked >= limit
@@ -87,48 +87,47 @@ module Jobs
       # embeddings, be it model or strategy version
       posts
         .where(<<~SQL)
-          #{table_name}.model_version < #{vector_rep.version}
+          #{table_name}.model_version < #{vector_def.version}
           OR
-          #{table_name}.strategy_version < #{strategy.version}
+          #{table_name}.strategy_version < #{vector_def.strategy_version}
         SQL
-        .find_in_batches do |batch|
-          vector_rep.gen_bulk_reprensentations(batch)
-          rebaked += batch.size
+        .limit(limit - rebaked)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
         end
 
       return if rebaked >= limit
 
       # Finally, we'll try to backfill embeddings for posts that have outdated
       # embeddings due to edits. Here we only do 10% of the limit
-      posts_batch_size = 1000
-
-      outdated_post_ids =
-        posts
-          .where("#{table_name}.updated_at < ?", 7.days.ago)
-          .order("random()")
-          .limit((limit - rebaked) / 10)
-          .pluck(:id)
-
-      outdated_post_ids.each_slice(posts_batch_size) do |batch|
-        vector_rep.gen_bulk_reprensentations(Post.where(id: batch))
-        rebaked += batch.length
-      end
+      posts
+        .where("#{table_name}.updated_at < ?", 7.days.ago)
+        .order("random()")
+        .limit((limit - rebaked) / 10)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
+        end
 
       rebaked
     end
 
     private
 
-    def populate_topic_embeddings(vector_rep, topics, force: false)
+    def populate_topic_embeddings(vector, topics, force: false)
       done = 0
 
-      topics = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL") if !force
+      topics =
+        topics.where("#{DiscourseAi::Embeddings::Schema::TOPICS_TABLE}.topic_id IS NULL") if !force
 
       ids = topics.pluck("topics.id")
       batch_size = 1000
 
       ids.each_slice(batch_size) do |batch|
-        vector_rep.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
+        vector.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
         done += batch.length
       end
 

app/jobs/scheduled/sentiment_backfill.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Jobs
+  class SentimentBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(_args)
+      return if !SiteSetting.ai_sentiment_enabled
+
+      base_budget = SiteSetting.ai_sentiment_backfill_maximum_posts_per_hour
+      return if base_budget.zero?
+      # Split budget in 12 intervals, but make sure is at least one.
+      #
+      # This is not exact as we don't have a way of tracking how many
+      # posts we classified in the current hour, but it's a good enough approximation.
+      limit_per_job = [base_budget, 12].max / 12
+
+      classificator = DiscourseAi::Sentiment::PostClassification.new
+      return if !classificator.has_classifiers?
+
+      posts =
+        DiscourseAi::Sentiment::PostClassification.backfill_query(
+          max_age_days: SiteSetting.ai_sentiment_backfill_post_max_age_days,
+        ).limit(limit_per_job)
+
+      classificator.bulk_classify!(posts)
+    end
+  end
+end

app/jobs/scheduled/summaries_backfill.rb

@@ -12,22 +12,27 @@ module ::Jobs
 
       system_user = Discourse.system_user
 
+      if SiteSetting.ai_summary_gists_enabled
+        gist_t = AiSummary.summary_types[:gist]
+
+        backfill_candidates(gist_t)
+          .limit(current_budget(gist_t))
+          .each do |topic|
+            DiscourseAi::Summarization.topic_gist(topic).force_summarize(system_user)
+          end
+      end
+
       complete_t = AiSummary.summary_types[:complete]
       backfill_candidates(complete_t)
         .limit(current_budget(complete_t))
         .each do |topic|
           DiscourseAi::Summarization.topic_summary(topic).force_summarize(system_user)
         end
-
-      return unless SiteSetting.ai_summary_gists_enabled
-
-      gist_t = AiSummary.summary_types[:gist]
-      backfill_candidates(gist_t)
-        .limit(current_budget(gist_t))
-        .each { |topic| DiscourseAi::Summarization.topic_gist(topic).force_summarize(system_user) }
     end
 
     def backfill_candidates(summary_type)
+      max_age_days = SiteSetting.ai_summary_backfill_topic_max_age_days
+
       Topic
         .where("topics.word_count >= ?", SiteSetting.ai_summary_backfill_minimum_word_count)
         .joins(<<~SQL)
@@ -36,9 +41,15 @@ module ::Jobs
                           ais.target_type = 'Topic' AND
                           ais.summary_type = '#{summary_type}'
         SQL
+        .where("topics.created_at > current_timestamp - INTERVAL '#{max_age_days.to_i} DAY'")
         .where(
-          "ais.id IS NULL OR UPPER(ais.content_range) < topics.highest_post_number + 1",
-        ) # (1..1) gets stored ad (1..2).
+          <<~SQL, # (1..1) gets stored ad (1..2).
+          ais.id IS NULL OR (
+            UPPER(ais.content_range) < topics.highest_post_number + 1 
+            AND ais.created_at < (current_timestamp - INTERVAL '5 minutes')
+          )
+        SQL
+        )
         .order("ais.created_at DESC NULLS FIRST, topics.last_posted_at DESC")
     end