FIX: Use COPY to avoid migration statement timeout

* DEV: Embedding tables' model_id has to be a bigint

* Drop old search_bit indexes

* copy rag fragment embeddings created during deploy window

.discourse-compatibility

@@ -1,3 +1,4 @@
+< 3.4.0.beta4-dev: 20612fde52d3f740cad64823ef8aadb0748b567f
 < 3.4.0.beta3-dev: decf1bb49d737ea15308400f22f89d1d1e71d13d
 < 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
 < 3.3.0.beta5-dev: 4d8090002f6dcd8e34d41033606bf131fa221475

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-edit.js

@@ -1,6 +1,6 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default class AdminPluginsShowDiscourseAiLlmsShow extends DiscourseRoute {
+export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
   async model(params) {
     const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
     const id = parseInt(params.id, 10);

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-personas-edit.js

@@ -1,6 +1,6 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default class AdminPluginsShowDiscourseAiPersonasShow extends DiscourseRoute {
+export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
   async model(params) {
     const allPersonas = this.modelFor(
       "adminPlugins.show.discourse-ai-personas"

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-spam.js

@@ -0,0 +1,11 @@
+import { service } from "@ember/service";
+import { ajax } from "discourse/lib/ajax";
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default class DiscourseAiSpamRoute extends DiscourseRoute {
+  @service store;
+
+  model() {
+    return ajax("/admin/plugins/discourse-ai/ai-spam.json");
+  }
+}

admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-tools-edit.js

@@ -1,6 +1,6 @@
 import DiscourseRoute from "discourse/routes/discourse";
 
-export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
+export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
   async model(params) {
     const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
     const id = parseInt(params.id, 10);

admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-spam.hbs

@@ -0,0 +1 @@
+<AiSpam @model={{this.model}} />

app/controllers/discourse_ai/admin/ai_llm_quotas_controller.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class AiLlmQuotasController < ::Admin::AdminController
+      requires_plugin ::DiscourseAi::PLUGIN_NAME
+
+      def index
+        quotas = LlmQuota.includes(:group)
+
+        render json: {
+                 quotas:
+                   ActiveModel::ArraySerializer.new(quotas, each_serializer: LlmQuotaSerializer),
+               }
+      end
+
+      def create
+        quota = LlmQuota.new(quota_params)
+
+        if quota.save
+          render json: LlmQuotaSerializer.new(quota), status: :created
+        else
+          render_json_error quota
+        end
+      end
+
+      def update
+        quota = LlmQuota.find(params[:id])
+
+        if quota.update(quota_params)
+          render json: LlmQuotaSerializer.new(quota)
+        else
+          render_json_error quota
+        end
+      end
+
+      def destroy
+        quota = LlmQuota.find(params[:id])
+        quota.destroy!
+
+        head :no_content
+      rescue ActiveRecord::RecordNotFound
+        render json: { error: I18n.t("not_found") }, status: 404
+      end
+
+      private
+
+      def quota_params
+        params.require(:quota).permit(
+          :group_id,
+          :llm_model_id,
+          :max_tokens,
+          :max_usages,
+          :duration_seconds,
+        )
+      end
+    end
+  end
+end

app/controllers/discourse_ai/admin/ai_llms_controller.rb

@@ -6,7 +6,7 @@ module DiscourseAi
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
       def index
-        llms = LlmModel.all.order(:display_name)
+        llms = LlmModel.all.includes(:llm_quotas).order(:display_name)
 
         render json: {
                  ai_llms:
@@ -30,13 +30,21 @@ module DiscourseAi
                }
       end
 
-      def show
+      def new
+      end
+
+      def edit
         llm_model = LlmModel.find(params[:id])
         render json: LlmModelSerializer.new(llm_model)
       end
 
       def create
         llm_model = LlmModel.new(ai_llm_params)
+
+        # we could do nested attributes but the mechanics are not ideal leading
+        # to lots of complex debugging, this is simpler
+        quota_params.each { |quota| llm_model.llm_quotas.build(quota) } if quota_params
+
         if llm_model.save
           llm_model.toggle_companion_user
           render json: LlmModelSerializer.new(llm_model), status: :created
@@ -48,6 +56,25 @@ module DiscourseAi
       def update
         llm_model = LlmModel.find(params[:id])
 
+        if params[:ai_llm].key?(:llm_quotas)
+          if quota_params
+            existing_quota_group_ids = llm_model.llm_quotas.pluck(:group_id)
+            new_quota_group_ids = quota_params.map { |q| q[:group_id] }
+
+            llm_model
+              .llm_quotas
+              .where(group_id: existing_quota_group_ids - new_quota_group_ids)
+              .destroy_all
+
+            quota_params.each do |quota_param|
+              quota = llm_model.llm_quotas.find_or_initialize_by(group_id: quota_param[:group_id])
+              quota.update!(quota_param)
+            end
+          else
+            llm_model.llm_quotas.destroy_all
+          end
+        end
+
         if llm_model.seeded?
           return render_json_error(I18n.t("discourse_ai.llm.cannot_edit_builtin"), status: 403)
         end
@@ -107,6 +134,19 @@ module DiscourseAi
 
       private
 
+      def quota_params
+        if params[:ai_llm][:llm_quotas].present?
+          params[:ai_llm][:llm_quotas].map do |quota|
+            mapped = {}
+            mapped[:group_id] = quota[:group_id].to_i
+            mapped[:max_tokens] = quota[:max_tokens].to_i if quota[:max_tokens].present?
+            mapped[:max_usages] = quota[:max_usages].to_i if quota[:max_usages].present?
+            mapped[:duration_seconds] = quota[:duration_seconds].to_i
+            mapped
+          end
+        end
+      end
+
       def ai_llm_params(updating: nil)
         return {} if params[:ai_llm].blank?
 

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -5,7 +5,7 @@ module DiscourseAi
     class AiPersonasController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_persona, only: %i[show update destroy create_user]
+      before_action :find_ai_persona, only: %i[edit update destroy create_user]
 
       def index
         ai_personas =
@@ -33,7 +33,10 @@ module DiscourseAi
         render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
       end
 
-      def show
+      def new
+      end
+
+      def edit
         render json: LocalizedAiPersonaSerializer.new(@ai_persona)
       end
 

app/controllers/discourse_ai/admin/ai_spam_controller.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class AiSpamController < ::Admin::AdminController
+      requires_plugin "discourse-ai"
+
+      def show
+        render json: AiSpamSerializer.new(spam_config, root: false)
+      end
+
+      def update
+        updated_params = {}
+        if allowed_params.key?(:llm_model_id)
+          llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
+          if llm_model_id.to_i < 0 &&
+               !SiteSetting.ai_spam_detection_model_allowed_seeded_models_map.include?(
+                 llm_model_id.to_s,
+               )
+            return(
+              render_json_error(
+                I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"),
+                status: 422,
+              )
+            )
+          end
+        end
+        updated_params[:data] = {
+          custom_instructions: allowed_params[:custom_instructions],
+        } if allowed_params.key?(:custom_instructions)
+
+        if updated_params.present?
+          # not using upsert cause we will not get the correct validation errors
+          if AiModerationSetting.spam
+            AiModerationSetting.spam.update!(updated_params)
+          else
+            AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
+          end
+        end
+
+        is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
+
+        if allowed_params.key?(:is_enabled)
+          if is_enabled && !AiModerationSetting.spam&.llm_model_id
+            return(
+              render_json_error(
+                I18n.t("discourse_ai.llm.configuration.must_select_model"),
+                status: 422,
+              )
+            )
+          end
+
+          SiteSetting.ai_spam_detection_enabled = is_enabled
+        end
+
+        render json: AiSpamSerializer.new(spam_config, root: false)
+      end
+
+      def test
+        url = params[:post_url].to_s
+        post = nil
+
+        if url.match?(/^\d+$/)
+          post_id = url.to_i
+          post = Post.find_by(id: post_id)
+        end
+
+        route = UrlHelper.rails_route_from_url(url) if !post
+
+        if route
+          if route[:controller] == "topics"
+            post_number = route[:post_number] || 1
+            post = Post.with_deleted.find_by(post_number: post_number, topic_id: route[:topic_id])
+          end
+        end
+
+        raise Discourse::NotFound if !post
+
+        result =
+          DiscourseAi::AiModeration::SpamScanner.test_post(
+            post,
+            custom_instructions: params[:custom_instructions],
+            llm_id: params[:llm_id],
+          )
+
+        render json: result
+      end
+
+      def fix_errors
+        case params[:error]
+        when "spam_scanner_not_admin"
+          begin
+            DiscourseAi::AiModeration::SpamScanner.fix_spam_scanner_not_admin
+            render json: success_json
+          rescue ActiveRecord::RecordInvalid
+            render_json_error(
+              I18n.t("discourse_ai.spam_detection.bot_user_update_failed"),
+              status: :unprocessable_entity,
+            )
+          rescue StandardError
+            render_json_error(
+              I18n.t("discourse_ai.spam_detection.unexpected"),
+              status: :internal_server_error,
+            )
+          end
+        else
+          render_json_error(
+            I18n.t("discourse_ai.spam_detection.invalid_error_type"),
+            status: :bad_request,
+          )
+        end
+      end
+
+      private
+
+      def allowed_params
+        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
+      end
+
+      def spam_config
+        spam_config = {
+          enabled: SiteSetting.ai_spam_detection_enabled,
+          settings: AiModerationSetting.spam,
+        }
+
+        spam_config[:stats] = DiscourseAi::AiModeration::SpamReport.generate(min_date: 1.week.ago)
+
+        if spam_config[:stats].scanned_count > 0
+          spam_config[
+            :flagging_username
+          ] = DiscourseAi::AiModeration::SpamScanner.flagging_user&.username
+        end
+        spam_config
+      end
+    end
+  end
+end

app/controllers/discourse_ai/admin/ai_tools_controller.rb

@@ -5,14 +5,17 @@ module DiscourseAi
     class AiToolsController < ::Admin::AdminController
       requires_plugin ::DiscourseAi::PLUGIN_NAME
 
-      before_action :find_ai_tool, only: %i[test show update destroy]
+      before_action :find_ai_tool, only: %i[test edit update destroy]
 
       def index
         ai_tools = AiTool.all
         render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
       end
 
-      def show
+      def new
+      end
+
+      def edit
         render_serialized(@ai_tool, AiCustomToolSerializer)
       end
 

app/controllers/discourse_ai/admin/ai_usage_controller.rb

@@ -6,6 +6,9 @@ module DiscourseAi
       requires_plugin "discourse-ai"
 
       def show
+      end
+
+      def report
         render json: AiUsageSerializer.new(create_report, root: false)
       end
 

app/controllers/discourse_ai/ai_bot/artifacts_controller.rb

@@ -19,22 +19,33 @@ module DiscourseAi
           raise Discourse::NotFound if !guardian.can_see?(post)
         end
 
+        name = artifact.name
+
+        if params[:version].present?
+          artifact = artifact.versions.find_by(version_number: params[:version])
+          raise Discourse::NotFound if !artifact
+        end
+
+        js = artifact.js || ""
+        if !js.match?(%r{\A\s*<script.*</script>}mi)
+          mod = ""
+          mod = " type=\"module\"" if js.match?(/\A\s*import.*/)
+          js = "<script#{mod}>\n#{js}\n</script>"
+        end
         # Prepare the inner (untrusted) HTML document
         untrusted_html = <<~HTML
           <!DOCTYPE html>
           <html>
             <head>
               <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(artifact.name)}</title>
+              <title>#{ERB::Util.html_escape(name)}</title>
               <style>
                 #{artifact.css}
               </style>
             </head>
             <body>
               #{artifact.html}
-              <script>
-                #{artifact.js}
-              </script>
+              #{js}
             </body>
           </html>
         HTML
@@ -45,7 +56,8 @@ module DiscourseAi
           <html>
             <head>
               <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(artifact.name)}</title>
+              <title>#{ERB::Util.html_escape(name)}</title>
+              <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, user-scalable=yes, viewport-fit=cover, interactive-widget=resizes-content">
               <style>
                 html, body, iframe {
                   margin: 0;
@@ -67,7 +79,9 @@ module DiscourseAi
         HTML
 
         response.headers.delete("X-Frame-Options")
-        response.headers["Content-Security-Policy"] = "script-src 'unsafe-inline';"
+        response.headers[
+          "Content-Security-Policy"
+        ] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://unpkg.com https://cdnjs.cloudflare.com https://ajax.googleapis.com https://cdn.jsdelivr.net;"
         response.headers["X-Robots-Tag"] = "noindex"
 
         # Render the content

app/jobs/regular/ai_spam_scan.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Jobs
+  class AiSpamScan < ::Jobs::Base
+    def execute(args)
+      return if !args[:post_id]
+      post = Post.find_by(id: args[:post_id])
+      return if !post
+
+      DiscourseAi::AiModeration::SpamScanner.perform_scan(post)
+    end
+  end
+end

app/jobs/regular/digest_rag_upload.rb

@@ -18,9 +18,7 @@ module ::Jobs
       target = target_type.constantize.find_by(id: target_id)
       return if !target
 
-      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+      vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
 
       tokenizer = vector_rep.tokenizer
       chunk_tokens = target.rag_chunk_tokens

app/jobs/regular/evaluate_post_uploads.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class EvaluatePostUploads < ::Jobs::Base
-    def execute(args)
-      return unless SiteSetting.ai_nsfw_detection_enabled
-      return if (post_id = args[:post_id]).blank?
-
-      post = Post.includes(:uploads).find_by_id(post_id)
-      return if post.nil? || post.uploads.empty?
-
-      return if post.uploads.none? { |u| FileHelper.is_supported_image?(u.url) }
-
-      DiscourseAi::PostClassificator.new(DiscourseAi::Nsfw::Classification.new).classify!(post)
-    end
-  end
-end

app/jobs/regular/fast_track_topic_gist.rb

@@ -14,7 +14,7 @@ module ::Jobs
 
       summarizer = DiscourseAi::Summarization.topic_gist(topic)
       gist = summarizer.existing_summary
-      return if gist.present? && !gist.outdated
+      return if gist.present? && (!gist.outdated || gist.created_at >= 5.minutes.ago)
 
       summarizer.force_summarize(Discourse.system_user)
     end

app/jobs/regular/generate_embeddings.rb

@@ -16,11 +16,7 @@ module Jobs
       return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
       return if post.raw.blank?
 
-      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
-
-      vector_rep.generate_representation_from(target)
+      DiscourseAi::Embeddings::Vector.instance.generate_representation_from(target)
     end
   end
 end

app/jobs/regular/generate_rag_embeddings.rb

@@ -8,13 +8,11 @@ module ::Jobs
     def execute(args)
       return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
 
-      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+      vector = DiscourseAi::Embeddings::Vector.instance
 
       # generate_representation_from checks compares the digest value to make sure
       # the embedding is only generated once per fragment unless something changes.
-      fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
+      fragments.map { |fragment| vector.generate_representation_from(fragment) }
 
       last_fragment = fragments.last
       target = last_fragment.target

app/jobs/regular/toxicity_classify_chat_message.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module ::Jobs
-  class ToxicityClassifyChatMessage < ::Jobs::Base
-    def execute(args)
-      return unless SiteSetting.ai_toxicity_enabled
-
-      return if (chat_message_id = args[:chat_message_id]).blank?
-
-      chat_message = ::Chat::Message.find_by(id: chat_message_id)
-      return if chat_message&.message.blank?
-
-      DiscourseAi::ChatMessageClassificator.new(
-        DiscourseAi::Toxicity::ToxicityClassification.new,
-      ).classify!(chat_message)
-    end
-  end
-end

app/jobs/regular/toxicity_classify_post.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module ::Jobs
-  class ToxicityClassifyPost < ::Jobs::Base
-    def execute(args)
-      return unless SiteSetting.ai_toxicity_enabled
-
-      post_id = args[:post_id]
-      return if post_id.blank?
-
-      post = Post.find_by(id: post_id, post_type: Post.types[:regular])
-      return if post&.raw.blank?
-
-      DiscourseAi::PostClassificator.new(
-        DiscourseAi::Toxicity::ToxicityClassification.new,
-      ).classify!(post)
-    end
-  end
-end

app/jobs/scheduled/embeddings_backfill.rb

@@ -20,10 +20,9 @@ module Jobs
 
       rebaked = 0
 
-      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
-      table_name = vector_rep.topic_table_name
+      vector = DiscourseAi::Embeddings::Vector.instance
+      vector_def = vector.vdef
+      table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
 
       topics =
         Topic
@@ -31,21 +30,20 @@ module Jobs
           .where(archetype: Archetype.default)
           .where(deleted_at: nil)
           .order("topics.bumped_at DESC")
-          .limit(limit - rebaked)
 
-      rebaked += populate_topic_embeddings(vector_rep, topics)
+      rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
 
       return if rebaked >= limit
 
       # Then, we'll try to backfill embeddings for topics that have outdated
       # embeddings, be it model or strategy version
-      relation = topics.where(<<~SQL)
-          #{table_name}.model_version < #{vector_rep.version}
+      relation = topics.where(<<~SQL).limit(limit - rebaked)
+          #{table_name}.model_version < #{vector_def.version}
           OR
-          #{table_name}.strategy_version < #{strategy.version}
+          #{table_name}.strategy_version < #{vector_def.strategy_version}
         SQL
 
-      rebaked += populate_topic_embeddings(vector_rep, relation)
+      rebaked += populate_topic_embeddings(vector, relation)
 
       return if rebaked >= limit
 
@@ -57,28 +55,30 @@ module Jobs
           .where("#{table_name}.updated_at < topics.updated_at")
           .limit((limit - rebaked) / 10)
 
-      populate_topic_embeddings(vector_rep, relation, force: true)
+      populate_topic_embeddings(vector, relation, force: true)
 
       return if rebaked >= limit
 
       return unless SiteSetting.ai_embeddings_per_post_enabled
 
       # Now for posts
-      table_name = vector_rep.post_table_name
+      table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
+      posts_batch_size = 1000
 
       posts =
         Post
           .joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
           .where(deleted_at: nil)
           .where(post_type: Post.types[:regular])
-          .limit(limit - rebaked)
 
       # First, we'll try to backfill embeddings for posts that have none
       posts
         .where("#{table_name}.post_id IS NULL")
-        .find_in_batches do |batch|
-          vector_rep.gen_bulk_reprensentations(batch)
-          rebaked += batch.size
+        .limit(limit - rebaked)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
         end
 
       return if rebaked >= limit
@@ -87,48 +87,47 @@ module Jobs
       # embeddings, be it model or strategy version
       posts
         .where(<<~SQL)
-          #{table_name}.model_version < #{vector_rep.version}
+          #{table_name}.model_version < #{vector_def.version}
           OR
-          #{table_name}.strategy_version < #{strategy.version}
+          #{table_name}.strategy_version < #{vector_def.strategy_version}
         SQL
-        .find_in_batches do |batch|
-          vector_rep.gen_bulk_reprensentations(batch)
-          rebaked += batch.size
+        .limit(limit - rebaked)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
         end
 
       return if rebaked >= limit
 
       # Finally, we'll try to backfill embeddings for posts that have outdated
       # embeddings due to edits. Here we only do 10% of the limit
-      posts_batch_size = 1000
-
-      outdated_post_ids =
-        posts
-          .where("#{table_name}.updated_at < ?", 7.days.ago)
-          .order("random()")
-          .limit((limit - rebaked) / 10)
-          .pluck(:id)
-
-      outdated_post_ids.each_slice(posts_batch_size) do |batch|
-        vector_rep.gen_bulk_reprensentations(Post.where(id: batch))
-        rebaked += batch.length
-      end
+      posts
+        .where("#{table_name}.updated_at < ?", 7.days.ago)
+        .order("random()")
+        .limit((limit - rebaked) / 10)
+        .pluck(:id)
+        .each_slice(posts_batch_size) do |batch|
+          vector.gen_bulk_reprensentations(Post.where(id: batch))
+          rebaked += batch.length
+        end
 
       rebaked
     end
 
     private
 
-    def populate_topic_embeddings(vector_rep, topics, force: false)
+    def populate_topic_embeddings(vector, topics, force: false)
       done = 0
 
-      topics = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL") if !force
+      topics =
+        topics.where("#{DiscourseAi::Embeddings::Schema::TOPICS_TABLE}.topic_id IS NULL") if !force
 
       ids = topics.pluck("topics.id")
       batch_size = 1000
 
       ids.each_slice(batch_size) do |batch|
-        vector_rep.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
+        vector.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
         done += batch.length
       end
 

app/jobs/scheduled/sentiment_backfill.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Jobs
+  class SentimentBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    cluster_concurrency 1
+
+    def execute(_args)
+      return if !SiteSetting.ai_sentiment_enabled
+
+      base_budget = SiteSetting.ai_sentiment_backfill_maximum_posts_per_hour
+      return if base_budget.zero?
+      # Split budget in 12 intervals, but make sure is at least one.
+      #
+      # This is not exact as we don't have a way of tracking how many
+      # posts we classified in the current hour, but it's a good enough approximation.
+      limit_per_job = [base_budget, 12].max / 12
+
+      classificator = DiscourseAi::Sentiment::PostClassification.new
+      return if !classificator.has_classifiers?
+
+      posts =
+        DiscourseAi::Sentiment::PostClassification.backfill_query(
+          max_age_days: SiteSetting.ai_sentiment_backfill_post_max_age_days,
+        ).limit(limit_per_job)
+
+      classificator.bulk_classify!(posts)
+    end
+  end
+end

app/jobs/scheduled/summaries_backfill.rb

@@ -12,22 +12,27 @@ module ::Jobs
 
       system_user = Discourse.system_user
 
+      if SiteSetting.ai_summary_gists_enabled
+        gist_t = AiSummary.summary_types[:gist]
+
+        backfill_candidates(gist_t)
+          .limit(current_budget(gist_t))
+          .each do |topic|
+            DiscourseAi::Summarization.topic_gist(topic).force_summarize(system_user)
+          end
+      end
+
       complete_t = AiSummary.summary_types[:complete]
       backfill_candidates(complete_t)
         .limit(current_budget(complete_t))
         .each do |topic|
           DiscourseAi::Summarization.topic_summary(topic).force_summarize(system_user)
         end
-
-      return unless SiteSetting.ai_summary_gists_enabled
-
-      gist_t = AiSummary.summary_types[:gist]
-      backfill_candidates(gist_t)
-        .limit(current_budget(gist_t))
-        .each { |topic| DiscourseAi::Summarization.topic_gist(topic).force_summarize(system_user) }
     end
 
     def backfill_candidates(summary_type)
+      max_age_days = SiteSetting.ai_summary_backfill_topic_max_age_days
+
       Topic
         .where("topics.word_count >= ?", SiteSetting.ai_summary_backfill_minimum_word_count)
         .joins(<<~SQL)
@@ -36,9 +41,15 @@ module ::Jobs
                           ais.target_type = 'Topic' AND
                           ais.summary_type = '#{summary_type}'
         SQL
+        .where("topics.created_at > current_timestamp - INTERVAL '#{max_age_days.to_i} DAY'")
         .where(
-          "ais.id IS NULL OR UPPER(ais.content_range) < topics.highest_post_number + 1",
-        ) # (1..1) gets stored ad (1..2).
+          <<~SQL, # (1..1) gets stored ad (1..2).
+          ais.id IS NULL OR (
+            UPPER(ais.content_range) < topics.highest_post_number + 1 
+            AND ais.created_at < (current_timestamp - INTERVAL '5 minutes')
+          )
+        SQL
+        )
         .order("ais.created_at DESC NULLS FIRST, topics.last_posted_at DESC")
     end
 

app/models/ai_api_audit_log.rb

@@ -15,6 +15,7 @@ class AiApiAuditLog < ActiveRecord::Base
     Ollama = 7
     SambaNova = 8
     Mistral = 9
+    OpenRouter = 10
   end
 
   def next_log_id

app/models/ai_artifact.rb

@@ -1,23 +1,29 @@
 # frozen_string_literal: true
 
 class AiArtifact < ActiveRecord::Base
+  has_many :versions, class_name: "AiArtifactVersion", dependent: :destroy
   belongs_to :user
   belongs_to :post
   validates :html, length: { maximum: 65_535 }
   validates :css, length: { maximum: 65_535 }
   validates :js, length: { maximum: 65_535 }
 
-  def self.iframe_for(id)
+  def self.iframe_for(id, version = nil)
     <<~HTML
       <div class='ai-artifact'>
-        <iframe src='#{url(id)}' frameborder="0" height="100%" width="100%"></iframe>
-        <a href='#{url(id)}' target='_blank'>#{I18n.t("discourse_ai.ai_artifact.link")}</a>
+        <iframe src='#{url(id, version)}' frameborder="0" height="100%" width="100%"></iframe>
+        <a href='#{url(id, version)}' target='_blank'>#{I18n.t("discourse_ai.ai_artifact.link")}</a>
       </div>
     HTML
   end
 
-  def self.url(id)
-    Discourse.base_url + "/discourse-ai/ai-bot/artifacts/#{id}"
+  def self.url(id, version = nil)
+    url = Discourse.base_url + "/discourse-ai/ai-bot/artifacts/#{id}"
+    if version
+      "#{url}/#{version}"
+    else
+      url
+    end
   end
 
   def self.share_publicly(id:, post:)
@@ -33,6 +39,37 @@ class AiArtifact < ActiveRecord::Base
   def url
     self.class.url(id)
   end
+
+  def apply_diff(html_diff: nil, css_diff: nil, js_diff: nil, change_description: nil)
+    differ = DiscourseAi::Utils::DiffUtils
+
+    html = html_diff ? differ.apply_hunk(self.html, html_diff) : self.html
+    css = css_diff ? differ.apply_hunk(self.css, css_diff) : self.css
+    js = js_diff ? differ.apply_hunk(self.js, js_diff) : self.js
+
+    create_new_version(html: html, css: css, js: js, change_description: change_description)
+  end
+
+  def create_new_version(html: nil, css: nil, js: nil, change_description: nil)
+    latest_version = versions.order(version_number: :desc).first
+    new_version_number = latest_version ? latest_version.version_number + 1 : 1
+    version = nil
+
+    transaction do
+      # Create the version record
+      version =
+        versions.create!(
+          version_number: new_version_number,
+          html: html || self.html,
+          css: css || self.css,
+          js: js || self.js,
+          change_description: change_description,
+        )
+      save!
+    end
+
+    version
+  end
 end
 
 # == Schema Information

app/models/ai_artifact_version.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+class AiArtifactVersion < ActiveRecord::Base
+  belongs_to :ai_artifact
+  validates :html, length: { maximum: 65_535 }
+  validates :css, length: { maximum: 65_535 }
+  validates :js, length: { maximum: 65_535 }
+end
+
+# == Schema Information
+#
+# Table name: ai_artifact_versions
+#
+#  id                 :bigint           not null, primary key
+#  ai_artifact_id     :bigint           not null
+#  version_number     :integer          not null
+#  html               :string(65535)
+#  css                :string(65535)
+#  js                 :string(65535)
+#  metadata           :jsonb
+#  change_description :string
+#  created_at         :datetime         not null
+#  updated_at         :datetime         not null
+#
+# Indexes
+#
+#  index_ai_artifact_versions_on_ai_artifact_id_and_version_number  (ai_artifact_id,version_number) UNIQUE
+#