DEV: Update libraries for Ruby 3.3 compat (#477) (#720)

Co-authored-by: Rafael dos Santos Silva <[email protected]>

.discourse-compatibility

@@ -0,0 +1,6 @@
+< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
+< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
+< 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
+3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
+< 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
+3.1.0.beta3: 057fbe1ce6

.eslintrc.cjs

@@ -0,0 +1 @@
+module.exports = require("@discourse/lint-configs/eslint");

.github/workflows/discourse-plugin.yml

@@ -0,0 +1,11 @@
+name: Discourse Plugin
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  ci:
+    uses: discourse/.github/.github/workflows/discourse-plugin.yml@v1

.gitignore

@@ -0,0 +1,3 @@
+node_modules
+/gems
+/auto_generated

.prettierignore

@@ -0,0 +1,21 @@
+app/assets/stylesheets/vendor/
+documentation/
+package.json
+config/locales/**/*.yml
+!config/locales/**/*.en*.yml
+script/import_scripts/**/*.yml
+
+plugins/**/lib/javascripts/locale
+public/
+!/app/assets/javascripts/discourse/public
+vendor/
+app/assets/javascripts/discourse/tests/fixtures
+spec/
+node_modules/
+dist/
+tmp/
+
+**/*.rb
+**/*.html
+**/*.json
+**/*.md

.prettierrc.cjs

@@ -0,0 +1 @@
+module.exports = require("@discourse/lint-configs/prettier");

.rubocop.yml

@@ -0,0 +1,4 @@
+inherit_gem:
+  rubocop-discourse: stree-compat.yml
+RSpec/NamedSubject:
+  Enabled: false

.streerc

@@ -0,0 +1,2 @@
+--print-width=100
+--plugins=plugin/trailing_comma,plugin/disable_auto_ternary

.template-lintrc.cjs

@@ -0,0 +1 @@
+module.exports = require("@discourse/lint-configs/template-lint");

Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+group :development do
+  gem "rubocop-discourse"
+  gem "syntax_tree"
+end

Gemfile.lock

@@ -0,0 +1,53 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    json (2.7.1)
+    language_server-protocol (3.17.0.3)
+    parallel (1.24.0)
+    parser (3.3.0.3)
+      ast (~> 2.4.1)
+      racc
+    prettier_print (1.2.1)
+    racc (1.7.3)
+    rainbow (3.1.1)
+    regexp_parser (2.9.0)
+    rexml (3.2.6)
+    rubocop (1.59.0)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.2.2.4)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.30.0)
+      parser (>= 3.2.1.0)
+    rubocop-capybara (2.20.0)
+      rubocop (~> 1.41)
+    rubocop-discourse (3.6.0)
+      rubocop (>= 1.59.0)
+      rubocop-rspec (>= 2.25.0)
+    rubocop-factory_bot (2.25.1)
+      rubocop (~> 1.41)
+    rubocop-rspec (2.26.1)
+      rubocop (~> 1.40)
+      rubocop-capybara (~> 2.17)
+      rubocop-factory_bot (~> 2.22)
+    ruby-progressbar (1.13.0)
+    syntax_tree (6.2.0)
+      prettier_print (>= 1.2.0)
+    unicode-display_width (2.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rubocop-discourse
+  syntax_tree
+
+BUNDLED WITH
+   2.5.4

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Civilized Discourse Construction Kit
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -1,3 +1,5 @@
-# Discourse AI is now shipped with Discourse core
+# **Discourse AI** Plugin
 
-https://github.com/discourse/discourse/tree/main/plugins/discourse-ai
+**Plugin Summary**
+
+For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco

app/controllers/discourse_ai/admin/ai_personas_controller.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class AiPersonasController < ::Admin::AdminController
+      before_action :find_ai_persona, only: %i[show update destroy create_user]
+
+      def index
+        ai_personas =
+          AiPersona.ordered.map do |persona|
+            # we use a special serializer here cause names and descriptions are
+            # localized for system personas
+            LocalizedAiPersonaSerializer.new(persona, root: false)
+          end
+        tools =
+          DiscourseAi::AiBot::Personas::Persona.all_available_tools.map do |tool|
+            AiToolSerializer.new(tool, root: false)
+          end
+        llms =
+          DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
+            { id: hash[:value], name: hash[:name] }
+          end
+        render json: { ai_personas: ai_personas, meta: { commands: tools, llms: llms } }
+      end
+
+      def show
+        render json: LocalizedAiPersonaSerializer.new(@ai_persona)
+      end
+
+      def create
+        ai_persona = AiPersona.new(ai_persona_params)
+        if ai_persona.save
+          render json: { ai_persona: ai_persona }, status: :created
+        else
+          render_json_error ai_persona
+        end
+      end
+
+      def create_user
+        user = @ai_persona.create_user!
+        render json: BasicUserSerializer.new(user, root: "user")
+      end
+
+      def update
+        if @ai_persona.update(ai_persona_params)
+          render json: @ai_persona
+        else
+          render_json_error @ai_persona
+        end
+      end
+
+      def destroy
+        if @ai_persona.destroy
+          head :no_content
+        else
+          render_json_error @ai_persona
+        end
+      end
+
+      private
+
+      def find_ai_persona
+        @ai_persona = AiPersona.find(params[:id])
+      end
+
+      def ai_persona_params
+        permitted =
+          params.require(:ai_persona).permit(
+            :name,
+            :description,
+            :enabled,
+            :system_prompt,
+            :priority,
+            :top_p,
+            :temperature,
+            :default_llm,
+            :user_id,
+            :mentionable,
+            :max_context_posts,
+            allowed_group_ids: [],
+          )
+
+        if commands = params.dig(:ai_persona, :commands)
+          permitted[:commands] = permit_commands(commands)
+        end
+
+        permitted
+      end
+
+      def permit_commands(commands)
+        return [] if !commands.is_a?(Array)
+
+        commands.filter_map do |command, options|
+          break nil if !command.is_a?(String)
+          options&.permit! if options && options.is_a?(ActionController::Parameters)
+
+          if options
+            [command, options]
+          else
+            command
+          end
+        end
+      end
+    end
+  end
+end

app/controllers/discourse_ai/admin/dashboard_controller.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class DashboardController < ::Admin::StaffController
+      requires_plugin DiscourseAi::PLUGIN_NAME
+
+      def sentiment
+      end
+    end
+  end
+end

app/controllers/discourse_ai/ai_bot/bot_controller.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module AiBot
+    class BotController < ::ApplicationController
+      requires_plugin ::DiscourseAi::PLUGIN_NAME
+      requires_login
+
+      def stop_streaming_response
+        post = Post.find(params[:post_id])
+        guardian.ensure_can_see!(post)
+
+        Discourse.redis.del("gpt_cancel:#{post.id}")
+
+        render json: {}, status: 200
+      end
+
+      def show_bot_username
+        bot_user_id = DiscourseAi::AiBot::EntryPoint.map_bot_model_to_user_id(params[:username])
+        raise Discourse::InvalidParameters.new(:username) if !bot_user_id
+
+        bot_username_lower = User.find(bot_user_id).username_lower
+
+        render json: { bot_username: bot_username_lower }, status: 200
+      end
+    end
+  end
+end

app/controllers/discourse_ai/ai_helper/assistant_controller.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module AiHelper
+    class AssistantController < ::ApplicationController
+      requires_plugin ::DiscourseAi::PLUGIN_NAME
+      requires_login
+      before_action :ensure_can_request_suggestions
+      before_action :rate_limiter_performed!, except: %i[prompts]
+
+      def suggest
+        input = get_text_param!
+
+        prompt = CompletionPrompt.find_by(id: params[:mode])
+
+        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
+
+        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+          raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
+
+          prompt.custom_instruction = params[:custom_prompt]
+        end
+
+        suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+
+        hijack do
+          render json:
+                   DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
+                     prompt,
+                     input,
+                     current_user,
+                   ),
+                 status: 200
+        end
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
+        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
+                          status: 502
+      end
+
+      def suggest_title
+        input = get_text_param!
+
+        prompt = CompletionPrompt.enabled_by_name("generate_titles")
+        raise Discourse::InvalidParameters.new(:mode) if !prompt
+
+        hijack do
+          render json:
+                   DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
+                     prompt,
+                     input,
+                     current_user,
+                   ),
+                 status: 200
+        end
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
+        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
+                          status: 502
+      end
+
+      def suggest_category
+        input = get_text_param!
+        input_hash = { text: input }
+
+        render json:
+                 DiscourseAi::AiHelper::SemanticCategorizer.new(
+                   input_hash,
+                   current_user,
+                 ).categories,
+               status: 200
+      end
+
+      def suggest_tags
+        input = get_text_param!
+        input_hash = { text: input }
+
+        render json: DiscourseAi::AiHelper::SemanticCategorizer.new(input_hash, current_user).tags,
+               status: 200
+      end
+
+      def suggest_thumbnails(input)
+        hijack do
+          thumbnails = DiscourseAi::AiHelper::Painter.new.commission_thumbnails(input, current_user)
+
+          render json: { thumbnails: thumbnails }, status: 200
+        end
+      end
+
+      def explain
+        post_id = get_post_param!
+        term_to_explain = get_text_param!
+        post = Post.includes(:topic).find_by(id: post_id)
+
+        raise Discourse::InvalidParameters.new(:post_id) unless post
+
+        Jobs.enqueue(
+          :stream_post_helper,
+          post_id: post.id,
+          user_id: current_user.id,
+          term_to_explain: term_to_explain,
+        )
+
+        render json: { success: true }, status: 200
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
+        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
+                          status: 502
+      end
+
+      private
+
+      def get_text_param!
+        params[:text].tap { |t| raise Discourse::InvalidParameters.new(:text) if t.blank? }
+      end
+
+      def get_post_param!
+        params[:post_id].tap { |t| raise Discourse::InvalidParameters.new(:post_id) if t.blank? }
+      end
+
+      def rate_limiter_performed!
+        RateLimiter.new(current_user, "ai_assistant", 6, 3.minutes).performed!
+      end
+
+      def ensure_can_request_suggestions
+        user_group_ids = current_user.group_ids
+
+        allowed =
+          SiteSetting.ai_helper_allowed_groups_map.any? do |group_id|
+            user_group_ids.include?(group_id)
+          end
+
+        raise Discourse::InvalidAccess if !allowed
+      end
+    end
+  end
+end

app/controllers/discourse_ai/embeddings/embeddings_controller.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Embeddings
+    class EmbeddingsController < ::ApplicationController
+      requires_plugin ::DiscourseAi::PLUGIN_NAME
+
+      SEMANTIC_SEARCH_TYPE = "semantic_search"
+
+      def search
+        query = params[:q].to_s
+
+        if query.length < SiteSetting.min_search_term_length
+          raise Discourse::InvalidParameters.new(:q)
+        end
+
+        grouped_results =
+          Search::GroupedSearchResults.new(
+            type_filter: SEMANTIC_SEARCH_TYPE,
+            term: query,
+            search_context: guardian,
+            use_pg_headlines_for_excerpt: false,
+          )
+
+        semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
+
+        if !semantic_search.cached_query?(query)
+          RateLimiter.new(current_user, "semantic-search", 4, 1.minutes).performed!
+        end
+
+        hijack do
+          semantic_search
+            .search_for_topics(query)
+            .each { |topic_post| grouped_results.add(topic_post) }
+
+          render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
+        end
+      end
+    end
+  end
+end

app/jobs/regular/create_ai_reply.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ::Jobs
+  class CreateAiReply < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return unless bot_user = User.find_by(id: args[:bot_user_id])
+      return unless post = Post.includes(:topic).find_by(id: args[:post_id])
+      persona_id = args[:persona_id]
+
+      begin
+        persona = DiscourseAi::AiBot::Personas::Persona.find_by(user: post.user, id: persona_id)
+        raise DiscourseAi::AiBot::Bot::BOT_NOT_FOUND if persona.nil?
+
+        bot = DiscourseAi::AiBot::Bot.as(bot_user, persona: persona.new)
+
+        DiscourseAi::AiBot::Playground.new(bot).reply_to(post)
+      rescue DiscourseAi::AiBot::Bot::BOT_NOT_FOUND
+        Rails.logger.warn(
+          "Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",
+        )
+      end
+    end
+  end
+end

app/jobs/regular/evaluate_post_uploads.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Jobs
+  class EvaluatePostUploads < ::Jobs::Base
+    def execute(args)
+      return unless SiteSetting.ai_nsfw_detection_enabled
+      return if (post_id = args[:post_id]).blank?
+
+      post = Post.includes(:uploads).find_by_id(post_id)
+      return if post.nil? || post.uploads.empty?
+
+      return if post.uploads.none? { |u| FileHelper.is_supported_image?(u.url) }
+
+      DiscourseAi::PostClassificator.new(DiscourseAi::Nsfw::Classification.new).classify!(post)
+    end
+  end
+end

app/jobs/regular/generate_chat_thread_title.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateChatThreadTitle < ::Jobs::Base
+    sidekiq_options queue: "low"
+
+    def execute(args)
+      return unless SiteSetting.ai_helper_automatic_chat_thread_title
+      return if (thread_id = args[:thread_id]).blank?
+
+      thread = ::Chat::Thread.find_by_id(thread_id)
+      return if thread.nil? || thread.title.present?
+
+      title = DiscourseAi::AiHelper::ChatThreadTitler.new(thread).suggested_title
+      return if title.blank?
+
+      # TODO use a proper API that will make the new title update live
+      thread.update!(title: title)
+    end
+  end
+end

app/jobs/regular/generate_embeddings.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateEmbeddings < ::Jobs::Base
+    sidekiq_options queue: "low"
+
+    def execute(args)
+      return unless SiteSetting.ai_embeddings_enabled
+      return if args[:target_type].blank? || args[:target_id].blank?
+      target = args[:target_type].constantize.find_by_id(args[:target_id])
+      return if target.nil? || target.deleted_at.present?
+
+      topic = target.is_a?(Topic) ? target : target.topic
+      post = target.is_a?(Post) ? target : target.first_post
+      return unless topic.present? && post.present?
+      return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
+      return if post.raw.blank?
+
+      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
+      vector_rep =
+        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
+
+      vector_rep.generate_representation_from(target)
+    end
+  end
+end

app/jobs/regular/post_sentiment_analysis.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ::Jobs
+  class PostSentimentAnalysis < ::Jobs::Base
+    def execute(args)
+      return unless SiteSetting.ai_sentiment_enabled
+      return if (post_id = args[:post_id]).blank?
+
+      post = Post.find_by(id: post_id, post_type: Post.types[:regular])
+      return if post&.raw.blank?
+
+      DiscourseAi::PostClassificator.new(
+        DiscourseAi::Sentiment::SentimentClassification.new,
+      ).classify!(post)
+    end
+  end
+end

app/jobs/regular/stream_post_helper.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Jobs
+  class StreamPostHelper < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return unless post = Post.includes(:topic).find_by(id: args[:post_id])
+      return unless user = User.find_by(id: args[:user_id])
+      return unless args[:term_to_explain]
+
+      topic = post.topic
+      reply_to = post.reply_to_post
+
+      guardian = Guardian.new(user)
+      return unless guardian.can_see?(post)
+
+      prompt = CompletionPrompt.enabled_by_name("explain")
+
+      input = <<~TEXT
+      <term>#{args[:term_to_explain]}</term>
+      <context>#{post.raw}</context>
+          <topic>#{topic.title}</topic>
+          #{reply_to ? "<replyTo>#{reply_to.raw}</replyTo>" : nil}
+        TEXT
+
+      DiscourseAi::AiHelper::Assistant.new.stream_prompt(
+        prompt,
+        input,
+        user,
+        "/discourse-ai/ai-helper/explain/#{post.id}",
+      )
+    end
+  end
+end

app/jobs/regular/toxicity_classify_chat_message.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module ::Jobs
+  class ToxicityClassifyChatMessage < ::Jobs::Base
+    def execute(args)
+      return unless SiteSetting.ai_toxicity_enabled
+
+      return if (chat_message_id = args[:chat_message_id]).blank?
+
+      chat_message = ::Chat::Message.find_by(id: chat_message_id)
+      return if chat_message&.message.blank?
+
+      DiscourseAi::ChatMessageClassificator.new(
+        DiscourseAi::Toxicity::ToxicityClassification.new,
+      ).classify!(chat_message)
+    end
+  end
+end

app/jobs/regular/toxicity_classify_post.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ::Jobs
+  class ToxicityClassifyPost < ::Jobs::Base
+    def execute(args)
+      return unless SiteSetting.ai_toxicity_enabled
+
+      post_id = args[:post_id]
+      return if post_id.blank?
+
+      post = Post.find_by(id: post_id, post_type: Post.types[:regular])
+      return if post&.raw.blank?
+
+      DiscourseAi::PostClassificator.new(
+        DiscourseAi::Toxicity::ToxicityClassification.new,
+      ).classify!(post)
+    end
+  end
+end

app/jobs/regular/update_ai_bot_pm_title.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ::Jobs
+  class UpdateAiBotPmTitle < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return unless bot_user = User.find_by(id: args[:bot_user_id])
+      return unless bot = DiscourseAi::AiBot::Bot.as(bot_user)
+      return unless post = Post.includes(:topic).find_by(id: args[:post_id])
+
+      return unless post.topic.custom_fields[DiscourseAi::AiBot::EntryPoint::REQUIRE_TITLE_UPDATE]
+
+      DiscourseAi::AiBot::Playground.new(bot).title_playground(post)
+    end
+  end
+end

app/jobs/scheduled/embeddings_backfill.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Jobs
+  class EmbeddingsBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    sidekiq_options queue: "low"
+    cluster_concurrency 1
+
+    def execute(args)
+      return unless SiteSetting.ai_embeddings_enabled
+
+      limit = SiteSetting.ai_embeddings_backfill_batch_size
+
+      if limit > 50_000
+        limit = 50_000
+        Rails.logger.warn(
+          "Limiting backfill batch size to 50,000 to avoid OOM errors, reduce ai_embeddings_backfill_batch_size to avoid this warning",
+        )
+      end
+
+      rebaked = 0
+
+      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
+      vector_rep =
+        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
+      table_name = vector_rep.topic_table_name
+
+      topics =
+        Topic
+          .joins("LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id")
+          .where(archetype: Archetype.default)
+          .where(deleted_at: nil)
+          .order("topics.bumped_at DESC")
+          .limit(limit - rebaked)
+
+      rebaked += populate_topic_embeddings(vector_rep, topics)
+
+      vector_rep.consider_indexing
+
+      return if rebaked >= limit
+
+      # Then, we'll try to backfill embeddings for topics that have outdated
+      # embeddings, be it model or strategy version
+      relation = topics.where(<<~SQL)
+          #{table_name}.model_version < #{vector_rep.version}
+          OR
+          #{table_name}.strategy_version < #{strategy.version}
+        SQL
+
+      rebaked += populate_topic_embeddings(vector_rep, relation)
+
+      return if rebaked >= limit
+
+      # Finally, we'll try to backfill embeddings for topics that have outdated
+      # embeddings due to edits or new replies. Here we only do 10% of the limit
+      relation =
+        topics.where("#{table_name}.updated_at < ?", 7.days.ago).limit((limit - rebaked) / 10)
+
+      populate_topic_embeddings(vector_rep, relation)
+
+      return if rebaked >= limit
+
+      return unless SiteSetting.ai_embeddings_per_post_enabled
+
+      # Now for posts
+      table_name = vector_rep.post_table_name
+
+      posts =
+        Post
+          .joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
+          .where(deleted_at: nil)
+          .limit(limit - rebaked)
+
+      # First, we'll try to backfill embeddings for posts that have none
+      posts
+        .where("#{table_name}.post_id IS NULL")
+        .find_each do |t|
+          vector_rep.generate_representation_from(t)
+          rebaked += 1
+        end
+
+      vector_rep.consider_indexing
+
+      return if rebaked >= limit
+
+      # Then, we'll try to backfill embeddings for posts that have outdated
+      # embeddings, be it model or strategy version
+      posts
+        .where(<<~SQL)
+          #{table_name}.model_version < #{vector_rep.version}
+          OR
+          #{table_name}.strategy_version < #{strategy.version}
+        SQL
+        .find_each do |t|
+          vector_rep.generate_representation_from(t)
+          rebaked += 1
+        end
+
+      return if rebaked >= limit
+
+      # Finally, we'll try to backfill embeddings for posts that have outdated
+      # embeddings due to edits. Here we only do 10% of the limit
+      posts
+        .where("#{table_name}.updated_at < ?", 7.days.ago)
+        .order("random()")
+        .limit((limit - rebaked) / 10)
+        .pluck(:id)
+        .each do |id|
+          vector_rep.generate_representation_from(Post.find_by(id: id))
+          rebaked += 1
+        end
+
+      rebaked
+    end
+
+    private
+
+    def populate_topic_embeddings(vector_rep, topics)
+      done = 0
+      ids = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL").pluck("topics.id")
+
+      ids.each do |id|
+        topic = Topic.find_by(id: id)
+        if topic
+          vector_rep.generate_representation_from(topic)
+          done += 1
+        end
+      end
+      done
+    end
+  end
+end

app/mailers/ai_report_mailer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AiReportMailer < ActionMailer::Base
+  include Email::BuildEmailHelper
+
+  def send_report(to_address, opts = {})
+    build_email(to_address, **opts)
+  end
+end

app/models/ai_api_audit_log.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class AiApiAuditLog < ActiveRecord::Base
+  module Provider
+    OpenAI = 1
+    Anthropic = 2
+    HuggingFaceTextGeneration = 3
+    Gemini = 4
+    Vllm = 5
+  end
+end
+
+# == Schema Information
+#
+# Table name: ai_api_audit_logs
+#
+#  id                   :bigint           not null, primary key
+#  provider_id          :integer          not null
+#  user_id              :integer
+#  topic_id             :integer
+#  post_id              :integer
+#  request_tokens       :integer
+#  response_tokens      :integer
+#  raw_request_payload  :string
+#  raw_response_payload :string
+#  created_at           :datetime         not null
+#  updated_at           :datetime         not null
+#