Fix lib directory

DEV: Update engine config to match skeleton
- adds `lib/` autoloading (and removes the plugin.rb workaround) - adds eager loading of scheduled jobs in development (otherwise they don't work at all)
2024-05-02 19:19:09 +01:00 · 2024-05-02 18:53:22 +01:00 · 2024-05-02 12:42:30 +10:00 · 2024-05-01 17:50:58 +10:00 · 2024-04-30 21:57:37 +02:00 · 2024-04-30 09:47:21 -04:00
598 changed files with 1129374 additions and 9522 deletions
@@ -1,3 +1,6 @@
+< 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
+< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
+< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
 < 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
 3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
 < 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
@@ -0,0 +1,21 @@
+app/assets/stylesheets/vendor/
+documentation/
+package.json
+config/locales/**/*.yml
+!config/locales/**/*.en*.yml
+script/import_scripts/**/*.yml
+
+plugins/**/lib/javascripts/locale
+public/
+!/app/assets/javascripts/discourse/public
+vendor/
+app/assets/javascripts/discourse/tests/fixtures
+spec/
+node_modules/
+dist/
+tmp/
+
+**/*.rb
+**/*.html
+**/*.json
+**/*.md
@@ -1,58 +1,77 @@
 GEM
  remote: https://rubygems.org/
  specs:
+    activesupport (7.1.3.2)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      mutex_m
+      tzinfo (~> 2.0)
    ast (2.4.2)
-    json (2.6.3)
+    base64 (0.2.0)
+    bigdecimal (3.1.6)
+    concurrent-ruby (1.2.3)
+    connection_pool (2.4.1)
+    drb (2.2.1)
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
+    json (2.7.1)
    language_server-protocol (3.17.0.3)
-    parallel (1.23.0)
-    parser (3.2.2.4)
+    minitest (5.22.2)
+    mutex_m (0.2.0)
+    parallel (1.24.0)
+    parser (3.3.0.5)
      ast (~> 2.4.1)
      racc
    prettier_print (1.2.1)
-    racc (1.7.2)
+    racc (1.7.3)
    rainbow (3.1.1)
-    regexp_parser (2.8.2)
+    regexp_parser (2.9.0)
    rexml (3.2.6)
-    rubocop (1.57.2)
+    rubocop (1.61.0)
      json (~> 2.3)
      language_server-protocol (>= 3.17.0)
      parallel (~> 1.10)
-      parser (>= 3.2.2.4)
+      parser (>= 3.3.0.2)
      rainbow (>= 2.2.2, < 4.0)
      regexp_parser (>= 1.8, < 3.0)
      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
      ruby-progressbar (~> 1.7)
      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.30.0)
-      parser (>= 3.2.1.0)
-    rubocop-capybara (2.19.0)
+    rubocop-ast (1.31.1)
+      parser (>= 3.3.0.4)
+    rubocop-capybara (2.20.0)
      rubocop (~> 1.41)
-    rubocop-discourse (3.4.0)
-      rubocop (>= 1.1.0)
-      rubocop-rspec (>= 2.0.0)
-    rubocop-factory_bot (2.24.0)
-      rubocop (~> 1.33)
-    rubocop-rspec (2.25.0)
+    rubocop-discourse (3.7.1)
+      activesupport (>= 6.1)
+      rubocop (>= 1.59.0)
+      rubocop-capybara (>= 2.0.0)
+      rubocop-factory_bot (>= 2.0.0)
+      rubocop-rspec (>= 2.25.0)
+    rubocop-factory_bot (2.25.1)
+      rubocop (~> 1.41)
+    rubocop-rspec (2.27.1)
      rubocop (~> 1.40)
      rubocop-capybara (~> 2.17)
      rubocop-factory_bot (~> 2.22)
    ruby-progressbar (1.13.0)
    syntax_tree (6.2.0)
      prettier_print (>= 1.2.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
    unicode-display_width (2.5.0)

 PLATFORMS
-  arm64-darwin-20
  ruby
-  x86_64-darwin-18
-  x86_64-darwin-19
-  x86_64-darwin-20
-  x86_64-linux

 DEPENDENCIES
  rubocop-discourse
  syntax_tree

 BUNDLED WITH
-   2.4.21
+   2.5.4
@@ -0,0 +1,23 @@
+import { AUTO_GROUPS } from "discourse/lib/constants";
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default DiscourseRoute.extend({
+  async model() {
+    const record = this.store.createRecord("ai-persona");
+    record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
+    record.set("rag_uploads", []);
+    // these match the defaults on the table
+    record.set("rag_chunk_tokens", 374);
+    record.set("rag_chunk_overlap_tokens", 10);
+    record.set("rag_conversation_chunks", 10);
+    return record;
+  },
+
+  setupController(controller, model) {
+    this._super(controller, model);
+    controller.set(
+      "allPersonas",
+      this.modelFor("adminPlugins.show.discourse-ai-personas")
+    );
+  },
+});
@@ -0,0 +1,19 @@
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default DiscourseRoute.extend({
+  async model(params) {
+    const allPersonas = this.modelFor(
+      "adminPlugins.show.discourse-ai-personas"
+    );
+    const id = parseInt(params.id, 10);
+    return allPersonas.findBy("id", id);
+  },
+
+  setupController(controller, model) {
+    this._super(controller, model);
+    controller.set(
+      "allPersonas",
+      this.modelFor("adminPlugins.show.discourse-ai-personas")
+    );
+  },
+});
@@ -0,0 +1,7 @@
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default class DiscourseAiAiPersonasRoute extends DiscourseRoute {
+  model() {
+    return this.store.findAll("ai-persona");
+  }
+}
@@ -0,0 +1 @@
+<AiPersonaListEditor @personas={{this.model}} />
@@ -0,0 +1,4 @@
+<AiPersonaListEditor
+  @personas={{this.allPersonas}}
+  @currentPersona={{this.model}}
+/>
@@ -0,0 +1,4 @@
+<AiPersonaListEditor
+  @personas={{this.allPersonas}}
+  @currentPersona={{this.model}}
+/>
@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class AiPersonasController < ::Admin::AdminController
+      requires_plugin ::DiscourseAi::PLUGIN_NAME
+
+      before_action :find_ai_persona,
+                    only: %i[show update destroy create_user indexing_status_check]
+
+      def index
+        ai_personas =
+          AiPersona.ordered.map do |persona|
+            # we use a special serializer here cause names and descriptions are
+            # localized for system personas
+            LocalizedAiPersonaSerializer.new(persona, root: false)
+          end
+        tools =
+          DiscourseAi::AiBot::Personas::Persona.all_available_tools.map do |tool|
+            AiToolSerializer.new(tool, root: false)
+          end
+        llms =
+          DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
+            { id: hash[:value], name: hash[:name] }
+          end
+        render json: { ai_personas: ai_personas, meta: { commands: tools, llms: llms } }
+      end
+
+      def show
+        render json: LocalizedAiPersonaSerializer.new(@ai_persona)
+      end
+
+      def create
+        ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
+        if ai_persona.save
+          RagDocumentFragment.link_persona_and_uploads(ai_persona, attached_upload_ids)
+
+          render json: { ai_persona: ai_persona }, status: :created
+        else
+          render_json_error ai_persona
+        end
+      end
+
+      def create_user
+        user = @ai_persona.create_user!
+        render json: BasicUserSerializer.new(user, root: "user")
+      end
+
+      def update
+        if @ai_persona.update(ai_persona_params.except(:rag_uploads))
+          RagDocumentFragment.update_persona_uploads(@ai_persona, attached_upload_ids)
+
+          render json: @ai_persona
+        else
+          render_json_error @ai_persona
+        end
+      end
+
+      def destroy
+        if @ai_persona.destroy
+          head :no_content
+        else
+          render_json_error @ai_persona
+        end
+      end
+
+      def upload_file
+        file = params[:file] || params[:files].first
+
+        if !SiteSetting.ai_embeddings_enabled?
+          raise Discourse::InvalidAccess.new("Embeddings not enabled")
+        end
+
+        validate_extension!(file.original_filename)
+        validate_file_size!(file.tempfile.size)
+
+        hijack do
+          upload =
+            UploadCreator.new(
+              file.tempfile,
+              file.original_filename,
+              type: "discourse_ai_rag_upload",
+              skip_validations: true,
+            ).create_for(current_user.id)
+
+          if upload.persisted?
+            render json: UploadSerializer.new(upload)
+          else
+            render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
+          end
+        end
+      end
+
+      def indexing_status_check
+        render json: RagDocumentFragment.indexing_status(@ai_persona, @ai_persona.uploads)
+      end
+
+      private
+
+      def find_ai_persona
+        @ai_persona = AiPersona.find(params[:id])
+      end
+
+      def attached_upload_ids
+        ai_persona_params[:rag_uploads].to_a.map { |h| h[:id] }
+      end
+
+      def ai_persona_params
+        permitted =
+          params.require(:ai_persona).permit(
+            :name,
+            :description,
+            :enabled,
+            :system_prompt,
+            :priority,
+            :top_p,
+            :temperature,
+            :default_llm,
+            :user_id,
+            :mentionable,
+            :max_context_posts,
+            :vision_enabled,
+            :vision_max_pixels,
+            :rag_chunk_tokens,
+            :rag_chunk_overlap_tokens,
+            :rag_conversation_chunks,
+            :question_consolidator_llm,
+            allowed_group_ids: [],
+            rag_uploads: [:id],
+          )
+
+        if commands = params.dig(:ai_persona, :commands)
+          permitted[:commands] = permit_commands(commands)
+        end
+
+        permitted
+      end
+
+      def permit_commands(commands)
+        return [] if !commands.is_a?(Array)
+
+        commands.filter_map do |command, options|
+          break nil if !command.is_a?(String)
+          options&.permit! if options && options.is_a?(ActionController::Parameters)
+
+          if options
+            [command, options]
+          else
+            command
+          end
+        end
+      end
+
+      def validate_extension!(filename)
+        extension = File.extname(filename)[1..-1] || ""
+        authorized_extensions = %w[txt md]
+        if !authorized_extensions.include?(extension)
+          raise Discourse::InvalidParameters.new(
+                  I18n.t(
+                    "upload.unauthorized",
+                    authorized_extensions: authorized_extensions.join(" "),
+                  ),
+                )
+        end
+      end
+
+      def validate_file_size!(filesize)
+        max_size_bytes = 20.megabytes
+        if filesize > max_size_bytes
+          raise Discourse::InvalidParameters.new(
+                  I18n.t(
+                    "upload.attachments.too_large_humanized",
+                    max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
+                  ),
+                )
+        end
+      end
+    end
+  end
+end
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class DashboardController < ::Admin::StaffController
+      requires_plugin DiscourseAi::PLUGIN_NAME
+
+      def sentiment
+      end
+    end
+  end
+end
@@ -6,6 +6,21 @@ module DiscourseAi
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      requires_login

+      def show_debug_info
+        post = Post.find(params[:post_id])
+        guardian.ensure_can_debug_ai_bot_conversation!(post)
+
+        posts =
+          Post
+            .where("post_number <= ?", post.post_number)
+            .where(topic_id: post.topic_id)
+            .order("post_number DESC")
+
+        debug_info = AiApiAuditLog.where(post: posts).order(created_at: :desc).first
+
+        render json: debug_info, status: 200
+      end
+
      def stop_streaming_response
        post = Post.find(params[:post_id])
        guardian.ensure_can_see!(post)
@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module AiBot
+    class SharedAiConversationsController < ::ApplicationController
+      requires_plugin ::DiscourseAi::PLUGIN_NAME
+      requires_login only: %i[create update destroy]
+      before_action :require_site_settings!
+
+      skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required, only: %i[show]
+
+      def create
+        ensure_allowed_create!
+
+        RateLimiter.new(current_user, "share-ai-conversation", 10, 1.minute).performed!
+
+        shared_conversation = SharedAiConversation.share_conversation(current_user, @topic)
+
+        if shared_conversation.persisted?
+          render json: success_json.merge(share_key: shared_conversation.share_key)
+        else
+          render json: failed_json.merge(error: I18n.t("discourse_ai.share_ai.failed_to_share")),
+                 status: :unprocessable_entity
+        end
+      end
+
+      def destroy
+        ensure_allowed_destroy!
+
+        SharedAiConversation.destroy_conversation(@shared_conversation)
+
+        render json:
+                 success_json.merge(message: I18n.t("discourse_ai.share_ai.conversation_deleted"))
+      end
+
+      def show
+        @shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
+        raise Discourse::NotFound if @shared_conversation.blank?
+
+        expires_in 1.minute, public: true
+        response.headers["X-Robots-Tag"] = "noindex"
+
+        if request.format.json?
+          render json: success_json.merge(@shared_conversation.to_json)
+        else
+          render "show", layout: false
+        end
+      end
+
+      def preview
+        ensure_allowed_preview!
+        data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
+        data[:error] = @error if @error
+        data[:share_key] = @shared_conversation.share_key if @shared_conversation
+        data[:topic_id] = @topic.id
+        render json: data
+      end
+
+      private
+
+      def require_site_settings!
+        if !SiteSetting.discourse_ai_enabled ||
+             !SiteSetting.ai_bot_public_sharing_allowed_groups_map.any? ||
+             !SiteSetting.ai_bot_enabled
+          raise Discourse::NotFound
+        end
+      end
+
+      def ensure_allowed_preview!
+        @topic = Topic.find_by(id: params[:topic_id])
+        raise Discourse::NotFound if !@topic
+
+        @shared_conversation = SharedAiConversation.find_by(target: @topic)
+
+        @error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
+        if @error == :not_allowed
+          raise Discourse::InvalidAccess.new(
+                  nil,
+                  nil,
+                  custom_message: "discourse_ai.share_ai.errors.#{@error}",
+                )
+        end
+      end
+
+      def ensure_allowed_destroy!
+        @shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
+
+        raise Discourse::InvalidAccess if @shared_conversation.blank?
+
+        guardian.ensure_can_destroy_shared_ai_bot_conversation!(@shared_conversation)
+      end
+
+      def ensure_allowed_create!
+        @topic = Topic.find_by(id: params[:topic_id])
+        raise Discourse::NotFound if !@topic
+
+        error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
+        if error
+          raise Discourse::InvalidAccess.new(
+                  nil,
+                  nil,
+                  custom_message: "discourse_ai.share_ai.errors.#{error}",
+                )
+        end
+      end
+    end
+  end
+end
@@ -8,14 +8,7 @@ module DiscourseAi
      before_action :ensure_can_request_suggestions
      before_action :rate_limiter_performed!, except: %i[prompts]

-      def prompts
-        render json:
-                 ActiveModel::ArraySerializer.new(
-                   DiscourseAi::AiHelper::LlmPrompt.new.available_prompts,
-                   root: false,
-                 ),
-               status: 200
-      end
+      include SecureUploadEndpointHelpers

      def suggest
        input = get_text_param!
@@ -23,45 +16,45 @@ module DiscourseAi
        prompt = CompletionPrompt.find_by(id: params[:mode])

        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
-        if prompt.prompt_type == "custom_prompt" && params[:custom_prompt].blank?
-          raise Discourse::InvalidParameters.new(:custom_prompt)
+
+        if prompt.id == CompletionPrompt::CUSTOM_PROMPT
+          raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
+
+          prompt.custom_instruction = params[:custom_prompt]
        end

+        suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+
        hijack do
          render json:
-                   DiscourseAi::AiHelper::LlmPrompt.new.generate_and_send_prompt(prompt, params),
+                   DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
+                     prompt,
+                     input,
+                     current_user,
+                   ),
                 status: 200
        end
-      rescue ::DiscourseAi::Inference::OpenAiCompletions::CompletionFailed,
-             ::DiscourseAi::Inference::HuggingFaceTextGeneration::CompletionFailed,
-             ::DiscourseAi::Inference::AnthropicCompletions::CompletionFailed => e
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                          status: 502
      end

      def suggest_title
        input = get_text_param!
-        input_hash = { text: input }

-        llm_prompt =
-          DiscourseAi::AiHelper::LlmPrompt
-            .new
-            .available_prompts(name_filter: "generate_titles")
-            .first
-        prompt = CompletionPrompt.find_by(id: llm_prompt[:id])
-        raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
+        prompt = CompletionPrompt.enabled_by_name("generate_titles")
+        raise Discourse::InvalidParameters.new(:mode) if !prompt

        hijack do
          render json:
-                   DiscourseAi::AiHelper::LlmPrompt.new.generate_and_send_prompt(
+                   DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
                     prompt,
-                     input_hash,
+                     input,
+                     current_user,
                   ),
                 status: 200
        end
-      rescue ::DiscourseAi::Inference::OpenAiCompletions::CompletionFailed,
-             ::DiscourseAi::Inference::HuggingFaceTextGeneration::CompletionFailed,
-             ::DiscourseAi::Inference::AnthropicCompletions::CompletionFailed => e
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                          status: 502
      end
@@ -86,9 +79,7 @@ module DiscourseAi
               status: 200
      end

-      def suggest_thumbnails
-        input = get_text_param!
-
+      def suggest_thumbnails(input)
        hijack do
          thumbnails = DiscourseAi::AiHelper::Painter.new.commission_thumbnails(input, current_user)

@@ -98,21 +89,45 @@ module DiscourseAi

      def explain
        post_id = get_post_param!
-        text = get_text_param!
-        post = Post.find_by(id: post_id)
+        term_to_explain = get_text_param!
+        post = Post.includes(:topic).find_by(id: post_id)

        raise Discourse::InvalidParameters.new(:post_id) unless post

-        render json:
-                 DiscourseAi::AiHelper::TopicHelper.new(
-                   { text: text },
-                   current_user,
-                   post: post,
-                 ).explain,
-               status: 200
-      rescue ::DiscourseAi::Inference::OpenAiCompletions::CompletionFailed,
-             ::DiscourseAi::Inference::HuggingFaceTextGeneration::CompletionFailed,
-             ::DiscourseAi::Inference::AnthropicCompletions::CompletionFailed => e
+        Jobs.enqueue(
+          :stream_post_helper,
+          post_id: post.id,
+          user_id: current_user.id,
+          term_to_explain: term_to_explain,
+        )
+
+        render json: { success: true }, status: 200
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
+        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
+                          status: 502
+      end
+
+      def caption_image
+        image_url = params[:image_url]
+        raise Discourse::InvalidParameters.new(:image_url) if !image_url
+
+        image = upload_from_full_url(image_url)
+        raise Discourse::NotFound if image.blank?
+        final_image_url = get_caption_url(image, image_url)
+
+        hijack do
+          caption =
+            DiscourseAi::AiHelper::Assistant.new.generate_image_caption(
+              final_image_url,
+              current_user,
+            )
+          render json: {
+                   caption:
+                     "#{caption} (#{I18n.t("discourse_ai.ai_helper.image_caption.attribution")})",
+                 },
+                 status: 200
+        end
+      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed, Net::HTTPBadResponse
        render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
                          status: 502
      end
@@ -132,14 +147,18 @@ module DiscourseAi
      end

      def ensure_can_request_suggestions
-        user_group_ids = current_user.group_ids
+        if !current_user.in_any_groups?(SiteSetting.ai_helper_allowed_groups_map)
+          raise Discourse::InvalidAccess
+        end
+      end

-        allowed =
-          SiteSetting.ai_helper_allowed_groups_map.any? do |group_id|
-            user_group_ids.include?(group_id)
-          end
+      def get_caption_url(image_upload, image_url)
+        if image_upload.secure?
+          check_secure_upload_permission(image_upload)
+          return Discourse.store.url_for(image_upload)
+        end

-        raise Discourse::InvalidAccess if !allowed
+        UrlHelper.absolute(image_url)
      end
    end
  end
@@ -36,6 +36,34 @@ module DiscourseAi
          render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
        end
      end
+
+      def quick_search
+        query = params[:q].to_s
+
+        if query.length < SiteSetting.min_search_term_length
+          raise Discourse::InvalidParameters.new(:q)
+        end
+
+        grouped_results =
+          Search::GroupedSearchResults.new(
+            type_filter: SEMANTIC_SEARCH_TYPE,
+            term: query,
+            search_context: guardian,
+            use_pg_headlines_for_excerpt: false,
+          )
+
+        semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
+
+        if !semantic_search.cached_query?(query)
+          RateLimiter.new(current_user, "semantic-search", 60, 1.minutes).performed!
+        end
+
+        hijack do
+          semantic_search.quick_search(query).each { |topic_post| grouped_results.add(topic_post) }
+
+          render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
+        end
+      end
    end
  end
 end
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+module DiscourseAi
+  module AiBot
+    module SharedAiConversationsHelper
+      # bump up version when assets change
+      # long term we may want to change this cause it is hard to remember
+      # to bump versions, but for now this does the job
+      VERSION = "1"
+
+      def share_asset_url(short_path)
+        ::UrlHelper.local_cdn_url("/plugins/discourse-ai/ai-share/#{short_path}?#{VERSION}")
+      end
+    end
+  end
+end
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ::Jobs
+  class CreateAiReply < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return unless bot_user = User.find_by(id: args[:bot_user_id])
+      return unless post = Post.includes(:topic).find_by(id: args[:post_id])
+      persona_id = args[:persona_id]
+
+      begin
+        persona = DiscourseAi::AiBot::Personas::Persona.find_by(user: post.user, id: persona_id)
+        raise DiscourseAi::AiBot::Bot::BOT_NOT_FOUND if persona.nil?
+
+        bot = DiscourseAi::AiBot::Bot.as(bot_user, persona: persona.new)
+
+        DiscourseAi::AiBot::Playground.new(bot).reply_to(post)
+      rescue DiscourseAi::AiBot::Bot::BOT_NOT_FOUND
+        Rails.logger.warn(
+          "Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",
+        )
+      end
+    end
+  end
+end
@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module ::Jobs
+  class DigestRagUpload < ::Jobs::Base
+    CHUNK_SIZE = 1024
+    CHUNK_OVERLAP = 64
+    MAX_FRAGMENTS = 100_000
+
+    # TODO(roman): Add a way to automatically recover from errors, resulting in unindexed uploads.
+    def execute(args)
+      return if (upload = Upload.find_by(id: args[:upload_id])).nil?
+      return if (ai_persona = AiPersona.find_by(id: args[:ai_persona_id])).nil?
+
+      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
+      vector_rep =
+        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+
+      tokenizer = vector_rep.tokenizer
+      chunk_tokens = ai_persona.rag_chunk_tokens
+      overlap_tokens = ai_persona.rag_chunk_overlap_tokens
+
+      fragment_ids = RagDocumentFragment.where(ai_persona: ai_persona, upload: upload).pluck(:id)
+
+      # Check if this is the first time we process this upload.
+      if fragment_ids.empty?
+        document = get_uploaded_file(upload)
+        return if document.nil?
+
+        RagDocumentFragment.publish_status(upload, { total: 0, indexed: 0, left: 0 })
+
+        fragment_ids = []
+        idx = 0
+
+        ActiveRecord::Base.transaction do
+          chunk_document(
+            file: document,
+            tokenizer: tokenizer,
+            chunk_tokens: chunk_tokens,
+            overlap_tokens: overlap_tokens,
+          ) do |chunk, metadata|
+            fragment_ids << RagDocumentFragment.create!(
+              ai_persona: ai_persona,
+              fragment: chunk,
+              fragment_number: idx + 1,
+              upload: upload,
+              metadata: metadata,
+            ).id
+
+            idx += 1
+
+            if idx > MAX_FRAGMENTS
+              Rails.logger.warn("Upload #{upload.id} has too many fragments, truncating.")
+              break
+            end
+          end
+        end
+      end
+
+      fragment_ids.each_slice(50) do |slice|
+        Jobs.enqueue(:generate_rag_embeddings, fragment_ids: slice)
+      end
+    end
+
+    private
+
+    def chunk_document(file:, tokenizer:, chunk_tokens:, overlap_tokens:)
+      buffer = +""
+      current_metadata = nil
+      done = false
+      overlap = ""
+
+      # generally this will be plenty
+      read_size = chunk_tokens * 10
+
+      while buffer.present? || !done
+        if buffer.length < read_size
+          read = file.read(read_size)
+          done = true if read.nil?
+
+          read = Encodings.to_utf8(read) if read
+
+          buffer << (read || "")
+        end
+
+        # at this point we unconditionally have 2x CHUNK_SIZE worth of data in the buffer
+        metadata_regex = /\[\[metadata (.*?)\]\]/m
+
+        before_metadata, new_metadata, after_metadata = buffer.split(metadata_regex)
+        to_chunk = nil
+
+        if before_metadata.present?
+          to_chunk = before_metadata
+        elsif after_metadata.present?
+          current_metadata = new_metadata
+          to_chunk = after_metadata
+          buffer = buffer.split(metadata_regex, 2).last
+          overlap = ""
+        else
+          current_metadata = new_metadata
+          buffer = buffer.split(metadata_regex, 2).last
+          overlap = ""
+          next
+        end
+
+        chunk, split_char = first_chunk(to_chunk, tokenizer: tokenizer, chunk_tokens: chunk_tokens)
+        buffer = buffer[chunk.length..-1]
+
+        processed_chunk = overlap + chunk
+
+        processed_chunk.strip!
+        processed_chunk.gsub!(/\n[\n]+/, "\n\n")
+
+        yield processed_chunk, current_metadata
+
+        current_chunk_tokens = tokenizer.encode(chunk)
+        overlap_token_ids = current_chunk_tokens[-overlap_tokens..-1] || current_chunk_tokens
+
+        overlap = ""
+
+        while overlap_token_ids.present?
+          begin
+            overlap = tokenizer.decode(overlap_token_ids) + split_char
+            break if overlap.encoding == Encoding::UTF_8
+          rescue StandardError
+            # it is possible that we truncated mid char
+          end
+          overlap_token_ids.shift
+        end
+
+        # remove first word it is probably truncated
+        overlap = overlap.split(" ", 2).last
+      end
+    end
+
+    def first_chunk(text, chunk_tokens:, tokenizer:, splitters: ["\n\n", "\n", ".", ""])
+      return text, " " if tokenizer.tokenize(text).length <= chunk_tokens
+
+      splitters = splitters.find_all { |s| text.include?(s) }.compact
+
+      buffer = +""
+      split_char = nil
+
+      splitters.each do |splitter|
+        split_char = splitter
+
+        text
+          .split(split_char)
+          .each do |part|
+            break if tokenizer.tokenize(buffer + split_char + part).length > chunk_tokens
+            buffer << split_char
+            buffer << part
+          end
+        break if buffer.length > 0
+      end
+
+      [buffer, split_char]
+    end
+
+    def get_uploaded_file(upload)
+      store = Discourse.store
+      @file ||=
+        if store.external?
+          # Upload#filesize could be approximate.
+          # add two extra Mbs to make sure that we'll be able to download the upload.
+          max_filesize = upload.filesize + 2.megabytes
+          store.download(upload, max_file_size_kb: max_filesize)
+        else
+          File.open(store.path_for(upload))
+        end
+    end
+  end
+end
@@ -11,7 +11,7 @@ module Jobs

      return if post.uploads.none? { |u| FileHelper.is_supported_image?(u.url) }

-      DiscourseAi::PostClassificator.new(DiscourseAi::NSFW::NSFWClassification.new).classify!(post)
+      DiscourseAi::PostClassificator.new(DiscourseAi::Nsfw::Classification.new).classify!(post)
    end
  end
 end
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Jobs
+  class GenerateEmbeddings < ::Jobs::Base
+    sidekiq_options queue: "low"
+
+    def execute(args)
+      return unless SiteSetting.ai_embeddings_enabled
+      return if args[:target_type].blank? || args[:target_id].blank?
+      target = args[:target_type].constantize.find_by_id(args[:target_id])
+      return if target.nil? || target.deleted_at.present?
+
+      topic = target.is_a?(Topic) ? target : target.topic
+      post = target.is_a?(Post) ? target : target.first_post
+      return unless topic.present? && post.present?
+      return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
+      return if post.raw.blank?
+
+      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
+      vector_rep =
+        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
+
+      vector_rep.generate_representation_from(target)
+    end
+  end
+end
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module ::Jobs
+  class GenerateRagEmbeddings < ::Jobs::Base
+    sidekiq_options queue: "ultra_low"
+    # we could also restrict concurrency but this takes so long if it is not concurrent
+
+    def execute(args)
+      return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
+
+      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
+      vector_rep =
+        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+
+      # generate_representation_from checks compares the digest value to make sure
+      # the embedding is only generated once per fragment unless something changes.
+      fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
+
+      last_fragment = fragments.last
+      ai_persona = last_fragment.ai_persona
+      upload = last_fragment.upload
+
+      indexing_status = RagDocumentFragment.indexing_status(ai_persona, [upload])[upload.id]
+      RagDocumentFragment.publish_status(upload, indexing_status)
+    end
+  end
+end
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module ::Jobs
+  class SharedConversationAdjustUploadSecurity < ::Jobs::Base
+    def execute(args)
+      if args[:conversation_id].present?
+        # The conversation context includes post cooked content so this
+        # must be updated when target uploads security changes.
+        update_conversation(args[:conversation_id])
+      elsif args[:target_id].present? && args[:target_type].present?
+        # If we deleted the conversation then we just need to update the target's
+        # uploads security, no need to update the conversation.
+        update_target(args[:target_id], args[:target_type])
+      end
+    end
+
+    private
+
+    def update_conversation(conversation_id)
+      conversation = SharedAiConversation.find_by(id: conversation_id)
+      return if conversation.blank?
+
+      # NOTE: Only Topics are supported for now, in future we will need a more flexible
+      # way of doing this.
+      if conversation.target_type == "Topic"
+        rebaked_posts = TopicUploadSecurityManager.new(conversation.target).run
+
+        if rebaked_posts.any?
+          new_context =
+            conversation.context.map do |context_post|
+              rebaked_post = rebaked_posts.find { |p| p.id == context_post["id"] }
+              context_post["cooked"] = rebaked_post.cooked if rebaked_post
+              context_post
+            end
+
+          conversation.update(context: new_context)
+        end
+      end
+    end
+
+    def update_target(target_id, target_type)
+      # NOTE: Only Topics are supported for now, in future we will need a more flexible
+      # way of doing this.
+      if target_type == "Topic"
+        topic = target_type.constantize.find_by(id: target_id)
+        return if topic.blank?
+        TopicUploadSecurityManager.new(topic).run
+      end
+    end
+  end
+end
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Jobs
+  class StreamPostHelper < ::Jobs::Base
+    sidekiq_options retry: false
+
+    def execute(args)
+      return unless post = Post.includes(:topic).find_by(id: args[:post_id])
+      return unless user = User.find_by(id: args[:user_id])
+      return unless args[:term_to_explain]
+
+      topic = post.topic
+      reply_to = post.reply_to_post
+
+      return unless user.guardian.can_see?(post)
+
+      prompt = CompletionPrompt.enabled_by_name("explain")
+
+      input = <<~TEXT
+      <term>#{args[:term_to_explain]}</term>
+      <context>#{post.raw}</context>
+          <topic>#{topic.title}</topic>
+          #{reply_to ? "<replyTo>#{reply_to.raw}</replyTo>" : nil}
+        TEXT
+
+      DiscourseAi::AiHelper::Assistant.new.stream_prompt(
+        prompt,
+        input,
+        user,
+        "/discourse-ai/ai-helper/explain/#{post.id}",
+      )
+    end
+  end
+end
@@ -6,12 +6,12 @@ module ::Jobs

    def execute(args)
      return unless bot_user = User.find_by(id: args[:bot_user_id])
-      return unless bot = DiscourseAi::AiBot::Bot.as(bot_user)
+      return unless bot = DiscourseAi::AiBot::Bot.as(bot_user, model: args[:model])
      return unless post = Post.includes(:topic).find_by(id: args[:post_id])

      return unless post.topic.custom_fields[DiscourseAi::AiBot::EntryPoint::REQUIRE_TITLE_UPDATE]

-      bot.update_pm_title(post)
+      DiscourseAi::AiBot::Playground.new(bot).title_playground(post)
    end
  end
 end
@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Jobs
+  class EmbeddingsBackfill < ::Jobs::Scheduled
+    every 5.minutes
+    sidekiq_options queue: "low"
+    cluster_concurrency 1
+
+    def execute(args)
+      return unless SiteSetting.ai_embeddings_enabled
+
+      limit = SiteSetting.ai_embeddings_backfill_batch_size
+
+      if limit > 50_000
+        limit = 50_000
+        Rails.logger.warn(
+          "Limiting backfill batch size to 50,000 to avoid OOM errors, reduce ai_embeddings_backfill_batch_size to avoid this warning",
+        )
+      end
+
+      rebaked = 0
+
+      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
+      vector_rep =
+        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
+      table_name = vector_rep.topic_table_name
+
+      topics =
+        Topic
+          .joins("LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id")
+          .where(archetype: Archetype.default)
+          .where(deleted_at: nil)
+          .order("topics.bumped_at DESC")
+          .limit(limit - rebaked)
+
+      rebaked += populate_topic_embeddings(vector_rep, topics)
+
+      vector_rep.consider_indexing
+
+      return if rebaked >= limit
+
+      # Then, we'll try to backfill embeddings for topics that have outdated
+      # embeddings, be it model or strategy version
+      relation = topics.where(<<~SQL)
+          #{table_name}.model_version < #{vector_rep.version}
+          OR
+          #{table_name}.strategy_version < #{strategy.version}
+        SQL
+
+      rebaked += populate_topic_embeddings(vector_rep, relation)
+
+      return if rebaked >= limit
+
+      # Finally, we'll try to backfill embeddings for topics that have outdated
+      # embeddings due to edits or new replies. Here we only do 10% of the limit
+      relation =
+        topics.where("#{table_name}.updated_at < ?", 7.days.ago).limit((limit - rebaked) / 10)
+
+      populate_topic_embeddings(vector_rep, relation)
+
+      return if rebaked >= limit
+
+      return unless SiteSetting.ai_embeddings_per_post_enabled
+
+      # Now for posts
+      table_name = vector_rep.post_table_name
+
+      posts =
+        Post
+          .joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
+          .where(deleted_at: nil)
+          .limit(limit - rebaked)
+
+      # First, we'll try to backfill embeddings for posts that have none
+      posts
+        .where("#{table_name}.post_id IS NULL")
+        .find_each do |t|
+          vector_rep.generate_representation_from(t)
+          rebaked += 1
+        end
+
+      vector_rep.consider_indexing
+
+      return if rebaked >= limit
+
+      # Then, we'll try to backfill embeddings for posts that have outdated
+      # embeddings, be it model or strategy version
+      posts
+        .where(<<~SQL)
+          #{table_name}.model_version < #{vector_rep.version}
+          OR
+          #{table_name}.strategy_version < #{strategy.version}
+        SQL
+        .find_each do |t|
+          vector_rep.generate_representation_from(t)
+          rebaked += 1
+        end
+
+      return if rebaked >= limit
+
+      # Finally, we'll try to backfill embeddings for posts that have outdated
+      # embeddings due to edits. Here we only do 10% of the limit
+      posts
+        .where("#{table_name}.updated_at < ?", 7.days.ago)
+        .order("random()")
+        .limit((limit - rebaked) / 10)
+        .pluck(:id)
+        .each do |id|
+          vector_rep.generate_representation_from(Post.find_by(id: id))
+          rebaked += 1
+        end
+
+      rebaked
+    end
+
+    private
+
+    def populate_topic_embeddings(vector_rep, topics)
+      done = 0
+      ids = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL").pluck("topics.id")
+
+      ids.each do |id|
+        topic = Topic.find_by(id: id)
+        if topic
+          vector_rep.generate_representation_from(topic)
+          done += 1
+        end
+      end
+      done
+    end
+  end
+end
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				`<AiPersonaListEditor @personas={{this.model}} />`