FIX: Final summary sometimes missing

This PR fixes an issue where the summary updater sometimes doesn't show the last update. Due to some timing issues with the summary updater and the component this seems to happen. By explicitly calling set raw after the cooked, we ensure the final summary will always be set. No tests as this timing issue is difficult to test.
2024-11-06 12:00:35 -08:00
492 changed files with 8074 additions and 24552 deletions
@@ -1,4 +1,3 @@
-< 3.4.0.beta4-dev: 20612fde52d3f740cad64823ef8aadb0748b567f
 < 3.4.0.beta3-dev: decf1bb49d737ea15308400f22f89d1d1e71d13d
 < 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
 < 3.3.0.beta5-dev: 4d8090002f6dcd8e34d41033606bf131fa221475
@@ -0,0 +1 @@
+module.exports = require("@discourse/lint-configs/eslint");
@@ -1,18 +1,18 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
-  queryParams = {
+export default DiscourseRoute.extend({
+  queryParams: {
    llmTemplate: { refreshModel: true },
-  };
+  },

  async model() {
    const record = this.store.createRecord("ai-llm");
    record.provider_params = {};
    return record;
-  }
+  },

  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allLlms",
      this.modelFor("adminPlugins.show.discourse-ai-llms")
@@ -21,5 +21,5 @@ export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
      "llmTemplate",
      this.paramsFor(this.routeName).llmTemplate || null
    );
-  }
-}
+  },
+});
@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
+export default DiscourseRoute.extend({
  async model(params) {
    const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
    const id = parseInt(params.id, 10);
    const record = allLlms.findBy("id", id);
    record.provider_params = record.provider_params || {};
    return record;
-  }
+  },

  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allLlms",
      this.modelFor("adminPlugins.show.discourse-ai-llms")
    );
-  }
-}
+  },
+});
@@ -1,7 +1,7 @@
 import { AUTO_GROUPS } from "discourse/lib/constants";
 import DiscourseRoute from "discourse/routes/discourse";

-export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
+export default DiscourseRoute.extend({
  async model() {
    const record = this.store.createRecord("ai-persona");
    record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
@@ -13,13 +13,13 @@ export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRou
    record.set("allow_personal_messages", true);
    record.set("tool_details", false);
    return record;
-  }
+  },

  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allPersonas",
      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
-  }
-}
+  },
+});
@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
+export default DiscourseRoute.extend({
  async model(params) {
    const allPersonas = this.modelFor(
      "adminPlugins.show.discourse-ai-personas"
    );
    const id = parseInt(params.id, 10);
    return allPersonas.findBy("id", id);
-  }
+  },

  setupController(controller, model) {
-    super.setupController(controller, model);
+    this._super(controller, model);
    controller.set(
      "allPersonas",
      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
-  }
-}
+  },
+});
@@ -1,11 +0,0 @@
-import { service } from "@ember/service";
-import { ajax } from "discourse/lib/ajax";
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiSpamRoute extends DiscourseRoute {
-  @service store;
-
-  model() {
-    return ajax("/admin/plugins/discourse-ai/ai-spam.json");
-  }
-}
@@ -1,6 +1,6 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
+export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
  async model(params) {
    const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
    const id = parseInt(params.id, 10);
@@ -1,11 +0,0 @@
-import { service } from "@ember/service";
-import { ajax } from "discourse/lib/ajax";
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiUsageRoute extends DiscourseRoute {
-  @service store;
-
-  model() {
-    return ajax("/admin/plugins/discourse-ai/ai-usage.json");
-  }
-}
@@ -1 +0,0 @@
-<AiSpam @model={{this.model}} />
@@ -1,7 +0,0 @@
-<section class="ai-persona-tool-editor__current admin-detail pull-left">
-  <AiToolEditor
-    @tools={{this.allTools}}
-    @model={{this.model}}
-    @presets={{this.presets}}
-  />
-</section>
@@ -1,7 +1,5 @@
-<section class="ai-persona-tool-editor__current admin-detail pull-left">
-  <AiToolEditor
-    @tools={{this.allTools}}
-    @model={{this.model}}
-    @presets={{this.presets}}
-  />
-</section>
+<AiToolEditor
+  @tools={{this.allTools}}
+  @model={{this.model}}
+  @presets={{this.presets}}
+/>
@@ -0,0 +1,5 @@
+<AiToolEditor
+  @tools={{this.allTools}}
+  @model={{this.model}}
+  @presets={{this.presets}}
+/>
@@ -1 +0,0 @@
-<AiUsage @model={{this.model}} />
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Admin
-    class AiLlmQuotasController < ::Admin::AdminController
-      requires_plugin ::DiscourseAi::PLUGIN_NAME
-
-      def index
-        quotas = LlmQuota.includes(:group)
-
-        render json: {
-                 quotas:
-                   ActiveModel::ArraySerializer.new(quotas, each_serializer: LlmQuotaSerializer),
-               }
-      end
-
-      def create
-        quota = LlmQuota.new(quota_params)
-
-        if quota.save
-          render json: LlmQuotaSerializer.new(quota), status: :created
-        else
-          render_json_error quota
-        end
-      end
-
-      def update
-        quota = LlmQuota.find(params[:id])
-
-        if quota.update(quota_params)
-          render json: LlmQuotaSerializer.new(quota)
-        else
-          render_json_error quota
-        end
-      end
-
-      def destroy
-        quota = LlmQuota.find(params[:id])
-        quota.destroy!
-
-        head :no_content
-      rescue ActiveRecord::RecordNotFound
-        render json: { error: I18n.t("not_found") }, status: 404
-      end
-
-      private
-
-      def quota_params
-        params.require(:quota).permit(
-          :group_id,
-          :llm_model_id,
-          :max_tokens,
-          :max_usages,
-          :duration_seconds,
-        )
-      end
-    end
-  end
-end
@@ -6,7 +6,7 @@ module DiscourseAi
      requires_plugin ::DiscourseAi::PLUGIN_NAME

      def index
-        llms = LlmModel.all.includes(:llm_quotas).order(:display_name)
+        llms = LlmModel.all.order(:display_name)

        render json: {
                 ai_llms:
@@ -30,21 +30,13 @@ module DiscourseAi
               }
      end

-      def new
-      end
-
-      def edit
+      def show
        llm_model = LlmModel.find(params[:id])
        render json: LlmModelSerializer.new(llm_model)
      end

      def create
        llm_model = LlmModel.new(ai_llm_params)
-
-        # we could do nested attributes but the mechanics are not ideal leading
-        # to lots of complex debugging, this is simpler
-        quota_params.each { |quota| llm_model.llm_quotas.build(quota) } if quota_params
-
        if llm_model.save
          llm_model.toggle_companion_user
          render json: LlmModelSerializer.new(llm_model), status: :created
@@ -56,25 +48,6 @@ module DiscourseAi
      def update
        llm_model = LlmModel.find(params[:id])

-        if params[:ai_llm].key?(:llm_quotas)
-          if quota_params
-            existing_quota_group_ids = llm_model.llm_quotas.pluck(:group_id)
-            new_quota_group_ids = quota_params.map { |q| q[:group_id] }
-
-            llm_model
-              .llm_quotas
-              .where(group_id: existing_quota_group_ids - new_quota_group_ids)
-              .destroy_all
-
-            quota_params.each do |quota_param|
-              quota = llm_model.llm_quotas.find_or_initialize_by(group_id: quota_param[:group_id])
-              quota.update!(quota_param)
-            end
-          else
-            llm_model.llm_quotas.destroy_all
-          end
-        end
-
        if llm_model.seeded?
          return render_json_error(I18n.t("discourse_ai.llm.cannot_edit_builtin"), status: 403)
        end
@@ -134,19 +107,6 @@ module DiscourseAi

      private

-      def quota_params
-        if params[:ai_llm][:llm_quotas].present?
-          params[:ai_llm][:llm_quotas].map do |quota|
-            mapped = {}
-            mapped[:group_id] = quota[:group_id].to_i
-            mapped[:max_tokens] = quota[:max_tokens].to_i if quota[:max_tokens].present?
-            mapped[:max_usages] = quota[:max_usages].to_i if quota[:max_usages].present?
-            mapped[:duration_seconds] = quota[:duration_seconds].to_i
-            mapped
-          end
-        end
-      end
-
      def ai_llm_params(updating: nil)
        return {} if params[:ai_llm].blank?

@@ -5,7 +5,7 @@ module DiscourseAi
    class AiPersonasController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME

-      before_action :find_ai_persona, only: %i[edit update destroy create_user]
+      before_action :find_ai_persona, only: %i[show update destroy create_user]

      def index
        ai_personas =
@@ -33,10 +33,7 @@ module DiscourseAi
        render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
      end

-      def new
-      end
-
-      def edit
+      def show
        render json: LocalizedAiPersonaSerializer.new(@ai_persona)
      end

@@ -1,137 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Admin
-    class AiSpamController < ::Admin::AdminController
-      requires_plugin "discourse-ai"
-
-      def show
-        render json: AiSpamSerializer.new(spam_config, root: false)
-      end
-
-      def update
-        updated_params = {}
-        if allowed_params.key?(:llm_model_id)
-          llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
-          if llm_model_id.to_i < 0 &&
-               !SiteSetting.ai_spam_detection_model_allowed_seeded_models_map.include?(
-                 llm_model_id.to_s,
-               )
-            return(
-              render_json_error(
-                I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"),
-                status: 422,
-              )
-            )
-          end
-        end
-        updated_params[:data] = {
-          custom_instructions: allowed_params[:custom_instructions],
-        } if allowed_params.key?(:custom_instructions)
-
-        if updated_params.present?
-          # not using upsert cause we will not get the correct validation errors
-          if AiModerationSetting.spam
-            AiModerationSetting.spam.update!(updated_params)
-          else
-            AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
-          end
-        end
-
-        is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
-
-        if allowed_params.key?(:is_enabled)
-          if is_enabled && !AiModerationSetting.spam&.llm_model_id
-            return(
-              render_json_error(
-                I18n.t("discourse_ai.llm.configuration.must_select_model"),
-                status: 422,
-              )
-            )
-          end
-
-          SiteSetting.ai_spam_detection_enabled = is_enabled
-        end
-
-        render json: AiSpamSerializer.new(spam_config, root: false)
-      end
-
-      def test
-        url = params[:post_url].to_s
-        post = nil
-
-        if url.match?(/^\d+$/)
-          post_id = url.to_i
-          post = Post.find_by(id: post_id)
-        end
-
-        route = UrlHelper.rails_route_from_url(url) if !post
-
-        if route
-          if route[:controller] == "topics"
-            post_number = route[:post_number] || 1
-            post = Post.with_deleted.find_by(post_number: post_number, topic_id: route[:topic_id])
-          end
-        end
-
-        raise Discourse::NotFound if !post
-
-        result =
-          DiscourseAi::AiModeration::SpamScanner.test_post(
-            post,
-            custom_instructions: params[:custom_instructions],
-            llm_id: params[:llm_id],
-          )
-
-        render json: result
-      end
-
-      def fix_errors
-        case params[:error]
-        when "spam_scanner_not_admin"
-          begin
-            DiscourseAi::AiModeration::SpamScanner.fix_spam_scanner_not_admin
-            render json: success_json
-          rescue ActiveRecord::RecordInvalid
-            render_json_error(
-              I18n.t("discourse_ai.spam_detection.bot_user_update_failed"),
-              status: :unprocessable_entity,
-            )
-          rescue StandardError
-            render_json_error(
-              I18n.t("discourse_ai.spam_detection.unexpected"),
-              status: :internal_server_error,
-            )
-          end
-        else
-          render_json_error(
-            I18n.t("discourse_ai.spam_detection.invalid_error_type"),
-            status: :bad_request,
-          )
-        end
-      end
-
-      private
-
-      def allowed_params
-        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
-      end
-
-      def spam_config
-        spam_config = {
-          enabled: SiteSetting.ai_spam_detection_enabled,
-          settings: AiModerationSetting.spam,
-        }
-
-        spam_config[:stats] = DiscourseAi::AiModeration::SpamReport.generate(min_date: 1.week.ago)
-
-        if spam_config[:stats].scanned_count > 0
-          spam_config[
-            :flagging_username
-          ] = DiscourseAi::AiModeration::SpamScanner.flagging_user&.username
-        end
-        spam_config
-      end
-    end
-  end
-end
@@ -5,17 +5,14 @@ module DiscourseAi
    class AiToolsController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME

-      before_action :find_ai_tool, only: %i[test edit update destroy]
+      before_action :find_ai_tool, only: %i[test show update destroy]

      def index
        ai_tools = AiTool.all
        render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
      end

-      def new
-      end
-
-      def edit
+      def show
        render_serialized(@ai_tool, AiCustomToolSerializer)
      end

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Admin
-    class AiUsageController < ::Admin::AdminController
-      requires_plugin "discourse-ai"
-
-      def show
-      end
-
-      def report
-        render json: AiUsageSerializer.new(create_report, root: false)
-      end
-
-      private
-
-      def create_report
-        report =
-          DiscourseAi::Completions::Report.new(
-            start_date: params[:start_date]&.to_date || 30.days.ago,
-            end_date: params[:end_date]&.to_date || Time.current,
-          )
-
-        report = report.filter_by_feature(params[:feature]) if params[:feature].present?
-        report = report.filter_by_model(params[:model]) if params[:model].present?
-        report
-      end
-    end
-  end
-end
@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module AiBot
-    class ArtifactsController < ApplicationController
-      requires_plugin DiscourseAi::PLUGIN_NAME
-      before_action :require_site_settings!
-
-      skip_before_action :preload_json, :check_xhr, only: %i[show]
-
-      def show
-        artifact = AiArtifact.find(params[:id])
-
-        post = Post.find_by(id: artifact.post_id)
-        if artifact.metadata&.dig("public")
-          # no guardian needed
-        else
-          raise Discourse::NotFound if !post&.topic&.private_message?
-          raise Discourse::NotFound if !guardian.can_see?(post)
-        end
-
-        name = artifact.name
-
-        if params[:version].present?
-          artifact = artifact.versions.find_by(version_number: params[:version])
-          raise Discourse::NotFound if !artifact
-        end
-
-        js = artifact.js || ""
-        if !js.match?(%r{\A\s*<script.*</script>}mi)
-          mod = ""
-          mod = " type=\"module\"" if js.match?(/\A\s*import.*/)
-          js = "<script#{mod}>\n#{js}\n</script>"
-        end
-        # Prepare the inner (untrusted) HTML document
-        untrusted_html = <<~HTML
-          <!DOCTYPE html>
-          <html>
-            <head>
-              <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(name)}</title>
-              <style>
-                #{artifact.css}
-              </style>
-            </head>
-            <body>
-              #{artifact.html}
-              #{js}
-            </body>
-          </html>
-        HTML
-
-        # Prepare the outer (trusted) HTML document
-        trusted_html = <<~HTML
-          <!DOCTYPE html>
-          <html>
-            <head>
-              <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(name)}</title>
-              <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, user-scalable=yes, viewport-fit=cover, interactive-widget=resizes-content">
-              <style>
-                html, body, iframe {
-                  margin: 0;
-                  padding: 0;
-                  width: 100%;
-                  height: 100%;
-                  border: 0;
-                  overflow: hidden;
-                }
-                iframe {
-                  overflow: auto;
-                }
-              </style>
-            </head>
-            <body>
-              <iframe sandbox="allow-scripts allow-forms" height="100%" width="100%" srcdoc="#{ERB::Util.html_escape(untrusted_html)}" frameborder="0"></iframe>
-            </body>
-          </html>
-        HTML
-
-        response.headers.delete("X-Frame-Options")
-        response.headers[
-          "Content-Security-Policy"
-        ] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://unpkg.com https://cdnjs.cloudflare.com https://ajax.googleapis.com https://cdn.jsdelivr.net;"
-        response.headers["X-Robots-Tag"] = "noindex"
-
-        # Render the content
-        render html: trusted_html.html_safe, layout: false, content_type: "text/html"
-      end
-
-      private
-
-      def require_site_settings!
-        if !SiteSetting.discourse_ai_enabled ||
-             !SiteSetting.ai_artifact_security.in?(%w[lax strict])
-          raise Discourse::NotFound
-        end
-      end
-    end
-  end
-end
@@ -6,14 +6,6 @@ module DiscourseAi
      requires_plugin ::DiscourseAi::PLUGIN_NAME
      requires_login

-      def show_debug_info_by_id
-        log = AiApiAuditLog.find(params[:id])
-        raise Discourse::NotFound if !log.topic
-
-        guardian.ensure_can_debug_ai_bot_conversation!(log.topic)
-        render json: AiApiAuditLogSerializer.new(log, root: false), status: 200
-      end
-
      def show_debug_info
        post = Post.find(params[:post_id])
        guardian.ensure_can_debug_ai_bot_conversation!(post)
@@ -7,8 +7,7 @@ module DiscourseAi
      requires_login only: %i[create update destroy]
      before_action :require_site_settings!

-      skip_before_action :preload_json, :check_xhr, only: %i[show asset]
-      skip_before_action :verify_authenticity_token, only: ["asset"]
+      skip_before_action :preload_json, :check_xhr, only: %i[show]

      def create
        ensure_allowed_create!
@@ -51,30 +50,6 @@ module DiscourseAi
        end
      end

-      def asset
-        no_cookies
-
-        name = params[:name]
-        path, content_type =
-          if name == "share"
-            %w[share.css text/css]
-          elsif name == "highlight"
-            %w[highlight.min.js application/javascript]
-          else
-            raise Discourse::NotFound
-          end
-
-        content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
-
-        # note, path contains a ":version" which automatically busts the cache
-        # based on file content, so this is safe
-        response.headers["Last-Modified"] = 10.years.ago.httpdate
-        response.headers["Content-Length"] = content.bytesize.to_s
-        immutable_for 1.year
-
-        render plain: content, disposition: :nil, content_type: content_type
-      end
-
      def preview
        ensure_allowed_preview!
        data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
@@ -35,7 +35,7 @@ module DiscourseAi
          prompt.custom_instruction = params[:custom_prompt]
        end

-        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+        suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST

        hijack do
          render json:
@@ -2,29 +2,13 @@
 module DiscourseAi
  module AiBot
    module SharedAiConversationsHelper
-      # keeping it here for caching
-      def self.share_asset_url(asset_name)
-        if !%w[share.css highlight.js].include?(asset_name)
-          raise StandardError, "unknown asset type #{asset_name}"
-        end
+      # bump up version when assets change
+      # long term we may want to change this cause it is hard to remember
+      # to bump versions, but for now this does the job
+      VERSION = "1"

-        @urls ||= {}
-        url = @urls[asset_name]
-        return url if url
-
-        path = asset_name
-        path = "highlight.min.js" if asset_name == "highlight.js"
-
-        content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
-        sha1 = Digest::SHA1.hexdigest(content)
-
-        url = "/discourse-ai/ai-bot/shared-ai-conversations/asset/#{sha1}/#{asset_name}"
-
-        @urls[asset_name] = GlobalPath.cdn_path(url)
-      end
-
-      def share_asset_url(asset_name)
-        DiscourseAi::AiBot::SharedAiConversationsHelper.share_asset_url(asset_name)
+      def share_asset_url(short_path)
+        ::UrlHelper.absolute("/plugins/discourse-ai/ai-share/#{short_path}?#{VERSION}")
      end
    end
  end
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class AiSpamScan < ::Jobs::Base
-    def execute(args)
-      return if !args[:post_id]
-      post = Post.find_by(id: args[:post_id])
-      return if !post
-
-      DiscourseAi::AiModeration::SpamScanner.perform_scan(post)
-    end
-  end
-end
@@ -18,7 +18,9 @@ module ::Jobs
      target = target_type.constantize.find_by(id: target_id)
      return if !target

-      vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
+      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
+      vector_rep =
+        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)

      tokenizer = vector_rep.tokenizer
      chunk_tokens = target.rag_chunk_tokens
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Jobs
+  class EvaluatePostUploads < ::Jobs::Base
+    def execute(args)
+      return unless SiteSetting.ai_nsfw_detection_enabled
+      return if (post_id = args[:post_id]).blank?
+
+      post = Post.includes(:uploads).find_by_id(post_id)
+      return if post.nil? || post.uploads.empty?
+
+      return if post.uploads.none? { |u| FileHelper.is_supported_image?(u.url) }
+
+      DiscourseAi::PostClassificator.new(DiscourseAi::Nsfw::Classification.new).classify!(post)
+    end
+  end
+end
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				`module.exports = require("@discourse/lint-configs/eslint");`