wip experiment with /filter route

Update translations (#1032 )
DEV: Change to use DPageSubheader (#1033 )
2024-12-20 15:55:49 -05:00 · 2024-12-18 15:19:47 +01:00 · 2024-12-18 17:39:31 +10:00 · 2024-12-17 11:02:18 +11:00 · 2024-12-16 15:00:05 -08:00 · 2024-12-16 14:30:11 -06:00
356 changed files with 13701 additions and 4229 deletions
@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default DiscourseRoute.extend({
+export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
  async model(params) {
    const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
    const id = parseInt(params.id, 10);
    const record = allLlms.findBy("id", id);
    record.provider_params = record.provider_params || {};
    return record;
-  },
+  }

  setupController(controller, model) {
-    this._super(controller, model);
+    super.setupController(controller, model);
    controller.set(
      "allLlms",
      this.modelFor("adminPlugins.show.discourse-ai-llms")
    );
-  },
-});
+  }
+}
@@ -1,18 +1,18 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default DiscourseRoute.extend({
-  queryParams: {
+export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
+  queryParams = {
    llmTemplate: { refreshModel: true },
-  },
+  };

  async model() {
    const record = this.store.createRecord("ai-llm");
    record.provider_params = {};
    return record;
-  },
+  }

  setupController(controller, model) {
-    this._super(controller, model);
+    super.setupController(controller, model);
    controller.set(
      "allLlms",
      this.modelFor("adminPlugins.show.discourse-ai-llms")
@@ -21,5 +21,5 @@ export default DiscourseRoute.extend({
      "llmTemplate",
      this.paramsFor(this.routeName).llmTemplate || null
    );
-  },
-});
+  }
+}
@@ -1,19 +1,19 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default DiscourseRoute.extend({
+export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
  async model(params) {
    const allPersonas = this.modelFor(
      "adminPlugins.show.discourse-ai-personas"
    );
    const id = parseInt(params.id, 10);
    return allPersonas.findBy("id", id);
-  },
+  }

  setupController(controller, model) {
-    this._super(controller, model);
+    super.setupController(controller, model);
    controller.set(
      "allPersonas",
      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
-  },
-});
+  }
+}
@@ -1,7 +1,7 @@
 import { AUTO_GROUPS } from "discourse/lib/constants";
 import DiscourseRoute from "discourse/routes/discourse";

-export default DiscourseRoute.extend({
+export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
  async model() {
    const record = this.store.createRecord("ai-persona");
    record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
@@ -13,13 +13,13 @@ export default DiscourseRoute.extend({
    record.set("allow_personal_messages", true);
    record.set("tool_details", false);
    return record;
-  },
+  }

  setupController(controller, model) {
-    this._super(controller, model);
+    super.setupController(controller, model);
    controller.set(
      "allPersonas",
      this.modelFor("adminPlugins.show.discourse-ai-personas")
    );
-  },
-});
+  }
+}
@@ -0,0 +1,11 @@
+import { service } from "@ember/service";
+import { ajax } from "discourse/lib/ajax";
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default class DiscourseAiSpamRoute extends DiscourseRoute {
+  @service store;
+
+  model() {
+    return ajax("/admin/plugins/discourse-ai/ai-spam.json");
+  }
+}
@@ -1,6 +1,6 @@
 import DiscourseRoute from "discourse/routes/discourse";

-export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
+export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
  async model(params) {
    const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
    const id = parseInt(params.id, 10);
@@ -0,0 +1,11 @@
+import { service } from "@ember/service";
+import { ajax } from "discourse/lib/ajax";
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default class DiscourseAiUsageRoute extends DiscourseRoute {
+  @service store;
+
+  model() {
+    return ajax("/admin/plugins/discourse-ai/ai-usage.json");
+  }
+}
@@ -0,0 +1 @@
+<AiSpam @model={{this.model}} />
@@ -0,0 +1,7 @@
+<section class="ai-persona-tool-editor__current admin-detail pull-left">
+  <AiToolEditor
+    @tools={{this.allTools}}
+    @model={{this.model}}
+    @presets={{this.presets}}
+  />
+</section>
@@ -1,5 +1,7 @@
-<AiToolEditor
-  @tools={{this.allTools}}
-  @model={{this.model}}
-  @presets={{this.presets}}
-/>
+<section class="ai-persona-tool-editor__current admin-detail pull-left">
+  <AiToolEditor
+    @tools={{this.allTools}}
+    @model={{this.model}}
+    @presets={{this.presets}}
+  />
+</section>
@@ -1,5 +0,0 @@
-<AiToolEditor
-  @tools={{this.allTools}}
-  @model={{this.model}}
-  @presets={{this.presets}}
-/>
@@ -0,0 +1 @@
+<AiUsage @model={{this.model}} />
@@ -30,7 +30,10 @@ module DiscourseAi
               }
      end

-      def show
+      def new
+      end
+
+      def edit
        llm_model = LlmModel.find(params[:id])
        render json: LlmModelSerializer.new(llm_model)
      end
@@ -5,7 +5,7 @@ module DiscourseAi
    class AiPersonasController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME

-      before_action :find_ai_persona, only: %i[show update destroy create_user]
+      before_action :find_ai_persona, only: %i[edit update destroy create_user]

      def index
        ai_personas =
@@ -33,7 +33,10 @@ module DiscourseAi
        render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
      end

-      def show
+      def new
+      end
+
+      def edit
        render json: LocalizedAiPersonaSerializer.new(@ai_persona)
      end

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class AiSpamController < ::Admin::AdminController
+      requires_plugin "discourse-ai"
+
+      def show
+        render json: AiSpamSerializer.new(spam_config, root: false)
+      end
+
+      def update
+        updated_params = {}
+        if allowed_params.key?(:llm_model_id)
+          llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
+          if llm_model_id.to_i < 0 &&
+               !SiteSetting.ai_spam_detection_model_allowed_seeded_models_map.include?(
+                 "custom:#{llm_model_id}",
+               )
+            return(
+              render_json_error(
+                I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"),
+                status: 422,
+              )
+            )
+          end
+        end
+        updated_params[:data] = {
+          custom_instructions: allowed_params[:custom_instructions],
+        } if allowed_params.key?(:custom_instructions)
+
+        if updated_params.present?
+          # not using upsert cause we will not get the correct validation errors
+          if AiModerationSetting.spam
+            AiModerationSetting.spam.update!(updated_params)
+          else
+            AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
+          end
+        end
+
+        is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
+
+        if allowed_params.key?(:is_enabled)
+          if is_enabled && !AiModerationSetting.spam&.llm_model_id
+            return(
+              render_json_error(
+                I18n.t("discourse_ai.llm.configuration.must_select_model"),
+                status: 422,
+              )
+            )
+          end
+
+          SiteSetting.ai_spam_detection_enabled = is_enabled
+        end
+
+        render json: AiSpamSerializer.new(spam_config, root: false)
+      end
+
+      def test
+        url = params[:post_url].to_s
+        post = nil
+
+        if url.match?(/^\d+$/)
+          post_id = url.to_i
+          post = Post.find_by(id: post_id)
+        end
+
+        route = UrlHelper.rails_route_from_url(url) if !post
+
+        if route
+          if route[:controller] == "topics"
+            post_number = route[:post_number] || 1
+            post = Post.with_deleted.find_by(post_number: post_number, topic_id: route[:topic_id])
+          end
+        end
+
+        raise Discourse::NotFound if !post
+
+        result =
+          DiscourseAi::AiModeration::SpamScanner.test_post(
+            post,
+            custom_instructions: params[:custom_instructions],
+            llm_id: params[:llm_id],
+          )
+
+        render json: result
+      end
+
+      private
+
+      def allowed_params
+        params.permit(:is_enabled, :llm_model_id, :custom_instructions)
+      end
+
+      def spam_config
+        spam_config = {
+          enabled: SiteSetting.ai_spam_detection_enabled,
+          settings: AiModerationSetting.spam,
+        }
+
+        spam_config[:stats] = DiscourseAi::AiModeration::SpamReport.generate(min_date: 1.week.ago)
+
+        if spam_config[:stats].scanned_count > 0
+          spam_config[
+            :flagging_username
+          ] = DiscourseAi::AiModeration::SpamScanner.flagging_user&.username
+        end
+        spam_config
+      end
+    end
+  end
+end
@@ -5,14 +5,17 @@ module DiscourseAi
    class AiToolsController < ::Admin::AdminController
      requires_plugin ::DiscourseAi::PLUGIN_NAME

-      before_action :find_ai_tool, only: %i[test show update destroy]
+      before_action :find_ai_tool, only: %i[test edit update destroy]

      def index
        ai_tools = AiTool.all
        render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
      end

-      def show
+      def new
+      end
+
+      def edit
        render_serialized(@ai_tool, AiCustomToolSerializer)
      end

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Admin
+    class AiUsageController < ::Admin::AdminController
+      requires_plugin "discourse-ai"
+
+      def show
+      end
+
+      def report
+        render json: AiUsageSerializer.new(create_report, root: false)
+      end
+
+      private
+
+      def create_report
+        report =
+          DiscourseAi::Completions::Report.new(
+            start_date: params[:start_date]&.to_date || 30.days.ago,
+            end_date: params[:end_date]&.to_date || Time.current,
+          )
+
+        report = report.filter_by_feature(params[:feature]) if params[:feature].present?
+        report = report.filter_by_model(params[:model]) if params[:model].present?
+        report
+      end
+    end
+  end
+end
@@ -19,22 +19,33 @@ module DiscourseAi
          raise Discourse::NotFound if !guardian.can_see?(post)
        end

+        name = artifact.name
+
+        if params[:version].present?
+          artifact = artifact.versions.find_by(version_number: params[:version])
+          raise Discourse::NotFound if !artifact
+        end
+
+        js = artifact.js || ""
+        if !js.match?(%r{\A\s*<script.*</script>}mi)
+          mod = ""
+          mod = " type=\"module\"" if js.match?(/\A\s*import.*/)
+          js = "<script#{mod}>\n#{js}\n</script>"
+        end
        # Prepare the inner (untrusted) HTML document
        untrusted_html = <<~HTML
          <!DOCTYPE html>
          <html>
            <head>
              <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(artifact.name)}</title>
+              <title>#{ERB::Util.html_escape(name)}</title>
              <style>
                #{artifact.css}
              </style>
            </head>
            <body>
              #{artifact.html}
-              <script>
-                #{artifact.js}
-              </script>
+              #{js}
            </body>
          </html>
        HTML
@@ -45,7 +56,7 @@ module DiscourseAi
          <html>
            <head>
              <meta charset="UTF-8">
-              <title>#{ERB::Util.html_escape(artifact.name)}</title>
+              <title>#{ERB::Util.html_escape(name)}</title>
              <style>
                html, body, iframe {
                  margin: 0;
@@ -67,7 +78,9 @@ module DiscourseAi
        HTML

        response.headers.delete("X-Frame-Options")
-        response.headers["Content-Security-Policy"] = "script-src 'unsafe-inline';"
+        response.headers[
+          "Content-Security-Policy"
+        ] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://unpkg.com https://cdnjs.cloudflare.com https://ajax.googleapis.com https://cdn.jsdelivr.net;"
        response.headers["X-Robots-Tag"] = "noindex"

        # Render the content
@@ -35,7 +35,7 @@ module DiscourseAi
          prompt.custom_instruction = params[:custom_prompt]
        end

-        suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
+        return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST

        hijack do
          render json:
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Jobs
+  class AiSpamScan < ::Jobs::Base
+    def execute(args)
+      return if !args[:post_id]
+      post = Post.find_by(id: args[:post_id])
+      return if !post
+
+      DiscourseAi::AiModeration::SpamScanner.perform_scan(post)
+    end
+  end
+end
@@ -18,9 +18,7 @@ module ::Jobs
      target = target_type.constantize.find_by(id: target_id)
      return if !target

-      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+      vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation

      tokenizer = vector_rep.tokenizer
      chunk_tokens = target.rag_chunk_tokens
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class EvaluatePostUploads < ::Jobs::Base
-    def execute(args)
-      return unless SiteSetting.ai_nsfw_detection_enabled
-      return if (post_id = args[:post_id]).blank?
-
-      post = Post.includes(:uploads).find_by_id(post_id)
-      return if post.nil? || post.uploads.empty?
-
-      return if post.uploads.none? { |u| FileHelper.is_supported_image?(u.url) }
-
-      DiscourseAi::PostClassificator.new(DiscourseAi::Nsfw::Classification.new).classify!(post)
-    end
-  end
-end
@@ -1,23 +1,20 @@
 # frozen_string_literal: true

 module ::Jobs
-  class UpdateHotTopicGist < ::Jobs::Base
+  class FastTrackTopicGist < ::Jobs::Base
    sidekiq_options retry: false

    def execute(args)
      return if !SiteSetting.discourse_ai_enabled
      return if !SiteSetting.ai_summarization_enabled
-      return if SiteSetting.ai_summarize_max_hot_topics_gists_per_batch.zero?
+      return if !SiteSetting.ai_summary_gists_enabled

      topic = Topic.find_by(id: args[:topic_id])
      return if topic.blank?

-      return if !TopicHotScore.where(topic: topic).exists?
-
      summarizer = DiscourseAi::Summarization.topic_gist(topic)
      gist = summarizer.existing_summary
-      return if gist.blank?
-      return if !gist.outdated
+      return if gist.present? && (!gist.outdated || gist.created_at >= 5.minutes.ago)

      summarizer.force_summarize(Discourse.system_user)
    end
@@ -16,11 +16,7 @@ module Jobs
      return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
      return if post.raw.blank?

-      strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
-
-      vector_rep.generate_representation_from(target)
+      DiscourseAi::Embeddings::Vector.instance.generate_representation_from(target)
    end
  end
 end
@@ -8,13 +8,11 @@ module ::Jobs
    def execute(args)
      return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?

-      truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
-      vector_rep =
-        DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
+      vector = DiscourseAi::Embeddings::Vector.instance

      # generate_representation_from checks compares the digest value to make sure
      # the embedding is only generated once per fragment unless something changes.
-      fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
+      fragments.map { |fragment| vector.generate_representation_from(fragment) }

      last_fragment = fragments.last
      target = last_fragment.target
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module ::Jobs
-  class HotTopicsGistBatch < ::Jobs::Base
-    def execute(_args)
-      return if !SiteSetting.discourse_ai_enabled
-      return if !SiteSetting.ai_summarization_enabled
-      return if SiteSetting.ai_summarize_max_hot_topics_gists_per_batch.zero?
-
-      Topic
-        .joins("JOIN topic_hot_scores on topics.id = topic_hot_scores.topic_id")
-        .order("topic_hot_scores.score DESC")
-        .limit(SiteSetting.ai_summarize_max_hot_topics_gists_per_batch)
-        .each do |topic|
-          summarizer = DiscourseAi::Summarization.topic_gist(topic)
-          gist = summarizer.existing_summary
-
-          summarizer.force_summarize(Discourse.system_user) if gist.blank? || gist.outdated
-        end
-    end
-  end
-end
@@ -9,9 +9,7 @@ module ::Jobs
      post = Post.find_by(id: post_id, post_type: Post.types[:regular])
      return if post&.raw.blank?

-      DiscourseAi::PostClassificator.new(
-        DiscourseAi::Sentiment::SentimentClassification.new,
-      ).classify!(post)
+      DiscourseAi::Sentiment::PostClassification.new.classify!(post)
    end
  end
 end
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module ::Jobs
-  class ToxicityClassifyChatMessage < ::Jobs::Base
-    def execute(args)
-      return unless SiteSetting.ai_toxicity_enabled
-
-      return if (chat_message_id = args[:chat_message_id]).blank?
-
-      chat_message = ::Chat::Message.find_by(id: chat_message_id)
-      return if chat_message&.message.blank?
-
-      DiscourseAi::ChatMessageClassificator.new(
-        DiscourseAi::Toxicity::ToxicityClassification.new,
-      ).classify!(chat_message)
-    end
-  end
-end
--- a/Show More
+++ b/Show More