FIX: Use COPY to avoid migration statement timeout

2025-01-14 12:23:49 -03:00
441 changed files with 4299 additions and 857070 deletions
@@ -1,5 +1,3 @@
-< 3.5.0.beta1-dev: bdef136080074a993e7c4f5ca562edc31a8ba756
-< 3.4.0.beta4-dev: a53719ab8eb071459f215227421b3ea4987e5f87
 < 3.4.0.beta4-dev: 20612fde52d3f740cad64823ef8aadb0748b567f
 < 3.4.0.beta3-dev: decf1bb49d737ea15308400f22f89d1d1e71d13d
 < 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
@@ -2,6 +2,3 @@ node_modules
 /gems
 /auto_generated
 .env
-evals/log
-evals/cases
-config/eval-llms.local.yml
@@ -2,6 +2,3 @@ inherit_gem:
  rubocop-discourse: stree-compat.yml
 RSpec/NamedSubject:
  Enabled: false
-
-Style/GlobalVars:
-  AllowedVariables: [$prometheus_client]
@@ -29,7 +29,7 @@ GEM
      racc
    prettier_print (1.2.1)
    racc (1.8.0)
-    rack (3.1.10)
+    rack (3.1.6)
    rainbow (3.1.1)
    regexp_parser (2.9.2)
    rexml (3.3.9)
@@ -1,6 +1,6 @@
 The MIT License (MIT)

-Copyright (c) Civilized Discourse Construction Kit, Inc.
+Copyright (c) 2021 Civilized Discourse Construction Kit

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -3,27 +3,3 @@
 **Plugin Summary**

 For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco
-
-### Evals
-
-The directory `evals` contains AI evals for the Discourse AI plugin. 
-You may create a local config by copying `config/eval-llms.yml` to `config/eval-llms.local.yml` and modifying the values.
-
-To run them use: 
-
-cd evals
-./run --help
-
-```
-Usage: evals/run [options]
-    -e, --eval NAME                  Name of the evaluation to run
-        --list-models                List models
-    -m, --model NAME                 Model to evaluate (will eval all models if not specified)
-    -l, --list                       List evals
-```
-
-To run evals you will need to configure API keys in your environment:
-
-OPENAI_API_KEY=your_openai_api_key
-ANTHROPIC_API_KEY=your_anthropic_api_key
-GEMINI_API_KEY=your_gemini_api_key
@@ -1,7 +0,0 @@
-{
-  "tests": {
-    "requiredPlugins": [
-      "https://github.com/discourse/discourse-prometheus"
-    ]
-  }
-}
@@ -1,21 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class AdminPluginsShowDiscourseAiEmbeddingsEdit extends DiscourseRoute {
-  async model(params) {
-    const allEmbeddings = this.modelFor(
-      "adminPlugins.show.discourse-ai-embeddings"
-    );
-    const id = parseInt(params.id, 10);
-    const record = allEmbeddings.findBy("id", id);
-    record.provider_params = record.provider_params || {};
-    return record;
-  }
-
-  setupController(controller, model) {
-    super.setupController(controller, model);
-    controller.set(
-      "allEmbeddings",
-      this.modelFor("adminPlugins.show.discourse-ai-embeddings")
-    );
-  }
-}
@@ -1,17 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class AdminPluginsShowDiscourseAiEmbeddingsNew extends DiscourseRoute {
-  async model() {
-    const record = this.store.createRecord("ai-embedding");
-    record.provider_params = {};
-    return record;
-  }
-
-  setupController(controller, model) {
-    super.setupController(controller, model);
-    controller.set(
-      "allEmbeddings",
-      this.modelFor("adminPlugins.show.discourse-ai-embeddings")
-    );
-  }
-}
@@ -1,7 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiAiEmbeddingsRoute extends DiscourseRoute {
-  model() {
-    return this.store.findAll("ai-embedding");
-  }
-}
@@ -14,7 +14,5 @@ export default class DiscourseAiToolsEditRoute extends DiscourseRoute {

    controller.set("allTools", toolsModel);
    controller.set("presets", toolsModel.resultSetMeta.presets);
-    controller.set("llms", toolsModel.resultSetMeta.llms);
-    controller.set("settings", toolsModel.resultSetMeta.settings);
  }
 }
@@ -11,7 +11,5 @@ export default class DiscourseAiToolsNewRoute extends DiscourseRoute {

    controller.set("allTools", toolsModel);
    controller.set("presets", toolsModel.resultSetMeta.presets);
-    controller.set("llms", toolsModel.resultSetMeta.llms);
-    controller.set("settings", toolsModel.resultSetMeta.settings);
  }
 }
@@ -1,4 +0,0 @@
-<AiEmbeddingsListEditor
-  @embeddings={{this.allEmbeddings}}
-  @currentEmbedding={{this.model}}
-/>
@@ -1 +0,0 @@
-<AiEmbeddingsListEditor @embeddings={{this.model}} />
@@ -1,4 +0,0 @@
-<AiEmbeddingsListEditor
-  @embeddings={{this.allEmbeddings}}
-  @currentEmbedding={{this.model}}
-/>
@@ -3,7 +3,5 @@
    @tools={{this.allTools}}
    @model={{this.model}}
    @presets={{this.presets}}
-    @llms={{this.llms}}
-    @settings={{this.settings}}
  />
 </section>
@@ -3,7 +3,5 @@
    @tools={{this.allTools}}
    @model={{this.model}}
    @presets={{this.presets}}
-    @llms={{this.llms}}
-    @settings={{this.settings}}
  />
 </section>
@@ -1,133 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Admin
-    class AiEmbeddingsController < ::Admin::AdminController
-      requires_plugin ::DiscourseAi::PLUGIN_NAME
-
-      def index
-        embedding_defs = EmbeddingDefinition.all.order(:display_name)
-
-        render json: {
-                 ai_embeddings:
-                   ActiveModel::ArraySerializer.new(
-                     embedding_defs,
-                     each_serializer: AiEmbeddingDefinitionSerializer,
-                     root: false,
-                   ).as_json,
-                 meta: {
-                   provider_params: EmbeddingDefinition.provider_params,
-                   providers: EmbeddingDefinition.provider_names,
-                   distance_functions: EmbeddingDefinition.distance_functions,
-                   tokenizers:
-                     EmbeddingDefinition.tokenizer_names.map { |tn|
-                       { id: tn, name: tn.split("::").last }
-                     },
-                   presets: EmbeddingDefinition.presets,
-                 },
-               }
-      end
-
-      def new
-      end
-
-      def edit
-        embedding_def = EmbeddingDefinition.find(params[:id])
-        render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
-      end
-
-      def create
-        embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
-
-        if embedding_def.save
-          render json: AiEmbeddingDefinitionSerializer.new(embedding_def), status: :created
-        else
-          render_json_error embedding_def
-        end
-      end
-
-      def update
-        embedding_def = EmbeddingDefinition.find(params[:id])
-
-        if embedding_def.seeded?
-          return(
-            render_json_error(I18n.t("discourse_ai.embeddings.cannot_edit_builtin"), status: 403)
-          )
-        end
-
-        if embedding_def.update(ai_embeddings_params.except(:dimensions))
-          render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
-        else
-          render_json_error embedding_def
-        end
-      end
-
-      def destroy
-        embedding_def = EmbeddingDefinition.find(params[:id])
-
-        if embedding_def.seeded?
-          return(
-            render_json_error(I18n.t("discourse_ai.embeddings.cannot_edit_builtin"), status: 403)
-          )
-        end
-
-        if embedding_def.id == SiteSetting.ai_embeddings_selected_model.to_i
-          return render_json_error(I18n.t("discourse_ai.embeddings.delete_failed"), status: 409)
-        end
-
-        if embedding_def.destroy
-          head :no_content
-        else
-          render_json_error embedding_def
-        end
-      end
-
-      def test
-        RateLimiter.new(
-          current_user,
-          "ai_embeddings_test_#{current_user.id}",
-          3,
-          1.minute,
-        ).performed!
-
-        embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
-        DiscourseAi::Embeddings::Vector.new(embedding_def).vector_from("this is a test")
-
-        render json: { success: true }
-      rescue Net::HTTPBadResponse => e
-        render json: { success: false, error: e.message }
-      end
-
-      private
-
-      def ai_embeddings_params
-        permitted =
-          params.require(:ai_embedding).permit(
-            :display_name,
-            :dimensions,
-            :max_sequence_length,
-            :pg_function,
-            :provider,
-            :url,
-            :api_key,
-            :tokenizer_class,
-            :embed_prompt,
-            :search_prompt,
-            :matryoshka_dimensions,
-          )
-
-        extra_field_names = EmbeddingDefinition.provider_params.dig(permitted[:provider]&.to_sym)
-        if extra_field_names.present?
-          received_prov_params =
-            params.dig(:ai_embedding, :provider_params)&.slice(*extra_field_names.keys)
-
-          if received_prov_params.present?
-            permitted[:provider_params] = received_prov_params.permit!
-          end
-        end
-
-        permitted
-      end
-    end
-  end
-end
@@ -23,28 +23,14 @@ module DiscourseAi
          .each do |tool|
            tools << {
              id: "custom-#{tool.id}",
-              name:
-                I18n.t(
-                  "discourse_ai.tools.custom_name",
-                  name: tool.name.capitalize,
-                  tool_name: tool.tool_name,
-                ),
+              name: I18n.t("discourse_ai.tools.custom_name", name: tool.name.capitalize),
            }
          end
        llms =
-          DiscourseAi::Configuration::LlmEnumerator.values_for_serialization(
-            allowed_seeded_llm_ids: SiteSetting.ai_bot_allowed_seeded_models_map,
-          )
-        render json: {
-                 ai_personas: ai_personas,
-                 meta: {
-                   tools: tools,
-                   llms: llms,
-                   settings: {
-                     rag_images_enabled: SiteSetting.ai_rag_images_enabled,
-                   },
-                 },
-               }
+          DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
+            { id: hash[:value], name: hash[:name] }
+          end
+        render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
      end

      def new
@@ -196,7 +182,7 @@ module DiscourseAi
            :priority,
            :top_p,
            :temperature,
-            :default_llm_id,
+            :default_llm,
            :user_id,
            :max_context_posts,
            :vision_enabled,
@@ -204,8 +190,7 @@ module DiscourseAi
            :rag_chunk_tokens,
            :rag_chunk_overlap_tokens,
            :rag_conversation_chunks,
-            :rag_llm_model_id,
-            :question_consolidator_llm_id,
+            :question_consolidator_llm,
            :allow_chat_channel_mentions,
            :allow_chat_direct_messages,
            :allow_topic_mentions,
@@ -84,13 +84,11 @@ module DiscourseAi
          .require(:ai_tool)
          .permit(
            :name,
-            :tool_name,
            :description,
            :script,
            :summary,
            :rag_chunk_tokens,
            :rag_chunk_overlap_tokens,
-            :rag_llm_model_id,
            rag_uploads: [:id],
            parameters: [:name, :type, :description, :required, enum: []],
          )
@@ -20,7 +20,7 @@ module DiscourseAi
      def upload_file
        file = params[:file] || params[:files].first

-        if !DiscourseAi::Embeddings.enabled?
+        if !SiteSetting.ai_embeddings_enabled?
          raise Discourse::InvalidAccess.new("Embeddings not enabled")
        end

@@ -48,8 +48,7 @@ module DiscourseAi

      def validate_extension!(filename)
        extension = File.extname(filename)[1..-1] || ""
-        authorized_extensions = %w[txt md pdf]
-        authorized_extensions.concat(%w[png jpg jpeg]) if SiteSetting.ai_rag_images_enabled
+        authorized_extensions = %w[txt md]
        if !authorized_extensions.include?(extension)
          raise Discourse::InvalidParameters.new(
                  I18n.t(
@@ -12,7 +12,7 @@ module DiscourseAi
        artifact = AiArtifact.find(params[:id])

        post = Post.find_by(id: artifact.post_id)
-        if artifact.public?
+        if artifact.metadata&.dig("public")
          # no guardian needed
        else
          raise Discourse::NotFound if !post&.topic&.private_message?
@@ -81,7 +81,7 @@ module DiscourseAi
        response.headers.delete("X-Frame-Options")
        response.headers[
          "Content-Security-Policy"
-        ] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' #{AiArtifact::ALLOWED_CDN_SOURCES.join(" ")};"
+        ] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://unpkg.com https://cdnjs.cloudflare.com https://ajax.googleapis.com https://cdn.jsdelivr.net;"
        response.headers["X-Robots-Tag"] = "noindex"

        # Render the content
@@ -44,31 +44,6 @@ module DiscourseAi

        render json: { bot_username: bot_user.username_lower }, status: 200
      end
-
-      def discover
-        ai_persona =
-          AiPersona.all_personas.find do |persona|
-            persona.id == SiteSetting.ai_bot_discover_persona.to_i
-          end
-
-        if ai_persona.nil? || !current_user.in_any_groups?(ai_persona.allowed_group_ids.to_a)
-          raise Discourse::InvalidAccess.new
-        end
-
-        if ai_persona.default_llm_id.blank?
-          render_json_error "Discover persona is missing a default LLM model.", status: 503
-          return
-        end
-
-        query = params[:query]
-        raise Discourse::InvalidParameters.new("Missing query to discover") if query.blank?
-
-        RateLimiter.new(current_user, "ai_bot_discover_#{current_user.id}", 3, 1.minute).performed!
-
-        Jobs.enqueue(:stream_discover_reply, user_id: current_user.id, query: query)
-
-        render json: {}, status: 200
-      end
    end
  end
 end
@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Sentiment
-    class SentimentController < ::Admin::StaffController
-      include Constants
-      requires_plugin ::DiscourseAi::PLUGIN_NAME
-
-      DEFAULT_POSTS_LIMIT = 50
-      MAX_POSTS_LIMIT = 100
-
-      def posts
-        group_by = params.required(:group_by)&.to_sym
-        group_value = params.required(:group_value).presence
-        start_date = params[:start_date].presence
-        end_date = params[:end_date]
-        threshold = SENTIMENT_THRESHOLD
-
-        raise Discourse::InvalidParameters if %i[category tag].exclude?(group_by)
-
-        limit = fetch_limit_from_params(default: DEFAULT_POSTS_LIMIT, max: MAX_POSTS_LIMIT)
-        offset = params[:offset].to_i || 0
-
-        case group_by
-        when :category
-          grouping_clause = "c.name"
-          grouping_join = "" # categories already joined
-        when :tag
-          grouping_clause = "tags.name"
-          grouping_join =
-            "INNER JOIN topic_tags tt ON tt.topic_id = p.topic_id INNER JOIN tags ON tags.id = tt.tag_id"
-        end
-
-        posts =
-          DB.query(
-            <<~SQL,
-          SELECT
-            p.id AS post_id,
-            p.topic_id,
-            t.fancy_title AS topic_title,
-            p.cooked as post_cooked,
-            p.user_id,
-            p.post_number,
-            p.created_at AS created_at,
-            u.username,
-            u.name,
-            u.uploaded_avatar_id,
-            c.id AS category_id,
-            c.name AS category_name,
-            c.color AS category_color,
-            c.slug AS category_slug,
-            c.description AS category_description,
-            (CASE 
-              WHEN (cr.classification::jsonb->'positive')::float > :threshold THEN 'positive'
-              WHEN (cr.classification::jsonb->'negative')::float > :threshold THEN 'negative'
-              ELSE 'neutral'
-            END) AS sentiment
-          FROM posts p
-          INNER JOIN topics t ON t.id = p.topic_id
-          INNER JOIN classification_results cr ON cr.target_id = p.id AND cr.target_type = 'Post'
-          LEFT JOIN users u ON u.id = p.user_id
-          LEFT JOIN categories c ON c.id = t.category_id
-          #{grouping_join}
-          WHERE
-            #{grouping_clause} = :group_value AND
-            t.archetype = 'regular' AND
-            p.user_id > 0 AND
-            cr.model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest' AND
-            ((:start_date IS NULL OR p.created_at > :start_date) AND (:end_date IS NULL OR p.created_at < :end_date))
-            AND p.deleted_at IS NULL
-          ORDER BY p.created_at DESC
-          LIMIT :limit OFFSET :offset
-        SQL
-            group_value: group_value,
-            start_date: start_date,
-            end_date: end_date,
-            threshold: threshold,
-            limit: limit + 1,
-            offset: offset,
-          )
-
-        has_more = posts.length > limit
-        posts.pop if has_more
-
-        render_json_dump(
-          posts:
-            serialize_data(
-              posts,
-              AiSentimentPostSerializer,
-              scope: guardian,
-              add_raw: true,
-              add_excerpt: true,
-              add_title: true,
-            ),
-          has_more: has_more,
-          next_offset: has_more ? offset + limit : nil,
-        )
-      end
-    end
-  end
-end
@@ -18,7 +18,7 @@ module ::Jobs
      target = target_type.constantize.find_by(id: target_id)
      return if !target

-      vector_rep = DiscourseAi::Embeddings::Vector.instance
+      vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation

      tokenizer = vector_rep.tokenizer
      chunk_tokens = target.rag_chunk_tokens
@@ -28,7 +28,7 @@ module ::Jobs

      # Check if this is the first time we process this upload.
      if fragment_ids.empty?
-        document = get_uploaded_file(upload: upload, target: target)
+        document = get_uploaded_file(upload)
        return if document.nil?

        RagDocumentFragment.publish_status(upload, { total: 0, indexed: 0, left: 0 })
@@ -163,32 +163,7 @@ module ::Jobs
      [buffer, split_char]
    end

-    def get_uploaded_file(upload:, target:)
-      if %w[png jpg jpeg].include?(upload.extension) && !SiteSetting.ai_rag_images_enabled
-        raise Discourse::InvalidAccess.new(
-                "The setting ai_rag_images_enabled is false, can not index images",
-              )
-      end
-      if upload.extension == "pdf"
-        return(
-          DiscourseAi::Utils::PdfToText.as_fake_file(
-            upload: upload,
-            llm_model: SiteSetting.ai_rag_images_enabled ? target.rag_llm_model : nil,
-            user: Discourse.system_user,
-          )
-        )
-      end
-
-      if %w[png jpg jpeg].include?(upload.extension)
-        return(
-          DiscourseAi::Utils::ImageToText.as_fake_file(
-            uploads: [upload],
-            llm_model: target.rag_llm_model,
-            user: Discourse.system_user,
-          )
-        )
-      end
-
+    def get_uploaded_file(upload)
      store = Discourse.store
      @file ||=
        if store.external?
@@ -16,7 +16,7 @@ module ::Jobs
      gist = summarizer.existing_summary
      return if gist.present? && (!gist.outdated || gist.created_at >= 5.minutes.ago)

-      summarizer.summarize(Discourse.system_user)
+      summarizer.force_summarize(Discourse.system_user)
    end
  end
 end
@@ -5,7 +5,7 @@ module Jobs
    sidekiq_options queue: "low"

    def execute(args)
-      return unless DiscourseAi::Embeddings.enabled?
+      return unless SiteSetting.ai_embeddings_enabled
      return if args[:target_type].blank? || args[:target_id].blank?
      target = args[:target_type].constantize.find_by_id(args[:target_id])
      return if target.nil? || target.deleted_at.present?
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module ::Jobs
-  class ManageEmbeddingDefSearchIndex < ::Jobs::Base
-    def execute(args)
-      embedding_def = EmbeddingDefinition.find_by(id: args[:id])
-      return if embedding_def.nil?
-      return if DiscourseAi::Embeddings::Schema.correctly_indexed?(embedding_def)
-
-      DiscourseAi::Embeddings::Schema.prepare_search_indexes(embedding_def)
-    end
-  end
-end
@@ -2,8 +2,6 @@

 module ::Jobs
  class PostSentimentAnalysis < ::Jobs::Base
-    sidekiq_options queue: "low"
-
    def execute(args)
      return unless SiteSetting.ai_sentiment_enabled
      return if (post_id = args[:post_id]).blank?
@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Jobs
-  class StreamDiscoverReply < ::Jobs::Base
-    sidekiq_options retry: false
-
-    def execute(args)
-      return if (user = User.find_by(id: args[:user_id])).nil?
-      return if (query = args[:query]).blank?
-
-      ai_persona_klass =
-        AiPersona.all_personas.find do |persona|
-          persona.id == SiteSetting.ai_bot_discover_persona.to_i
-        end
-
-      if ai_persona_klass.nil? || !user.in_any_groups?(ai_persona_klass.allowed_group_ids.to_a)
-        return
-      end
-      return if (llm_model = LlmModel.find_by(id: ai_persona_klass.default_llm_id)).nil?
-
-      bot =
-        DiscourseAi::AiBot::Bot.as(
-          Discourse.system_user,
-          persona: ai_persona_klass.new,
-          model: llm_model,
-        )
-
-      streamed_reply = +""
-      start = Time.now
-
-      base = { query: query, model_used: llm_model.display_name }
-
-      bot.reply(
-        { conversation_context: [{ type: :user, content: query }], skip_tool_details: true },
-      ) do |partial|
-        streamed_reply << partial
-
-        # Throttle updates.
-        if (Time.now - start > 0.3) || Rails.env.test?
-          payload = base.merge(done: false, ai_discover_reply: streamed_reply)
-          publish_update(user, payload)
-          start = Time.now
-        end
-      end
-
-      publish_update(user, base.merge(done: true, ai_discover_reply: streamed_reply))
-    end
-
-    def publish_update(user, payload)
-      MessageBus.publish("/discourse-ai/ai-bot/discover", payload, user_ids: [user.id])
-    end
-  end
-end
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`<AiEmbeddingsListEditor @embeddings={{this.model}} />`