work in progress adding roles to personas

# Conflicts: # app/models/ai_persona.rb
minor fix to read ... complete is just confusing
2024-05-01 15:39:13 +10:00 · 2024-04-30 17:08:23 +10:00 · 2024-04-30 17:08:23 +10:00 · 2024-04-30 17:08:21 +10:00 · 2024-04-30 17:07:22 +10:00
256 changed files with 97448 additions and 416076 deletions
@@ -1,4 +1,3 @@
-< 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
 < 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
 < 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
 < 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
@@ -31,8 +31,7 @@ GEM
    racc (1.7.3)
    rainbow (3.1.1)
    regexp_parser (2.9.0)
-    rexml (3.2.8)
-      strscan (>= 3.0.9)
+    rexml (3.2.6)
    rubocop (1.61.0)
      json (~> 2.3)
      language_server-protocol (>= 3.17.0)
@@ -61,7 +60,6 @@ GEM
      rubocop-capybara (~> 2.17)
      rubocop-factory_bot (~> 2.22)
    ruby-progressbar (1.13.0)
-    strscan (3.1.0)
    syntax_tree (6.2.0)
      prettier_print (>= 1.2.0)
    tzinfo (2.0.6)
@@ -0,0 +1,7 @@
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default DiscourseRoute.extend({
+  model() {
+    return this.modelFor("adminPlugins.show.discourse-ai.ai-personas");
+  },
+});
@@ -17,7 +17,7 @@ export default DiscourseRoute.extend({
    this._super(controller, model);
    controller.set(
      "allPersonas",
-      this.modelFor("adminPlugins.show.discourse-ai-personas")
+      this.modelFor("adminPlugins.show.discourse-ai.ai-personas")
    );
  },
 });
@@ -3,7 +3,7 @@ import DiscourseRoute from "discourse/routes/discourse";
 export default DiscourseRoute.extend({
  async model(params) {
    const allPersonas = this.modelFor(
-      "adminPlugins.show.discourse-ai-personas"
+      "adminPlugins.show.discourse-ai.ai-personas"
    );
    const id = parseInt(params.id, 10);
    return allPersonas.findBy("id", id);
@@ -13,7 +13,7 @@ export default DiscourseRoute.extend({
    this._super(controller, model);
    controller.set(
      "allPersonas",
-      this.modelFor("adminPlugins.show.discourse-ai-personas")
+      this.modelFor("adminPlugins.show.discourse-ai.ai-personas")
    );
  },
 });
@@ -0,0 +1,12 @@
+import { inject as service } from "@ember/service";
+import DiscourseRoute from "discourse/routes/discourse";
+
+export default class DiscourseAiIndexRoute extends DiscourseRoute {
+  @service router;
+
+  beforeModel() {
+    return this.router.transitionTo(
+      "adminPlugins.show.discourse-ai.ai-personas.index"
+    );
+  }
+}
@@ -1,16 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default DiscourseRoute.extend({
-  async model() {
-    const record = this.store.createRecord("ai-llm");
-    return record;
-  },
-
-  setupController(controller, model) {
-    this._super(controller, model);
-    controller.set(
-      "allLlms",
-      this.modelFor("adminPlugins.show.discourse-ai-llms")
-    );
-  },
-});
@@ -1,17 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default DiscourseRoute.extend({
-  async model(params) {
-    const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
-    const id = parseInt(params.id, 10);
-    return allLlms.findBy("id", id);
-  },
-
-  setupController(controller, model) {
-    this._super(controller, model);
-    controller.set(
-      "allLlms",
-      this.modelFor("adminPlugins.show.discourse-ai-llms")
-    );
-  },
-});
@@ -1,7 +0,0 @@
-import DiscourseRoute from "discourse/routes/discourse";
-
-export default class DiscourseAiAiLlmsRoute extends DiscourseRoute {
-  model() {
-    return this.store.findAll("ai-llm");
-  }
-}
@@ -1 +0,0 @@
-<AiLlmsListEditor @llms={{this.model}} />
@@ -1 +0,0 @@
-<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
-<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1,10 +0,0 @@
-<DBreadcrumbsItem as |linkClass|>
-  <LinkTo @route="adminPlugins.show.discourse-ai-personas" class={{linkClass}}>
-    {{@model.plugin.nameTitleized}}
-  </LinkTo>
-</DBreadcrumbsItem>
-
-<AiPersonaListEditor
-  @personas={{this.allPersonas}}
-  @currentPersona={{this.model}}
-/>
@@ -0,0 +1,4 @@
+<AiPersonaListEditor
+  @personas={{this.allPersonas}}
+  @currentPersona={{this.model}}
+/>
@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-
-module DiscourseAi
-  module Admin
-    class AiLlmsController < ::Admin::AdminController
-      requires_plugin ::DiscourseAi::PLUGIN_NAME
-
-      def index
-        llms = LlmModel.all
-
-        render json: {
-                 ai_llms:
-                   ActiveModel::ArraySerializer.new(
-                     llms,
-                     each_serializer: LlmModelSerializer,
-                     root: false,
-                   ).as_json,
-                 meta: {
-                   providers: DiscourseAi::Completions::Llm.provider_names,
-                   tokenizers:
-                     DiscourseAi::Completions::Llm.tokenizer_names.map { |tn|
-                       { id: tn, name: tn.split("::").last }
-                     },
-                 },
-               }
-      end
-
-      def show
-        llm_model = LlmModel.find(params[:id])
-        render json: LlmModelSerializer.new(llm_model)
-      end
-
-      def create
-        llm_model = LlmModel.new(ai_llm_params)
-        if llm_model.save
-          render json: { ai_persona: llm_model }, status: :created
-        else
-          render_json_error llm_model
-        end
-      end
-
-      def update
-        llm_model = LlmModel.find(params[:id])
-
-        if llm_model.update(ai_llm_params)
-          render json: llm_model
-        else
-          render_json_error llm_model
-        end
-      end
-
-      def test
-        RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!
-
-        llm_model = LlmModel.new(ai_llm_params)
-
-        DiscourseAi::Completions::Llm.proxy_from_obj(llm_model).generate(
-          "How much is 1 + 1?",
-          user: current_user,
-          feature_name: "llm_validator",
-        )
-
-        render json: { success: true }
-      rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
-        render json: { success: false, error: e.message }
-      end
-
-      private
-
-      def ai_llm_params
-        params.require(:ai_llm).permit(
-          :display_name,
-          :name,
-          :provider,
-          :tokenizer,
-          :max_prompt_tokens,
-          :url,
-          :api_key,
-        )
-      end
-    end
-  end
-end
@@ -125,7 +125,7 @@ module DiscourseAi
            :rag_chunk_overlap_tokens,
            :rag_conversation_chunks,
            :question_consolidator_llm,
-            :allow_chat,
+            :role,
            allowed_group_ids: [],
            rag_uploads: [:id],
          )
@@ -19,8 +19,7 @@ module DiscourseAi
        if shared_conversation.persisted?
          render json: success_json.merge(share_key: shared_conversation.share_key)
        else
-          render json:
-                   failed_json.merge(error: I18n.t("discourse_ai.share_ai.errors.failed_to_share")),
+          render json: failed_json.merge(error: I18n.t("discourse_ai.share_ai.failed_to_share")),
                 status: :unprocessable_entity
        end
      end
@@ -31,9 +30,7 @@ module DiscourseAi
        SharedAiConversation.destroy_conversation(@shared_conversation)

        render json:
-                 success_json.merge(
-                   message: I18n.t("discourse_ai.share_ai.errors.conversation_deleted"),
-                 )
+                 success_json.merge(message: I18n.t("discourse_ai.share_ai.conversation_deleted"))
      end

      def show
@@ -107,50 +107,20 @@ module DiscourseAi
                          status: 502
      end

-      def random_caption
-        captions = [
-          "A beautiful landscape",
-          "An adorable puppy",
-          "A delicious meal",
-          "A cozy fireplace",
-          "A stunning sunset",
-          "A charming cityscape",
-          "A peaceful garden",
-          "A majestic mountain range",
-          "A captivating work of art",
-        ]
-        captions.sample
-      end
-
      def caption_image
        image_url = params[:image_url]
-        image_url_type = params[:image_url_type]
-
        raise Discourse::InvalidParameters.new(:image_url) if !image_url
-        raise Discourse::InvalidParameters.new(:image_url) if !image_url_type
-
-        if image_url_type == "short_path"
-          image = Upload.find_by(sha1: Upload.sha1_from_short_path(image_url))
-        elsif image_url_type == "short_url"
-          image = Upload.find_by(sha1: Upload.sha1_from_short_url(image_url))
-        else
-          image = upload_from_full_url(image_url)
-        end

+        image = upload_from_full_url(image_url)
        raise Discourse::NotFound if image.blank?
        final_image_url = get_caption_url(image, image_url)

        hijack do
-          if Rails.env.development?
-            sleep 2 # Simulate a delay of 2 seconds
-            caption = random_caption
-          else
-            caption =
-              DiscourseAi::AiHelper::Assistant.new.generate_image_caption(
-                final_image_url,
-                current_user,
-              )
-          end
+          caption =
+            DiscourseAi::AiHelper::Assistant.new.generate_image_caption(
+              final_image_url,
+              current_user,
+            )
          render json: {
                   caption:
                     "#{caption} (#{I18n.t("discourse_ai.ai_helper.image_caption.attribution")})",
@@ -20,7 +20,6 @@ module DiscourseAi
            term: query,
            search_context: guardian,
            use_pg_headlines_for_excerpt: false,
-            can_lazy_load_categories: guardian.can_lazy_load_categories?,
          )

        semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module ::Jobs
-  class CreateAiChatReply < ::Jobs::Base
-    sidekiq_options retry: false
-
-    def execute(args)
-      channel = ::Chat::Channel.find_by(id: args[:channel_id])
-      return if channel.blank?
-
-      message = ::Chat::Message.find_by(id: args[:message_id])
-      return if message.blank?
-
-      personaClass =
-        DiscourseAi::AiBot::Personas::Persona.find_by(id: args[:persona_id], user: message.user)
-      return if personaClass.blank?
-
-      user = User.find_by(id: personaClass.user_id)
-      bot = DiscourseAi::AiBot::Bot.as(user, persona: personaClass.new)
-
-      DiscourseAi::AiBot::Playground.new(bot).reply_to_chat_message(
-        message,
-        channel,
-        args[:context_post_ids],
-      )
-    end
-  end
-end
@@ -10,7 +10,13 @@ module ::Jobs
      persona_id = args[:persona_id]

      begin
-        persona = DiscourseAi::AiBot::Personas::Persona.find_by(user: post.user, id: persona_id)
+        persona =
+          if args[:skip_persona_security_check]
+            persona = AiPersona.all_personas.find { |persona| persona.id == persona_id }
+          else
+            persona = DiscourseAi::AiBot::Personas::Persona.find_by(user: post.user, id: persona_id)
+          end
+
        raise DiscourseAi::AiBot::Bot::BOT_NOT_FOUND if persona.nil?

        bot = DiscourseAi::AiBot::Bot.as(bot_user, persona: persona.new)
@@ -11,7 +11,6 @@ class AiApiAuditLog < ActiveRecord::Base
    Gemini = 4
    Vllm = 5
    Cohere = 6
-    Ollama = 7
  end
 end

@@ -22,14 +21,12 @@ end
 #  id                   :bigint           not null, primary key
 #  provider_id          :integer          not null
 #  user_id              :integer
+#  topic_id             :integer
+#  post_id              :integer
 #  request_tokens       :integer
 #  response_tokens      :integer
 #  raw_request_payload  :string
 #  raw_response_payload :string
 #  created_at           :datetime         not null
 #  updated_at           :datetime         not null
-#  topic_id             :integer
-#  post_id              :integer
-#  feature_name         :string(255)
-#  language_model       :string(255)
 #
@@ -8,7 +8,6 @@ class AiPersona < ActiveRecord::Base
  validates :description, presence: true, length: { maximum: 2000 }
  validates :system_prompt, presence: true, length: { maximum: 10_000_000 }
  validate :system_persona_unchangeable, on: :update, if: :system
-  validate :chat_preconditions
  validates :max_context_posts, numericality: { greater_than: 0 }, allow_nil: true
  # leaves some room for growth but sets a maximum to avoid memory issues
  # we may want to revisit this in the future
@@ -47,15 +46,19 @@ class AiPersona < ActiveRecord::Base
      .map(&:class_instance)
  end

+  def self.persona_class_by_id(id)
+    AiPersona.all_personas.find { |persona| persona.id == id } if id
+  end
+
  def self.persona_users(user: nil)
    persona_users =
      persona_cache[:persona_users] ||= AiPersona
        .where(enabled: true)
        .joins(:user)
        .pluck(
-          "ai_personas.id, users.id, users.username_lower, allowed_group_ids, default_llm, mentionable, allow_chat",
+          "ai_personas.id, users.id, users.username_lower, allowed_group_ids, default_llm, mentionable",
        )
-        .map do |id, user_id, username, allowed_group_ids, default_llm, mentionable, allow_chat|
+        .map do |id, user_id, username, allowed_group_ids, default_llm, mentionable|
          {
            id: id,
            user_id: user_id,
@@ -63,7 +66,6 @@ class AiPersona < ActiveRecord::Base
            allowed_group_ids: allowed_group_ids,
            default_llm: default_llm,
            mentionable: mentionable,
-            allow_chat: allow_chat,
          }
        end

@@ -74,20 +76,59 @@ class AiPersona < ActiveRecord::Base
    end
  end

-  def self.allowed_chat(user: nil)
-    personas = persona_cache[:allowed_chat] ||= persona_users.select { |u| u[:allow_chat] }
-    if user
-      personas.select { |u| user.in_any_groups?(u[:allowed_group_ids]) }
+  def self.topic_responder_for(category_id:)
+    return nil if !category_id
+
+    all_responders =
+      persona_cache[:topic_responders] ||= AiPersona
+        .where(role: "topic_responder")
+        .where(enabled: true)
+        .pluck(:id, :role_category_ids)
+
+    id, _ = all_responders.find { |id, role_category_ids| role_category_ids.include?(category_id) }
+
+    if id
+      { id: id }
    else
-      personas
+      nil
+    end
+  end
+
+  def self.message_responder_for(group_id: nil)
+    return nil if !group_id
+
+    all_responders =
+      persona_cache[:message_responders] ||= AiPersona
+        .where(role: "message_responder")
+        .where(enabled: true)
+        .pluck(:id, :role_group_ids)
+
+    id, _ = all_responders.find { |id, role_group_ids| role_group_ids.include?(group_id) }
+
+    if id
+      { id: id }
+    else
+      nil
    end
  end

  def self.mentionables(user: nil)
    all_mentionables =
-      persona_cache[:mentionables] ||= persona_users.select do |mentionable|
-        mentionable[:mentionable]
-      end
+      persona_cache[:mentionable_usernames] ||= AiPersona
+        .where(mentionable: true)
+        .where(enabled: true)
+        .joins(:user)
+        .pluck("ai_personas.id, users.id, users.username_lower, allowed_group_ids, default_llm")
+        .map do |id, user_id, username, allowed_group_ids, default_llm|
+          {
+            id: id,
+            user_id: user_id,
+            username: username,
+            allowed_group_ids: allowed_group_ids,
+            default_llm: default_llm,
+          }
+        end
+
    if user
      all_mentionables.select { |mentionable| user.in_any_groups?(mentionable[:allowed_group_ids]) }
    else
@@ -113,7 +154,8 @@ class AiPersona < ActiveRecord::Base
    vision_max_pixels = self.vision_max_pixels
    rag_conversation_chunks = self.rag_conversation_chunks
    question_consolidator_llm = self.question_consolidator_llm
-    allow_chat = self.allow_chat
+    role = self.role
+    role_whispers = self.role_whispers

    persona_class = DiscourseAi::AiBot::Personas::Persona.system_personas_by_id[self.id]
    if persona_class
@@ -133,10 +175,6 @@ class AiPersona < ActiveRecord::Base
        user_id
      end

-      persona_class.define_singleton_method :allow_chat do
-        allow_chat
-      end
-
      persona_class.define_singleton_method :mentionable do
        mentionable
      end
@@ -165,6 +203,14 @@ class AiPersona < ActiveRecord::Base
        rag_conversation_chunks
      end

+      persona_class.define_singleton_method :role_whispers do
+        role_whispers
+      end
+
+      persona_class.define_singleton_method :role do
+        role
+      end
+
      return persona_class
    end

@@ -256,8 +302,12 @@ class AiPersona < ActiveRecord::Base
        question_consolidator_llm
      end

-      define_singleton_method :allow_chat do
-        allow_chat
+      define_singleton_method :role do
+        role
+      end
+
+      define_singleton_method :role_whispers do
+        role_whispers
      end

      define_singleton_method :to_s do
@@ -350,15 +400,9 @@ class AiPersona < ActiveRecord::Base

  private

-  def chat_preconditions
-    if allow_chat && !default_llm
-      errors.add(:default_llm, I18n.t("discourse_ai.ai_bot.personas.default_llm_required"))
-    end
-  end
-
  def system_persona_unchangeable
-    if top_p_changed? || temperature_changed? || system_prompt_changed? || commands_changed? ||
-         name_changed? || description_changed?
+    if role_changed? || top_p_changed? || temperature_changed? || system_prompt_changed? ||
+         commands_changed? || name_changed? || description_changed?
      errors.add(:base, I18n.t("discourse_ai.ai_bot.personas.cannot_edit_system_persona"))
    end
  end
@@ -400,14 +444,13 @@ end
 #  rag_chunk_tokens            :integer          default(374), not null
 #  rag_chunk_overlap_tokens    :integer          default(10), not null
 #  rag_conversation_chunks     :integer          default(10), not null
+#  question_consolidator_llm   :text
 #  role                        :enum             default("bot"), not null
 #  role_category_ids           :integer          default([]), not null, is an Array
 #  role_tags                   :string           default([]), not null, is an Array
 #  role_group_ids              :integer          default([]), not null, is an Array
 #  role_whispers               :boolean          default(FALSE), not null
 #  role_max_responses_per_hour :integer          default(50), not null
-#  question_consolidator_llm   :text
-#  allow_chat                  :boolean          default(FALSE), not null
 #
 # Indexes
 #
@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-class ChatMessageCustomPrompt < ActiveRecord::Base
-  # belongs_to chat message but going to avoid the cross dependency for now
-end
-
-# == Schema Information
-#
-# Table name: message_custom_prompts
-#
-#  id            :bigint           not null, primary key
-#  message_id    :bigint           not null
-#  custom_prompt :json             not null
-#  created_at    :datetime         not null
-#  updated_at    :datetime         not null
-#
-# Indexes
-#
-#  index_message_custom_prompts_on_message_id  (message_id) UNIQUE
-#
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-class LlmModel < ActiveRecord::Base
-  def tokenizer_class
-    tokenizer.constantize
-  end
-end
-
-# == Schema Information
-#
-# Table name: llm_models
-#
-#  id                :bigint           not null, primary key
-#  display_name      :string
-#  name              :string           not null
-#  provider          :string           not null
-#  tokenizer         :string           not null
-#  max_prompt_tokens :integer          not null
-#  created_at        :datetime         not null
-#  updated_at        :datetime         not null
-#  url               :string
-#  api_key           :string
-#
@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-class LlmModelSerializer < ApplicationSerializer
-  root "llm"
-
-  attributes :id, :display_name, :name, :provider, :max_prompt_tokens, :tokenizer, :api_key, :url
-end
@@ -24,7 +24,11 @@ class LocalizedAiPersonaSerializer < ApplicationSerializer
             :rag_chunk_overlap_tokens,
             :rag_conversation_chunks,
             :question_consolidator_llm,
-             :allow_chat
+             :role,
+             :role_tags,
+             :role_category_ids,
+             :role_whispers,
+             :role_max_responses_per_hour

  has_one :user, serializer: BasicUserSerializer, embed: :object
  has_many :rag_uploads, serializer: UploadSerializer, embed: :object
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Sam Saffron	6351c401f8	work in progress adding roles to personas # Conflicts: # app/models/ai_persona.rb	2024-05-01 15:39:13 +10:00
Sam Saffron	acd3ac2371	minor fix to read ... complete is just confusing	2024-04-30 17:08:23 +10:00
Sam Saffron	c61af76d86	Responder for categories	2024-04-30 17:08:23 +10:00
Sam Saffron	cd7790f39a	message responder backend now works	2024-04-30 17:08:21 +10:00
Sam Saffron	4fcf12287f	work in progress adding roles to personas	2024-04-30 17:07:22 +10:00
				`@@ -1 +0,0 @@`
				`<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />`