Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Jordan Vidrine
|
47fdb1acbf |
@@ -1,6 +1,3 @@
|
||||
< 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
|
||||
< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
|
||||
< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
|
||||
< 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
|
||||
3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
|
||||
< 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
app/assets/stylesheets/vendor/
|
||||
documentation/
|
||||
package.json
|
||||
config/locales/**/*.yml
|
||||
!config/locales/**/*.en*.yml
|
||||
script/import_scripts/**/*.yml
|
||||
|
||||
plugins/**/lib/javascripts/locale
|
||||
public/
|
||||
!/app/assets/javascripts/discourse/public
|
||||
vendor/
|
||||
app/assets/javascripts/discourse/tests/fixtures
|
||||
spec/
|
||||
node_modules/
|
||||
dist/
|
||||
tmp/
|
||||
|
||||
**/*.rb
|
||||
**/*.html
|
||||
**/*.json
|
||||
**/*.md
|
||||
+23
-42
@@ -1,77 +1,58 @@
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
activesupport (7.1.3.2)
|
||||
base64
|
||||
bigdecimal
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
connection_pool (>= 2.2.5)
|
||||
drb
|
||||
i18n (>= 1.6, < 2)
|
||||
minitest (>= 5.1)
|
||||
mutex_m
|
||||
tzinfo (~> 2.0)
|
||||
ast (2.4.2)
|
||||
base64 (0.2.0)
|
||||
bigdecimal (3.1.6)
|
||||
concurrent-ruby (1.2.3)
|
||||
connection_pool (2.4.1)
|
||||
drb (2.2.1)
|
||||
i18n (1.14.1)
|
||||
concurrent-ruby (~> 1.0)
|
||||
json (2.7.1)
|
||||
json (2.6.3)
|
||||
language_server-protocol (3.17.0.3)
|
||||
minitest (5.22.2)
|
||||
mutex_m (0.2.0)
|
||||
parallel (1.24.0)
|
||||
parser (3.3.0.5)
|
||||
parallel (1.23.0)
|
||||
parser (3.2.2.4)
|
||||
ast (~> 2.4.1)
|
||||
racc
|
||||
prettier_print (1.2.1)
|
||||
racc (1.7.3)
|
||||
racc (1.7.2)
|
||||
rainbow (3.1.1)
|
||||
regexp_parser (2.9.0)
|
||||
regexp_parser (2.8.2)
|
||||
rexml (3.2.6)
|
||||
rubocop (1.61.0)
|
||||
rubocop (1.57.2)
|
||||
json (~> 2.3)
|
||||
language_server-protocol (>= 3.17.0)
|
||||
parallel (~> 1.10)
|
||||
parser (>= 3.3.0.2)
|
||||
parser (>= 3.2.2.4)
|
||||
rainbow (>= 2.2.2, < 4.0)
|
||||
regexp_parser (>= 1.8, < 3.0)
|
||||
rexml (>= 3.2.5, < 4.0)
|
||||
rubocop-ast (>= 1.30.0, < 2.0)
|
||||
rubocop-ast (>= 1.28.1, < 2.0)
|
||||
ruby-progressbar (~> 1.7)
|
||||
unicode-display_width (>= 2.4.0, < 3.0)
|
||||
rubocop-ast (1.31.1)
|
||||
parser (>= 3.3.0.4)
|
||||
rubocop-capybara (2.20.0)
|
||||
rubocop-ast (1.30.0)
|
||||
parser (>= 3.2.1.0)
|
||||
rubocop-capybara (2.19.0)
|
||||
rubocop (~> 1.41)
|
||||
rubocop-discourse (3.7.1)
|
||||
activesupport (>= 6.1)
|
||||
rubocop (>= 1.59.0)
|
||||
rubocop-capybara (>= 2.0.0)
|
||||
rubocop-factory_bot (>= 2.0.0)
|
||||
rubocop-rspec (>= 2.25.0)
|
||||
rubocop-factory_bot (2.25.1)
|
||||
rubocop (~> 1.41)
|
||||
rubocop-rspec (2.27.1)
|
||||
rubocop-discourse (3.4.0)
|
||||
rubocop (>= 1.1.0)
|
||||
rubocop-rspec (>= 2.0.0)
|
||||
rubocop-factory_bot (2.24.0)
|
||||
rubocop (~> 1.33)
|
||||
rubocop-rspec (2.25.0)
|
||||
rubocop (~> 1.40)
|
||||
rubocop-capybara (~> 2.17)
|
||||
rubocop-factory_bot (~> 2.22)
|
||||
ruby-progressbar (1.13.0)
|
||||
syntax_tree (6.2.0)
|
||||
prettier_print (>= 1.2.0)
|
||||
tzinfo (2.0.6)
|
||||
concurrent-ruby (~> 1.0)
|
||||
unicode-display_width (2.5.0)
|
||||
|
||||
PLATFORMS
|
||||
arm64-darwin-20
|
||||
ruby
|
||||
x86_64-darwin-18
|
||||
x86_64-darwin-19
|
||||
x86_64-darwin-20
|
||||
x86_64-linux
|
||||
|
||||
DEPENDENCIES
|
||||
rubocop-discourse
|
||||
syntax_tree
|
||||
|
||||
BUNDLED WITH
|
||||
2.5.4
|
||||
2.4.21
|
||||
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
import { AUTO_GROUPS } from "discourse/lib/constants";
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default DiscourseRoute.extend({
|
||||
async model() {
|
||||
const record = this.store.createRecord("ai-persona");
|
||||
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
|
||||
record.set("rag_uploads", []);
|
||||
// these match the defaults on the table
|
||||
record.set("rag_chunk_tokens", 374);
|
||||
record.set("rag_chunk_overlap_tokens", 10);
|
||||
record.set("rag_conversation_chunks", 10);
|
||||
return record;
|
||||
},
|
||||
|
||||
setupController(controller, model) {
|
||||
this._super(controller, model);
|
||||
controller.set(
|
||||
"allPersonas",
|
||||
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
||||
);
|
||||
},
|
||||
});
|
||||
-19
@@ -1,19 +0,0 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default DiscourseRoute.extend({
|
||||
async model(params) {
|
||||
const allPersonas = this.modelFor(
|
||||
"adminPlugins.show.discourse-ai-personas"
|
||||
);
|
||||
const id = parseInt(params.id, 10);
|
||||
return allPersonas.findBy("id", id);
|
||||
},
|
||||
|
||||
setupController(controller, model) {
|
||||
this._super(controller, model);
|
||||
controller.set(
|
||||
"allPersonas",
|
||||
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
||||
);
|
||||
},
|
||||
});
|
||||
@@ -1,7 +0,0 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class DiscourseAiAiPersonasRoute extends DiscourseRoute {
|
||||
model() {
|
||||
return this.store.findAll("ai-persona");
|
||||
}
|
||||
}
|
||||
-1
@@ -1 +0,0 @@
|
||||
<AiPersonaListEditor @personas={{this.model}} />
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
<AiPersonaListEditor
|
||||
@personas={{this.allPersonas}}
|
||||
@currentPersona={{this.model}}
|
||||
/>
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
<AiPersonaListEditor
|
||||
@personas={{this.allPersonas}}
|
||||
@currentPersona={{this.model}}
|
||||
/>
|
||||
@@ -1,180 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Admin
|
||||
class AiPersonasController < ::Admin::AdminController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
before_action :find_ai_persona,
|
||||
only: %i[show update destroy create_user indexing_status_check]
|
||||
|
||||
def index
|
||||
ai_personas =
|
||||
AiPersona.ordered.map do |persona|
|
||||
# we use a special serializer here cause names and descriptions are
|
||||
# localized for system personas
|
||||
LocalizedAiPersonaSerializer.new(persona, root: false)
|
||||
end
|
||||
tools =
|
||||
DiscourseAi::AiBot::Personas::Persona.all_available_tools.map do |tool|
|
||||
AiToolSerializer.new(tool, root: false)
|
||||
end
|
||||
llms =
|
||||
DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
|
||||
{ id: hash[:value], name: hash[:name] }
|
||||
end
|
||||
render json: { ai_personas: ai_personas, meta: { commands: tools, llms: llms } }
|
||||
end
|
||||
|
||||
def show
|
||||
render json: LocalizedAiPersonaSerializer.new(@ai_persona)
|
||||
end
|
||||
|
||||
def create
|
||||
ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
|
||||
if ai_persona.save
|
||||
RagDocumentFragment.link_persona_and_uploads(ai_persona, attached_upload_ids)
|
||||
|
||||
render json: { ai_persona: ai_persona }, status: :created
|
||||
else
|
||||
render_json_error ai_persona
|
||||
end
|
||||
end
|
||||
|
||||
def create_user
|
||||
user = @ai_persona.create_user!
|
||||
render json: BasicUserSerializer.new(user, root: "user")
|
||||
end
|
||||
|
||||
def update
|
||||
if @ai_persona.update(ai_persona_params.except(:rag_uploads))
|
||||
RagDocumentFragment.update_persona_uploads(@ai_persona, attached_upload_ids)
|
||||
|
||||
render json: @ai_persona
|
||||
else
|
||||
render_json_error @ai_persona
|
||||
end
|
||||
end
|
||||
|
||||
def destroy
|
||||
if @ai_persona.destroy
|
||||
head :no_content
|
||||
else
|
||||
render_json_error @ai_persona
|
||||
end
|
||||
end
|
||||
|
||||
def upload_file
|
||||
file = params[:file] || params[:files].first
|
||||
|
||||
if !SiteSetting.ai_embeddings_enabled?
|
||||
raise Discourse::InvalidAccess.new("Embeddings not enabled")
|
||||
end
|
||||
|
||||
validate_extension!(file.original_filename)
|
||||
validate_file_size!(file.tempfile.size)
|
||||
|
||||
hijack do
|
||||
upload =
|
||||
UploadCreator.new(
|
||||
file.tempfile,
|
||||
file.original_filename,
|
||||
type: "discourse_ai_rag_upload",
|
||||
skip_validations: true,
|
||||
).create_for(current_user.id)
|
||||
|
||||
if upload.persisted?
|
||||
render json: UploadSerializer.new(upload)
|
||||
else
|
||||
render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def indexing_status_check
|
||||
render json: RagDocumentFragment.indexing_status(@ai_persona, @ai_persona.uploads)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def find_ai_persona
|
||||
@ai_persona = AiPersona.find(params[:id])
|
||||
end
|
||||
|
||||
def attached_upload_ids
|
||||
ai_persona_params[:rag_uploads].to_a.map { |h| h[:id] }
|
||||
end
|
||||
|
||||
def ai_persona_params
|
||||
permitted =
|
||||
params.require(:ai_persona).permit(
|
||||
:name,
|
||||
:description,
|
||||
:enabled,
|
||||
:system_prompt,
|
||||
:priority,
|
||||
:top_p,
|
||||
:temperature,
|
||||
:default_llm,
|
||||
:user_id,
|
||||
:mentionable,
|
||||
:max_context_posts,
|
||||
:vision_enabled,
|
||||
:vision_max_pixels,
|
||||
:rag_chunk_tokens,
|
||||
:rag_chunk_overlap_tokens,
|
||||
:rag_conversation_chunks,
|
||||
:question_consolidator_llm,
|
||||
allowed_group_ids: [],
|
||||
rag_uploads: [:id],
|
||||
)
|
||||
|
||||
if commands = params.dig(:ai_persona, :commands)
|
||||
permitted[:commands] = permit_commands(commands)
|
||||
end
|
||||
|
||||
permitted
|
||||
end
|
||||
|
||||
def permit_commands(commands)
|
||||
return [] if !commands.is_a?(Array)
|
||||
|
||||
commands.filter_map do |command, options|
|
||||
break nil if !command.is_a?(String)
|
||||
options&.permit! if options && options.is_a?(ActionController::Parameters)
|
||||
|
||||
if options
|
||||
[command, options]
|
||||
else
|
||||
command
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def validate_extension!(filename)
|
||||
extension = File.extname(filename)[1..-1] || ""
|
||||
authorized_extensions = %w[txt md]
|
||||
if !authorized_extensions.include?(extension)
|
||||
raise Discourse::InvalidParameters.new(
|
||||
I18n.t(
|
||||
"upload.unauthorized",
|
||||
authorized_extensions: authorized_extensions.join(" "),
|
||||
),
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def validate_file_size!(filesize)
|
||||
max_size_bytes = 20.megabytes
|
||||
if filesize > max_size_bytes
|
||||
raise Discourse::InvalidParameters.new(
|
||||
I18n.t(
|
||||
"upload.attachments.too_large_humanized",
|
||||
max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
|
||||
),
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,12 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Admin
|
||||
class DashboardController < ::Admin::StaffController
|
||||
requires_plugin DiscourseAi::PLUGIN_NAME
|
||||
|
||||
def sentiment
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -6,21 +6,6 @@ module DiscourseAi
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
requires_login
|
||||
|
||||
def show_debug_info
|
||||
post = Post.find(params[:post_id])
|
||||
guardian.ensure_can_debug_ai_bot_conversation!(post)
|
||||
|
||||
posts =
|
||||
Post
|
||||
.where("post_number <= ?", post.post_number)
|
||||
.where(topic_id: post.topic_id)
|
||||
.order("post_number DESC")
|
||||
|
||||
debug_info = AiApiAuditLog.where(post: posts).order(created_at: :desc).first
|
||||
|
||||
render json: debug_info, status: 200
|
||||
end
|
||||
|
||||
def stop_streaming_response
|
||||
post = Post.find(params[:post_id])
|
||||
guardian.ensure_can_see!(post)
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module AiBot
|
||||
class SharedAiConversationsController < ::ApplicationController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
requires_login only: %i[create update destroy]
|
||||
before_action :require_site_settings!
|
||||
|
||||
skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required, only: %i[show]
|
||||
|
||||
def create
|
||||
ensure_allowed_create!
|
||||
|
||||
RateLimiter.new(current_user, "share-ai-conversation", 10, 1.minute).performed!
|
||||
|
||||
shared_conversation = SharedAiConversation.share_conversation(current_user, @topic)
|
||||
|
||||
if shared_conversation.persisted?
|
||||
render json: success_json.merge(share_key: shared_conversation.share_key)
|
||||
else
|
||||
render json: failed_json.merge(error: I18n.t("discourse_ai.share_ai.failed_to_share")),
|
||||
status: :unprocessable_entity
|
||||
end
|
||||
end
|
||||
|
||||
def destroy
|
||||
ensure_allowed_destroy!
|
||||
|
||||
SharedAiConversation.destroy_conversation(@shared_conversation)
|
||||
|
||||
render json:
|
||||
success_json.merge(message: I18n.t("discourse_ai.share_ai.conversation_deleted"))
|
||||
end
|
||||
|
||||
def show
|
||||
@shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
|
||||
raise Discourse::NotFound if @shared_conversation.blank?
|
||||
|
||||
expires_in 1.minute, public: true
|
||||
response.headers["X-Robots-Tag"] = "noindex"
|
||||
|
||||
if request.format.json?
|
||||
render json: success_json.merge(@shared_conversation.to_json)
|
||||
else
|
||||
render "show", layout: false
|
||||
end
|
||||
end
|
||||
|
||||
def preview
|
||||
ensure_allowed_preview!
|
||||
data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
|
||||
data[:error] = @error if @error
|
||||
data[:share_key] = @shared_conversation.share_key if @shared_conversation
|
||||
data[:topic_id] = @topic.id
|
||||
render json: data
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def require_site_settings!
|
||||
if !SiteSetting.discourse_ai_enabled ||
|
||||
!SiteSetting.ai_bot_public_sharing_allowed_groups_map.any? ||
|
||||
!SiteSetting.ai_bot_enabled
|
||||
raise Discourse::NotFound
|
||||
end
|
||||
end
|
||||
|
||||
def ensure_allowed_preview!
|
||||
@topic = Topic.find_by(id: params[:topic_id])
|
||||
raise Discourse::NotFound if !@topic
|
||||
|
||||
@shared_conversation = SharedAiConversation.find_by(target: @topic)
|
||||
|
||||
@error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
|
||||
if @error == :not_allowed
|
||||
raise Discourse::InvalidAccess.new(
|
||||
nil,
|
||||
nil,
|
||||
custom_message: "discourse_ai.share_ai.errors.#{@error}",
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def ensure_allowed_destroy!
|
||||
@shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
|
||||
|
||||
raise Discourse::InvalidAccess if @shared_conversation.blank?
|
||||
|
||||
guardian.ensure_can_destroy_shared_ai_bot_conversation!(@shared_conversation)
|
||||
end
|
||||
|
||||
def ensure_allowed_create!
|
||||
@topic = Topic.find_by(id: params[:topic_id])
|
||||
raise Discourse::NotFound if !@topic
|
||||
|
||||
error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
|
||||
if error
|
||||
raise Discourse::InvalidAccess.new(
|
||||
nil,
|
||||
nil,
|
||||
custom_message: "discourse_ai.share_ai.errors.#{error}",
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -8,7 +8,14 @@ module DiscourseAi
|
||||
before_action :ensure_can_request_suggestions
|
||||
before_action :rate_limiter_performed!, except: %i[prompts]
|
||||
|
||||
include SecureUploadEndpointHelpers
|
||||
def prompts
|
||||
render json:
|
||||
ActiveModel::ArraySerializer.new(
|
||||
DiscourseAi::AiHelper::LlmPrompt.new.available_prompts,
|
||||
root: false,
|
||||
),
|
||||
status: 200
|
||||
end
|
||||
|
||||
def suggest
|
||||
input = get_text_param!
|
||||
@@ -16,45 +23,45 @@ module DiscourseAi
|
||||
prompt = CompletionPrompt.find_by(id: params[:mode])
|
||||
|
||||
raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
|
||||
|
||||
if prompt.id == CompletionPrompt::CUSTOM_PROMPT
|
||||
raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
|
||||
|
||||
prompt.custom_instruction = params[:custom_prompt]
|
||||
if prompt.prompt_type == "custom_prompt" && params[:custom_prompt].blank?
|
||||
raise Discourse::InvalidParameters.new(:custom_prompt)
|
||||
end
|
||||
|
||||
suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
|
||||
|
||||
hijack do
|
||||
render json:
|
||||
DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
|
||||
prompt,
|
||||
input,
|
||||
current_user,
|
||||
),
|
||||
DiscourseAi::AiHelper::LlmPrompt.new.generate_and_send_prompt(prompt, params),
|
||||
status: 200
|
||||
end
|
||||
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
|
||||
rescue ::DiscourseAi::Inference::OpenAiCompletions::CompletionFailed,
|
||||
::DiscourseAi::Inference::HuggingFaceTextGeneration::CompletionFailed,
|
||||
::DiscourseAi::Inference::AnthropicCompletions::CompletionFailed => e
|
||||
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
|
||||
status: 502
|
||||
end
|
||||
|
||||
def suggest_title
|
||||
input = get_text_param!
|
||||
input_hash = { text: input }
|
||||
|
||||
prompt = CompletionPrompt.enabled_by_name("generate_titles")
|
||||
raise Discourse::InvalidParameters.new(:mode) if !prompt
|
||||
llm_prompt =
|
||||
DiscourseAi::AiHelper::LlmPrompt
|
||||
.new
|
||||
.available_prompts(name_filter: "generate_titles")
|
||||
.first
|
||||
prompt = CompletionPrompt.find_by(id: llm_prompt[:id])
|
||||
raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
|
||||
|
||||
hijack do
|
||||
render json:
|
||||
DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
|
||||
DiscourseAi::AiHelper::LlmPrompt.new.generate_and_send_prompt(
|
||||
prompt,
|
||||
input,
|
||||
current_user,
|
||||
input_hash,
|
||||
),
|
||||
status: 200
|
||||
end
|
||||
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
|
||||
rescue ::DiscourseAi::Inference::OpenAiCompletions::CompletionFailed,
|
||||
::DiscourseAi::Inference::HuggingFaceTextGeneration::CompletionFailed,
|
||||
::DiscourseAi::Inference::AnthropicCompletions::CompletionFailed => e
|
||||
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
|
||||
status: 502
|
||||
end
|
||||
@@ -79,7 +86,9 @@ module DiscourseAi
|
||||
status: 200
|
||||
end
|
||||
|
||||
def suggest_thumbnails(input)
|
||||
def suggest_thumbnails
|
||||
input = get_text_param!
|
||||
|
||||
hijack do
|
||||
thumbnails = DiscourseAi::AiHelper::Painter.new.commission_thumbnails(input, current_user)
|
||||
|
||||
@@ -89,45 +98,21 @@ module DiscourseAi
|
||||
|
||||
def explain
|
||||
post_id = get_post_param!
|
||||
term_to_explain = get_text_param!
|
||||
post = Post.includes(:topic).find_by(id: post_id)
|
||||
text = get_text_param!
|
||||
post = Post.find_by(id: post_id)
|
||||
|
||||
raise Discourse::InvalidParameters.new(:post_id) unless post
|
||||
|
||||
Jobs.enqueue(
|
||||
:stream_post_helper,
|
||||
post_id: post.id,
|
||||
user_id: current_user.id,
|
||||
term_to_explain: term_to_explain,
|
||||
)
|
||||
|
||||
render json: { success: true }, status: 200
|
||||
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
|
||||
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
|
||||
status: 502
|
||||
end
|
||||
|
||||
def caption_image
|
||||
image_url = params[:image_url]
|
||||
raise Discourse::InvalidParameters.new(:image_url) if !image_url
|
||||
|
||||
image = upload_from_full_url(image_url)
|
||||
raise Discourse::NotFound if image.blank?
|
||||
final_image_url = get_caption_url(image, image_url)
|
||||
|
||||
hijack do
|
||||
caption =
|
||||
DiscourseAi::AiHelper::Assistant.new.generate_image_caption(
|
||||
final_image_url,
|
||||
current_user,
|
||||
)
|
||||
render json: {
|
||||
caption:
|
||||
"#{caption} (#{I18n.t("discourse_ai.ai_helper.image_caption.attribution")})",
|
||||
},
|
||||
status: 200
|
||||
end
|
||||
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed, Net::HTTPBadResponse
|
||||
render json:
|
||||
DiscourseAi::AiHelper::TopicHelper.new(
|
||||
{ text: text },
|
||||
current_user,
|
||||
post: post,
|
||||
).explain,
|
||||
status: 200
|
||||
rescue ::DiscourseAi::Inference::OpenAiCompletions::CompletionFailed,
|
||||
::DiscourseAi::Inference::HuggingFaceTextGeneration::CompletionFailed,
|
||||
::DiscourseAi::Inference::AnthropicCompletions::CompletionFailed => e
|
||||
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
|
||||
status: 502
|
||||
end
|
||||
@@ -147,18 +132,14 @@ module DiscourseAi
|
||||
end
|
||||
|
||||
def ensure_can_request_suggestions
|
||||
if !current_user.in_any_groups?(SiteSetting.ai_helper_allowed_groups_map)
|
||||
raise Discourse::InvalidAccess
|
||||
end
|
||||
end
|
||||
user_group_ids = current_user.group_ids
|
||||
|
||||
def get_caption_url(image_upload, image_url)
|
||||
if image_upload.secure?
|
||||
check_secure_upload_permission(image_upload)
|
||||
return Discourse.store.url_for(image_upload)
|
||||
end
|
||||
allowed =
|
||||
SiteSetting.ai_helper_allowed_groups_map.any? do |group_id|
|
||||
user_group_ids.include?(group_id)
|
||||
end
|
||||
|
||||
UrlHelper.absolute(image_url)
|
||||
raise Discourse::InvalidAccess if !allowed
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -36,34 +36,6 @@ module DiscourseAi
|
||||
render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
|
||||
end
|
||||
end
|
||||
|
||||
def quick_search
|
||||
query = params[:q].to_s
|
||||
|
||||
if query.length < SiteSetting.min_search_term_length
|
||||
raise Discourse::InvalidParameters.new(:q)
|
||||
end
|
||||
|
||||
grouped_results =
|
||||
Search::GroupedSearchResults.new(
|
||||
type_filter: SEMANTIC_SEARCH_TYPE,
|
||||
term: query,
|
||||
search_context: guardian,
|
||||
use_pg_headlines_for_excerpt: false,
|
||||
)
|
||||
|
||||
semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
|
||||
|
||||
if !semantic_search.cached_query?(query)
|
||||
RateLimiter.new(current_user, "semantic-search", 60, 1.minutes).performed!
|
||||
end
|
||||
|
||||
hijack do
|
||||
semantic_search.quick_search(query).each { |topic_post| grouped_results.add(topic_post) }
|
||||
|
||||
render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
module DiscourseAi
|
||||
module AiBot
|
||||
module SharedAiConversationsHelper
|
||||
# bump up version when assets change
|
||||
# long term we may want to change this cause it is hard to remember
|
||||
# to bump versions, but for now this does the job
|
||||
VERSION = "1"
|
||||
|
||||
def share_asset_url(short_path)
|
||||
::UrlHelper.local_cdn_url("/plugins/discourse-ai/ai-share/#{short_path}?#{VERSION}")
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,26 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module ::Jobs
|
||||
class CreateAiReply < ::Jobs::Base
|
||||
sidekiq_options retry: false
|
||||
|
||||
def execute(args)
|
||||
return unless bot_user = User.find_by(id: args[:bot_user_id])
|
||||
return unless post = Post.includes(:topic).find_by(id: args[:post_id])
|
||||
persona_id = args[:persona_id]
|
||||
|
||||
begin
|
||||
persona = DiscourseAi::AiBot::Personas::Persona.find_by(user: post.user, id: persona_id)
|
||||
raise DiscourseAi::AiBot::Bot::BOT_NOT_FOUND if persona.nil?
|
||||
|
||||
bot = DiscourseAi::AiBot::Bot.as(bot_user, persona: persona.new)
|
||||
|
||||
DiscourseAi::AiBot::Playground.new(bot).reply_to(post)
|
||||
rescue DiscourseAi::AiBot::Bot::BOT_NOT_FOUND
|
||||
Rails.logger.warn(
|
||||
"Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,172 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module ::Jobs
|
||||
class DigestRagUpload < ::Jobs::Base
|
||||
CHUNK_SIZE = 1024
|
||||
CHUNK_OVERLAP = 64
|
||||
MAX_FRAGMENTS = 100_000
|
||||
|
||||
# TODO(roman): Add a way to automatically recover from errors, resulting in unindexed uploads.
|
||||
def execute(args)
|
||||
return if (upload = Upload.find_by(id: args[:upload_id])).nil?
|
||||
return if (ai_persona = AiPersona.find_by(id: args[:ai_persona_id])).nil?
|
||||
|
||||
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
|
||||
|
||||
tokenizer = vector_rep.tokenizer
|
||||
chunk_tokens = ai_persona.rag_chunk_tokens
|
||||
overlap_tokens = ai_persona.rag_chunk_overlap_tokens
|
||||
|
||||
fragment_ids = RagDocumentFragment.where(ai_persona: ai_persona, upload: upload).pluck(:id)
|
||||
|
||||
# Check if this is the first time we process this upload.
|
||||
if fragment_ids.empty?
|
||||
document = get_uploaded_file(upload)
|
||||
return if document.nil?
|
||||
|
||||
RagDocumentFragment.publish_status(upload, { total: 0, indexed: 0, left: 0 })
|
||||
|
||||
fragment_ids = []
|
||||
idx = 0
|
||||
|
||||
ActiveRecord::Base.transaction do
|
||||
chunk_document(
|
||||
file: document,
|
||||
tokenizer: tokenizer,
|
||||
chunk_tokens: chunk_tokens,
|
||||
overlap_tokens: overlap_tokens,
|
||||
) do |chunk, metadata|
|
||||
fragment_ids << RagDocumentFragment.create!(
|
||||
ai_persona: ai_persona,
|
||||
fragment: chunk,
|
||||
fragment_number: idx + 1,
|
||||
upload: upload,
|
||||
metadata: metadata,
|
||||
).id
|
||||
|
||||
idx += 1
|
||||
|
||||
if idx > MAX_FRAGMENTS
|
||||
Rails.logger.warn("Upload #{upload.id} has too many fragments, truncating.")
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
fragment_ids.each_slice(50) do |slice|
|
||||
Jobs.enqueue(:generate_rag_embeddings, fragment_ids: slice)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def chunk_document(file:, tokenizer:, chunk_tokens:, overlap_tokens:)
|
||||
buffer = +""
|
||||
current_metadata = nil
|
||||
done = false
|
||||
overlap = ""
|
||||
|
||||
# generally this will be plenty
|
||||
read_size = chunk_tokens * 10
|
||||
|
||||
while buffer.present? || !done
|
||||
if buffer.length < read_size
|
||||
read = file.read(read_size)
|
||||
done = true if read.nil?
|
||||
|
||||
read = Encodings.to_utf8(read) if read
|
||||
|
||||
buffer << (read || "")
|
||||
end
|
||||
|
||||
# at this point we unconditionally have 2x CHUNK_SIZE worth of data in the buffer
|
||||
metadata_regex = /\[\[metadata (.*?)\]\]/m
|
||||
|
||||
before_metadata, new_metadata, after_metadata = buffer.split(metadata_regex)
|
||||
to_chunk = nil
|
||||
|
||||
if before_metadata.present?
|
||||
to_chunk = before_metadata
|
||||
elsif after_metadata.present?
|
||||
current_metadata = new_metadata
|
||||
to_chunk = after_metadata
|
||||
buffer = buffer.split(metadata_regex, 2).last
|
||||
overlap = ""
|
||||
else
|
||||
current_metadata = new_metadata
|
||||
buffer = buffer.split(metadata_regex, 2).last
|
||||
overlap = ""
|
||||
next
|
||||
end
|
||||
|
||||
chunk, split_char = first_chunk(to_chunk, tokenizer: tokenizer, chunk_tokens: chunk_tokens)
|
||||
buffer = buffer[chunk.length..-1]
|
||||
|
||||
processed_chunk = overlap + chunk
|
||||
|
||||
processed_chunk.strip!
|
||||
processed_chunk.gsub!(/\n[\n]+/, "\n\n")
|
||||
|
||||
yield processed_chunk, current_metadata
|
||||
|
||||
current_chunk_tokens = tokenizer.encode(chunk)
|
||||
overlap_token_ids = current_chunk_tokens[-overlap_tokens..-1] || current_chunk_tokens
|
||||
|
||||
overlap = ""
|
||||
|
||||
while overlap_token_ids.present?
|
||||
begin
|
||||
overlap = tokenizer.decode(overlap_token_ids) + split_char
|
||||
break if overlap.encoding == Encoding::UTF_8
|
||||
rescue StandardError
|
||||
# it is possible that we truncated mid char
|
||||
end
|
||||
overlap_token_ids.shift
|
||||
end
|
||||
|
||||
# remove first word it is probably truncated
|
||||
overlap = overlap.split(" ", 2).last
|
||||
end
|
||||
end
|
||||
|
||||
def first_chunk(text, chunk_tokens:, tokenizer:, splitters: ["\n\n", "\n", ".", ""])
|
||||
return text, " " if tokenizer.tokenize(text).length <= chunk_tokens
|
||||
|
||||
splitters = splitters.find_all { |s| text.include?(s) }.compact
|
||||
|
||||
buffer = +""
|
||||
split_char = nil
|
||||
|
||||
splitters.each do |splitter|
|
||||
split_char = splitter
|
||||
|
||||
text
|
||||
.split(split_char)
|
||||
.each do |part|
|
||||
break if tokenizer.tokenize(buffer + split_char + part).length > chunk_tokens
|
||||
buffer << split_char
|
||||
buffer << part
|
||||
end
|
||||
break if buffer.length > 0
|
||||
end
|
||||
|
||||
[buffer, split_char]
|
||||
end
|
||||
|
||||
def get_uploaded_file(upload)
|
||||
store = Discourse.store
|
||||
@file ||=
|
||||
if store.external?
|
||||
# Upload#filesize could be approximate.
|
||||
# add two extra Mbs to make sure that we'll be able to download the upload.
|
||||
max_filesize = upload.filesize + 2.megabytes
|
||||
store.download(upload, max_file_size_kb: max_filesize)
|
||||
else
|
||||
File.open(store.path_for(upload))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,26 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Jobs
|
||||
class GenerateEmbeddings < ::Jobs::Base
|
||||
sidekiq_options queue: "low"
|
||||
|
||||
def execute(args)
|
||||
return unless SiteSetting.ai_embeddings_enabled
|
||||
return if args[:target_type].blank? || args[:target_id].blank?
|
||||
target = args[:target_type].constantize.find_by_id(args[:target_id])
|
||||
return if target.nil? || target.deleted_at.present?
|
||||
|
||||
topic = target.is_a?(Topic) ? target : target.topic
|
||||
post = target.is_a?(Post) ? target : target.first_post
|
||||
return unless topic.present? && post.present?
|
||||
return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
|
||||
return if post.raw.blank?
|
||||
|
||||
strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
|
||||
|
||||
vector_rep.generate_representation_from(target)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,27 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module ::Jobs
|
||||
class GenerateRagEmbeddings < ::Jobs::Base
|
||||
sidekiq_options queue: "ultra_low"
|
||||
# we could also restrict concurrency but this takes so long if it is not concurrent
|
||||
|
||||
def execute(args)
|
||||
return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
|
||||
|
||||
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
|
||||
|
||||
# generate_representation_from checks compares the digest value to make sure
|
||||
# the embedding is only generated once per fragment unless something changes.
|
||||
fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
|
||||
|
||||
last_fragment = fragments.last
|
||||
ai_persona = last_fragment.ai_persona
|
||||
upload = last_fragment.upload
|
||||
|
||||
indexing_status = RagDocumentFragment.indexing_status(ai_persona, [upload])[upload.id]
|
||||
RagDocumentFragment.publish_status(upload, indexing_status)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,51 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module ::Jobs
|
||||
class SharedConversationAdjustUploadSecurity < ::Jobs::Base
|
||||
def execute(args)
|
||||
if args[:conversation_id].present?
|
||||
# The conversation context includes post cooked content so this
|
||||
# must be updated when target uploads security changes.
|
||||
update_conversation(args[:conversation_id])
|
||||
elsif args[:target_id].present? && args[:target_type].present?
|
||||
# If we deleted the conversation then we just need to update the target's
|
||||
# uploads security, no need to update the conversation.
|
||||
update_target(args[:target_id], args[:target_type])
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def update_conversation(conversation_id)
|
||||
conversation = SharedAiConversation.find_by(id: conversation_id)
|
||||
return if conversation.blank?
|
||||
|
||||
# NOTE: Only Topics are supported for now, in future we will need a more flexible
|
||||
# way of doing this.
|
||||
if conversation.target_type == "Topic"
|
||||
rebaked_posts = TopicUploadSecurityManager.new(conversation.target).run
|
||||
|
||||
if rebaked_posts.any?
|
||||
new_context =
|
||||
conversation.context.map do |context_post|
|
||||
rebaked_post = rebaked_posts.find { |p| p.id == context_post["id"] }
|
||||
context_post["cooked"] = rebaked_post.cooked if rebaked_post
|
||||
context_post
|
||||
end
|
||||
|
||||
conversation.update(context: new_context)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def update_target(target_id, target_type)
|
||||
# NOTE: Only Topics are supported for now, in future we will need a more flexible
|
||||
# way of doing this.
|
||||
if target_type == "Topic"
|
||||
topic = target_type.constantize.find_by(id: target_id)
|
||||
return if topic.blank?
|
||||
TopicUploadSecurityManager.new(topic).run
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,34 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Jobs
|
||||
class StreamPostHelper < ::Jobs::Base
|
||||
sidekiq_options retry: false
|
||||
|
||||
def execute(args)
|
||||
return unless post = Post.includes(:topic).find_by(id: args[:post_id])
|
||||
return unless user = User.find_by(id: args[:user_id])
|
||||
return unless args[:term_to_explain]
|
||||
|
||||
topic = post.topic
|
||||
reply_to = post.reply_to_post
|
||||
|
||||
return unless user.guardian.can_see?(post)
|
||||
|
||||
prompt = CompletionPrompt.enabled_by_name("explain")
|
||||
|
||||
input = <<~TEXT
|
||||
<term>#{args[:term_to_explain]}</term>
|
||||
<context>#{post.raw}</context>
|
||||
<topic>#{topic.title}</topic>
|
||||
#{reply_to ? "<replyTo>#{reply_to.raw}</replyTo>" : nil}
|
||||
TEXT
|
||||
|
||||
DiscourseAi::AiHelper::Assistant.new.stream_prompt(
|
||||
prompt,
|
||||
input,
|
||||
user,
|
||||
"/discourse-ai/ai-helper/explain/#{post.id}",
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,132 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Jobs
|
||||
class EmbeddingsBackfill < ::Jobs::Scheduled
|
||||
every 5.minutes
|
||||
sidekiq_options queue: "low"
|
||||
cluster_concurrency 1
|
||||
|
||||
def execute(args)
|
||||
return unless SiteSetting.ai_embeddings_enabled
|
||||
|
||||
limit = SiteSetting.ai_embeddings_backfill_batch_size
|
||||
|
||||
if limit > 50_000
|
||||
limit = 50_000
|
||||
Rails.logger.warn(
|
||||
"Limiting backfill batch size to 50,000 to avoid OOM errors, reduce ai_embeddings_backfill_batch_size to avoid this warning",
|
||||
)
|
||||
end
|
||||
|
||||
rebaked = 0
|
||||
|
||||
strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
|
||||
table_name = vector_rep.topic_table_name
|
||||
|
||||
topics =
|
||||
Topic
|
||||
.joins("LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id")
|
||||
.where(archetype: Archetype.default)
|
||||
.where(deleted_at: nil)
|
||||
.order("topics.bumped_at DESC")
|
||||
.limit(limit - rebaked)
|
||||
|
||||
rebaked += populate_topic_embeddings(vector_rep, topics)
|
||||
|
||||
vector_rep.consider_indexing
|
||||
|
||||
return if rebaked >= limit
|
||||
|
||||
# Then, we'll try to backfill embeddings for topics that have outdated
|
||||
# embeddings, be it model or strategy version
|
||||
relation = topics.where(<<~SQL)
|
||||
#{table_name}.model_version < #{vector_rep.version}
|
||||
OR
|
||||
#{table_name}.strategy_version < #{strategy.version}
|
||||
SQL
|
||||
|
||||
rebaked += populate_topic_embeddings(vector_rep, relation)
|
||||
|
||||
return if rebaked >= limit
|
||||
|
||||
# Finally, we'll try to backfill embeddings for topics that have outdated
|
||||
# embeddings due to edits or new replies. Here we only do 10% of the limit
|
||||
relation =
|
||||
topics.where("#{table_name}.updated_at < ?", 7.days.ago).limit((limit - rebaked) / 10)
|
||||
|
||||
populate_topic_embeddings(vector_rep, relation)
|
||||
|
||||
return if rebaked >= limit
|
||||
|
||||
return unless SiteSetting.ai_embeddings_per_post_enabled
|
||||
|
||||
# Now for posts
|
||||
table_name = vector_rep.post_table_name
|
||||
|
||||
posts =
|
||||
Post
|
||||
.joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
|
||||
.where(deleted_at: nil)
|
||||
.limit(limit - rebaked)
|
||||
|
||||
# First, we'll try to backfill embeddings for posts that have none
|
||||
posts
|
||||
.where("#{table_name}.post_id IS NULL")
|
||||
.find_each do |t|
|
||||
vector_rep.generate_representation_from(t)
|
||||
rebaked += 1
|
||||
end
|
||||
|
||||
vector_rep.consider_indexing
|
||||
|
||||
return if rebaked >= limit
|
||||
|
||||
# Then, we'll try to backfill embeddings for posts that have outdated
|
||||
# embeddings, be it model or strategy version
|
||||
posts
|
||||
.where(<<~SQL)
|
||||
#{table_name}.model_version < #{vector_rep.version}
|
||||
OR
|
||||
#{table_name}.strategy_version < #{strategy.version}
|
||||
SQL
|
||||
.find_each do |t|
|
||||
vector_rep.generate_representation_from(t)
|
||||
rebaked += 1
|
||||
end
|
||||
|
||||
return if rebaked >= limit
|
||||
|
||||
# Finally, we'll try to backfill embeddings for posts that have outdated
|
||||
# embeddings due to edits. Here we only do 10% of the limit
|
||||
posts
|
||||
.where("#{table_name}.updated_at < ?", 7.days.ago)
|
||||
.order("random()")
|
||||
.limit((limit - rebaked) / 10)
|
||||
.pluck(:id)
|
||||
.each do |id|
|
||||
vector_rep.generate_representation_from(Post.find_by(id: id))
|
||||
rebaked += 1
|
||||
end
|
||||
|
||||
rebaked
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def populate_topic_embeddings(vector_rep, topics)
|
||||
done = 0
|
||||
ids = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL").pluck("topics.id")
|
||||
|
||||
ids.each do |id|
|
||||
topic = Topic.find_by(id: id)
|
||||
if topic
|
||||
vector_rep.generate_representation_from(topic)
|
||||
done += 1
|
||||
end
|
||||
end
|
||||
done
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,9 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class AiReportMailer < ActionMailer::Base
|
||||
include Email::BuildEmailHelper
|
||||
|
||||
def send_report(to_address, opts = {})
|
||||
build_email(to_address, **opts)
|
||||
end
|
||||
end
|
||||
@@ -1,16 +1,10 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class AiApiAuditLog < ActiveRecord::Base
|
||||
belongs_to :post
|
||||
belongs_to :topic
|
||||
|
||||
module Provider
|
||||
OpenAI = 1
|
||||
Anthropic = 2
|
||||
HuggingFaceTextGeneration = 3
|
||||
Gemini = 4
|
||||
Vllm = 5
|
||||
Cohere = 6
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,60 +1,40 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class CompletionPrompt < ActiveRecord::Base
|
||||
# TODO(roman): Remove may 2024.
|
||||
self.ignored_columns = ["provider"]
|
||||
|
||||
TRANSLATE = -301
|
||||
GENERATE_TITLES = -307
|
||||
PROOFREAD = -303
|
||||
MARKDOWN_TABLE = -304
|
||||
CUSTOM_PROMPT = -305
|
||||
EXPLAIN = -306
|
||||
ILLUSTRATE_POST = -308
|
||||
# TODO(roman): Remove sept 2023.
|
||||
self.ignored_columns = ["value"]
|
||||
|
||||
enum :prompt_type, { text: 0, list: 1, diff: 2 }
|
||||
|
||||
validates :messages, length: { maximum: 20 }
|
||||
validate :each_message_length
|
||||
|
||||
after_commit { DiscourseAi::AiHelper::Assistant.clear_prompt_cache! }
|
||||
def messages_with_user_input(user_input)
|
||||
return messages unless user_input.present?
|
||||
|
||||
def self.enabled_by_name(name)
|
||||
where(enabled: true).find_by(name: name)
|
||||
end
|
||||
|
||||
attr_accessor :custom_instruction
|
||||
|
||||
def messages_with_input(input)
|
||||
return unless input
|
||||
|
||||
user_input =
|
||||
if id == CUSTOM_PROMPT && custom_instruction.present?
|
||||
"#{custom_instruction}:\n#{input}"
|
||||
if user_input[:custom_prompt].present?
|
||||
case ::DiscourseAi::AiHelper::LlmPrompt.new.enabled_provider
|
||||
when "huggingface"
|
||||
self.messages.each { |msg| msg.sub!("{{custom_prompt}}", user_input[:custom_prompt]) }
|
||||
else
|
||||
input
|
||||
self.messages.each do |msg|
|
||||
msg["content"].sub!("{{custom_prompt}}", user_input[:custom_prompt])
|
||||
end
|
||||
end
|
||||
|
||||
instructions = [messages_hash[:insts], messages_hash[:post_insts].to_s].join("\n")
|
||||
|
||||
prompt = DiscourseAi::Completions::Prompt.new(instructions)
|
||||
|
||||
messages_hash[:examples].to_a do |example_pair|
|
||||
prompt.push(type: :user, content: example_pair.first)
|
||||
prompt.push(type: :model, content: example_pair.second)
|
||||
end
|
||||
|
||||
prompt.push(type: :user, content: "<input>#{user_input}</input>")
|
||||
|
||||
prompt
|
||||
case ::DiscourseAi::AiHelper::LlmPrompt.new.enabled_provider
|
||||
when "openai"
|
||||
self.messages << { role: "user", content: user_input[:text] }
|
||||
when "anthropic"
|
||||
self.messages << { "role" => "Input", "content" => "<input>#{user_input[:text]}</input>" }
|
||||
when "huggingface"
|
||||
self.messages.first.sub("{{user_input}}", user_input[:text])
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def messages_hash
|
||||
@messages_hash ||= messages.symbolize_keys!
|
||||
end
|
||||
|
||||
def each_message_length
|
||||
messages.each_with_index do |msg, idx|
|
||||
next if msg["content"].length <= 1000
|
||||
@@ -76,8 +56,7 @@ end
|
||||
# created_at :datetime not null
|
||||
# updated_at :datetime not null
|
||||
# messages :jsonb
|
||||
# temperature :integer
|
||||
# stop_sequences :string is an Array
|
||||
# provider :text
|
||||
#
|
||||
# Indexes
|
||||
#
|
||||
|
||||
@@ -7,18 +7,3 @@ end
|
||||
class ::Post
|
||||
has_one :post_custom_prompt, dependent: :destroy
|
||||
end
|
||||
|
||||
# == Schema Information
|
||||
#
|
||||
# Table name: post_custom_prompts
|
||||
#
|
||||
# id :bigint not null, primary key
|
||||
# post_id :integer not null
|
||||
# custom_prompt :json not null
|
||||
# created_at :datetime not null
|
||||
# updated_at :datetime not null
|
||||
#
|
||||
# Indexes
|
||||
#
|
||||
# index_post_custom_prompts_on_post_id (post_id) UNIQUE
|
||||
#
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class RagDocumentFragment < ActiveRecord::Base
|
||||
belongs_to :upload
|
||||
belongs_to :ai_persona
|
||||
|
||||
class << self
|
||||
def link_persona_and_uploads(persona, upload_ids)
|
||||
return if persona.blank?
|
||||
return if upload_ids.blank?
|
||||
return if !SiteSetting.ai_embeddings_enabled?
|
||||
|
||||
UploadReference.ensure_exist!(upload_ids: upload_ids, target: persona)
|
||||
|
||||
upload_ids.each do |upload_id|
|
||||
Jobs.enqueue(:digest_rag_upload, ai_persona_id: persona.id, upload_id: upload_id)
|
||||
end
|
||||
end
|
||||
|
||||
def update_persona_uploads(persona, upload_ids)
|
||||
return if persona.blank?
|
||||
return if !SiteSetting.ai_embeddings_enabled?
|
||||
|
||||
if upload_ids.blank?
|
||||
RagDocumentFragment.where(ai_persona: persona).destroy_all
|
||||
UploadReference.where(target: persona).destroy_all
|
||||
else
|
||||
RagDocumentFragment.where(ai_persona: persona).where.not(upload_id: upload_ids).destroy_all
|
||||
link_persona_and_uploads(persona, upload_ids)
|
||||
end
|
||||
end
|
||||
|
||||
def indexing_status(persona, uploads)
|
||||
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
|
||||
|
||||
embeddings_table = vector_rep.rag_fragments_table_name
|
||||
|
||||
results = DB.query(<<~SQL, persona_id: persona.id, upload_ids: uploads.map(&:id))
|
||||
SELECT
|
||||
uploads.id,
|
||||
SUM(CASE WHEN (rdf.upload_id IS NOT NULL) THEN 1 ELSE 0 END) AS total,
|
||||
SUM(CASE WHEN (eft.rag_document_fragment_id IS NOT NULL) THEN 1 ELSE 0 END) as indexed,
|
||||
SUM(CASE WHEN (rdf.upload_id IS NOT NULL AND eft.rag_document_fragment_id IS NULL) THEN 1 ELSE 0 END) as left
|
||||
FROM uploads
|
||||
LEFT OUTER JOIN rag_document_fragments rdf ON uploads.id = rdf.upload_id AND rdf.ai_persona_id = :persona_id
|
||||
LEFT OUTER JOIN #{embeddings_table} eft ON rdf.id = eft.rag_document_fragment_id
|
||||
WHERE uploads.id IN (:upload_ids)
|
||||
GROUP BY uploads.id
|
||||
SQL
|
||||
|
||||
results.reduce({}) do |acc, r|
|
||||
acc[r.id] = { total: r.total, indexed: r.indexed, left: r.left }
|
||||
acc
|
||||
end
|
||||
end
|
||||
|
||||
def publish_status(upload, status)
|
||||
MessageBus.publish(
|
||||
"/discourse-ai/ai-persona-rag/#{upload.id}",
|
||||
status,
|
||||
user_ids: [upload.user_id],
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# == Schema Information
|
||||
#
|
||||
# Table name: rag_document_fragments
|
||||
#
|
||||
# id :bigint not null, primary key
|
||||
# fragment :text not null
|
||||
# upload_id :integer not null
|
||||
# ai_persona_id :integer not null
|
||||
# fragment_number :integer not null
|
||||
# created_at :datetime not null
|
||||
# updated_at :datetime not null
|
||||
# metadata :text
|
||||
#
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user