Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Keegan George
|
3ea1e8efd8 | ||
|
Keegan George
|
2b98e84acf |
@@ -1,7 +1,8 @@
|
||||
< 3.4.0.beta3-dev: decf1bb49d737ea15308400f22f89d1d1e71d13d
|
||||
< 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
|
||||
< 3.3.0.beta5-dev: 4d8090002f6dcd8e34d41033606bf131fa221475
|
||||
< 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
|
||||
< 3.3.0.beta1-dev: d94234d89f50733678a5c62809445198a9fda74b
|
||||
3.2.999: d94234d89f50733678a5c62809445198a9fda74b
|
||||
< 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
|
||||
< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
|
||||
< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
|
||||
< 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
|
||||
3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
|
||||
< 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
|
||||
3.1.0.beta3: 057fbe1ce6
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
node_modules
|
||||
/gems
|
||||
/auto_generated
|
||||
.env
|
||||
|
||||
+27
-34
@@ -1,7 +1,7 @@
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
activesupport (7.1.3.4)
|
||||
activesupport (7.1.3.2)
|
||||
base64
|
||||
bigdecimal
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
@@ -13,27 +13,27 @@ GEM
|
||||
tzinfo (~> 2.0)
|
||||
ast (2.4.2)
|
||||
base64 (0.2.0)
|
||||
bigdecimal (3.1.8)
|
||||
concurrent-ruby (1.3.3)
|
||||
bigdecimal (3.1.6)
|
||||
concurrent-ruby (1.2.3)
|
||||
connection_pool (2.4.1)
|
||||
drb (2.2.1)
|
||||
i18n (1.14.5)
|
||||
i18n (1.14.1)
|
||||
concurrent-ruby (~> 1.0)
|
||||
json (2.7.2)
|
||||
json (2.7.1)
|
||||
language_server-protocol (3.17.0.3)
|
||||
minitest (5.24.1)
|
||||
minitest (5.22.2)
|
||||
mutex_m (0.2.0)
|
||||
parallel (1.25.1)
|
||||
parser (3.3.3.0)
|
||||
parallel (1.24.0)
|
||||
parser (3.3.0.5)
|
||||
ast (~> 2.4.1)
|
||||
racc
|
||||
prettier_print (1.2.1)
|
||||
racc (1.8.0)
|
||||
rack (3.1.6)
|
||||
racc (1.7.3)
|
||||
rainbow (3.1.1)
|
||||
regexp_parser (2.9.2)
|
||||
rexml (3.3.9)
|
||||
rubocop (1.64.1)
|
||||
regexp_parser (2.9.0)
|
||||
rexml (3.2.8)
|
||||
strscan (>= 3.0.9)
|
||||
rubocop (1.61.0)
|
||||
json (~> 2.3)
|
||||
language_server-protocol (>= 3.17.0)
|
||||
parallel (~> 1.10)
|
||||
@@ -41,34 +41,27 @@ GEM
|
||||
rainbow (>= 2.2.2, < 4.0)
|
||||
regexp_parser (>= 1.8, < 3.0)
|
||||
rexml (>= 3.2.5, < 4.0)
|
||||
rubocop-ast (>= 1.31.1, < 2.0)
|
||||
rubocop-ast (>= 1.30.0, < 2.0)
|
||||
ruby-progressbar (~> 1.7)
|
||||
unicode-display_width (>= 2.4.0, < 3.0)
|
||||
rubocop-ast (1.31.3)
|
||||
parser (>= 3.3.1.0)
|
||||
rubocop-capybara (2.21.0)
|
||||
rubocop-ast (1.31.1)
|
||||
parser (>= 3.3.0.4)
|
||||
rubocop-capybara (2.20.0)
|
||||
rubocop (~> 1.41)
|
||||
rubocop-discourse (3.8.1)
|
||||
rubocop-discourse (3.7.1)
|
||||
activesupport (>= 6.1)
|
||||
rubocop (>= 1.59.0)
|
||||
rubocop-capybara (>= 2.0.0)
|
||||
rubocop-factory_bot (>= 2.0.0)
|
||||
rubocop-rails (>= 2.25.0)
|
||||
rubocop-rspec (>= 3.0.1)
|
||||
rubocop-rspec_rails (>= 2.30.0)
|
||||
rubocop-factory_bot (2.26.1)
|
||||
rubocop (~> 1.61)
|
||||
rubocop-rails (2.25.1)
|
||||
activesupport (>= 4.2.0)
|
||||
rack (>= 1.1)
|
||||
rubocop (>= 1.33.0, < 2.0)
|
||||
rubocop-ast (>= 1.31.1, < 2.0)
|
||||
rubocop-rspec (3.0.2)
|
||||
rubocop (~> 1.61)
|
||||
rubocop-rspec_rails (2.30.0)
|
||||
rubocop (~> 1.61)
|
||||
rubocop-rspec (~> 3, >= 3.0.1)
|
||||
rubocop-rspec (>= 2.25.0)
|
||||
rubocop-factory_bot (2.25.1)
|
||||
rubocop (~> 1.41)
|
||||
rubocop-rspec (2.27.1)
|
||||
rubocop (~> 1.40)
|
||||
rubocop-capybara (~> 2.17)
|
||||
rubocop-factory_bot (~> 2.22)
|
||||
ruby-progressbar (1.13.0)
|
||||
strscan (3.1.0)
|
||||
syntax_tree (6.2.0)
|
||||
prettier_print (>= 1.2.0)
|
||||
tzinfo (2.0.6)
|
||||
@@ -83,4 +76,4 @@ DEPENDENCIES
|
||||
syntax_tree
|
||||
|
||||
BUNDLED WITH
|
||||
2.5.10
|
||||
2.5.4
|
||||
|
||||
@@ -1,13 +1,8 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default DiscourseRoute.extend({
|
||||
queryParams: {
|
||||
llmTemplate: { refreshModel: true },
|
||||
},
|
||||
|
||||
async model() {
|
||||
const record = this.store.createRecord("ai-llm");
|
||||
record.provider_params = {};
|
||||
return record;
|
||||
},
|
||||
|
||||
@@ -17,9 +12,5 @@ export default DiscourseRoute.extend({
|
||||
"allLlms",
|
||||
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
||||
);
|
||||
controller.set(
|
||||
"llmTemplate",
|
||||
this.paramsFor(this.routeName).llmTemplate || null
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
+1
-3
@@ -4,9 +4,7 @@ export default DiscourseRoute.extend({
|
||||
async model(params) {
|
||||
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
|
||||
const id = parseInt(params.id, 10);
|
||||
const record = allLlms.findBy("id", id);
|
||||
record.provider_params = record.provider_params || {};
|
||||
return record;
|
||||
return allLlms.findBy("id", id);
|
||||
},
|
||||
|
||||
setupController(controller, model) {
|
||||
|
||||
-2
@@ -10,8 +10,6 @@ export default DiscourseRoute.extend({
|
||||
record.set("rag_chunk_tokens", 374);
|
||||
record.set("rag_chunk_overlap_tokens", 10);
|
||||
record.set("rag_conversation_chunks", 10);
|
||||
record.set("allow_personal_messages", true);
|
||||
record.set("tool_details", false);
|
||||
return record;
|
||||
},
|
||||
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class DiscourseAiToolsNewRoute extends DiscourseRoute {
|
||||
async model() {
|
||||
return this.store.createRecord("ai-tool");
|
||||
}
|
||||
|
||||
setupController(controller) {
|
||||
super.setupController(...arguments);
|
||||
const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
|
||||
|
||||
controller.set("allTools", toolsModel);
|
||||
controller.set("presets", toolsModel.resultSetMeta.presets);
|
||||
}
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
|
||||
async model(params) {
|
||||
const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
|
||||
const id = parseInt(params.id, 10);
|
||||
|
||||
return allTools.find((tool) => tool.id === id);
|
||||
}
|
||||
|
||||
setupController(controller) {
|
||||
super.setupController(...arguments);
|
||||
const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
|
||||
|
||||
controller.set("allTools", toolsModel);
|
||||
controller.set("presets", toolsModel.resultSetMeta.presets);
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
import { service } from "@ember/service";
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class DiscourseAiToolsRoute extends DiscourseRoute {
|
||||
@service store;
|
||||
|
||||
model() {
|
||||
return this.store.findAll("ai-tool");
|
||||
}
|
||||
}
|
||||
+1
-5
@@ -1,5 +1 @@
|
||||
<AiLlmsListEditor
|
||||
@llms={{this.allLlms}}
|
||||
@currentLlm={{this.model}}
|
||||
@llmTemplate={{this.llmTemplate}}
|
||||
/>
|
||||
<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
|
||||
+6
@@ -1,3 +1,9 @@
|
||||
<DBreadcrumbsItem as |linkClass|>
|
||||
<LinkTo @route="adminPlugins.show.discourse-ai-personas" class={{linkClass}}>
|
||||
{{@model.plugin.nameTitleized}}
|
||||
</LinkTo>
|
||||
</DBreadcrumbsItem>
|
||||
|
||||
<AiPersonaListEditor
|
||||
@personas={{this.allPersonas}}
|
||||
@currentPersona={{this.model}}
|
||||
|
||||
-1
@@ -1 +0,0 @@
|
||||
<AiToolListEditor @tools={{this.model}} />
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
<AiToolEditor
|
||||
@tools={{this.allTools}}
|
||||
@model={{this.model}}
|
||||
@presets={{this.presets}}
|
||||
/>
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
<AiToolEditor
|
||||
@tools={{this.allTools}}
|
||||
@model={{this.model}}
|
||||
@presets={{this.presets}}
|
||||
/>
|
||||
@@ -6,7 +6,7 @@ module DiscourseAi
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
def index
|
||||
llms = LlmModel.all.order(:display_name)
|
||||
llms = LlmModel.all
|
||||
|
||||
render json: {
|
||||
ai_llms:
|
||||
@@ -14,13 +14,8 @@ module DiscourseAi
|
||||
llms,
|
||||
each_serializer: LlmModelSerializer,
|
||||
root: false,
|
||||
scope: {
|
||||
llm_usage: DiscourseAi::Configuration::LlmEnumerator.global_usage,
|
||||
},
|
||||
).as_json,
|
||||
meta: {
|
||||
provider_params: LlmModel.provider_params,
|
||||
presets: DiscourseAi::Completions::Llm.presets,
|
||||
providers: DiscourseAi::Completions::Llm.provider_names,
|
||||
tokenizers:
|
||||
DiscourseAi::Completions::Llm.tokenizer_names.map { |tn|
|
||||
@@ -38,8 +33,7 @@ module DiscourseAi
|
||||
def create
|
||||
llm_model = LlmModel.new(ai_llm_params)
|
||||
if llm_model.save
|
||||
llm_model.toggle_companion_user
|
||||
render json: LlmModelSerializer.new(llm_model), status: :created
|
||||
render json: { ai_persona: llm_model }, status: :created
|
||||
else
|
||||
render_json_error llm_model
|
||||
end
|
||||
@@ -48,46 +42,8 @@ module DiscourseAi
|
||||
def update
|
||||
llm_model = LlmModel.find(params[:id])
|
||||
|
||||
if llm_model.seeded?
|
||||
return render_json_error(I18n.t("discourse_ai.llm.cannot_edit_builtin"), status: 403)
|
||||
end
|
||||
|
||||
if llm_model.update(ai_llm_params(updating: llm_model))
|
||||
llm_model.toggle_companion_user
|
||||
render json: LlmModelSerializer.new(llm_model)
|
||||
else
|
||||
render_json_error llm_model
|
||||
end
|
||||
end
|
||||
|
||||
def destroy
|
||||
llm_model = LlmModel.find(params[:id])
|
||||
|
||||
if llm_model.seeded?
|
||||
return render_json_error(I18n.t("discourse_ai.llm.cannot_delete_builtin"), status: 403)
|
||||
end
|
||||
|
||||
in_use_by = DiscourseAi::Configuration::LlmValidator.new.modules_using(llm_model)
|
||||
|
||||
if !in_use_by.empty?
|
||||
return(
|
||||
render_json_error(
|
||||
I18n.t(
|
||||
"discourse_ai.llm.delete_failed",
|
||||
settings: in_use_by.join(", "),
|
||||
count: in_use_by.length,
|
||||
),
|
||||
status: 409,
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
# Clean up companion users
|
||||
llm_model.enabled_chat_bot = false
|
||||
llm_model.toggle_companion_user
|
||||
|
||||
if llm_model.destroy
|
||||
head :no_content
|
||||
if llm_model.update(ai_llm_params)
|
||||
render json: llm_model
|
||||
else
|
||||
render_json_error llm_model
|
||||
end
|
||||
@@ -98,7 +54,11 @@ module DiscourseAi
|
||||
|
||||
llm_model = LlmModel.new(ai_llm_params)
|
||||
|
||||
DiscourseAi::Configuration::LlmValidator.new.run_test(llm_model)
|
||||
DiscourseAi::Completions::Llm.proxy_from_obj(llm_model).generate(
|
||||
"How much is 1 + 1?",
|
||||
user: current_user,
|
||||
feature_name: "llm_validator",
|
||||
)
|
||||
|
||||
render json: { success: true }
|
||||
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
|
||||
@@ -107,44 +67,16 @@ module DiscourseAi
|
||||
|
||||
private
|
||||
|
||||
def ai_llm_params(updating: nil)
|
||||
return {} if params[:ai_llm].blank?
|
||||
|
||||
permitted =
|
||||
params.require(:ai_llm).permit(
|
||||
:display_name,
|
||||
:name,
|
||||
:provider,
|
||||
:tokenizer,
|
||||
:max_prompt_tokens,
|
||||
:api_key,
|
||||
:enabled_chat_bot,
|
||||
:vision_enabled,
|
||||
)
|
||||
|
||||
provider = updating ? updating.provider : permitted[:provider]
|
||||
permit_url = provider != LlmModel::BEDROCK_PROVIDER_NAME
|
||||
|
||||
new_url = params.dig(:ai_llm, :url)
|
||||
permitted[:url] = new_url if permit_url && new_url
|
||||
|
||||
extra_field_names = LlmModel.provider_params.dig(provider&.to_sym)
|
||||
if extra_field_names.present?
|
||||
received_prov_params =
|
||||
params.dig(:ai_llm, :provider_params)&.slice(*extra_field_names.keys)
|
||||
|
||||
if received_prov_params.present?
|
||||
received_prov_params.each do |pname, value|
|
||||
if extra_field_names[pname.to_sym] == :checkbox
|
||||
received_prov_params[pname] = ActiveModel::Type::Boolean.new.cast(value)
|
||||
end
|
||||
end
|
||||
|
||||
permitted[:provider_params] = received_prov_params.permit!
|
||||
end
|
||||
end
|
||||
|
||||
permitted
|
||||
def ai_llm_params
|
||||
params.require(:ai_llm).permit(
|
||||
:display_name,
|
||||
:name,
|
||||
:provider,
|
||||
:tokenizer,
|
||||
:max_prompt_tokens,
|
||||
:url,
|
||||
:api_key,
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -5,7 +5,8 @@ module DiscourseAi
|
||||
class AiPersonasController < ::Admin::AdminController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
before_action :find_ai_persona, only: %i[show update destroy create_user]
|
||||
before_action :find_ai_persona,
|
||||
only: %i[show update destroy create_user indexing_status_check]
|
||||
|
||||
def index
|
||||
ai_personas =
|
||||
@@ -18,19 +19,11 @@ module DiscourseAi
|
||||
DiscourseAi::AiBot::Personas::Persona.all_available_tools.map do |tool|
|
||||
AiToolSerializer.new(tool, root: false)
|
||||
end
|
||||
AiTool
|
||||
.where(enabled: true)
|
||||
.each do |tool|
|
||||
tools << {
|
||||
id: "custom-#{tool.id}",
|
||||
name: I18n.t("discourse_ai.tools.custom_name", name: tool.name.capitalize),
|
||||
}
|
||||
end
|
||||
llms =
|
||||
DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
|
||||
{ id: hash[:value], name: hash[:name] }
|
||||
end
|
||||
render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
|
||||
render json: { ai_personas: ai_personas, meta: { commands: tools, llms: llms } }
|
||||
end
|
||||
|
||||
def show
|
||||
@@ -40,12 +33,9 @@ module DiscourseAi
|
||||
def create
|
||||
ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
|
||||
if ai_persona.save
|
||||
RagDocumentFragment.link_target_and_uploads(ai_persona, attached_upload_ids)
|
||||
RagDocumentFragment.link_persona_and_uploads(ai_persona, attached_upload_ids)
|
||||
|
||||
render json: {
|
||||
ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
|
||||
},
|
||||
status: :created
|
||||
render json: { ai_persona: ai_persona }, status: :created
|
||||
else
|
||||
render_json_error ai_persona
|
||||
end
|
||||
@@ -58,9 +48,9 @@ module DiscourseAi
|
||||
|
||||
def update
|
||||
if @ai_persona.update(ai_persona_params.except(:rag_uploads))
|
||||
RagDocumentFragment.update_target_uploads(@ai_persona, attached_upload_ids)
|
||||
RagDocumentFragment.update_persona_uploads(@ai_persona, attached_upload_ids)
|
||||
|
||||
render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
|
||||
render json: @ai_persona
|
||||
else
|
||||
render_json_error @ai_persona
|
||||
end
|
||||
@@ -74,93 +64,39 @@ module DiscourseAi
|
||||
end
|
||||
end
|
||||
|
||||
def stream_reply
|
||||
persona =
|
||||
AiPersona.find_by(name: params[:persona_name]) ||
|
||||
AiPersona.find_by(id: params[:persona_id])
|
||||
return render_json_error(I18n.t("discourse_ai.errors.persona_not_found")) if persona.nil?
|
||||
def upload_file
|
||||
file = params[:file] || params[:files].first
|
||||
|
||||
return render_json_error(I18n.t("discourse_ai.errors.persona_disabled")) if !persona.enabled
|
||||
|
||||
if persona.default_llm.blank?
|
||||
return render_json_error(I18n.t("discourse_ai.errors.no_default_llm"))
|
||||
if !SiteSetting.ai_embeddings_enabled?
|
||||
raise Discourse::InvalidAccess.new("Embeddings not enabled")
|
||||
end
|
||||
|
||||
if params[:query].blank?
|
||||
return render_json_error(I18n.t("discourse_ai.errors.no_query_specified"))
|
||||
end
|
||||
validate_extension!(file.original_filename)
|
||||
validate_file_size!(file.tempfile.size)
|
||||
|
||||
if !persona.user_id
|
||||
return render_json_error(I18n.t("discourse_ai.errors.no_user_for_persona"))
|
||||
end
|
||||
hijack do
|
||||
upload =
|
||||
UploadCreator.new(
|
||||
file.tempfile,
|
||||
file.original_filename,
|
||||
type: "discourse_ai_rag_upload",
|
||||
skip_validations: true,
|
||||
).create_for(current_user.id)
|
||||
|
||||
if !params[:username] && !params[:user_unique_id]
|
||||
return render_json_error(I18n.t("discourse_ai.errors.no_user_specified"))
|
||||
end
|
||||
|
||||
user = nil
|
||||
|
||||
if params[:username]
|
||||
user = User.find_by_username(params[:username])
|
||||
return render_json_error(I18n.t("discourse_ai.errors.user_not_found")) if user.nil?
|
||||
elsif params[:user_unique_id]
|
||||
user = stage_user
|
||||
end
|
||||
|
||||
raise Discourse::NotFound if user.nil?
|
||||
|
||||
topic_id = params[:topic_id].to_i
|
||||
topic = nil
|
||||
|
||||
if topic_id > 0
|
||||
topic = Topic.find(topic_id)
|
||||
|
||||
if topic.topic_allowed_users.where(user_id: user.id).empty?
|
||||
return render_json_error(I18n.t("discourse_ai.errors.user_not_allowed"))
|
||||
if upload.persisted?
|
||||
render json: UploadSerializer.new(upload)
|
||||
else
|
||||
render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
hijack = request.env["rack.hijack"]
|
||||
io = hijack.call
|
||||
|
||||
DiscourseAi::AiBot::ResponseHttpStreamer.queue_streamed_reply(
|
||||
io: io,
|
||||
persona: persona,
|
||||
user: user,
|
||||
topic: topic,
|
||||
query: params[:query].to_s,
|
||||
custom_instructions: params[:custom_instructions].to_s,
|
||||
current_user: current_user,
|
||||
)
|
||||
def indexing_status_check
|
||||
render json: RagDocumentFragment.indexing_status(@ai_persona, @ai_persona.uploads)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
AI_STREAM_CONVERSATION_UNIQUE_ID = "ai-stream-conversation-unique-id"
|
||||
|
||||
def stage_user
|
||||
unique_id = params[:user_unique_id].to_s
|
||||
field = UserCustomField.find_by(name: AI_STREAM_CONVERSATION_UNIQUE_ID, value: unique_id)
|
||||
|
||||
if field
|
||||
field.user
|
||||
else
|
||||
preferred_username = params[:preferred_username]
|
||||
username = UserNameSuggester.suggest(preferred_username || unique_id)
|
||||
|
||||
user =
|
||||
User.new(
|
||||
username: username,
|
||||
email: "#{SecureRandom.hex}@invalid.com",
|
||||
staged: true,
|
||||
active: false,
|
||||
)
|
||||
user.custom_fields[AI_STREAM_CONVERSATION_UNIQUE_ID] = unique_id
|
||||
user.save!
|
||||
user
|
||||
end
|
||||
end
|
||||
|
||||
def find_ai_persona
|
||||
@ai_persona = AiPersona.find(params[:id])
|
||||
end
|
||||
@@ -181,6 +117,7 @@ module DiscourseAi
|
||||
:temperature,
|
||||
:default_llm,
|
||||
:user_id,
|
||||
:mentionable,
|
||||
:max_context_posts,
|
||||
:vision_enabled,
|
||||
:vision_max_pixels,
|
||||
@@ -188,33 +125,55 @@ module DiscourseAi
|
||||
:rag_chunk_overlap_tokens,
|
||||
:rag_conversation_chunks,
|
||||
:question_consolidator_llm,
|
||||
:allow_chat_channel_mentions,
|
||||
:allow_chat_direct_messages,
|
||||
:allow_topic_mentions,
|
||||
:allow_personal_messages,
|
||||
:tool_details,
|
||||
:forced_tool_count,
|
||||
:force_default_llm,
|
||||
:allow_chat,
|
||||
allowed_group_ids: [],
|
||||
rag_uploads: [:id],
|
||||
)
|
||||
|
||||
if tools = params.dig(:ai_persona, :tools)
|
||||
permitted[:tools] = permit_tools(tools)
|
||||
if commands = params.dig(:ai_persona, :commands)
|
||||
permitted[:commands] = permit_commands(commands)
|
||||
end
|
||||
|
||||
permitted
|
||||
end
|
||||
|
||||
def permit_tools(tools)
|
||||
return [] if !tools.is_a?(Array)
|
||||
def permit_commands(commands)
|
||||
return [] if !commands.is_a?(Array)
|
||||
|
||||
tools.filter_map do |tool, options, force_tool|
|
||||
break nil if !tool.is_a?(String)
|
||||
commands.filter_map do |command, options|
|
||||
break nil if !command.is_a?(String)
|
||||
options&.permit! if options && options.is_a?(ActionController::Parameters)
|
||||
|
||||
# this is simpler from a storage perspective, 1 way to store tools
|
||||
[tool, options, !!force_tool]
|
||||
if options
|
||||
[command, options]
|
||||
else
|
||||
command
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def validate_extension!(filename)
|
||||
extension = File.extname(filename)[1..-1] || ""
|
||||
authorized_extensions = %w[txt md]
|
||||
if !authorized_extensions.include?(extension)
|
||||
raise Discourse::InvalidParameters.new(
|
||||
I18n.t(
|
||||
"upload.unauthorized",
|
||||
authorized_extensions: authorized_extensions.join(" "),
|
||||
),
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def validate_file_size!(filesize)
|
||||
max_size_bytes = 20.megabytes
|
||||
if filesize > max_size_bytes
|
||||
raise Discourse::InvalidParameters.new(
|
||||
I18n.t(
|
||||
"upload.attachments.too_large_humanized",
|
||||
max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
|
||||
),
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Admin
|
||||
class AiToolsController < ::Admin::AdminController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
before_action :find_ai_tool, only: %i[test show update destroy]
|
||||
|
||||
def index
|
||||
ai_tools = AiTool.all
|
||||
render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
|
||||
end
|
||||
|
||||
def show
|
||||
render_serialized(@ai_tool, AiCustomToolSerializer)
|
||||
end
|
||||
|
||||
def create
|
||||
ai_tool = AiTool.new(ai_tool_params)
|
||||
ai_tool.created_by_id = current_user.id
|
||||
|
||||
if ai_tool.save
|
||||
RagDocumentFragment.link_target_and_uploads(ai_tool, attached_upload_ids)
|
||||
render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
|
||||
else
|
||||
render_json_error ai_tool
|
||||
end
|
||||
end
|
||||
|
||||
def update
|
||||
if @ai_tool.update(ai_tool_params)
|
||||
RagDocumentFragment.update_target_uploads(@ai_tool, attached_upload_ids)
|
||||
render_serialized(@ai_tool, AiCustomToolSerializer)
|
||||
else
|
||||
render_json_error @ai_tool
|
||||
end
|
||||
end
|
||||
|
||||
def destroy
|
||||
if @ai_tool.destroy
|
||||
head :no_content
|
||||
else
|
||||
render_json_error @ai_tool
|
||||
end
|
||||
end
|
||||
|
||||
def test
|
||||
@ai_tool.assign_attributes(ai_tool_params) if params[:ai_tool]
|
||||
parameters = params[:parameters].to_unsafe_h
|
||||
|
||||
# we need an llm so we have a tokenizer
|
||||
# but will do without if none is available
|
||||
llm = LlmModel.first&.to_llm
|
||||
runner = @ai_tool.runner(parameters, llm: llm, bot_user: current_user, context: {})
|
||||
result = runner.invoke
|
||||
|
||||
if result.is_a?(Hash) && result[:error]
|
||||
render_json_error result[:error]
|
||||
else
|
||||
render json: { output: result }
|
||||
end
|
||||
rescue ActiveRecord::RecordNotFound => e
|
||||
render_json_error e.message, status: 400
|
||||
rescue => e
|
||||
render_json_error "Error executing the tool: #{e.message}", status: 400
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def attached_upload_ids
|
||||
params[:ai_tool][:rag_uploads].to_a.map { |h| h[:id] }
|
||||
end
|
||||
|
||||
def find_ai_tool
|
||||
@ai_tool = AiTool.find(params[:id].to_i)
|
||||
end
|
||||
|
||||
def ai_tool_params
|
||||
params
|
||||
.require(:ai_tool)
|
||||
.permit(
|
||||
:name,
|
||||
:description,
|
||||
:script,
|
||||
:summary,
|
||||
:rag_chunk_tokens,
|
||||
:rag_chunk_overlap_tokens,
|
||||
rag_uploads: [:id],
|
||||
parameters: [:name, :type, :description, :required, enum: []],
|
||||
)
|
||||
.except(:rag_uploads)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,75 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Admin
|
||||
class RagDocumentFragmentsController < ::Admin::AdminController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
def indexing_status_check
|
||||
if params[:target_type] == "AiPersona"
|
||||
@target = AiPersona.find(params[:target_id])
|
||||
elsif params[:target_type] == "AiTool"
|
||||
@target = AiTool.find(params[:target_id])
|
||||
else
|
||||
raise Discourse::InvalidParameters.new("Invalid target type")
|
||||
end
|
||||
|
||||
render json: RagDocumentFragment.indexing_status(@target, @target.uploads)
|
||||
end
|
||||
|
||||
def upload_file
|
||||
file = params[:file] || params[:files].first
|
||||
|
||||
if !SiteSetting.ai_embeddings_enabled?
|
||||
raise Discourse::InvalidAccess.new("Embeddings not enabled")
|
||||
end
|
||||
|
||||
validate_extension!(file.original_filename)
|
||||
validate_file_size!(file.tempfile.size)
|
||||
|
||||
hijack do
|
||||
upload =
|
||||
UploadCreator.new(
|
||||
file.tempfile,
|
||||
file.original_filename,
|
||||
type: "discourse_ai_rag_upload",
|
||||
skip_validations: true,
|
||||
).create_for(current_user.id)
|
||||
|
||||
if upload.persisted?
|
||||
render json: UploadSerializer.new(upload)
|
||||
else
|
||||
render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def validate_extension!(filename)
|
||||
extension = File.extname(filename)[1..-1] || ""
|
||||
authorized_extensions = %w[txt md]
|
||||
if !authorized_extensions.include?(extension)
|
||||
raise Discourse::InvalidParameters.new(
|
||||
I18n.t(
|
||||
"upload.unauthorized",
|
||||
authorized_extensions: authorized_extensions.join(" "),
|
||||
),
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def validate_file_size!(filesize)
|
||||
max_size_bytes = 20.megabytes
|
||||
if filesize > max_size_bytes
|
||||
raise Discourse::InvalidParameters.new(
|
||||
I18n.t(
|
||||
"upload.attachments.too_large_humanized",
|
||||
max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
|
||||
),
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -6,14 +6,6 @@ module DiscourseAi
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
requires_login
|
||||
|
||||
def show_debug_info_by_id
|
||||
log = AiApiAuditLog.find(params[:id])
|
||||
raise Discourse::NotFound if !log.topic
|
||||
|
||||
guardian.ensure_can_debug_ai_bot_conversation!(log.topic)
|
||||
render json: AiApiAuditLogSerializer.new(log, root: false), status: 200
|
||||
end
|
||||
|
||||
def show_debug_info
|
||||
post = Post.find(params[:post_id])
|
||||
guardian.ensure_can_debug_ai_bot_conversation!(post)
|
||||
@@ -26,7 +18,7 @@ module DiscourseAi
|
||||
|
||||
debug_info = AiApiAuditLog.where(post: posts).order(created_at: :desc).first
|
||||
|
||||
render json: AiApiAuditLogSerializer.new(debug_info, root: false), status: 200
|
||||
render json: debug_info, status: 200
|
||||
end
|
||||
|
||||
def stop_streaming_response
|
||||
@@ -39,10 +31,12 @@ module DiscourseAi
|
||||
end
|
||||
|
||||
def show_bot_username
|
||||
bot_user = DiscourseAi::AiBot::EntryPoint.find_user_from_model(params[:username])
|
||||
raise Discourse::InvalidParameters.new(:username) if !bot_user
|
||||
bot_user_id = DiscourseAi::AiBot::EntryPoint.map_bot_model_to_user_id(params[:username])
|
||||
raise Discourse::InvalidParameters.new(:username) if !bot_user_id
|
||||
|
||||
render json: { bot_username: bot_user.username_lower }, status: 200
|
||||
bot_username_lower = User.find(bot_user_id).username_lower
|
||||
|
||||
render json: { bot_username: bot_username_lower }, status: 200
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -7,7 +7,7 @@ module DiscourseAi
|
||||
requires_login only: %i[create update destroy]
|
||||
before_action :require_site_settings!
|
||||
|
||||
skip_before_action :preload_json, :check_xhr, only: %i[show]
|
||||
skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required, only: %i[show]
|
||||
|
||||
def create
|
||||
ensure_allowed_create!
|
||||
|
||||
@@ -6,24 +6,12 @@ module DiscourseAi
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
requires_login
|
||||
before_action :ensure_can_request_suggestions
|
||||
before_action :rate_limiter_performed!
|
||||
before_action :rate_limiter_performed!, except: %i[prompts]
|
||||
|
||||
include SecureUploadEndpointHelpers
|
||||
|
||||
RATE_LIMITS = {
|
||||
"default" => {
|
||||
amount: 6,
|
||||
interval: 3.minutes,
|
||||
},
|
||||
"caption_image" => {
|
||||
amount: 20,
|
||||
interval: 1.minute,
|
||||
},
|
||||
}.freeze
|
||||
|
||||
def suggest
|
||||
input = get_text_param!
|
||||
force_default_locale = params[:force_default_locale] || false
|
||||
|
||||
prompt = CompletionPrompt.find_by(id: params[:mode])
|
||||
|
||||
@@ -43,7 +31,6 @@ module DiscourseAi
|
||||
prompt,
|
||||
input,
|
||||
current_user,
|
||||
force_default_locale,
|
||||
),
|
||||
status: 200
|
||||
end
|
||||
@@ -100,26 +87,18 @@ module DiscourseAi
|
||||
end
|
||||
end
|
||||
|
||||
def stream_suggestion
|
||||
def explain
|
||||
post_id = get_post_param!
|
||||
text = get_text_param!
|
||||
term_to_explain = get_text_param!
|
||||
post = Post.includes(:topic).find_by(id: post_id)
|
||||
prompt = CompletionPrompt.find_by(id: params[:mode])
|
||||
|
||||
raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
|
||||
raise Discourse::InvalidParameters.new(:post_id) unless post
|
||||
|
||||
if prompt.id == CompletionPrompt::CUSTOM_PROMPT
|
||||
raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
|
||||
end
|
||||
|
||||
Jobs.enqueue(
|
||||
:stream_post_helper,
|
||||
post_id: post.id,
|
||||
user_id: current_user.id,
|
||||
text: text,
|
||||
prompt: prompt.name,
|
||||
custom_prompt: params[:custom_prompt],
|
||||
term_to_explain: term_to_explain,
|
||||
)
|
||||
|
||||
render json: { success: true }, status: 200
|
||||
@@ -128,6 +107,21 @@ module DiscourseAi
|
||||
status: 502
|
||||
end
|
||||
|
||||
def random_caption
|
||||
captions = [
|
||||
"A beautiful landscape",
|
||||
"An adorable puppy",
|
||||
"A delicious meal",
|
||||
"A cozy fireplace",
|
||||
"A stunning sunset",
|
||||
"A charming cityscape",
|
||||
"A peaceful garden",
|
||||
"A majestic mountain range",
|
||||
"A captivating work of art",
|
||||
]
|
||||
captions.sample
|
||||
end
|
||||
|
||||
def caption_image
|
||||
image_url = params[:image_url]
|
||||
image_url_type = params[:image_url_type]
|
||||
@@ -144,12 +138,19 @@ module DiscourseAi
|
||||
end
|
||||
|
||||
raise Discourse::NotFound if image.blank?
|
||||
|
||||
check_secure_upload_permission(image) if image.secure?
|
||||
user = current_user
|
||||
final_image_url = get_caption_url(image, image_url)
|
||||
|
||||
hijack do
|
||||
caption = DiscourseAi::AiHelper::Assistant.new.generate_image_caption(image, user)
|
||||
if Rails.env.development?
|
||||
sleep 2 # Simulate a delay of 2 seconds
|
||||
caption = random_caption
|
||||
else
|
||||
caption =
|
||||
DiscourseAi::AiHelper::Assistant.new.generate_image_caption(
|
||||
final_image_url,
|
||||
current_user,
|
||||
)
|
||||
end
|
||||
render json: {
|
||||
caption:
|
||||
"#{caption} (#{I18n.t("discourse_ai.ai_helper.image_caption.attribution")})",
|
||||
@@ -172,23 +173,22 @@ module DiscourseAi
|
||||
end
|
||||
|
||||
def rate_limiter_performed!
|
||||
action_rate_limit = RATE_LIMITS[action_name] || RATE_LIMITS["default"]
|
||||
RateLimiter.new(
|
||||
current_user,
|
||||
"ai_assistant",
|
||||
action_rate_limit[:amount],
|
||||
action_rate_limit[:interval],
|
||||
).performed!
|
||||
RateLimiter.new(current_user, "ai_assistant", 6, 3.minutes).performed!
|
||||
end
|
||||
|
||||
def ensure_can_request_suggestions
|
||||
allowed_groups =
|
||||
(
|
||||
SiteSetting.composer_ai_helper_allowed_groups_map |
|
||||
SiteSetting.post_ai_helper_allowed_groups_map
|
||||
)
|
||||
if !current_user.in_any_groups?(SiteSetting.ai_helper_allowed_groups_map)
|
||||
raise Discourse::InvalidAccess
|
||||
end
|
||||
end
|
||||
|
||||
raise Discourse::InvalidAccess if !current_user.in_any_groups?(allowed_groups)
|
||||
def get_caption_url(image_upload, image_url)
|
||||
if image_upload.secure?
|
||||
check_secure_upload_permission(image_upload)
|
||||
return Discourse.store.url_for(image_upload)
|
||||
end
|
||||
|
||||
UrlHelper.absolute(image_url)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Discord
|
||||
class BotController < ::ApplicationController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
skip_before_action :verify_authenticity_token
|
||||
|
||||
def interactions
|
||||
# Request signature verification
|
||||
begin
|
||||
verify_request!
|
||||
rescue Ed25519::VerifyError
|
||||
return head :unauthorized
|
||||
end
|
||||
|
||||
body = request.body.read
|
||||
interaction = JSON.parse(body, object_class: OpenStruct)
|
||||
|
||||
if interaction.type == 1
|
||||
# Respond to Discord PING request
|
||||
render json: { type: 1 }
|
||||
else
|
||||
if !SiteSetting.ai_discord_allowed_guilds_map.include?(interaction.guild_id)
|
||||
return head :forbidden
|
||||
end
|
||||
|
||||
response = { type: 5, data: { content: "Searching..." } }
|
||||
hijack { render json: response }
|
||||
|
||||
# Respond to Discord command
|
||||
Jobs.enqueue(:stream_discord_reply, interaction: body)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def verify_request!
|
||||
signature = request.headers["X-Signature-Ed25519"]
|
||||
timestamp = request.headers["X-Signature-Timestamp"]
|
||||
verify_key.verify([signature].pack("H*"), "#{timestamp}#{request.raw_post}")
|
||||
end
|
||||
|
||||
def verify_key
|
||||
Ed25519::VerifyKey.new([SiteSetting.ai_discord_app_public_key].pack("H*")).freeze
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -7,12 +7,8 @@ module DiscourseAi
|
||||
|
||||
SEMANTIC_SEARCH_TYPE = "semantic_search"
|
||||
|
||||
MAX_HYDE_SEARCHES_PER_MINUTE = 4
|
||||
MAX_SEARCHES_PER_MINUTE = 100
|
||||
|
||||
def search
|
||||
query = params[:q].to_s
|
||||
skip_hyde = params[:hyde].to_s.downcase == "false" || params[:hyde].to_s == "0"
|
||||
|
||||
if query.length < SiteSetting.min_search_term_length
|
||||
raise Discourse::InvalidParameters.new(:q)
|
||||
@@ -29,25 +25,13 @@ module DiscourseAi
|
||||
|
||||
semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
|
||||
|
||||
if !skip_hyde && !semantic_search.cached_query?(query)
|
||||
RateLimiter.new(
|
||||
current_user,
|
||||
"semantic-search",
|
||||
MAX_HYDE_SEARCHES_PER_MINUTE,
|
||||
1.minutes,
|
||||
).performed!
|
||||
else
|
||||
RateLimiter.new(
|
||||
current_user,
|
||||
"semantic-search-non-hyde",
|
||||
MAX_SEARCHES_PER_MINUTE,
|
||||
1.minutes,
|
||||
).performed!
|
||||
if !semantic_search.cached_query?(query)
|
||||
RateLimiter.new(current_user, "semantic-search", 4, 1.minutes).performed!
|
||||
end
|
||||
|
||||
hijack do
|
||||
semantic_search
|
||||
.search_for_topics(query, _page = 1, hyde: !skip_hyde)
|
||||
.search_for_topics(query)
|
||||
.each { |topic_post| grouped_results.add(topic_post) }
|
||||
|
||||
render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
|
||||
@@ -55,9 +39,6 @@ module DiscourseAi
|
||||
end
|
||||
|
||||
def quick_search
|
||||
# this search function searches posts (vs: topics)
|
||||
# it requires post embeddings and a reranker
|
||||
# it will not perform a hyde expantion
|
||||
query = params[:q].to_s
|
||||
|
||||
if query.length < SiteSetting.min_search_term_length
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Summarization
|
||||
class ChatSummaryController < ::Chat::ApiController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
requires_plugin ::Chat::PLUGIN_NAME
|
||||
|
||||
VALID_SINCE_VALUES = [1, 3, 6, 12, 24, 72, 168]
|
||||
|
||||
def show
|
||||
since = params[:since].to_i
|
||||
raise Discourse::InvalidParameters.new(:since) if !VALID_SINCE_VALUES.include?(since)
|
||||
|
||||
channel = ::Chat::Channel.find(params[:channel_id])
|
||||
guardian.ensure_can_join_chat_channel!(channel)
|
||||
|
||||
summarizer = DiscourseAi::Summarization.chat_channel_summary(channel, since)
|
||||
raise Discourse::NotFound.new unless summarizer
|
||||
|
||||
guardian.ensure_can_request_summary!
|
||||
|
||||
RateLimiter.new(current_user, "channel_summary", 6, 5.minutes).performed!
|
||||
|
||||
hijack do
|
||||
strategy = DiscourseAi::Summarization::Strategies::ChatMessages.new(channel, since)
|
||||
|
||||
summarized_text =
|
||||
if strategy.targets_data.empty?
|
||||
I18n.t("discourse_ai.summarization.chat.no_targets")
|
||||
else
|
||||
summarizer.summarize(current_user)&.summarized_text
|
||||
end
|
||||
|
||||
render json: { summary: summarized_text }
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,46 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Summarization
|
||||
class SummaryController < ::ApplicationController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
def show
|
||||
topic = Topic.find(params[:topic_id])
|
||||
guardian.ensure_can_see!(topic)
|
||||
|
||||
raise Discourse::NotFound if !guardian.can_see_summary?(topic)
|
||||
|
||||
RateLimiter.new(current_user, "summary", 6, 5.minutes).performed! if current_user
|
||||
|
||||
opts = params.permit(:skip_age_check)
|
||||
skip_age_check = opts[:skip_age_check] == "true"
|
||||
|
||||
summarization_service = DiscourseAi::TopicSummarization.for(topic, current_user)
|
||||
|
||||
if params[:stream] && current_user
|
||||
cached_summary = summarization_service.cached_summary
|
||||
|
||||
if cached_summary && !skip_age_check
|
||||
render_serialized(cached_summary, AiTopicSummarySerializer)
|
||||
return
|
||||
end
|
||||
|
||||
Jobs.enqueue(
|
||||
:stream_topic_ai_summary,
|
||||
topic_id: topic.id,
|
||||
user_id: current_user.id,
|
||||
skip_age_check: skip_age_check,
|
||||
)
|
||||
|
||||
render json: success_json
|
||||
else
|
||||
hijack do
|
||||
summary = summarization_service.summarize(skip_age_check: skip_age_check)
|
||||
render_serialized(summary, AiTopicSummarySerializer)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -8,7 +8,7 @@ module DiscourseAi
|
||||
VERSION = "1"
|
||||
|
||||
def share_asset_url(short_path)
|
||||
::UrlHelper.absolute("/plugins/discourse-ai/ai-share/#{short_path}?#{VERSION}")
|
||||
::UrlHelper.local_cdn_url("/plugins/discourse-ai/ai-share/#{short_path}?#{VERSION}")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -9,24 +9,17 @@ module ::Jobs
|
||||
# TODO(roman): Add a way to automatically recover from errors, resulting in unindexed uploads.
|
||||
def execute(args)
|
||||
return if (upload = Upload.find_by(id: args[:upload_id])).nil?
|
||||
|
||||
target_type = args[:target_type]
|
||||
target_id = args[:target_id]
|
||||
|
||||
return if !target_type || !target_id
|
||||
|
||||
target = target_type.constantize.find_by(id: target_id)
|
||||
return if !target
|
||||
return if (ai_persona = AiPersona.find_by(id: args[:ai_persona_id])).nil?
|
||||
|
||||
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
|
||||
|
||||
tokenizer = vector_rep.tokenizer
|
||||
chunk_tokens = target.rag_chunk_tokens
|
||||
overlap_tokens = target.rag_chunk_overlap_tokens
|
||||
chunk_tokens = ai_persona.rag_chunk_tokens
|
||||
overlap_tokens = ai_persona.rag_chunk_overlap_tokens
|
||||
|
||||
fragment_ids = RagDocumentFragment.where(target: target, upload: upload).pluck(:id)
|
||||
fragment_ids = RagDocumentFragment.where(ai_persona: ai_persona, upload: upload).pluck(:id)
|
||||
|
||||
# Check if this is the first time we process this upload.
|
||||
if fragment_ids.empty?
|
||||
@@ -46,7 +39,7 @@ module ::Jobs
|
||||
overlap_tokens: overlap_tokens,
|
||||
) do |chunk, metadata|
|
||||
fragment_ids << RagDocumentFragment.create!(
|
||||
target: target,
|
||||
ai_persona: ai_persona,
|
||||
fragment: chunk,
|
||||
fragment_number: idx + 1,
|
||||
upload: upload,
|
||||
@@ -126,9 +119,7 @@ module ::Jobs
|
||||
|
||||
while overlap_token_ids.present?
|
||||
begin
|
||||
padding = split_char
|
||||
padding = " " if padding.empty?
|
||||
overlap = tokenizer.decode(overlap_token_ids) + padding
|
||||
overlap = tokenizer.decode(overlap_token_ids) + split_char
|
||||
break if overlap.encoding == Encoding::UTF_8
|
||||
rescue StandardError
|
||||
# it is possible that we truncated mid char
|
||||
@@ -137,7 +128,7 @@ module ::Jobs
|
||||
end
|
||||
|
||||
# remove first word it is probably truncated
|
||||
overlap = overlap.split(/\s/, 2).last.to_s.lstrip
|
||||
overlap = overlap.split(" ", 2).last
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ module Jobs
|
||||
|
||||
topic = target.is_a?(Topic) ? target : target.topic
|
||||
post = target.is_a?(Post) ? target : target.first_post
|
||||
return if topic.blank? || post.blank?
|
||||
return unless topic.present? && post.present?
|
||||
return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
|
||||
return if post.raw.blank?
|
||||
|
||||
|
||||
@@ -17,10 +17,10 @@ module ::Jobs
|
||||
fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
|
||||
|
||||
last_fragment = fragments.last
|
||||
target = last_fragment.target
|
||||
ai_persona = last_fragment.ai_persona
|
||||
upload = last_fragment.upload
|
||||
|
||||
indexing_status = RagDocumentFragment.indexing_status(target, [upload])[upload.id]
|
||||
indexing_status = RagDocumentFragment.indexing_status(ai_persona, [upload])[upload.id]
|
||||
RagDocumentFragment.publish_status(upload, indexing_status)
|
||||
end
|
||||
end
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user