Compare commits

..
Author SHA1 Message Date
Sam Saffron 6351c401f8 work in progress adding roles to personas
# Conflicts:
#	app/models/ai_persona.rb
2024-05-01 15:39:13 +10:00
Sam Saffron acd3ac2371 minor fix to read ... complete is just confusing 2024-04-30 17:08:23 +10:00
Sam Saffron c61af76d86 Responder for categories 2024-04-30 17:08:23 +10:00
Sam Saffron cd7790f39a message responder backend now works 2024-04-30 17:08:21 +10:00
Sam Saffron 4fcf12287f work in progress adding roles to personas 2024-04-30 17:07:22 +10:00
434 changed files with 99749 additions and 426233 deletions
-1
View File
@@ -1,4 +1,3 @@
< 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
< 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
+25 -36
View File
@@ -1,7 +1,7 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (7.1.3.4)
activesupport (7.1.3.2)
base64
bigdecimal
concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -13,28 +13,26 @@ GEM
tzinfo (~> 2.0)
ast (2.4.2)
base64 (0.2.0)
bigdecimal (3.1.8)
concurrent-ruby (1.3.3)
bigdecimal (3.1.6)
concurrent-ruby (1.2.3)
connection_pool (2.4.1)
drb (2.2.1)
i18n (1.14.5)
i18n (1.14.1)
concurrent-ruby (~> 1.0)
json (2.7.2)
json (2.7.1)
language_server-protocol (3.17.0.3)
minitest (5.24.1)
minitest (5.22.2)
mutex_m (0.2.0)
parallel (1.25.1)
parser (3.3.3.0)
parallel (1.24.0)
parser (3.3.0.5)
ast (~> 2.4.1)
racc
prettier_print (1.2.1)
racc (1.8.0)
rack (3.1.6)
racc (1.7.3)
rainbow (3.1.1)
regexp_parser (2.9.2)
rexml (3.3.1)
strscan
rubocop (1.64.1)
regexp_parser (2.9.0)
rexml (3.2.6)
rubocop (1.61.0)
json (~> 2.3)
language_server-protocol (>= 3.17.0)
parallel (~> 1.10)
@@ -42,35 +40,26 @@ GEM
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0)
rubocop-ast (>= 1.31.1, < 2.0)
rubocop-ast (>= 1.30.0, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 3.0)
rubocop-ast (1.31.3)
parser (>= 3.3.1.0)
rubocop-capybara (2.21.0)
rubocop-ast (1.31.1)
parser (>= 3.3.0.4)
rubocop-capybara (2.20.0)
rubocop (~> 1.41)
rubocop-discourse (3.8.1)
rubocop-discourse (3.7.1)
activesupport (>= 6.1)
rubocop (>= 1.59.0)
rubocop-capybara (>= 2.0.0)
rubocop-factory_bot (>= 2.0.0)
rubocop-rails (>= 2.25.0)
rubocop-rspec (>= 3.0.1)
rubocop-rspec_rails (>= 2.30.0)
rubocop-factory_bot (2.26.1)
rubocop (~> 1.61)
rubocop-rails (2.25.1)
activesupport (>= 4.2.0)
rack (>= 1.1)
rubocop (>= 1.33.0, < 2.0)
rubocop-ast (>= 1.31.1, < 2.0)
rubocop-rspec (3.0.2)
rubocop (~> 1.61)
rubocop-rspec_rails (2.30.0)
rubocop (~> 1.61)
rubocop-rspec (~> 3, >= 3.0.1)
rubocop-rspec (>= 2.25.0)
rubocop-factory_bot (2.25.1)
rubocop (~> 1.41)
rubocop-rspec (2.27.1)
rubocop (~> 1.40)
rubocop-capybara (~> 2.17)
rubocop-factory_bot (~> 2.22)
ruby-progressbar (1.13.0)
strscan (3.1.0)
syntax_tree (6.2.0)
prettier_print (>= 1.2.0)
tzinfo (2.0.6)
@@ -85,4 +74,4 @@ DEPENDENCIES
syntax_tree
BUNDLED WITH
2.5.10
2.5.4
@@ -0,0 +1,7 @@
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
model() {
return this.modelFor("adminPlugins.show.discourse-ai.ai-personas");
},
});
@@ -17,7 +17,7 @@ export default DiscourseRoute.extend({
this._super(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
this.modelFor("adminPlugins.show.discourse-ai.ai-personas")
);
},
});
@@ -3,7 +3,7 @@ import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model(params) {
const allPersonas = this.modelFor(
"adminPlugins.show.discourse-ai-personas"
"adminPlugins.show.discourse-ai.ai-personas"
);
const id = parseInt(params.id, 10);
return allPersonas.findBy("id", id);
@@ -13,7 +13,7 @@ export default DiscourseRoute.extend({
this._super(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
this.modelFor("adminPlugins.show.discourse-ai.ai-personas")
);
},
});
@@ -0,0 +1,12 @@
import { inject as service } from "@ember/service";
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiIndexRoute extends DiscourseRoute {
@service router;
beforeModel() {
return this.router.transitionTo(
"adminPlugins.show.discourse-ai.ai-personas.index"
);
}
}
@@ -1,16 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model() {
const record = this.store.createRecord("ai-llm");
return record;
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
);
},
});
@@ -1,17 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model(params) {
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const id = parseInt(params.id, 10);
return allLlms.findBy("id", id);
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
);
},
});
@@ -1,7 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiAiLlmsRoute extends DiscourseRoute {
model() {
return this.store.findAll("ai-llm");
}
}
@@ -1,15 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiToolsNewRoute extends DiscourseRoute {
async model() {
return this.store.createRecord("ai-tool");
}
setupController(controller) {
super.setupController(...arguments);
const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
controller.set("allTools", toolsModel);
controller.set("presets", toolsModel.resultSetMeta.presets);
}
}
@@ -1,18 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
async model(params) {
const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
const id = parseInt(params.id, 10);
return allTools.find((tool) => tool.id === id);
}
setupController(controller) {
super.setupController(...arguments);
const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
controller.set("allTools", toolsModel);
controller.set("presets", toolsModel.resultSetMeta.presets);
}
}
@@ -1,10 +0,0 @@
import { service } from "@ember/service";
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiToolsRoute extends DiscourseRoute {
@service store;
model() {
return this.store.findAll("ai-tool");
}
}
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.model}} />
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
<AiToolListEditor @tools={{this.model}} />
@@ -1,5 +0,0 @@
<AiToolEditor
@tools={{this.allTools}}
@model={{this.model}}
@presets={{this.presets}}
/>
@@ -1,5 +0,0 @@
<AiToolEditor
@tools={{this.allTools}}
@model={{this.model}}
@presets={{this.presets}}
/>
@@ -1,128 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiLlmsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
llms = LlmModel.all.order(:display_name)
render json: {
ai_llms:
ActiveModel::ArraySerializer.new(
llms,
each_serializer: LlmModelSerializer,
root: false,
).as_json,
meta: {
provider_params: LlmModel.provider_params,
presets: DiscourseAi::Completions::Llm.presets,
providers: DiscourseAi::Completions::Llm.provider_names,
tokenizers:
DiscourseAi::Completions::Llm.tokenizer_names.map { |tn|
{ id: tn, name: tn.split("::").last }
},
},
}
end
def show
llm_model = LlmModel.find(params[:id])
render json: LlmModelSerializer.new(llm_model)
end
def create
llm_model = LlmModel.new(ai_llm_params)
if llm_model.save
llm_model.toggle_companion_user
render json: { ai_persona: LlmModelSerializer.new(llm_model) }, status: :created
else
render_json_error LlmModelSerializer.new(llm_model)
end
end
def update
llm_model = LlmModel.find(params[:id])
if llm_model.update(ai_llm_params(updating: llm_model))
llm_model.toggle_companion_user
render json: LlmModelSerializer.new(llm_model)
else
render_json_error LlmModelSerializer.new(llm_model)
end
end
def destroy
llm_model = LlmModel.find(params[:id])
in_use_by = DiscourseAi::Configuration::LlmValidator.new.modules_using(llm_model)
if !in_use_by.empty?
return(
render_json_error(
I18n.t(
"discourse_ai.llm.delete_failed",
settings: in_use_by.join(", "),
count: in_use_by.length,
),
status: 409,
)
)
end
# Clean up companion users
llm_model.enabled_chat_bot = false
llm_model.toggle_companion_user
if llm_model.destroy
head :no_content
else
render_json_error llm_model
end
end
def test
RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!
llm_model = LlmModel.new(ai_llm_params)
DiscourseAi::Configuration::LlmValidator.new.run_test(llm_model)
render json: { success: true }
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
render json: { success: false, error: e.message }
end
private
def ai_llm_params(updating: nil)
permitted =
params.require(:ai_llm).permit(
:display_name,
:name,
:provider,
:tokenizer,
:max_prompt_tokens,
:api_key,
:enabled_chat_bot,
)
provider = updating ? updating.provider : permitted[:provider]
permit_url =
(updating && updating.url != LlmModel::RESERVED_VLLM_SRV_URL) ||
provider != LlmModel::BEDROCK_PROVIDER_NAME
permitted[:url] = params.dig(:ai_llm, :url) if permit_url
extra_field_names = LlmModel.provider_params.dig(provider&.to_sym, :fields).to_a
received_prov_params = params.dig(:ai_llm, :provider_params)
permitted[:provider_params] = received_prov_params.slice(
*extra_field_names,
).permit! if !extra_field_names.empty? && received_prov_params.present?
permitted
end
end
end
end
@@ -19,19 +19,11 @@ module DiscourseAi
DiscourseAi::AiBot::Personas::Persona.all_available_tools.map do |tool|
AiToolSerializer.new(tool, root: false)
end
AiTool
.where(enabled: true)
.each do |tool|
tools << {
id: "custom-#{tool.id}",
name: I18n.t("discourse_ai.tools.custom_name", name: tool.name.capitalize),
}
end
llms =
DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
{ id: hash[:value], name: hash[:name] }
end
render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
render json: { ai_personas: ai_personas, meta: { commands: tools, llms: llms } }
end
def show
@@ -43,10 +35,7 @@ module DiscourseAi
if ai_persona.save
RagDocumentFragment.link_persona_and_uploads(ai_persona, attached_upload_ids)
render json: {
ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
},
status: :created
render json: { ai_persona: ai_persona }, status: :created
else
render_json_error ai_persona
end
@@ -61,9 +50,9 @@ module DiscourseAi
if @ai_persona.update(ai_persona_params.except(:rag_uploads))
RagDocumentFragment.update_persona_uploads(@ai_persona, attached_upload_ids)
render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
render json: @ai_persona
else
render_json_error LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
render_json_error @ai_persona
end
end
@@ -71,7 +60,7 @@ module DiscourseAi
if @ai_persona.destroy
head :no_content
else
render_json_error LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
render_json_error @ai_persona
end
end
@@ -136,30 +125,29 @@ module DiscourseAi
:rag_chunk_overlap_tokens,
:rag_conversation_chunks,
:question_consolidator_llm,
:allow_chat,
:tool_details,
:role,
allowed_group_ids: [],
rag_uploads: [:id],
)
if tools = params.dig(:ai_persona, :tools)
permitted[:tools] = permit_tools(tools)
if commands = params.dig(:ai_persona, :commands)
permitted[:commands] = permit_commands(commands)
end
permitted
end
def permit_tools(tools)
return [] if !tools.is_a?(Array)
def permit_commands(commands)
return [] if !commands.is_a?(Array)
tools.filter_map do |tool, options|
break nil if !tool.is_a?(String)
commands.filter_map do |command, options|
break nil if !command.is_a?(String)
options&.permit! if options && options.is_a?(ActionController::Parameters)
if options
[tool, options]
[command, options]
else
tool
command
end
end
end
@@ -1,89 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiToolsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
before_action :find_ai_tool, only: %i[show update destroy]
def index
ai_tools = AiTool.all
render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
end
def show
render_serialized(@ai_tool, AiCustomToolSerializer)
end
def create
ai_tool = AiTool.new(ai_tool_params)
ai_tool.created_by_id = current_user.id
if ai_tool.save
render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
else
render_json_error ai_tool
end
end
def update
if @ai_tool.update(ai_tool_params)
render_serialized(@ai_tool, AiCustomToolSerializer)
else
render_json_error @ai_tool
end
end
def destroy
if @ai_tool.destroy
head :no_content
else
render_json_error @ai_tool
end
end
def test
if params[:id].present?
ai_tool = AiTool.find(params[:id])
else
ai_tool = AiTool.new(ai_tool_params)
end
parameters = params[:parameters].to_unsafe_h
# we need an llm so we have a tokenizer
# but will do without if none is available
llm = LlmModel.first&.to_llm
runner = ai_tool.runner(parameters, llm: llm, bot_user: current_user, context: {})
result = runner.invoke
if result.is_a?(Hash) && result[:error]
render_json_error result[:error]
else
render json: { output: result }
end
rescue ActiveRecord::RecordNotFound => e
render_json_error e.message, status: 400
rescue => e
render_json_error "Error executing the tool: #{e.message}", status: 400
end
private
def find_ai_tool
@ai_tool = AiTool.find(params[:id])
end
def ai_tool_params
params.require(:ai_tool).permit(
:name,
:description,
:script,
:summary,
parameters: [:name, :type, :description, :required, :enum, enum_values: []],
)
end
end
end
end
@@ -18,7 +18,7 @@ module DiscourseAi
debug_info = AiApiAuditLog.where(post: posts).order(created_at: :desc).first
render json: AiApiAuditLogSerializer.new(debug_info, root: false), status: 200
render json: debug_info, status: 200
end
def stop_streaming_response
@@ -31,10 +31,12 @@ module DiscourseAi
end
def show_bot_username
bot_user = DiscourseAi::AiBot::EntryPoint.find_user_from_model(params[:username])
raise Discourse::InvalidParameters.new(:username) if !bot_user
bot_user_id = DiscourseAi::AiBot::EntryPoint.map_bot_model_to_user_id(params[:username])
raise Discourse::InvalidParameters.new(:username) if !bot_user_id
render json: { bot_username: bot_user.username_lower }, status: 200
bot_username_lower = User.find(bot_user_id).username_lower
render json: { bot_username: bot_username_lower }, status: 200
end
end
end
@@ -7,7 +7,7 @@ module DiscourseAi
requires_login only: %i[create update destroy]
before_action :require_site_settings!
skip_before_action :preload_json, :check_xhr, only: %i[show]
skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required, only: %i[show]
def create
ensure_allowed_create!
@@ -19,8 +19,7 @@ module DiscourseAi
if shared_conversation.persisted?
render json: success_json.merge(share_key: shared_conversation.share_key)
else
render json:
failed_json.merge(error: I18n.t("discourse_ai.share_ai.errors.failed_to_share")),
render json: failed_json.merge(error: I18n.t("discourse_ai.share_ai.failed_to_share")),
status: :unprocessable_entity
end
end
@@ -31,9 +30,7 @@ module DiscourseAi
SharedAiConversation.destroy_conversation(@shared_conversation)
render json:
success_json.merge(
message: I18n.t("discourse_ai.share_ai.errors.conversation_deleted"),
)
success_json.merge(message: I18n.t("discourse_ai.share_ai.conversation_deleted"))
end
def show
@@ -12,7 +12,6 @@ module DiscourseAi
def suggest
input = get_text_param!
force_default_locale = params[:force_default_locale] || false
prompt = CompletionPrompt.find_by(id: params[:mode])
@@ -32,7 +31,6 @@ module DiscourseAi
prompt,
input,
current_user,
force_default_locale,
),
status: 200
end
@@ -111,26 +109,18 @@ module DiscourseAi
def caption_image
image_url = params[:image_url]
image_url_type = params[:image_url_type]
raise Discourse::InvalidParameters.new(:image_url) if !image_url
raise Discourse::InvalidParameters.new(:image_url) if !image_url_type
if image_url_type == "short_path"
image = Upload.find_by(sha1: Upload.sha1_from_short_path(image_url))
elsif image_url_type == "short_url"
image = Upload.find_by(sha1: Upload.sha1_from_short_url(image_url))
else
image = upload_from_full_url(image_url)
end
image = upload_from_full_url(image_url)
raise Discourse::NotFound if image.blank?
check_secure_upload_permission(image) if image.secure?
user = current_user
final_image_url = get_caption_url(image, image_url)
hijack do
caption = DiscourseAi::AiHelper::Assistant.new.generate_image_caption(image, user)
caption =
DiscourseAi::AiHelper::Assistant.new.generate_image_caption(
final_image_url,
current_user,
)
render json: {
caption:
"#{caption} (#{I18n.t("discourse_ai.ai_helper.image_caption.attribution")})",
@@ -161,6 +151,15 @@ module DiscourseAi
raise Discourse::InvalidAccess
end
end
def get_caption_url(image_upload, image_url)
if image_upload.secure?
check_secure_upload_permission(image_upload)
return Discourse.store.url_for(image_upload)
end
UrlHelper.absolute(image_url)
end
end
end
end
@@ -20,7 +20,6 @@ module DiscourseAi
term: query,
search_context: guardian,
use_pg_headlines_for_excerpt: false,
can_lazy_load_categories: guardian.can_lazy_load_categories?,
)
semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
@@ -1,48 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Summarization
class ChatSummaryController < ::Chat::ApiController
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_plugin ::Chat::PLUGIN_NAME
VALID_SINCE_VALUES = [1, 3, 6, 12, 24, 72, 168]
def show
since = params[:since].to_i
raise Discourse::InvalidParameters.new(:since) if !VALID_SINCE_VALUES.include?(since)
channel = ::Chat::Channel.find(params[:channel_id])
guardian.ensure_can_join_chat_channel!(channel)
strategy = DiscourseAi::Summarization.default_strategy
raise Discourse::NotFound.new unless strategy
guardian.ensure_can_request_summary!
RateLimiter.new(current_user, "channel_summary", 6, 5.minutes).performed!
hijack do
content = { content_title: channel.name }
content[:contents] = channel
.chat_messages
.where("chat_messages.created_at > ?", since.hours.ago)
.includes(:user)
.order(created_at: :asc)
.pluck(:id, :username_lower, :message)
.map { { id: _1, poster: _2, text: _3 } }
summarized_text =
if content[:contents].empty?
I18n.t("discourse_ai.summarization.chat.no_targets")
else
strategy.summarize(content, current_user).dig(:summary)
end
render json: { summary: summarized_text }
end
end
end
end
end

Some files were not shown because too many files have changed in this diff Show More