Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 awesomerobot
|
522c1aabb8 |
@@ -1,6 +1,3 @@
|
||||
< 3.5.0.beta1-dev: bdef136080074a993e7c4f5ca562edc31a8ba756
|
||||
< 3.4.0.beta4-dev: a53719ab8eb071459f215227421b3ea4987e5f87
|
||||
< 3.4.0.beta4-dev: 20612fde52d3f740cad64823ef8aadb0748b567f
|
||||
< 3.4.0.beta3-dev: decf1bb49d737ea15308400f22f89d1d1e71d13d
|
||||
< 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
|
||||
< 3.3.0.beta5-dev: 4d8090002f6dcd8e34d41033606bf131fa221475
|
||||
|
||||
@@ -2,6 +2,3 @@ node_modules
|
||||
/gems
|
||||
/auto_generated
|
||||
.env
|
||||
evals/log
|
||||
evals/cases
|
||||
config/eval-llms.local.yml
|
||||
|
||||
@@ -2,6 +2,3 @@ inherit_gem:
|
||||
rubocop-discourse: stree-compat.yml
|
||||
RSpec/NamedSubject:
|
||||
Enabled: false
|
||||
|
||||
Style/GlobalVars:
|
||||
AllowedVariables: [$prometheus_client]
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ GEM
|
||||
racc
|
||||
prettier_print (1.2.1)
|
||||
racc (1.8.0)
|
||||
rack (3.1.10)
|
||||
rack (3.1.6)
|
||||
rainbow (3.1.1)
|
||||
regexp_parser (2.9.2)
|
||||
rexml (3.3.9)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) Civilized Discourse Construction Kit, Inc.
|
||||
Copyright (c) 2021 Civilized Discourse Construction Kit
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -3,27 +3,3 @@
|
||||
**Plugin Summary**
|
||||
|
||||
For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco
|
||||
|
||||
### Evals
|
||||
|
||||
The directory `evals` contains AI evals for the Discourse AI plugin.
|
||||
You may create a local config by copying `config/eval-llms.yml` to `config/eval-llms.local.yml` and modifying the values.
|
||||
|
||||
To run them use:
|
||||
|
||||
cd evals
|
||||
./run --help
|
||||
|
||||
```
|
||||
Usage: evals/run [options]
|
||||
-e, --eval NAME Name of the evaluation to run
|
||||
--list-models List models
|
||||
-m, --model NAME Model to evaluate (will eval all models if not specified)
|
||||
-l, --list List evals
|
||||
```
|
||||
|
||||
To run evals you will need to configure API keys in your environment:
|
||||
|
||||
OPENAI_API_KEY=your_openai_api_key
|
||||
ANTHROPIC_API_KEY=your_anthropic_api_key
|
||||
GEMINI_API_KEY=your_gemini_api_key
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"tests": {
|
||||
"requiredPlugins": [
|
||||
"https://github.com/discourse/discourse-prometheus"
|
||||
]
|
||||
}
|
||||
}
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class AdminPluginsShowDiscourseAiEmbeddingsEdit extends DiscourseRoute {
|
||||
async model(params) {
|
||||
const allEmbeddings = this.modelFor(
|
||||
"adminPlugins.show.discourse-ai-embeddings"
|
||||
);
|
||||
const id = parseInt(params.id, 10);
|
||||
const record = allEmbeddings.findBy("id", id);
|
||||
record.provider_params = record.provider_params || {};
|
||||
return record;
|
||||
}
|
||||
|
||||
setupController(controller, model) {
|
||||
super.setupController(controller, model);
|
||||
controller.set(
|
||||
"allEmbeddings",
|
||||
this.modelFor("adminPlugins.show.discourse-ai-embeddings")
|
||||
);
|
||||
}
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class AdminPluginsShowDiscourseAiEmbeddingsNew extends DiscourseRoute {
|
||||
async model() {
|
||||
const record = this.store.createRecord("ai-embedding");
|
||||
record.provider_params = {};
|
||||
return record;
|
||||
}
|
||||
|
||||
setupController(controller, model) {
|
||||
super.setupController(controller, model);
|
||||
controller.set(
|
||||
"allEmbeddings",
|
||||
this.modelFor("adminPlugins.show.discourse-ai-embeddings")
|
||||
);
|
||||
}
|
||||
}
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class DiscourseAiAiEmbeddingsRoute extends DiscourseRoute {
|
||||
model() {
|
||||
return this.store.findAll("ai-embedding");
|
||||
}
|
||||
}
|
||||
-2
@@ -14,7 +14,5 @@ export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
|
||||
|
||||
controller.set("allTools", toolsModel);
|
||||
controller.set("presets", toolsModel.resultSetMeta.presets);
|
||||
controller.set("llms", toolsModel.resultSetMeta.llms);
|
||||
controller.set("settings", toolsModel.resultSetMeta.settings);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,5 @@ export default class DiscourseAiToolsNewRoute extends DiscourseRoute {
|
||||
|
||||
controller.set("allTools", toolsModel);
|
||||
controller.set("presets", toolsModel.resultSetMeta.presets);
|
||||
controller.set("llms", toolsModel.resultSetMeta.llms);
|
||||
controller.set("settings", toolsModel.resultSetMeta.settings);
|
||||
}
|
||||
}
|
||||
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
<AiEmbeddingsListEditor
|
||||
@embeddings={{this.allEmbeddings}}
|
||||
@currentEmbedding={{this.model}}
|
||||
/>
|
||||
-1
@@ -1 +0,0 @@
|
||||
<AiEmbeddingsListEditor @embeddings={{this.model}} />
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
<AiEmbeddingsListEditor
|
||||
@embeddings={{this.allEmbeddings}}
|
||||
@currentEmbedding={{this.model}}
|
||||
/>
|
||||
-2
@@ -3,7 +3,5 @@
|
||||
@tools={{this.allTools}}
|
||||
@model={{this.model}}
|
||||
@presets={{this.presets}}
|
||||
@llms={{this.llms}}
|
||||
@settings={{this.settings}}
|
||||
/>
|
||||
</section>
|
||||
-2
@@ -3,7 +3,5 @@
|
||||
@tools={{this.allTools}}
|
||||
@model={{this.model}}
|
||||
@presets={{this.presets}}
|
||||
@llms={{this.llms}}
|
||||
@settings={{this.settings}}
|
||||
/>
|
||||
</section>
|
||||
@@ -1,133 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Admin
|
||||
class AiEmbeddingsController < ::Admin::AdminController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
def index
|
||||
embedding_defs = EmbeddingDefinition.all.order(:display_name)
|
||||
|
||||
render json: {
|
||||
ai_embeddings:
|
||||
ActiveModel::ArraySerializer.new(
|
||||
embedding_defs,
|
||||
each_serializer: AiEmbeddingDefinitionSerializer,
|
||||
root: false,
|
||||
).as_json,
|
||||
meta: {
|
||||
provider_params: EmbeddingDefinition.provider_params,
|
||||
providers: EmbeddingDefinition.provider_names,
|
||||
distance_functions: EmbeddingDefinition.distance_functions,
|
||||
tokenizers:
|
||||
EmbeddingDefinition.tokenizer_names.map { |tn|
|
||||
{ id: tn, name: tn.split("::").last }
|
||||
},
|
||||
presets: EmbeddingDefinition.presets,
|
||||
},
|
||||
}
|
||||
end
|
||||
|
||||
def new
|
||||
end
|
||||
|
||||
def edit
|
||||
embedding_def = EmbeddingDefinition.find(params[:id])
|
||||
render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
|
||||
end
|
||||
|
||||
def create
|
||||
embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
|
||||
|
||||
if embedding_def.save
|
||||
render json: AiEmbeddingDefinitionSerializer.new(embedding_def), status: :created
|
||||
else
|
||||
render_json_error embedding_def
|
||||
end
|
||||
end
|
||||
|
||||
def update
|
||||
embedding_def = EmbeddingDefinition.find(params[:id])
|
||||
|
||||
if embedding_def.seeded?
|
||||
return(
|
||||
render_json_error(I18n.t("discourse_ai.embeddings.cannot_edit_builtin"), status: 403)
|
||||
)
|
||||
end
|
||||
|
||||
if embedding_def.update(ai_embeddings_params.except(:dimensions))
|
||||
render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
|
||||
else
|
||||
render_json_error embedding_def
|
||||
end
|
||||
end
|
||||
|
||||
def destroy
|
||||
embedding_def = EmbeddingDefinition.find(params[:id])
|
||||
|
||||
if embedding_def.seeded?
|
||||
return(
|
||||
render_json_error(I18n.t("discourse_ai.embeddings.cannot_edit_builtin"), status: 403)
|
||||
)
|
||||
end
|
||||
|
||||
if embedding_def.id == SiteSetting.ai_embeddings_selected_model.to_i
|
||||
return render_json_error(I18n.t("discourse_ai.embeddings.delete_failed"), status: 409)
|
||||
end
|
||||
|
||||
if embedding_def.destroy
|
||||
head :no_content
|
||||
else
|
||||
render_json_error embedding_def
|
||||
end
|
||||
end
|
||||
|
||||
def test
|
||||
RateLimiter.new(
|
||||
current_user,
|
||||
"ai_embeddings_test_#{current_user.id}",
|
||||
3,
|
||||
1.minute,
|
||||
).performed!
|
||||
|
||||
embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
|
||||
DiscourseAi::Embeddings::Vector.new(embedding_def).vector_from("this is a test")
|
||||
|
||||
render json: { success: true }
|
||||
rescue Net::HTTPBadResponse => e
|
||||
render json: { success: false, error: e.message }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def ai_embeddings_params
|
||||
permitted =
|
||||
params.require(:ai_embedding).permit(
|
||||
:display_name,
|
||||
:dimensions,
|
||||
:max_sequence_length,
|
||||
:pg_function,
|
||||
:provider,
|
||||
:url,
|
||||
:api_key,
|
||||
:tokenizer_class,
|
||||
:embed_prompt,
|
||||
:search_prompt,
|
||||
:matryoshka_dimensions,
|
||||
)
|
||||
|
||||
extra_field_names = EmbeddingDefinition.provider_params.dig(permitted[:provider]&.to_sym)
|
||||
if extra_field_names.present?
|
||||
received_prov_params =
|
||||
params.dig(:ai_embedding, :provider_params)&.slice(*extra_field_names.keys)
|
||||
|
||||
if received_prov_params.present?
|
||||
permitted[:provider_params] = received_prov_params.permit!
|
||||
end
|
||||
end
|
||||
|
||||
permitted
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,59 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Admin
|
||||
class AiLlmQuotasController < ::Admin::AdminController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
def index
|
||||
quotas = LlmQuota.includes(:group)
|
||||
|
||||
render json: {
|
||||
quotas:
|
||||
ActiveModel::ArraySerializer.new(quotas, each_serializer: LlmQuotaSerializer),
|
||||
}
|
||||
end
|
||||
|
||||
def create
|
||||
quota = LlmQuota.new(quota_params)
|
||||
|
||||
if quota.save
|
||||
render json: LlmQuotaSerializer.new(quota), status: :created
|
||||
else
|
||||
render_json_error quota
|
||||
end
|
||||
end
|
||||
|
||||
def update
|
||||
quota = LlmQuota.find(params[:id])
|
||||
|
||||
if quota.update(quota_params)
|
||||
render json: LlmQuotaSerializer.new(quota)
|
||||
else
|
||||
render_json_error quota
|
||||
end
|
||||
end
|
||||
|
||||
def destroy
|
||||
quota = LlmQuota.find(params[:id])
|
||||
quota.destroy!
|
||||
|
||||
head :no_content
|
||||
rescue ActiveRecord::RecordNotFound
|
||||
render json: { error: I18n.t("not_found") }, status: 404
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def quota_params
|
||||
params.require(:quota).permit(
|
||||
:group_id,
|
||||
:llm_model_id,
|
||||
:max_tokens,
|
||||
:max_usages,
|
||||
:duration_seconds,
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -6,7 +6,7 @@ module DiscourseAi
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
def index
|
||||
llms = LlmModel.all.includes(:llm_quotas).order(:display_name)
|
||||
llms = LlmModel.all.order(:display_name)
|
||||
|
||||
render json: {
|
||||
ai_llms:
|
||||
@@ -40,11 +40,6 @@ module DiscourseAi
|
||||
|
||||
def create
|
||||
llm_model = LlmModel.new(ai_llm_params)
|
||||
|
||||
# we could do nested attributes but the mechanics are not ideal leading
|
||||
# to lots of complex debugging, this is simpler
|
||||
quota_params.each { |quota| llm_model.llm_quotas.build(quota) } if quota_params
|
||||
|
||||
if llm_model.save
|
||||
llm_model.toggle_companion_user
|
||||
render json: LlmModelSerializer.new(llm_model), status: :created
|
||||
@@ -56,25 +51,6 @@ module DiscourseAi
|
||||
def update
|
||||
llm_model = LlmModel.find(params[:id])
|
||||
|
||||
if params[:ai_llm].key?(:llm_quotas)
|
||||
if quota_params
|
||||
existing_quota_group_ids = llm_model.llm_quotas.pluck(:group_id)
|
||||
new_quota_group_ids = quota_params.map { |q| q[:group_id] }
|
||||
|
||||
llm_model
|
||||
.llm_quotas
|
||||
.where(group_id: existing_quota_group_ids - new_quota_group_ids)
|
||||
.destroy_all
|
||||
|
||||
quota_params.each do |quota_param|
|
||||
quota = llm_model.llm_quotas.find_or_initialize_by(group_id: quota_param[:group_id])
|
||||
quota.update!(quota_param)
|
||||
end
|
||||
else
|
||||
llm_model.llm_quotas.destroy_all
|
||||
end
|
||||
end
|
||||
|
||||
if llm_model.seeded?
|
||||
return render_json_error(I18n.t("discourse_ai.llm.cannot_edit_builtin"), status: 403)
|
||||
end
|
||||
@@ -134,19 +110,6 @@ module DiscourseAi
|
||||
|
||||
private
|
||||
|
||||
def quota_params
|
||||
if params[:ai_llm][:llm_quotas].present?
|
||||
params[:ai_llm][:llm_quotas].map do |quota|
|
||||
mapped = {}
|
||||
mapped[:group_id] = quota[:group_id].to_i
|
||||
mapped[:max_tokens] = quota[:max_tokens].to_i if quota[:max_tokens].present?
|
||||
mapped[:max_usages] = quota[:max_usages].to_i if quota[:max_usages].present?
|
||||
mapped[:duration_seconds] = quota[:duration_seconds].to_i
|
||||
mapped
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def ai_llm_params(updating: nil)
|
||||
return {} if params[:ai_llm].blank?
|
||||
|
||||
|
||||
@@ -23,28 +23,14 @@ module DiscourseAi
|
||||
.each do |tool|
|
||||
tools << {
|
||||
id: "custom-#{tool.id}",
|
||||
name:
|
||||
I18n.t(
|
||||
"discourse_ai.tools.custom_name",
|
||||
name: tool.name.capitalize,
|
||||
tool_name: tool.tool_name,
|
||||
),
|
||||
name: I18n.t("discourse_ai.tools.custom_name", name: tool.name.capitalize),
|
||||
}
|
||||
end
|
||||
llms =
|
||||
DiscourseAi::Configuration::LlmEnumerator.values_for_serialization(
|
||||
allowed_seeded_llm_ids: SiteSetting.ai_bot_allowed_seeded_models_map,
|
||||
)
|
||||
render json: {
|
||||
ai_personas: ai_personas,
|
||||
meta: {
|
||||
tools: tools,
|
||||
llms: llms,
|
||||
settings: {
|
||||
rag_images_enabled: SiteSetting.ai_rag_images_enabled,
|
||||
},
|
||||
},
|
||||
}
|
||||
DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
|
||||
{ id: hash[:value], name: hash[:name] }
|
||||
end
|
||||
render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
|
||||
end
|
||||
|
||||
def new
|
||||
@@ -196,7 +182,7 @@ module DiscourseAi
|
||||
:priority,
|
||||
:top_p,
|
||||
:temperature,
|
||||
:default_llm_id,
|
||||
:default_llm,
|
||||
:user_id,
|
||||
:max_context_posts,
|
||||
:vision_enabled,
|
||||
@@ -204,8 +190,7 @@ module DiscourseAi
|
||||
:rag_chunk_tokens,
|
||||
:rag_chunk_overlap_tokens,
|
||||
:rag_conversation_chunks,
|
||||
:rag_llm_model_id,
|
||||
:question_consolidator_llm_id,
|
||||
:question_consolidator_llm,
|
||||
:allow_chat_channel_mentions,
|
||||
:allow_chat_direct_messages,
|
||||
:allow_topic_mentions,
|
||||
|
||||
@@ -15,7 +15,7 @@ module DiscourseAi
|
||||
llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
|
||||
if llm_model_id.to_i < 0 &&
|
||||
!SiteSetting.ai_spam_detection_model_allowed_seeded_models_map.include?(
|
||||
llm_model_id.to_s,
|
||||
"custom:#{llm_model_id}",
|
||||
)
|
||||
return(
|
||||
render_json_error(
|
||||
@@ -86,31 +86,6 @@ module DiscourseAi
|
||||
render json: result
|
||||
end
|
||||
|
||||
def fix_errors
|
||||
case params[:error]
|
||||
when "spam_scanner_not_admin"
|
||||
begin
|
||||
DiscourseAi::AiModeration::SpamScanner.fix_spam_scanner_not_admin
|
||||
render json: success_json
|
||||
rescue ActiveRecord::RecordInvalid
|
||||
render_json_error(
|
||||
I18n.t("discourse_ai.spam_detection.bot_user_update_failed"),
|
||||
status: :unprocessable_entity,
|
||||
)
|
||||
rescue StandardError
|
||||
render_json_error(
|
||||
I18n.t("discourse_ai.spam_detection.unexpected"),
|
||||
status: :internal_server_error,
|
||||
)
|
||||
end
|
||||
else
|
||||
render_json_error(
|
||||
I18n.t("discourse_ai.spam_detection.invalid_error_type"),
|
||||
status: :bad_request,
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def allowed_params
|
||||
|
||||
@@ -84,13 +84,11 @@ module DiscourseAi
|
||||
.require(:ai_tool)
|
||||
.permit(
|
||||
:name,
|
||||
:tool_name,
|
||||
:description,
|
||||
:script,
|
||||
:summary,
|
||||
:rag_chunk_tokens,
|
||||
:rag_chunk_overlap_tokens,
|
||||
:rag_llm_model_id,
|
||||
rag_uploads: [:id],
|
||||
parameters: [:name, :type, :description, :required, enum: []],
|
||||
)
|
||||
|
||||
@@ -20,7 +20,7 @@ module DiscourseAi
|
||||
def upload_file
|
||||
file = params[:file] || params[:files].first
|
||||
|
||||
if !DiscourseAi::Embeddings.enabled?
|
||||
if !SiteSetting.ai_embeddings_enabled?
|
||||
raise Discourse::InvalidAccess.new("Embeddings not enabled")
|
||||
end
|
||||
|
||||
@@ -48,8 +48,7 @@ module DiscourseAi
|
||||
|
||||
def validate_extension!(filename)
|
||||
extension = File.extname(filename)[1..-1] || ""
|
||||
authorized_extensions = %w[txt md pdf]
|
||||
authorized_extensions.concat(%w[png jpg jpeg]) if SiteSetting.ai_rag_images_enabled
|
||||
authorized_extensions = %w[txt md]
|
||||
if !authorized_extensions.include?(extension)
|
||||
raise Discourse::InvalidParameters.new(
|
||||
I18n.t(
|
||||
|
||||
@@ -12,7 +12,7 @@ module DiscourseAi
|
||||
artifact = AiArtifact.find(params[:id])
|
||||
|
||||
post = Post.find_by(id: artifact.post_id)
|
||||
if artifact.public?
|
||||
if artifact.metadata&.dig("public")
|
||||
# no guardian needed
|
||||
else
|
||||
raise Discourse::NotFound if !post&.topic&.private_message?
|
||||
@@ -57,7 +57,6 @@ module DiscourseAi
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>#{ERB::Util.html_escape(name)}</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, user-scalable=yes, viewport-fit=cover, interactive-widget=resizes-content">
|
||||
<style>
|
||||
html, body, iframe {
|
||||
margin: 0;
|
||||
@@ -81,7 +80,7 @@ module DiscourseAi
|
||||
response.headers.delete("X-Frame-Options")
|
||||
response.headers[
|
||||
"Content-Security-Policy"
|
||||
] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' #{AiArtifact::ALLOWED_CDN_SOURCES.join(" ")};"
|
||||
] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://unpkg.com https://cdnjs.cloudflare.com https://ajax.googleapis.com https://cdn.jsdelivr.net;"
|
||||
response.headers["X-Robots-Tag"] = "noindex"
|
||||
|
||||
# Render the content
|
||||
|
||||
@@ -44,31 +44,6 @@ module DiscourseAi
|
||||
|
||||
render json: { bot_username: bot_user.username_lower }, status: 200
|
||||
end
|
||||
|
||||
def discover
|
||||
ai_persona =
|
||||
AiPersona.all_personas.find do |persona|
|
||||
persona.id == SiteSetting.ai_bot_discover_persona.to_i
|
||||
end
|
||||
|
||||
if ai_persona.nil? || !current_user.in_any_groups?(ai_persona.allowed_group_ids.to_a)
|
||||
raise Discourse::InvalidAccess.new
|
||||
end
|
||||
|
||||
if ai_persona.default_llm_id.blank?
|
||||
render_json_error "Discover persona is missing a default LLM model.", status: 503
|
||||
return
|
||||
end
|
||||
|
||||
query = params[:query]
|
||||
raise Discourse::InvalidParameters.new("Missing query to discover") if query.blank?
|
||||
|
||||
RateLimiter.new(current_user, "ai_bot_discover_#{current_user.id}", 3, 1.minute).performed!
|
||||
|
||||
Jobs.enqueue(:stream_discover_reply, user_id: current_user.id, query: query)
|
||||
|
||||
render json: {}, status: 200
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Sentiment
|
||||
class SentimentController < ::Admin::StaffController
|
||||
include Constants
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
DEFAULT_POSTS_LIMIT = 50
|
||||
MAX_POSTS_LIMIT = 100
|
||||
|
||||
def posts
|
||||
group_by = params.required(:group_by)&.to_sym
|
||||
group_value = params.required(:group_value).presence
|
||||
start_date = params[:start_date].presence
|
||||
end_date = params[:end_date]
|
||||
threshold = SENTIMENT_THRESHOLD
|
||||
|
||||
raise Discourse::InvalidParameters if %i[category tag].exclude?(group_by)
|
||||
|
||||
limit = fetch_limit_from_params(default: DEFAULT_POSTS_LIMIT, max: MAX_POSTS_LIMIT)
|
||||
offset = params[:offset].to_i || 0
|
||||
|
||||
case group_by
|
||||
when :category
|
||||
grouping_clause = "c.name"
|
||||
grouping_join = "" # categories already joined
|
||||
when :tag
|
||||
grouping_clause = "tags.name"
|
||||
grouping_join =
|
||||
"INNER JOIN topic_tags tt ON tt.topic_id = p.topic_id INNER JOIN tags ON tags.id = tt.tag_id"
|
||||
end
|
||||
|
||||
posts =
|
||||
DB.query(
|
||||
<<~SQL,
|
||||
SELECT
|
||||
p.id AS post_id,
|
||||
p.topic_id,
|
||||
t.fancy_title AS topic_title,
|
||||
p.cooked as post_cooked,
|
||||
p.user_id,
|
||||
p.post_number,
|
||||
p.created_at AS created_at,
|
||||
u.username,
|
||||
u.name,
|
||||
u.uploaded_avatar_id,
|
||||
c.id AS category_id,
|
||||
c.name AS category_name,
|
||||
c.color AS category_color,
|
||||
c.slug AS category_slug,
|
||||
c.description AS category_description,
|
||||
(CASE
|
||||
WHEN (cr.classification::jsonb->'positive')::float > :threshold THEN 'positive'
|
||||
WHEN (cr.classification::jsonb->'negative')::float > :threshold THEN 'negative'
|
||||
ELSE 'neutral'
|
||||
END) AS sentiment
|
||||
FROM posts p
|
||||
INNER JOIN topics t ON t.id = p.topic_id
|
||||
INNER JOIN classification_results cr ON cr.target_id = p.id AND cr.target_type = 'Post'
|
||||
LEFT JOIN users u ON u.id = p.user_id
|
||||
LEFT JOIN categories c ON c.id = t.category_id
|
||||
#{grouping_join}
|
||||
WHERE
|
||||
#{grouping_clause} = :group_value AND
|
||||
t.archetype = 'regular' AND
|
||||
p.user_id > 0 AND
|
||||
cr.model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest' AND
|
||||
((:start_date IS NULL OR p.created_at > :start_date) AND (:end_date IS NULL OR p.created_at < :end_date))
|
||||
AND p.deleted_at IS NULL
|
||||
ORDER BY p.created_at DESC
|
||||
LIMIT :limit OFFSET :offset
|
||||
SQL
|
||||
group_value: group_value,
|
||||
start_date: start_date,
|
||||
end_date: end_date,
|
||||
threshold: threshold,
|
||||
limit: limit + 1,
|
||||
offset: offset,
|
||||
)
|
||||
|
||||
has_more = posts.length > limit
|
||||
posts.pop if has_more
|
||||
|
||||
render_json_dump(
|
||||
posts:
|
||||
serialize_data(
|
||||
posts,
|
||||
AiSentimentPostSerializer,
|
||||
scope: guardian,
|
||||
add_raw: true,
|
||||
add_excerpt: true,
|
||||
add_title: true,
|
||||
),
|
||||
has_more: has_more,
|
||||
next_offset: has_more ? offset + limit : nil,
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -18,7 +18,7 @@ module ::Jobs
|
||||
target = target_type.constantize.find_by(id: target_id)
|
||||
return if !target
|
||||
|
||||
vector_rep = DiscourseAi::Embeddings::Vector.instance
|
||||
vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
|
||||
|
||||
tokenizer = vector_rep.tokenizer
|
||||
chunk_tokens = target.rag_chunk_tokens
|
||||
@@ -28,7 +28,7 @@ module ::Jobs
|
||||
|
||||
# Check if this is the first time we process this upload.
|
||||
if fragment_ids.empty?
|
||||
document = get_uploaded_file(upload: upload, target: target)
|
||||
document = get_uploaded_file(upload)
|
||||
return if document.nil?
|
||||
|
||||
RagDocumentFragment.publish_status(upload, { total: 0, indexed: 0, left: 0 })
|
||||
@@ -163,32 +163,7 @@ module ::Jobs
|
||||
[buffer, split_char]
|
||||
end
|
||||
|
||||
def get_uploaded_file(upload:, target:)
|
||||
if %w[png jpg jpeg].include?(upload.extension) && !SiteSetting.ai_rag_images_enabled
|
||||
raise Discourse::InvalidAccess.new(
|
||||
"The setting ai_rag_images_enabled is false, can not index images",
|
||||
)
|
||||
end
|
||||
if upload.extension == "pdf"
|
||||
return(
|
||||
DiscourseAi::Utils::PdfToText.as_fake_file(
|
||||
upload: upload,
|
||||
llm_model: SiteSetting.ai_rag_images_enabled ? target.rag_llm_model : nil,
|
||||
user: Discourse.system_user,
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
if %w[png jpg jpeg].include?(upload.extension)
|
||||
return(
|
||||
DiscourseAi::Utils::ImageToText.as_fake_file(
|
||||
uploads: [upload],
|
||||
llm_model: target.rag_llm_model,
|
||||
user: Discourse.system_user,
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def get_uploaded_file(upload)
|
||||
store = Discourse.store
|
||||
@file ||=
|
||||
if store.external?
|
||||
|
||||
@@ -16,7 +16,7 @@ module ::Jobs
|
||||
gist = summarizer.existing_summary
|
||||
return if gist.present? && (!gist.outdated || gist.created_at >= 5.minutes.ago)
|
||||
|
||||
summarizer.summarize(Discourse.system_user)
|
||||
summarizer.force_summarize(Discourse.system_user)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -5,7 +5,7 @@ module Jobs
|
||||
sidekiq_options queue: "low"
|
||||
|
||||
def execute(args)
|
||||
return unless DiscourseAi::Embeddings.enabled?
|
||||
return unless SiteSetting.ai_embeddings_enabled
|
||||
return if args[:target_type].blank? || args[:target_id].blank?
|
||||
target = args[:target_type].constantize.find_by_id(args[:target_id])
|
||||
return if target.nil? || target.deleted_at.present?
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user