Compare commits

..
Author SHA1 Message Date
Nat f10af7b167 DEV: Remove the short-lived prompt for locale detection 2024-11-28 20:18:55 +08:00
320 changed files with 3538 additions and 12362 deletions
@@ -1,18 +1,18 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
queryParams = {
export default DiscourseRoute.extend({
queryParams: {
llmTemplate: { refreshModel: true },
};
},
async model() {
const record = this.store.createRecord("ai-llm");
record.provider_params = {};
return record;
}
},
setupController(controller, model) {
super.setupController(controller, model);
this._super(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
@@ -21,5 +21,5 @@ export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
"llmTemplate",
this.paramsFor(this.routeName).llmTemplate || null
);
}
}
},
});
@@ -1,19 +1,19 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
export default DiscourseRoute.extend({
async model(params) {
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const id = parseInt(params.id, 10);
const record = allLlms.findBy("id", id);
record.provider_params = record.provider_params || {};
return record;
}
},
setupController(controller, model) {
super.setupController(controller, model);
this._super(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
);
}
}
},
});
@@ -1,7 +1,7 @@
import { AUTO_GROUPS } from "discourse/lib/constants";
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
export default DiscourseRoute.extend({
async model() {
const record = this.store.createRecord("ai-persona");
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
@@ -13,13 +13,13 @@ export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRou
record.set("allow_personal_messages", true);
record.set("tool_details", false);
return record;
}
},
setupController(controller, model) {
super.setupController(controller, model);
this._super(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
);
}
}
},
});
@@ -1,19 +1,19 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
export default DiscourseRoute.extend({
async model(params) {
const allPersonas = this.modelFor(
"adminPlugins.show.discourse-ai-personas"
);
const id = parseInt(params.id, 10);
return allPersonas.findBy("id", id);
}
},
setupController(controller, model) {
super.setupController(controller, model);
this._super(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
);
}
}
},
});
@@ -1,11 +0,0 @@
import { service } from "@ember/service";
import { ajax } from "discourse/lib/ajax";
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiSpamRoute extends DiscourseRoute {
@service store;
model() {
return ajax("/admin/plugins/discourse-ai/ai-spam.json");
}
}
@@ -1,6 +1,6 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
async model(params) {
const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
const id = parseInt(params.id, 10);
@@ -1,11 +0,0 @@
import { service } from "@ember/service";
import { ajax } from "discourse/lib/ajax";
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiUsageRoute extends DiscourseRoute {
@service store;
model() {
return ajax("/admin/plugins/discourse-ai/ai-usage.json");
}
}
@@ -1 +0,0 @@
<AiSpam @model={{this.model}} />
@@ -1 +0,0 @@
<AiUsage @model={{this.model}} />
@@ -30,10 +30,7 @@ module DiscourseAi
}
end
def new
end
def edit
def show
llm_model = LlmModel.find(params[:id])
render json: LlmModelSerializer.new(llm_model)
end
@@ -5,7 +5,7 @@ module DiscourseAi
class AiPersonasController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
before_action :find_ai_persona, only: %i[edit update destroy create_user]
before_action :find_ai_persona, only: %i[show update destroy create_user]
def index
ai_personas =
@@ -33,10 +33,7 @@ module DiscourseAi
render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
end
def new
end
def edit
def show
render json: LocalizedAiPersonaSerializer.new(@ai_persona)
end
@@ -1,112 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiSpamController < ::Admin::AdminController
requires_plugin "discourse-ai"
def show
render json: AiSpamSerializer.new(spam_config, root: false)
end
def update
updated_params = {}
if allowed_params.key?(:llm_model_id)
llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
if llm_model_id.to_i < 0 &&
!SiteSetting.ai_spam_detection_model_allowed_seeded_models_map.include?(
"custom:#{llm_model_id}",
)
return(
render_json_error(
I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"),
status: 422,
)
)
end
end
updated_params[:data] = {
custom_instructions: allowed_params[:custom_instructions],
} if allowed_params.key?(:custom_instructions)
if updated_params.present?
# not using upsert cause we will not get the correct validation errors
if AiModerationSetting.spam
AiModerationSetting.spam.update!(updated_params)
else
AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
end
end
is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
if allowed_params.key?(:is_enabled)
if is_enabled && !AiModerationSetting.spam&.llm_model_id
return(
render_json_error(
I18n.t("discourse_ai.llm.configuration.must_select_model"),
status: 422,
)
)
end
SiteSetting.ai_spam_detection_enabled = is_enabled
end
render json: AiSpamSerializer.new(spam_config, root: false)
end
def test
url = params[:post_url].to_s
post = nil
if url.match?(/^\d+$/)
post_id = url.to_i
post = Post.find_by(id: post_id)
end
route = UrlHelper.rails_route_from_url(url) if !post
if route
if route[:controller] == "topics"
post_number = route[:post_number] || 1
post = Post.with_deleted.find_by(post_number: post_number, topic_id: route[:topic_id])
end
end
raise Discourse::NotFound if !post
result =
DiscourseAi::AiModeration::SpamScanner.test_post(
post,
custom_instructions: params[:custom_instructions],
llm_id: params[:llm_id],
)
render json: result
end
private
def allowed_params
params.permit(:is_enabled, :llm_model_id, :custom_instructions)
end
def spam_config
spam_config = {
enabled: SiteSetting.ai_spam_detection_enabled,
settings: AiModerationSetting.spam,
}
spam_config[:stats] = DiscourseAi::AiModeration::SpamReport.generate(min_date: 1.week.ago)
if spam_config[:stats].scanned_count > 0
spam_config[
:flagging_username
] = DiscourseAi::AiModeration::SpamScanner.flagging_user&.username
end
spam_config
end
end
end
end
@@ -5,17 +5,14 @@ module DiscourseAi
class AiToolsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
before_action :find_ai_tool, only: %i[test edit update destroy]
before_action :find_ai_tool, only: %i[test show update destroy]
def index
ai_tools = AiTool.all
render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
end
def new
end
def edit
def show
render_serialized(@ai_tool, AiCustomToolSerializer)
end
@@ -1,30 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiUsageController < ::Admin::AdminController
requires_plugin "discourse-ai"
def show
end
def report
render json: AiUsageSerializer.new(create_report, root: false)
end
private
def create_report
report =
DiscourseAi::Completions::Report.new(
start_date: params[:start_date]&.to_date || 30.days.ago,
end_date: params[:end_date]&.to_date || Time.current,
)
report = report.filter_by_feature(params[:feature]) if params[:feature].present?
report = report.filter_by_model(params[:model]) if params[:model].present?
report
end
end
end
end
@@ -19,33 +19,22 @@ module DiscourseAi
raise Discourse::NotFound if !guardian.can_see?(post)
end
name = artifact.name
if params[:version].present?
artifact = artifact.versions.find_by(version_number: params[:version])
raise Discourse::NotFound if !artifact
end
js = artifact.js || ""
if !js.match?(%r{\A\s*<script.*</script>}mi)
mod = ""
mod = " type=\"module\"" if js.match?(/\A\s*import.*/)
js = "<script#{mod}>\n#{js}\n</script>"
end
# Prepare the inner (untrusted) HTML document
untrusted_html = <<~HTML
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>#{ERB::Util.html_escape(name)}</title>
<title>#{ERB::Util.html_escape(artifact.name)}</title>
<style>
#{artifact.css}
</style>
</head>
<body>
#{artifact.html}
#{js}
<script>
#{artifact.js}
</script>
</body>
</html>
HTML
@@ -56,7 +45,7 @@ module DiscourseAi
<html>
<head>
<meta charset="UTF-8">
<title>#{ERB::Util.html_escape(name)}</title>
<title>#{ERB::Util.html_escape(artifact.name)}</title>
<style>
html, body, iframe {
margin: 0;
@@ -78,9 +67,7 @@ module DiscourseAi
HTML
response.headers.delete("X-Frame-Options")
response.headers[
"Content-Security-Policy"
] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://unpkg.com https://cdnjs.cloudflare.com https://ajax.googleapis.com https://cdn.jsdelivr.net;"
response.headers["Content-Security-Policy"] = "script-src 'unsafe-inline';"
response.headers["X-Robots-Tag"] = "noindex"
# Render the content
-13
View File
@@ -1,13 +0,0 @@
# frozen_string_literal: true
module Jobs
class AiSpamScan < ::Jobs::Base
def execute(args)
return if !args[:post_id]
post = Post.find_by(id: args[:post_id])
return if !post
DiscourseAi::AiModeration::SpamScanner.perform_scan(post)
end
end
end
+3 -1
View File
@@ -18,7 +18,9 @@ module ::Jobs
target = target_type.constantize.find_by(id: target_id)
return if !target
vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
vector_rep =
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
tokenizer = vector_rep.tokenizer
chunk_tokens = target.rag_chunk_tokens
+17
View File
@@ -0,0 +1,17 @@
# frozen_string_literal: true
module Jobs
class EvaluatePostUploads < ::Jobs::Base
def execute(args)
return unless SiteSetting.ai_nsfw_detection_enabled
return if (post_id = args[:post_id]).blank?
post = Post.includes(:uploads).find_by_id(post_id)
return if post.nil? || post.uploads.empty?
return if post.uploads.none? { |u| FileHelper.is_supported_image?(u.url) }
DiscourseAi::PostClassificator.new(DiscourseAi::Nsfw::Classification.new).classify!(post)
end
end
end
+1 -1
View File
@@ -14,7 +14,7 @@ module ::Jobs
summarizer = DiscourseAi::Summarization.topic_gist(topic)
gist = summarizer.existing_summary
return if gist.present? && (!gist.outdated || gist.created_at >= 5.minutes.ago)
return if gist.present? && !gist.outdated
summarizer.force_summarize(Discourse.system_user)
end
+5 -1
View File
@@ -16,7 +16,11 @@ module Jobs
return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
return if post.raw.blank?
DiscourseAi::Embeddings::Vector.instance.generate_representation_from(target)
strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
vector_rep =
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
vector_rep.generate_representation_from(target)
end
end
end
+4 -2
View File
@@ -8,11 +8,13 @@ module ::Jobs
def execute(args)
return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
vector = DiscourseAi::Embeddings::Vector.instance
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
vector_rep =
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
# generate_representation_from checks compares the digest value to make sure
# the embedding is only generated once per fragment unless something changes.
fragments.map { |fragment| vector.generate_representation_from(fragment) }
fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
last_fragment = fragments.last
target = last_fragment.target
+3 -1
View File
@@ -9,7 +9,9 @@ module ::Jobs
post = Post.find_by(id: post_id, post_type: Post.types[:regular])
return if post&.raw.blank?
DiscourseAi::Sentiment::PostClassification.new.classify!(post)
DiscourseAi::PostClassificator.new(
DiscourseAi::Sentiment::SentimentClassification.new,
).classify!(post)
end
end
end
@@ -0,0 +1,18 @@
# frozen_string_literal: true
module ::Jobs
class ToxicityClassifyChatMessage < ::Jobs::Base
def execute(args)
return unless SiteSetting.ai_toxicity_enabled
return if (chat_message_id = args[:chat_message_id]).blank?
chat_message = ::Chat::Message.find_by(id: chat_message_id)
return if chat_message&.message.blank?
DiscourseAi::ChatMessageClassificator.new(
DiscourseAi::Toxicity::ToxicityClassification.new,
).classify!(chat_message)
end
end
end
@@ -0,0 +1,19 @@
# frozen_string_literal: true
module ::Jobs
class ToxicityClassifyPost < ::Jobs::Base
def execute(args)
return unless SiteSetting.ai_toxicity_enabled
post_id = args[:post_id]
return if post_id.blank?
post = Post.find_by(id: post_id, post_type: Post.types[:regular])
return if post&.raw.blank?
DiscourseAi::PostClassificator.new(
DiscourseAi::Toxicity::ToxicityClassification.new,
).classify!(post)
end
end
end
+37 -36
View File
@@ -20,9 +20,10 @@ module Jobs
rebaked = 0
vector = DiscourseAi::Embeddings::Vector.instance
vector_def = vector.vdef
table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
vector_rep =
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
table_name = vector_rep.topic_table_name
topics =
Topic
@@ -30,20 +31,21 @@ module Jobs
.where(archetype: Archetype.default)
.where(deleted_at: nil)
.order("topics.bumped_at DESC")
.limit(limit - rebaked)
rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
rebaked += populate_topic_embeddings(vector_rep, topics)
return if rebaked >= limit
# Then, we'll try to backfill embeddings for topics that have outdated
# embeddings, be it model or strategy version
relation = topics.where(<<~SQL).limit(limit - rebaked)
#{table_name}.model_version < #{vector_def.version}
relation = topics.where(<<~SQL)
#{table_name}.model_version < #{vector_rep.version}
OR
#{table_name}.strategy_version < #{vector_def.strategy_version}
#{table_name}.strategy_version < #{strategy.version}
SQL
rebaked += populate_topic_embeddings(vector, relation)
rebaked += populate_topic_embeddings(vector_rep, relation)
return if rebaked >= limit
@@ -55,30 +57,28 @@ module Jobs
.where("#{table_name}.updated_at < topics.updated_at")
.limit((limit - rebaked) / 10)
populate_topic_embeddings(vector, relation, force: true)
populate_topic_embeddings(vector_rep, relation, force: true)
return if rebaked >= limit
return unless SiteSetting.ai_embeddings_per_post_enabled
# Now for posts
table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
posts_batch_size = 1000
table_name = vector_rep.post_table_name
posts =
Post
.joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
.where(deleted_at: nil)
.where(post_type: Post.types[:regular])
.limit(limit - rebaked)
# First, we'll try to backfill embeddings for posts that have none
posts
.where("#{table_name}.post_id IS NULL")
.limit(limit - rebaked)
.pluck(:id)
.each_slice(posts_batch_size) do |batch|
vector.gen_bulk_reprensentations(Post.where(id: batch))
rebaked += batch.length
.find_in_batches do |batch|
vector_rep.gen_bulk_reprensentations(batch)
rebaked += batch.size
end
return if rebaked >= limit
@@ -87,47 +87,48 @@ module Jobs
# embeddings, be it model or strategy version
posts
.where(<<~SQL)
#{table_name}.model_version < #{vector_def.version}
#{table_name}.model_version < #{vector_rep.version}
OR
#{table_name}.strategy_version < #{vector_def.strategy_version}
#{table_name}.strategy_version < #{strategy.version}
SQL
.limit(limit - rebaked)
.pluck(:id)
.each_slice(posts_batch_size) do |batch|
vector.gen_bulk_reprensentations(Post.where(id: batch))
rebaked += batch.length
.find_in_batches do |batch|
vector_rep.gen_bulk_reprensentations(batch)
rebaked += batch.size
end
return if rebaked >= limit
# Finally, we'll try to backfill embeddings for posts that have outdated
# embeddings due to edits. Here we only do 10% of the limit
posts
.where("#{table_name}.updated_at < ?", 7.days.ago)
.order("random()")
.limit((limit - rebaked) / 10)
.pluck(:id)
.each_slice(posts_batch_size) do |batch|
vector.gen_bulk_reprensentations(Post.where(id: batch))
rebaked += batch.length
end
posts_batch_size = 1000
outdated_post_ids =
posts
.where("#{table_name}.updated_at < ?", 7.days.ago)
.order("random()")
.limit((limit - rebaked) / 10)
.pluck(:id)
outdated_post_ids.each_slice(posts_batch_size) do |batch|
vector_rep.gen_bulk_reprensentations(Post.where(id: batch))
rebaked += batch.length
end
rebaked
end
private
def populate_topic_embeddings(vector, topics, force: false)
def populate_topic_embeddings(vector_rep, topics, force: false)
done = 0
topics =
topics.where("#{DiscourseAi::Embeddings::Schema::TOPICS_TABLE}.topic_id IS NULL") if !force
topics = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL") if !force
ids = topics.pluck("topics.id")
batch_size = 1000
ids.each_slice(batch_size) do |batch|
vector.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
vector_rep.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
done += batch.length
end
-30
View File
@@ -1,30 +0,0 @@
# frozen_string_literal: true
module Jobs
class SentimentBackfill < ::Jobs::Scheduled
every 5.minutes
cluster_concurrency 1
def execute(_args)
return if !SiteSetting.ai_sentiment_enabled
base_budget = SiteSetting.ai_sentiment_backfill_maximum_posts_per_hour
return if base_budget.zero?
# Split budget in 12 intervals, but make sure is at least one.
#
# This is not exact as we don't have a way of tracking how many
# posts we classified in the current hour, but it's a good enough approximation.
limit_per_job = [base_budget, 12].max / 12
classificator = DiscourseAi::Sentiment::PostClassification.new
return if !classificator.has_classifiers?
posts =
DiscourseAi::Sentiment::PostClassification.backfill_query(
max_age_days: SiteSetting.ai_sentiment_backfill_post_max_age_days,
).limit(limit_per_job)
classificator.bulk_classify!(posts)
end
end
end
+9 -20
View File
@@ -12,27 +12,22 @@ module ::Jobs
system_user = Discourse.system_user
if SiteSetting.ai_summary_gists_enabled
gist_t = AiSummary.summary_types[:gist]
backfill_candidates(gist_t)
.limit(current_budget(gist_t))
.each do |topic|
DiscourseAi::Summarization.topic_gist(topic).force_summarize(system_user)
end
end
complete_t = AiSummary.summary_types[:complete]
backfill_candidates(complete_t)
.limit(current_budget(complete_t))
.each do |topic|
DiscourseAi::Summarization.topic_summary(topic).force_summarize(system_user)
end
return unless SiteSetting.ai_summary_gists_enabled
gist_t = AiSummary.summary_types[:gist]
backfill_candidates(gist_t)
.limit(current_budget(gist_t))
.each { |topic| DiscourseAi::Summarization.topic_gist(topic).force_summarize(system_user) }
end
def backfill_candidates(summary_type)
max_age_days = SiteSetting.ai_summary_backfill_topic_max_age_days
Topic
.where("topics.word_count >= ?", SiteSetting.ai_summary_backfill_minimum_word_count)
.joins(<<~SQL)
@@ -41,15 +36,9 @@ module ::Jobs
ais.target_type = 'Topic' AND
ais.summary_type = '#{summary_type}'
SQL
.where("topics.created_at > current_timestamp - INTERVAL '#{max_age_days.to_i} DAY'")
.where(
<<~SQL, # (1..1) gets stored ad (1..2).
ais.id IS NULL OR (
UPPER(ais.content_range) < topics.highest_post_number + 1
AND ais.created_at < (current_timestamp - INTERVAL '5 minutes')
)
SQL
)
"ais.id IS NULL OR UPPER(ais.content_range) < topics.highest_post_number + 1",
) # (1..1) gets stored ad (1..2).
.order("ais.created_at DESC NULLS FIRST, topics.last_posted_at DESC")
end

Some files were not shown because too many files have changed in this diff Show More