Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f10af7b167 |
+7
-7
@@ -1,18 +1,18 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
|
||||
queryParams = {
|
||||
export default DiscourseRoute.extend({
|
||||
queryParams: {
|
||||
llmTemplate: { refreshModel: true },
|
||||
};
|
||||
},
|
||||
|
||||
async model() {
|
||||
const record = this.store.createRecord("ai-llm");
|
||||
record.provider_params = {};
|
||||
return record;
|
||||
}
|
||||
},
|
||||
|
||||
setupController(controller, model) {
|
||||
super.setupController(controller, model);
|
||||
this._super(controller, model);
|
||||
controller.set(
|
||||
"allLlms",
|
||||
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
||||
@@ -21,5 +21,5 @@ export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
|
||||
"llmTemplate",
|
||||
this.paramsFor(this.routeName).llmTemplate || null
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
+5
-5
@@ -1,19 +1,19 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
|
||||
export default DiscourseRoute.extend({
|
||||
async model(params) {
|
||||
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
|
||||
const id = parseInt(params.id, 10);
|
||||
const record = allLlms.findBy("id", id);
|
||||
record.provider_params = record.provider_params || {};
|
||||
return record;
|
||||
}
|
||||
},
|
||||
|
||||
setupController(controller, model) {
|
||||
super.setupController(controller, model);
|
||||
this._super(controller, model);
|
||||
controller.set(
|
||||
"allLlms",
|
||||
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
+5
-5
@@ -1,7 +1,7 @@
|
||||
import { AUTO_GROUPS } from "discourse/lib/constants";
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
|
||||
export default DiscourseRoute.extend({
|
||||
async model() {
|
||||
const record = this.store.createRecord("ai-persona");
|
||||
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
|
||||
@@ -13,13 +13,13 @@ export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRou
|
||||
record.set("allow_personal_messages", true);
|
||||
record.set("tool_details", false);
|
||||
return record;
|
||||
}
|
||||
},
|
||||
|
||||
setupController(controller, model) {
|
||||
super.setupController(controller, model);
|
||||
this._super(controller, model);
|
||||
controller.set(
|
||||
"allPersonas",
|
||||
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
+5
-5
@@ -1,19 +1,19 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
|
||||
export default DiscourseRoute.extend({
|
||||
async model(params) {
|
||||
const allPersonas = this.modelFor(
|
||||
"adminPlugins.show.discourse-ai-personas"
|
||||
);
|
||||
const id = parseInt(params.id, 10);
|
||||
return allPersonas.findBy("id", id);
|
||||
}
|
||||
},
|
||||
|
||||
setupController(controller, model) {
|
||||
super.setupController(controller, model);
|
||||
this._super(controller, model);
|
||||
controller.set(
|
||||
"allPersonas",
|
||||
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -1,11 +0,0 @@
|
||||
import { service } from "@ember/service";
|
||||
import { ajax } from "discourse/lib/ajax";
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class DiscourseAiSpamRoute extends DiscourseRoute {
|
||||
@service store;
|
||||
|
||||
model() {
|
||||
return ajax("/admin/plugins/discourse-ai/ai-spam.json");
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
|
||||
export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
|
||||
async model(params) {
|
||||
const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
|
||||
const id = parseInt(params.id, 10);
|
||||
@@ -1,11 +0,0 @@
|
||||
import { service } from "@ember/service";
|
||||
import { ajax } from "discourse/lib/ajax";
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
|
||||
export default class DiscourseAiUsageRoute extends DiscourseRoute {
|
||||
@service store;
|
||||
|
||||
model() {
|
||||
return ajax("/admin/plugins/discourse-ai/ai-usage.json");
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
<AiSpam @model={{this.model}} />
|
||||
@@ -1 +0,0 @@
|
||||
<AiUsage @model={{this.model}} />
|
||||
@@ -30,10 +30,7 @@ module DiscourseAi
|
||||
}
|
||||
end
|
||||
|
||||
def new
|
||||
end
|
||||
|
||||
def edit
|
||||
def show
|
||||
llm_model = LlmModel.find(params[:id])
|
||||
render json: LlmModelSerializer.new(llm_model)
|
||||
end
|
||||
|
||||
@@ -5,7 +5,7 @@ module DiscourseAi
|
||||
class AiPersonasController < ::Admin::AdminController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
before_action :find_ai_persona, only: %i[edit update destroy create_user]
|
||||
before_action :find_ai_persona, only: %i[show update destroy create_user]
|
||||
|
||||
def index
|
||||
ai_personas =
|
||||
@@ -33,10 +33,7 @@ module DiscourseAi
|
||||
render json: { ai_personas: ai_personas, meta: { tools: tools, llms: llms } }
|
||||
end
|
||||
|
||||
def new
|
||||
end
|
||||
|
||||
def edit
|
||||
def show
|
||||
render json: LocalizedAiPersonaSerializer.new(@ai_persona)
|
||||
end
|
||||
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Admin
|
||||
class AiSpamController < ::Admin::AdminController
|
||||
requires_plugin "discourse-ai"
|
||||
|
||||
def show
|
||||
render json: AiSpamSerializer.new(spam_config, root: false)
|
||||
end
|
||||
|
||||
def update
|
||||
updated_params = {}
|
||||
if allowed_params.key?(:llm_model_id)
|
||||
llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
|
||||
if llm_model_id.to_i < 0 &&
|
||||
!SiteSetting.ai_spam_detection_model_allowed_seeded_models_map.include?(
|
||||
"custom:#{llm_model_id}",
|
||||
)
|
||||
return(
|
||||
render_json_error(
|
||||
I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"),
|
||||
status: 422,
|
||||
)
|
||||
)
|
||||
end
|
||||
end
|
||||
updated_params[:data] = {
|
||||
custom_instructions: allowed_params[:custom_instructions],
|
||||
} if allowed_params.key?(:custom_instructions)
|
||||
|
||||
if updated_params.present?
|
||||
# not using upsert cause we will not get the correct validation errors
|
||||
if AiModerationSetting.spam
|
||||
AiModerationSetting.spam.update!(updated_params)
|
||||
else
|
||||
AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
|
||||
end
|
||||
end
|
||||
|
||||
is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
|
||||
|
||||
if allowed_params.key?(:is_enabled)
|
||||
if is_enabled && !AiModerationSetting.spam&.llm_model_id
|
||||
return(
|
||||
render_json_error(
|
||||
I18n.t("discourse_ai.llm.configuration.must_select_model"),
|
||||
status: 422,
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
SiteSetting.ai_spam_detection_enabled = is_enabled
|
||||
end
|
||||
|
||||
render json: AiSpamSerializer.new(spam_config, root: false)
|
||||
end
|
||||
|
||||
def test
|
||||
url = params[:post_url].to_s
|
||||
post = nil
|
||||
|
||||
if url.match?(/^\d+$/)
|
||||
post_id = url.to_i
|
||||
post = Post.find_by(id: post_id)
|
||||
end
|
||||
|
||||
route = UrlHelper.rails_route_from_url(url) if !post
|
||||
|
||||
if route
|
||||
if route[:controller] == "topics"
|
||||
post_number = route[:post_number] || 1
|
||||
post = Post.with_deleted.find_by(post_number: post_number, topic_id: route[:topic_id])
|
||||
end
|
||||
end
|
||||
|
||||
raise Discourse::NotFound if !post
|
||||
|
||||
result =
|
||||
DiscourseAi::AiModeration::SpamScanner.test_post(
|
||||
post,
|
||||
custom_instructions: params[:custom_instructions],
|
||||
llm_id: params[:llm_id],
|
||||
)
|
||||
|
||||
render json: result
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def allowed_params
|
||||
params.permit(:is_enabled, :llm_model_id, :custom_instructions)
|
||||
end
|
||||
|
||||
def spam_config
|
||||
spam_config = {
|
||||
enabled: SiteSetting.ai_spam_detection_enabled,
|
||||
settings: AiModerationSetting.spam,
|
||||
}
|
||||
|
||||
spam_config[:stats] = DiscourseAi::AiModeration::SpamReport.generate(min_date: 1.week.ago)
|
||||
|
||||
if spam_config[:stats].scanned_count > 0
|
||||
spam_config[
|
||||
:flagging_username
|
||||
] = DiscourseAi::AiModeration::SpamScanner.flagging_user&.username
|
||||
end
|
||||
spam_config
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -5,17 +5,14 @@ module DiscourseAi
|
||||
class AiToolsController < ::Admin::AdminController
|
||||
requires_plugin ::DiscourseAi::PLUGIN_NAME
|
||||
|
||||
before_action :find_ai_tool, only: %i[test edit update destroy]
|
||||
before_action :find_ai_tool, only: %i[test show update destroy]
|
||||
|
||||
def index
|
||||
ai_tools = AiTool.all
|
||||
render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
|
||||
end
|
||||
|
||||
def new
|
||||
end
|
||||
|
||||
def edit
|
||||
def show
|
||||
render_serialized(@ai_tool, AiCustomToolSerializer)
|
||||
end
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module DiscourseAi
|
||||
module Admin
|
||||
class AiUsageController < ::Admin::AdminController
|
||||
requires_plugin "discourse-ai"
|
||||
|
||||
def show
|
||||
end
|
||||
|
||||
def report
|
||||
render json: AiUsageSerializer.new(create_report, root: false)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def create_report
|
||||
report =
|
||||
DiscourseAi::Completions::Report.new(
|
||||
start_date: params[:start_date]&.to_date || 30.days.ago,
|
||||
end_date: params[:end_date]&.to_date || Time.current,
|
||||
)
|
||||
|
||||
report = report.filter_by_feature(params[:feature]) if params[:feature].present?
|
||||
report = report.filter_by_model(params[:model]) if params[:model].present?
|
||||
report
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -19,33 +19,22 @@ module DiscourseAi
|
||||
raise Discourse::NotFound if !guardian.can_see?(post)
|
||||
end
|
||||
|
||||
name = artifact.name
|
||||
|
||||
if params[:version].present?
|
||||
artifact = artifact.versions.find_by(version_number: params[:version])
|
||||
raise Discourse::NotFound if !artifact
|
||||
end
|
||||
|
||||
js = artifact.js || ""
|
||||
if !js.match?(%r{\A\s*<script.*</script>}mi)
|
||||
mod = ""
|
||||
mod = " type=\"module\"" if js.match?(/\A\s*import.*/)
|
||||
js = "<script#{mod}>\n#{js}\n</script>"
|
||||
end
|
||||
# Prepare the inner (untrusted) HTML document
|
||||
untrusted_html = <<~HTML
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>#{ERB::Util.html_escape(name)}</title>
|
||||
<title>#{ERB::Util.html_escape(artifact.name)}</title>
|
||||
<style>
|
||||
#{artifact.css}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
#{artifact.html}
|
||||
#{js}
|
||||
<script>
|
||||
#{artifact.js}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
HTML
|
||||
@@ -56,7 +45,7 @@ module DiscourseAi
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>#{ERB::Util.html_escape(name)}</title>
|
||||
<title>#{ERB::Util.html_escape(artifact.name)}</title>
|
||||
<style>
|
||||
html, body, iframe {
|
||||
margin: 0;
|
||||
@@ -78,9 +67,7 @@ module DiscourseAi
|
||||
HTML
|
||||
|
||||
response.headers.delete("X-Frame-Options")
|
||||
response.headers[
|
||||
"Content-Security-Policy"
|
||||
] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' https://unpkg.com https://cdnjs.cloudflare.com https://ajax.googleapis.com https://cdn.jsdelivr.net;"
|
||||
response.headers["Content-Security-Policy"] = "script-src 'unsafe-inline';"
|
||||
response.headers["X-Robots-Tag"] = "noindex"
|
||||
|
||||
# Render the content
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Jobs
|
||||
class AiSpamScan < ::Jobs::Base
|
||||
def execute(args)
|
||||
return if !args[:post_id]
|
||||
post = Post.find_by(id: args[:post_id])
|
||||
return if !post
|
||||
|
||||
DiscourseAi::AiModeration::SpamScanner.perform_scan(post)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -18,7 +18,9 @@ module ::Jobs
|
||||
target = target_type.constantize.find_by(id: target_id)
|
||||
return if !target
|
||||
|
||||
vector_rep = DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation
|
||||
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
|
||||
|
||||
tokenizer = vector_rep.tokenizer
|
||||
chunk_tokens = target.rag_chunk_tokens
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Jobs
|
||||
class EvaluatePostUploads < ::Jobs::Base
|
||||
def execute(args)
|
||||
return unless SiteSetting.ai_nsfw_detection_enabled
|
||||
return if (post_id = args[:post_id]).blank?
|
||||
|
||||
post = Post.includes(:uploads).find_by_id(post_id)
|
||||
return if post.nil? || post.uploads.empty?
|
||||
|
||||
return if post.uploads.none? { |u| FileHelper.is_supported_image?(u.url) }
|
||||
|
||||
DiscourseAi::PostClassificator.new(DiscourseAi::Nsfw::Classification.new).classify!(post)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -14,7 +14,7 @@ module ::Jobs
|
||||
|
||||
summarizer = DiscourseAi::Summarization.topic_gist(topic)
|
||||
gist = summarizer.existing_summary
|
||||
return if gist.present? && (!gist.outdated || gist.created_at >= 5.minutes.ago)
|
||||
return if gist.present? && !gist.outdated
|
||||
|
||||
summarizer.force_summarize(Discourse.system_user)
|
||||
end
|
||||
|
||||
@@ -16,7 +16,11 @@ module Jobs
|
||||
return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
|
||||
return if post.raw.blank?
|
||||
|
||||
DiscourseAi::Embeddings::Vector.instance.generate_representation_from(target)
|
||||
strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
|
||||
|
||||
vector_rep.generate_representation_from(target)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -8,11 +8,13 @@ module ::Jobs
|
||||
def execute(args)
|
||||
return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
|
||||
|
||||
vector = DiscourseAi::Embeddings::Vector.instance
|
||||
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
|
||||
|
||||
# generate_representation_from checks compares the digest value to make sure
|
||||
# the embedding is only generated once per fragment unless something changes.
|
||||
fragments.map { |fragment| vector.generate_representation_from(fragment) }
|
||||
fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
|
||||
|
||||
last_fragment = fragments.last
|
||||
target = last_fragment.target
|
||||
|
||||
@@ -9,7 +9,9 @@ module ::Jobs
|
||||
post = Post.find_by(id: post_id, post_type: Post.types[:regular])
|
||||
return if post&.raw.blank?
|
||||
|
||||
DiscourseAi::Sentiment::PostClassification.new.classify!(post)
|
||||
DiscourseAi::PostClassificator.new(
|
||||
DiscourseAi::Sentiment::SentimentClassification.new,
|
||||
).classify!(post)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module ::Jobs
|
||||
class ToxicityClassifyChatMessage < ::Jobs::Base
|
||||
def execute(args)
|
||||
return unless SiteSetting.ai_toxicity_enabled
|
||||
|
||||
return if (chat_message_id = args[:chat_message_id]).blank?
|
||||
|
||||
chat_message = ::Chat::Message.find_by(id: chat_message_id)
|
||||
return if chat_message&.message.blank?
|
||||
|
||||
DiscourseAi::ChatMessageClassificator.new(
|
||||
DiscourseAi::Toxicity::ToxicityClassification.new,
|
||||
).classify!(chat_message)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,19 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module ::Jobs
|
||||
class ToxicityClassifyPost < ::Jobs::Base
|
||||
def execute(args)
|
||||
return unless SiteSetting.ai_toxicity_enabled
|
||||
|
||||
post_id = args[:post_id]
|
||||
return if post_id.blank?
|
||||
|
||||
post = Post.find_by(id: post_id, post_type: Post.types[:regular])
|
||||
return if post&.raw.blank?
|
||||
|
||||
DiscourseAi::PostClassificator.new(
|
||||
DiscourseAi::Toxicity::ToxicityClassification.new,
|
||||
).classify!(post)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -20,9 +20,10 @@ module Jobs
|
||||
|
||||
rebaked = 0
|
||||
|
||||
vector = DiscourseAi::Embeddings::Vector.instance
|
||||
vector_def = vector.vdef
|
||||
table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
|
||||
strategy = DiscourseAi::Embeddings::Strategies::Truncation.new
|
||||
vector_rep =
|
||||
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(strategy)
|
||||
table_name = vector_rep.topic_table_name
|
||||
|
||||
topics =
|
||||
Topic
|
||||
@@ -30,20 +31,21 @@ module Jobs
|
||||
.where(archetype: Archetype.default)
|
||||
.where(deleted_at: nil)
|
||||
.order("topics.bumped_at DESC")
|
||||
.limit(limit - rebaked)
|
||||
|
||||
rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
|
||||
rebaked += populate_topic_embeddings(vector_rep, topics)
|
||||
|
||||
return if rebaked >= limit
|
||||
|
||||
# Then, we'll try to backfill embeddings for topics that have outdated
|
||||
# embeddings, be it model or strategy version
|
||||
relation = topics.where(<<~SQL).limit(limit - rebaked)
|
||||
#{table_name}.model_version < #{vector_def.version}
|
||||
relation = topics.where(<<~SQL)
|
||||
#{table_name}.model_version < #{vector_rep.version}
|
||||
OR
|
||||
#{table_name}.strategy_version < #{vector_def.strategy_version}
|
||||
#{table_name}.strategy_version < #{strategy.version}
|
||||
SQL
|
||||
|
||||
rebaked += populate_topic_embeddings(vector, relation)
|
||||
rebaked += populate_topic_embeddings(vector_rep, relation)
|
||||
|
||||
return if rebaked >= limit
|
||||
|
||||
@@ -55,30 +57,28 @@ module Jobs
|
||||
.where("#{table_name}.updated_at < topics.updated_at")
|
||||
.limit((limit - rebaked) / 10)
|
||||
|
||||
populate_topic_embeddings(vector, relation, force: true)
|
||||
populate_topic_embeddings(vector_rep, relation, force: true)
|
||||
|
||||
return if rebaked >= limit
|
||||
|
||||
return unless SiteSetting.ai_embeddings_per_post_enabled
|
||||
|
||||
# Now for posts
|
||||
table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
|
||||
posts_batch_size = 1000
|
||||
table_name = vector_rep.post_table_name
|
||||
|
||||
posts =
|
||||
Post
|
||||
.joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
|
||||
.where(deleted_at: nil)
|
||||
.where(post_type: Post.types[:regular])
|
||||
.limit(limit - rebaked)
|
||||
|
||||
# First, we'll try to backfill embeddings for posts that have none
|
||||
posts
|
||||
.where("#{table_name}.post_id IS NULL")
|
||||
.limit(limit - rebaked)
|
||||
.pluck(:id)
|
||||
.each_slice(posts_batch_size) do |batch|
|
||||
vector.gen_bulk_reprensentations(Post.where(id: batch))
|
||||
rebaked += batch.length
|
||||
.find_in_batches do |batch|
|
||||
vector_rep.gen_bulk_reprensentations(batch)
|
||||
rebaked += batch.size
|
||||
end
|
||||
|
||||
return if rebaked >= limit
|
||||
@@ -87,47 +87,48 @@ module Jobs
|
||||
# embeddings, be it model or strategy version
|
||||
posts
|
||||
.where(<<~SQL)
|
||||
#{table_name}.model_version < #{vector_def.version}
|
||||
#{table_name}.model_version < #{vector_rep.version}
|
||||
OR
|
||||
#{table_name}.strategy_version < #{vector_def.strategy_version}
|
||||
#{table_name}.strategy_version < #{strategy.version}
|
||||
SQL
|
||||
.limit(limit - rebaked)
|
||||
.pluck(:id)
|
||||
.each_slice(posts_batch_size) do |batch|
|
||||
vector.gen_bulk_reprensentations(Post.where(id: batch))
|
||||
rebaked += batch.length
|
||||
.find_in_batches do |batch|
|
||||
vector_rep.gen_bulk_reprensentations(batch)
|
||||
rebaked += batch.size
|
||||
end
|
||||
|
||||
return if rebaked >= limit
|
||||
|
||||
# Finally, we'll try to backfill embeddings for posts that have outdated
|
||||
# embeddings due to edits. Here we only do 10% of the limit
|
||||
posts
|
||||
.where("#{table_name}.updated_at < ?", 7.days.ago)
|
||||
.order("random()")
|
||||
.limit((limit - rebaked) / 10)
|
||||
.pluck(:id)
|
||||
.each_slice(posts_batch_size) do |batch|
|
||||
vector.gen_bulk_reprensentations(Post.where(id: batch))
|
||||
rebaked += batch.length
|
||||
end
|
||||
posts_batch_size = 1000
|
||||
|
||||
outdated_post_ids =
|
||||
posts
|
||||
.where("#{table_name}.updated_at < ?", 7.days.ago)
|
||||
.order("random()")
|
||||
.limit((limit - rebaked) / 10)
|
||||
.pluck(:id)
|
||||
|
||||
outdated_post_ids.each_slice(posts_batch_size) do |batch|
|
||||
vector_rep.gen_bulk_reprensentations(Post.where(id: batch))
|
||||
rebaked += batch.length
|
||||
end
|
||||
|
||||
rebaked
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def populate_topic_embeddings(vector, topics, force: false)
|
||||
def populate_topic_embeddings(vector_rep, topics, force: false)
|
||||
done = 0
|
||||
|
||||
topics =
|
||||
topics.where("#{DiscourseAi::Embeddings::Schema::TOPICS_TABLE}.topic_id IS NULL") if !force
|
||||
topics = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL") if !force
|
||||
|
||||
ids = topics.pluck("topics.id")
|
||||
batch_size = 1000
|
||||
|
||||
ids.each_slice(batch_size) do |batch|
|
||||
vector.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
|
||||
vector_rep.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
|
||||
done += batch.length
|
||||
end
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Jobs
|
||||
class SentimentBackfill < ::Jobs::Scheduled
|
||||
every 5.minutes
|
||||
cluster_concurrency 1
|
||||
|
||||
def execute(_args)
|
||||
return if !SiteSetting.ai_sentiment_enabled
|
||||
|
||||
base_budget = SiteSetting.ai_sentiment_backfill_maximum_posts_per_hour
|
||||
return if base_budget.zero?
|
||||
# Split budget in 12 intervals, but make sure is at least one.
|
||||
#
|
||||
# This is not exact as we don't have a way of tracking how many
|
||||
# posts we classified in the current hour, but it's a good enough approximation.
|
||||
limit_per_job = [base_budget, 12].max / 12
|
||||
|
||||
classificator = DiscourseAi::Sentiment::PostClassification.new
|
||||
return if !classificator.has_classifiers?
|
||||
|
||||
posts =
|
||||
DiscourseAi::Sentiment::PostClassification.backfill_query(
|
||||
max_age_days: SiteSetting.ai_sentiment_backfill_post_max_age_days,
|
||||
).limit(limit_per_job)
|
||||
|
||||
classificator.bulk_classify!(posts)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -12,27 +12,22 @@ module ::Jobs
|
||||
|
||||
system_user = Discourse.system_user
|
||||
|
||||
if SiteSetting.ai_summary_gists_enabled
|
||||
gist_t = AiSummary.summary_types[:gist]
|
||||
|
||||
backfill_candidates(gist_t)
|
||||
.limit(current_budget(gist_t))
|
||||
.each do |topic|
|
||||
DiscourseAi::Summarization.topic_gist(topic).force_summarize(system_user)
|
||||
end
|
||||
end
|
||||
|
||||
complete_t = AiSummary.summary_types[:complete]
|
||||
backfill_candidates(complete_t)
|
||||
.limit(current_budget(complete_t))
|
||||
.each do |topic|
|
||||
DiscourseAi::Summarization.topic_summary(topic).force_summarize(system_user)
|
||||
end
|
||||
|
||||
return unless SiteSetting.ai_summary_gists_enabled
|
||||
|
||||
gist_t = AiSummary.summary_types[:gist]
|
||||
backfill_candidates(gist_t)
|
||||
.limit(current_budget(gist_t))
|
||||
.each { |topic| DiscourseAi::Summarization.topic_gist(topic).force_summarize(system_user) }
|
||||
end
|
||||
|
||||
def backfill_candidates(summary_type)
|
||||
max_age_days = SiteSetting.ai_summary_backfill_topic_max_age_days
|
||||
|
||||
Topic
|
||||
.where("topics.word_count >= ?", SiteSetting.ai_summary_backfill_minimum_word_count)
|
||||
.joins(<<~SQL)
|
||||
@@ -41,15 +36,9 @@ module ::Jobs
|
||||
ais.target_type = 'Topic' AND
|
||||
ais.summary_type = '#{summary_type}'
|
||||
SQL
|
||||
.where("topics.created_at > current_timestamp - INTERVAL '#{max_age_days.to_i} DAY'")
|
||||
.where(
|
||||
<<~SQL, # (1..1) gets stored ad (1..2).
|
||||
ais.id IS NULL OR (
|
||||
UPPER(ais.content_range) < topics.highest_post_number + 1
|
||||
AND ais.created_at < (current_timestamp - INTERVAL '5 minutes')
|
||||
)
|
||||
SQL
|
||||
)
|
||||
"ais.id IS NULL OR UPPER(ais.content_range) < topics.highest_post_number + 1",
|
||||
) # (1..1) gets stored ad (1..2).
|
||||
.order("ais.created_at DESC NULLS FIRST, topics.last_posted_at DESC")
|
||||
end
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user