Compare commits
67
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
593e8dc2a5 | ||
|
|
0abd4b1244 | ||
|
|
120a20c5cd | ||
|
|
0cb2c413ba | ||
|
|
80adefa1c1 | ||
|
|
bc0657f478 | ||
|
|
c980c34d77 | ||
|
|
6456a4f44a | ||
|
|
eacc4ed945 | ||
|
|
4a2369356f | ||
|
|
4980a4b2f7 | ||
|
|
54ebbbe769 | ||
|
|
8b33ba3d19 | ||
|
|
f1c7ee8624 | ||
|
|
4da033c667 | ||
|
|
dabef02919 | ||
|
|
fbcc8e493a | ||
|
|
6b7d7c1179 | ||
|
|
0d3e6b2726 | ||
|
|
be05e28679 | ||
|
|
2f7895bb91 | ||
|
|
7a094cde18 | ||
|
|
251628bfa1 | ||
|
|
ef07fcb308 | ||
|
|
ddf2bf7034 | ||
|
|
23193ee6f2 | ||
|
|
54f2d34ccb | ||
|
|
616b990894 | ||
|
|
6c25718a7f | ||
|
|
79021252e9 | ||
|
|
aeba72441d | ||
|
|
95762723de | ||
|
|
5fb1177f7b | ||
|
|
ea24328415 | ||
|
|
f8231d259b | ||
|
|
e54f2da1a5 | ||
|
|
1921e4e90d | ||
|
|
2cc8115b48 | ||
|
|
86cf4ccba7 | ||
|
|
2c78961bed | ||
|
|
8e00e036e1 | ||
|
|
d56ed53eb1 | ||
|
|
d83248cf68 | ||
|
|
872e43e849 | ||
|
|
2900d2c87d | ||
|
|
b68a4ca718 | ||
|
|
1a10680818 | ||
|
|
52c644798d | ||
|
|
2652716398 | ||
|
|
a0aec48606 | ||
|
|
f09e74c05e | ||
|
|
2c8d81827f | ||
|
|
a9afa04329 | ||
|
|
530a795d43 | ||
|
|
dcde94a393 | ||
|
|
3c91f374ac | ||
|
|
fb80d776d8 | ||
|
|
48d08dedd4 | ||
|
|
b10be23533 | ||
|
|
6b9c66054c | ||
|
|
3ae1e4eaf0 | ||
|
|
755b63f31f | ||
|
|
0d7f353284 | ||
|
|
4fb686a548 | ||
|
|
5026ab52d0 | ||
|
|
823e8ef490 | ||
|
|
f75b13c4fa |
@@ -1 +0,0 @@
|
|||||||
module.exports = require("@discourse/lint-configs/eslint");
|
|
||||||
+7
-7
@@ -1,18 +1,18 @@
|
|||||||
import DiscourseRoute from "discourse/routes/discourse";
|
import DiscourseRoute from "discourse/routes/discourse";
|
||||||
|
|
||||||
export default DiscourseRoute.extend({
|
export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
|
||||||
queryParams: {
|
queryParams = {
|
||||||
llmTemplate: { refreshModel: true },
|
llmTemplate: { refreshModel: true },
|
||||||
},
|
};
|
||||||
|
|
||||||
async model() {
|
async model() {
|
||||||
const record = this.store.createRecord("ai-llm");
|
const record = this.store.createRecord("ai-llm");
|
||||||
record.provider_params = {};
|
record.provider_params = {};
|
||||||
return record;
|
return record;
|
||||||
},
|
}
|
||||||
|
|
||||||
setupController(controller, model) {
|
setupController(controller, model) {
|
||||||
this._super(controller, model);
|
super.setupController(controller, model);
|
||||||
controller.set(
|
controller.set(
|
||||||
"allLlms",
|
"allLlms",
|
||||||
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
||||||
@@ -21,5 +21,5 @@ export default DiscourseRoute.extend({
|
|||||||
"llmTemplate",
|
"llmTemplate",
|
||||||
this.paramsFor(this.routeName).llmTemplate || null
|
this.paramsFor(this.routeName).llmTemplate || null
|
||||||
);
|
);
|
||||||
},
|
}
|
||||||
});
|
}
|
||||||
|
|||||||
+5
-5
@@ -1,19 +1,19 @@
|
|||||||
import DiscourseRoute from "discourse/routes/discourse";
|
import DiscourseRoute from "discourse/routes/discourse";
|
||||||
|
|
||||||
export default DiscourseRoute.extend({
|
export default class AdminPluginsShowDiscourseAiLlmsShow extends DiscourseRoute {
|
||||||
async model(params) {
|
async model(params) {
|
||||||
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
|
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
|
||||||
const id = parseInt(params.id, 10);
|
const id = parseInt(params.id, 10);
|
||||||
const record = allLlms.findBy("id", id);
|
const record = allLlms.findBy("id", id);
|
||||||
record.provider_params = record.provider_params || {};
|
record.provider_params = record.provider_params || {};
|
||||||
return record;
|
return record;
|
||||||
},
|
}
|
||||||
|
|
||||||
setupController(controller, model) {
|
setupController(controller, model) {
|
||||||
this._super(controller, model);
|
super.setupController(controller, model);
|
||||||
controller.set(
|
controller.set(
|
||||||
"allLlms",
|
"allLlms",
|
||||||
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
||||||
);
|
);
|
||||||
},
|
}
|
||||||
});
|
}
|
||||||
|
|||||||
+5
-5
@@ -1,7 +1,7 @@
|
|||||||
import { AUTO_GROUPS } from "discourse/lib/constants";
|
import { AUTO_GROUPS } from "discourse/lib/constants";
|
||||||
import DiscourseRoute from "discourse/routes/discourse";
|
import DiscourseRoute from "discourse/routes/discourse";
|
||||||
|
|
||||||
export default DiscourseRoute.extend({
|
export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
|
||||||
async model() {
|
async model() {
|
||||||
const record = this.store.createRecord("ai-persona");
|
const record = this.store.createRecord("ai-persona");
|
||||||
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
|
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
|
||||||
@@ -13,13 +13,13 @@ export default DiscourseRoute.extend({
|
|||||||
record.set("allow_personal_messages", true);
|
record.set("allow_personal_messages", true);
|
||||||
record.set("tool_details", false);
|
record.set("tool_details", false);
|
||||||
return record;
|
return record;
|
||||||
},
|
}
|
||||||
|
|
||||||
setupController(controller, model) {
|
setupController(controller, model) {
|
||||||
this._super(controller, model);
|
super.setupController(controller, model);
|
||||||
controller.set(
|
controller.set(
|
||||||
"allPersonas",
|
"allPersonas",
|
||||||
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
||||||
);
|
);
|
||||||
},
|
}
|
||||||
});
|
}
|
||||||
|
|||||||
+5
-5
@@ -1,19 +1,19 @@
|
|||||||
import DiscourseRoute from "discourse/routes/discourse";
|
import DiscourseRoute from "discourse/routes/discourse";
|
||||||
|
|
||||||
export default DiscourseRoute.extend({
|
export default class AdminPluginsShowDiscourseAiPersonasShow extends DiscourseRoute {
|
||||||
async model(params) {
|
async model(params) {
|
||||||
const allPersonas = this.modelFor(
|
const allPersonas = this.modelFor(
|
||||||
"adminPlugins.show.discourse-ai-personas"
|
"adminPlugins.show.discourse-ai-personas"
|
||||||
);
|
);
|
||||||
const id = parseInt(params.id, 10);
|
const id = parseInt(params.id, 10);
|
||||||
return allPersonas.findBy("id", id);
|
return allPersonas.findBy("id", id);
|
||||||
},
|
}
|
||||||
|
|
||||||
setupController(controller, model) {
|
setupController(controller, model) {
|
||||||
this._super(controller, model);
|
super.setupController(controller, model);
|
||||||
controller.set(
|
controller.set(
|
||||||
"allPersonas",
|
"allPersonas",
|
||||||
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
||||||
);
|
);
|
||||||
},
|
}
|
||||||
});
|
}
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import { service } from "@ember/service";
|
||||||
|
import { ajax } from "discourse/lib/ajax";
|
||||||
|
import DiscourseRoute from "discourse/routes/discourse";
|
||||||
|
|
||||||
|
export default class DiscourseAiUsageRoute extends DiscourseRoute {
|
||||||
|
@service store;
|
||||||
|
|
||||||
|
model() {
|
||||||
|
return ajax("/admin/plugins/discourse-ai/ai-usage.json");
|
||||||
|
}
|
||||||
|
}
|
||||||
+7
-5
@@ -1,5 +1,7 @@
|
|||||||
<AiToolEditor
|
<section class="ai-persona-tool-editor__current admin-detail pull-left">
|
||||||
@tools={{this.allTools}}
|
<AiToolEditor
|
||||||
@model={{this.model}}
|
@tools={{this.allTools}}
|
||||||
@presets={{this.presets}}
|
@model={{this.model}}
|
||||||
/>
|
@presets={{this.presets}}
|
||||||
|
/>
|
||||||
|
</section>
|
||||||
+7
-5
@@ -1,5 +1,7 @@
|
|||||||
<AiToolEditor
|
<section class="ai-persona-tool-editor__current admin-detail pull-left">
|
||||||
@tools={{this.allTools}}
|
<AiToolEditor
|
||||||
@model={{this.model}}
|
@tools={{this.allTools}}
|
||||||
@presets={{this.presets}}
|
@model={{this.model}}
|
||||||
/>
|
@presets={{this.presets}}
|
||||||
|
/>
|
||||||
|
</section>
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
<AiUsage @model={{this.model}} />
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
module DiscourseAi
|
||||||
|
module Admin
|
||||||
|
class AiUsageController < ::Admin::AdminController
|
||||||
|
requires_plugin "discourse-ai"
|
||||||
|
|
||||||
|
def show
|
||||||
|
render json: AiUsageSerializer.new(create_report, root: false)
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def create_report
|
||||||
|
report =
|
||||||
|
DiscourseAi::Completions::Report.new(
|
||||||
|
start_date: params[:start_date]&.to_date || 30.days.ago,
|
||||||
|
end_date: params[:end_date]&.to_date || Time.current,
|
||||||
|
)
|
||||||
|
|
||||||
|
report = report.filter_by_feature(params[:feature]) if params[:feature].present?
|
||||||
|
report = report.filter_by_model(params[:model]) if params[:model].present?
|
||||||
|
report
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
module DiscourseAi
|
||||||
|
module AiBot
|
||||||
|
class ArtifactsController < ApplicationController
|
||||||
|
requires_plugin DiscourseAi::PLUGIN_NAME
|
||||||
|
before_action :require_site_settings!
|
||||||
|
|
||||||
|
skip_before_action :preload_json, :check_xhr, only: %i[show]
|
||||||
|
|
||||||
|
def show
|
||||||
|
artifact = AiArtifact.find(params[:id])
|
||||||
|
|
||||||
|
post = Post.find_by(id: artifact.post_id)
|
||||||
|
if artifact.metadata&.dig("public")
|
||||||
|
# no guardian needed
|
||||||
|
else
|
||||||
|
raise Discourse::NotFound if !post&.topic&.private_message?
|
||||||
|
raise Discourse::NotFound if !guardian.can_see?(post)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Prepare the inner (untrusted) HTML document
|
||||||
|
untrusted_html = <<~HTML
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<title>#{ERB::Util.html_escape(artifact.name)}</title>
|
||||||
|
<style>
|
||||||
|
#{artifact.css}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
#{artifact.html}
|
||||||
|
<script>
|
||||||
|
#{artifact.js}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
HTML
|
||||||
|
|
||||||
|
# Prepare the outer (trusted) HTML document
|
||||||
|
trusted_html = <<~HTML
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<title>#{ERB::Util.html_escape(artifact.name)}</title>
|
||||||
|
<style>
|
||||||
|
html, body, iframe {
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
border: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
iframe {
|
||||||
|
overflow: auto;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<iframe sandbox="allow-scripts allow-forms" height="100%" width="100%" srcdoc="#{ERB::Util.html_escape(untrusted_html)}" frameborder="0"></iframe>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
HTML
|
||||||
|
|
||||||
|
response.headers.delete("X-Frame-Options")
|
||||||
|
response.headers["Content-Security-Policy"] = "script-src 'unsafe-inline';"
|
||||||
|
response.headers["X-Robots-Tag"] = "noindex"
|
||||||
|
|
||||||
|
# Render the content
|
||||||
|
render html: trusted_html.html_safe, layout: false, content_type: "text/html"
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def require_site_settings!
|
||||||
|
if !SiteSetting.discourse_ai_enabled ||
|
||||||
|
!SiteSetting.ai_artifact_security.in?(%w[lax strict])
|
||||||
|
raise Discourse::NotFound
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -7,7 +7,8 @@ module DiscourseAi
|
|||||||
requires_login only: %i[create update destroy]
|
requires_login only: %i[create update destroy]
|
||||||
before_action :require_site_settings!
|
before_action :require_site_settings!
|
||||||
|
|
||||||
skip_before_action :preload_json, :check_xhr, only: %i[show]
|
skip_before_action :preload_json, :check_xhr, only: %i[show asset]
|
||||||
|
skip_before_action :verify_authenticity_token, only: ["asset"]
|
||||||
|
|
||||||
def create
|
def create
|
||||||
ensure_allowed_create!
|
ensure_allowed_create!
|
||||||
@@ -50,6 +51,30 @@ module DiscourseAi
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def asset
|
||||||
|
no_cookies
|
||||||
|
|
||||||
|
name = params[:name]
|
||||||
|
path, content_type =
|
||||||
|
if name == "share"
|
||||||
|
%w[share.css text/css]
|
||||||
|
elsif name == "highlight"
|
||||||
|
%w[highlight.min.js application/javascript]
|
||||||
|
else
|
||||||
|
raise Discourse::NotFound
|
||||||
|
end
|
||||||
|
|
||||||
|
content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
|
||||||
|
|
||||||
|
# note, path contains a ":version" which automatically busts the cache
|
||||||
|
# based on file content, so this is safe
|
||||||
|
response.headers["Last-Modified"] = 10.years.ago.httpdate
|
||||||
|
response.headers["Content-Length"] = content.bytesize.to_s
|
||||||
|
immutable_for 1.year
|
||||||
|
|
||||||
|
render plain: content, disposition: :nil, content_type: content_type
|
||||||
|
end
|
||||||
|
|
||||||
def preview
|
def preview
|
||||||
ensure_allowed_preview!
|
ensure_allowed_preview!
|
||||||
data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
|
data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ module DiscourseAi
|
|||||||
prompt.custom_instruction = params[:custom_prompt]
|
prompt.custom_instruction = params[:custom_prompt]
|
||||||
end
|
end
|
||||||
|
|
||||||
suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
|
return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
|
||||||
|
|
||||||
hijack do
|
hijack do
|
||||||
render json:
|
render json:
|
||||||
|
|||||||
@@ -2,13 +2,29 @@
|
|||||||
module DiscourseAi
|
module DiscourseAi
|
||||||
module AiBot
|
module AiBot
|
||||||
module SharedAiConversationsHelper
|
module SharedAiConversationsHelper
|
||||||
# bump up version when assets change
|
# keeping it here for caching
|
||||||
# long term we may want to change this cause it is hard to remember
|
def self.share_asset_url(asset_name)
|
||||||
# to bump versions, but for now this does the job
|
if !%w[share.css highlight.js].include?(asset_name)
|
||||||
VERSION = "1"
|
raise StandardError, "unknown asset type #{asset_name}"
|
||||||
|
end
|
||||||
|
|
||||||
def share_asset_url(short_path)
|
@urls ||= {}
|
||||||
::UrlHelper.absolute("/plugins/discourse-ai/ai-share/#{short_path}?#{VERSION}")
|
url = @urls[asset_name]
|
||||||
|
return url if url
|
||||||
|
|
||||||
|
path = asset_name
|
||||||
|
path = "highlight.min.js" if asset_name == "highlight.js"
|
||||||
|
|
||||||
|
content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
|
||||||
|
sha1 = Digest::SHA1.hexdigest(content)
|
||||||
|
|
||||||
|
url = "/discourse-ai/ai-bot/shared-ai-conversations/asset/#{sha1}/#{asset_name}"
|
||||||
|
|
||||||
|
@urls[asset_name] = GlobalPath.cdn_path(url)
|
||||||
|
end
|
||||||
|
|
||||||
|
def share_asset_url(asset_name)
|
||||||
|
DiscourseAi::AiBot::SharedAiConversationsHelper.share_asset_url(asset_name)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
+3
-6
@@ -1,23 +1,20 @@
|
|||||||
# frozen_string_literal: true
|
# frozen_string_literal: true
|
||||||
|
|
||||||
module ::Jobs
|
module ::Jobs
|
||||||
class UpdateHotTopicGist < ::Jobs::Base
|
class FastTrackTopicGist < ::Jobs::Base
|
||||||
sidekiq_options retry: false
|
sidekiq_options retry: false
|
||||||
|
|
||||||
def execute(args)
|
def execute(args)
|
||||||
return if !SiteSetting.discourse_ai_enabled
|
return if !SiteSetting.discourse_ai_enabled
|
||||||
return if !SiteSetting.ai_summarization_enabled
|
return if !SiteSetting.ai_summarization_enabled
|
||||||
return if SiteSetting.ai_summarize_max_hot_topics_gists_per_batch.zero?
|
return if !SiteSetting.ai_summary_gists_enabled
|
||||||
|
|
||||||
topic = Topic.find_by(id: args[:topic_id])
|
topic = Topic.find_by(id: args[:topic_id])
|
||||||
return if topic.blank?
|
return if topic.blank?
|
||||||
|
|
||||||
return if !TopicHotScore.where(topic: topic).exists?
|
|
||||||
|
|
||||||
summarizer = DiscourseAi::Summarization.topic_gist(topic)
|
summarizer = DiscourseAi::Summarization.topic_gist(topic)
|
||||||
gist = summarizer.existing_summary
|
gist = summarizer.existing_summary
|
||||||
return if gist.blank?
|
return if gist.present? && !gist.outdated
|
||||||
return if !gist.outdated
|
|
||||||
|
|
||||||
summarizer.force_summarize(Discourse.system_user)
|
summarizer.force_summarize(Discourse.system_user)
|
||||||
end
|
end
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
# frozen_string_literal: true
|
|
||||||
|
|
||||||
module ::Jobs
|
|
||||||
class HotTopicsGistBatch < ::Jobs::Base
|
|
||||||
def execute(_args)
|
|
||||||
return if !SiteSetting.discourse_ai_enabled
|
|
||||||
return if !SiteSetting.ai_summarization_enabled
|
|
||||||
return if SiteSetting.ai_summarize_max_hot_topics_gists_per_batch.zero?
|
|
||||||
|
|
||||||
Topic
|
|
||||||
.joins("JOIN topic_hot_scores on topics.id = topic_hot_scores.topic_id")
|
|
||||||
.order("topic_hot_scores.score DESC")
|
|
||||||
.limit(SiteSetting.ai_summarize_max_hot_topics_gists_per_batch)
|
|
||||||
.each do |topic|
|
|
||||||
summarizer = DiscourseAi::Summarization.topic_gist(topic)
|
|
||||||
gist = summarizer.existing_summary
|
|
||||||
|
|
||||||
summarizer.force_summarize(Discourse.system_user) if gist.blank? || gist.outdated
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
@@ -9,9 +9,7 @@ module ::Jobs
|
|||||||
post = Post.find_by(id: post_id, post_type: Post.types[:regular])
|
post = Post.find_by(id: post_id, post_type: Post.types[:regular])
|
||||||
return if post&.raw.blank?
|
return if post&.raw.blank?
|
||||||
|
|
||||||
DiscourseAi::PostClassificator.new(
|
DiscourseAi::Sentiment::PostClassification.new.classify!(post)
|
||||||
DiscourseAi::Sentiment::SentimentClassification.new,
|
|
||||||
).classify!(post)
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -70,14 +70,15 @@ module Jobs
|
|||||||
Post
|
Post
|
||||||
.joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
|
.joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
|
||||||
.where(deleted_at: nil)
|
.where(deleted_at: nil)
|
||||||
|
.where(post_type: Post.types[:regular])
|
||||||
.limit(limit - rebaked)
|
.limit(limit - rebaked)
|
||||||
|
|
||||||
# First, we'll try to backfill embeddings for posts that have none
|
# First, we'll try to backfill embeddings for posts that have none
|
||||||
posts
|
posts
|
||||||
.where("#{table_name}.post_id IS NULL")
|
.where("#{table_name}.post_id IS NULL")
|
||||||
.find_each do |t|
|
.find_in_batches do |batch|
|
||||||
vector_rep.generate_representation_from(t)
|
vector_rep.gen_bulk_reprensentations(batch)
|
||||||
rebaked += 1
|
rebaked += batch.size
|
||||||
end
|
end
|
||||||
|
|
||||||
return if rebaked >= limit
|
return if rebaked >= limit
|
||||||
@@ -90,24 +91,28 @@ module Jobs
|
|||||||
OR
|
OR
|
||||||
#{table_name}.strategy_version < #{strategy.version}
|
#{table_name}.strategy_version < #{strategy.version}
|
||||||
SQL
|
SQL
|
||||||
.find_each do |t|
|
.find_in_batches do |batch|
|
||||||
vector_rep.generate_representation_from(t)
|
vector_rep.gen_bulk_reprensentations(batch)
|
||||||
rebaked += 1
|
rebaked += batch.size
|
||||||
end
|
end
|
||||||
|
|
||||||
return if rebaked >= limit
|
return if rebaked >= limit
|
||||||
|
|
||||||
# Finally, we'll try to backfill embeddings for posts that have outdated
|
# Finally, we'll try to backfill embeddings for posts that have outdated
|
||||||
# embeddings due to edits. Here we only do 10% of the limit
|
# embeddings due to edits. Here we only do 10% of the limit
|
||||||
posts
|
posts_batch_size = 1000
|
||||||
.where("#{table_name}.updated_at < ?", 7.days.ago)
|
|
||||||
.order("random()")
|
outdated_post_ids =
|
||||||
.limit((limit - rebaked) / 10)
|
posts
|
||||||
.pluck(:id)
|
.where("#{table_name}.updated_at < ?", 7.days.ago)
|
||||||
.each do |id|
|
.order("random()")
|
||||||
vector_rep.generate_representation_from(Post.find_by(id: id))
|
.limit((limit - rebaked) / 10)
|
||||||
rebaked += 1
|
.pluck(:id)
|
||||||
end
|
|
||||||
|
outdated_post_ids.each_slice(posts_batch_size) do |batch|
|
||||||
|
vector_rep.gen_bulk_reprensentations(Post.where(id: batch))
|
||||||
|
rebaked += batch.length
|
||||||
|
end
|
||||||
|
|
||||||
rebaked
|
rebaked
|
||||||
end
|
end
|
||||||
@@ -120,14 +125,13 @@ module Jobs
|
|||||||
topics = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL") if !force
|
topics = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL") if !force
|
||||||
|
|
||||||
ids = topics.pluck("topics.id")
|
ids = topics.pluck("topics.id")
|
||||||
|
batch_size = 1000
|
||||||
|
|
||||||
ids.each do |id|
|
ids.each_slice(batch_size) do |batch|
|
||||||
topic = Topic.find_by(id: id)
|
vector_rep.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
|
||||||
if topic
|
done += batch.length
|
||||||
vector_rep.generate_representation_from(topic)
|
|
||||||
done += 1
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
||||||
done
|
done
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ module ::Jobs
|
|||||||
DiscourseAi::Summarization.topic_summary(topic).force_summarize(system_user)
|
DiscourseAi::Summarization.topic_summary(topic).force_summarize(system_user)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
return unless SiteSetting.ai_summary_gists_enabled
|
||||||
|
|
||||||
gist_t = AiSummary.summary_types[:gist]
|
gist_t = AiSummary.summary_types[:gist]
|
||||||
backfill_candidates(gist_t)
|
backfill_candidates(gist_t)
|
||||||
.limit(current_budget(gist_t))
|
.limit(current_budget(gist_t))
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
class AiApiAuditLog < ActiveRecord::Base
|
class AiApiAuditLog < ActiveRecord::Base
|
||||||
belongs_to :post
|
belongs_to :post
|
||||||
belongs_to :topic
|
belongs_to :topic
|
||||||
|
belongs_to :user
|
||||||
|
|
||||||
module Provider
|
module Provider
|
||||||
OpenAI = 1
|
OpenAI = 1
|
||||||
@@ -13,6 +14,7 @@ class AiApiAuditLog < ActiveRecord::Base
|
|||||||
Cohere = 6
|
Cohere = 6
|
||||||
Ollama = 7
|
Ollama = 7
|
||||||
SambaNova = 8
|
SambaNova = 8
|
||||||
|
Mistral = 9
|
||||||
end
|
end
|
||||||
|
|
||||||
def next_log_id
|
def next_log_id
|
||||||
@@ -42,3 +44,10 @@ end
|
|||||||
# feature_name :string(255)
|
# feature_name :string(255)
|
||||||
# language_model :string(255)
|
# language_model :string(255)
|
||||||
# feature_context :jsonb
|
# feature_context :jsonb
|
||||||
|
# cached_tokens :integer
|
||||||
|
#
|
||||||
|
# Indexes
|
||||||
|
#
|
||||||
|
# index_ai_api_audit_logs_on_created_at_and_feature_name (created_at,feature_name)
|
||||||
|
# index_ai_api_audit_logs_on_created_at_and_language_model (created_at,language_model)
|
||||||
|
#
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
class AiArtifact < ActiveRecord::Base
|
||||||
|
belongs_to :user
|
||||||
|
belongs_to :post
|
||||||
|
validates :html, length: { maximum: 65_535 }
|
||||||
|
validates :css, length: { maximum: 65_535 }
|
||||||
|
validates :js, length: { maximum: 65_535 }
|
||||||
|
|
||||||
|
def self.iframe_for(id)
|
||||||
|
<<~HTML
|
||||||
|
<div class='ai-artifact'>
|
||||||
|
<iframe src='#{url(id)}' frameborder="0" height="100%" width="100%"></iframe>
|
||||||
|
<a href='#{url(id)}' target='_blank'>#{I18n.t("discourse_ai.ai_artifact.link")}</a>
|
||||||
|
</div>
|
||||||
|
HTML
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.url(id)
|
||||||
|
Discourse.base_url + "/discourse-ai/ai-bot/artifacts/#{id}"
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.share_publicly(id:, post:)
|
||||||
|
artifact = AiArtifact.find_by(id: id)
|
||||||
|
artifact.update!(metadata: { public: true }) if artifact&.post&.topic&.id == post.topic.id
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.unshare_publicly(id:)
|
||||||
|
artifact = AiArtifact.find_by(id: id)
|
||||||
|
artifact&.update!(metadata: { public: false })
|
||||||
|
end
|
||||||
|
|
||||||
|
def url
|
||||||
|
self.class.url(id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# == Schema Information
|
||||||
|
#
|
||||||
|
# Table name: ai_artifacts
|
||||||
|
#
|
||||||
|
# id :bigint not null, primary key
|
||||||
|
# user_id :integer not null
|
||||||
|
# post_id :integer not null
|
||||||
|
# name :string(255) not null
|
||||||
|
# html :string(65535)
|
||||||
|
# css :string(65535)
|
||||||
|
# js :string(65535)
|
||||||
|
# metadata :jsonb
|
||||||
|
# created_at :datetime not null
|
||||||
|
# updated_at :datetime not null
|
||||||
|
#
|
||||||
+19
-10
@@ -9,15 +9,23 @@ class AiSummary < ActiveRecord::Base
|
|||||||
def self.store!(strategy, llm_model, summary, og_content, human:)
|
def self.store!(strategy, llm_model, summary, og_content, human:)
|
||||||
content_ids = og_content.map { |c| c[:id] }
|
content_ids = og_content.map { |c| c[:id] }
|
||||||
|
|
||||||
AiSummary.create!(
|
AiSummary
|
||||||
target: strategy.target,
|
.upsert(
|
||||||
algorithm: llm_model.name,
|
{
|
||||||
content_range: (content_ids.first..content_ids.last),
|
target_id: strategy.target.id,
|
||||||
summarized_text: summary,
|
target_type: strategy.target.class.name,
|
||||||
original_content_sha: build_sha(content_ids.join),
|
algorithm: llm_model.name,
|
||||||
summary_type: strategy.type,
|
content_range: (content_ids.first..content_ids.last),
|
||||||
origin: !!human ? origins[:human] : origins[:system],
|
summarized_text: summary,
|
||||||
)
|
original_content_sha: build_sha(content_ids.join),
|
||||||
|
summary_type: strategy.type,
|
||||||
|
origin: !!human ? origins[:human] : origins[:system],
|
||||||
|
},
|
||||||
|
unique_by: %i[target_id target_type summary_type],
|
||||||
|
update_only: %i[summarized_text original_content_sha algorithm origin content_range],
|
||||||
|
)
|
||||||
|
.first
|
||||||
|
.then { AiSummary.find_by(id: _1["id"]) }
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.build_sha(joined_ids)
|
def self.build_sha(joined_ids)
|
||||||
@@ -51,5 +59,6 @@ end
|
|||||||
#
|
#
|
||||||
# Indexes
|
# Indexes
|
||||||
#
|
#
|
||||||
# index_ai_summaries_on_target_type_and_target_id (target_type,target_id)
|
# idx_on_target_id_target_type_summary_type_3355609fbb (target_id,target_type,summary_type) UNIQUE
|
||||||
|
# index_ai_summaries_on_target_type_and_target_id (target_type,target_id)
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ class CompletionPrompt < ActiveRecord::Base
|
|||||||
CUSTOM_PROMPT = -305
|
CUSTOM_PROMPT = -305
|
||||||
EXPLAIN = -306
|
EXPLAIN = -306
|
||||||
ILLUSTRATE_POST = -308
|
ILLUSTRATE_POST = -308
|
||||||
|
DETECT_TEXT_LOCALE = -309
|
||||||
|
|
||||||
enum :prompt_type, { text: 0, list: 1, diff: 2 }
|
enum :prompt_type, { text: 0, list: 1, diff: 2 }
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,16 @@ class LlmModel < ActiveRecord::Base
|
|||||||
},
|
},
|
||||||
open_ai: {
|
open_ai: {
|
||||||
organization: :text,
|
organization: :text,
|
||||||
|
disable_native_tools: :checkbox,
|
||||||
|
},
|
||||||
|
mistral: {
|
||||||
|
disable_native_tools: :checkbox,
|
||||||
|
},
|
||||||
|
google: {
|
||||||
|
disable_native_tools: :checkbox,
|
||||||
|
},
|
||||||
|
azure: {
|
||||||
|
disable_native_tools: :checkbox,
|
||||||
},
|
},
|
||||||
hugging_face: {
|
hugging_face: {
|
||||||
disable_system_prompt: :checkbox,
|
disable_system_prompt: :checkbox,
|
||||||
|
|||||||
@@ -172,6 +172,7 @@ end
|
|||||||
# updated_at :datetime not null
|
# updated_at :datetime not null
|
||||||
# force_review :boolean default(FALSE), not null
|
# force_review :boolean default(FALSE), not null
|
||||||
# reject_reason :text
|
# reject_reason :text
|
||||||
|
# potentially_illegal :boolean default(FALSE)
|
||||||
#
|
#
|
||||||
# Indexes
|
# Indexes
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -229,6 +229,7 @@ end
|
|||||||
# updated_at :datetime not null
|
# updated_at :datetime not null
|
||||||
# force_review :boolean default(FALSE), not null
|
# force_review :boolean default(FALSE), not null
|
||||||
# reject_reason :text
|
# reject_reason :text
|
||||||
|
# potentially_illegal :boolean default(FALSE)
|
||||||
#
|
#
|
||||||
# Indexes
|
# Indexes
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -34,6 +34,12 @@ class SharedAiConversation < ActiveRecord::Base
|
|||||||
|
|
||||||
def self.destroy_conversation(conversation)
|
def self.destroy_conversation(conversation)
|
||||||
conversation.destroy
|
conversation.destroy
|
||||||
|
|
||||||
|
maybe_topic = conversation.target
|
||||||
|
if maybe_topic.is_a?(Topic)
|
||||||
|
AiArtifact.where(post: maybe_topic.posts).update_all(metadata: { public: false })
|
||||||
|
end
|
||||||
|
|
||||||
::Jobs.enqueue(
|
::Jobs.enqueue(
|
||||||
:shared_conversation_adjust_upload_security,
|
:shared_conversation_adjust_upload_security,
|
||||||
target_id: conversation.target_id,
|
target_id: conversation.target_id,
|
||||||
@@ -165,7 +171,7 @@ class SharedAiConversation < ActiveRecord::Base
|
|||||||
id: post.id,
|
id: post.id,
|
||||||
user_id: post.user_id,
|
user_id: post.user_id,
|
||||||
created_at: post.created_at,
|
created_at: post.created_at,
|
||||||
cooked: post.cooked,
|
cooked: cook_artifacts(post),
|
||||||
}
|
}
|
||||||
|
|
||||||
mapped[:persona] = persona if ai_bot_participant&.id == post.user_id
|
mapped[:persona] = persona if ai_bot_participant&.id == post.user_id
|
||||||
@@ -175,6 +181,24 @@ class SharedAiConversation < ActiveRecord::Base
|
|||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def self.cook_artifacts(post)
|
||||||
|
html = post.cooked
|
||||||
|
return html if !%w[lax strict].include?(SiteSetting.ai_artifact_security)
|
||||||
|
|
||||||
|
doc = Nokogiri::HTML5.fragment(html)
|
||||||
|
doc
|
||||||
|
.css("div.ai-artifact")
|
||||||
|
.each do |node|
|
||||||
|
id = node["data-ai-artifact-id"].to_i
|
||||||
|
if id > 0
|
||||||
|
AiArtifact.share_publicly(id: id, post: post)
|
||||||
|
node.replace(AiArtifact.iframe_for(id))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
doc.to_s
|
||||||
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def populate_user_info!(posts)
|
def populate_user_info!(posts)
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
class AiUsageSerializer < ApplicationSerializer
|
||||||
|
attributes :data, :features, :models, :users, :summary, :period
|
||||||
|
|
||||||
|
def data
|
||||||
|
object.tokens_by_period.as_json(
|
||||||
|
only: %i[period total_tokens total_cached_tokens total_request_tokens total_response_tokens],
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def period
|
||||||
|
object.guess_period
|
||||||
|
end
|
||||||
|
|
||||||
|
def features
|
||||||
|
object.feature_breakdown.as_json(
|
||||||
|
only: %i[
|
||||||
|
feature_name
|
||||||
|
usage_count
|
||||||
|
total_tokens
|
||||||
|
total_cached_tokens
|
||||||
|
total_request_tokens
|
||||||
|
total_response_tokens
|
||||||
|
],
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def models
|
||||||
|
object.model_breakdown.as_json(
|
||||||
|
only: %i[
|
||||||
|
llm
|
||||||
|
usage_count
|
||||||
|
total_tokens
|
||||||
|
total_cached_tokens
|
||||||
|
total_request_tokens
|
||||||
|
total_response_tokens
|
||||||
|
],
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def users
|
||||||
|
object.user_breakdown.map do |user|
|
||||||
|
{
|
||||||
|
avatar_template: User.avatar_template(user.username, user.uploaded_avatar_id),
|
||||||
|
username: user.username,
|
||||||
|
usage_count: user.usage_count,
|
||||||
|
total_tokens: user.total_tokens,
|
||||||
|
total_cached_tokens: user.total_cached_tokens,
|
||||||
|
total_request_tokens: user.total_request_tokens,
|
||||||
|
total_response_tokens: user.total_response_tokens,
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def summary
|
||||||
|
{
|
||||||
|
total_tokens: object.total_tokens,
|
||||||
|
total_cached_tokens: object.total_cached_tokens,
|
||||||
|
total_request_tokens: object.total_request_tokens,
|
||||||
|
total_response_tokens: object.total_response_tokens,
|
||||||
|
total_requests: object.total_requests,
|
||||||
|
date_range: {
|
||||||
|
start: object.start_date,
|
||||||
|
end: object.end_date,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -10,7 +10,7 @@
|
|||||||
<meta name="twitter:title" content="<%= I18n.t("discourse_ai.share_ai.title", title: @shared_conversation.title, site_name: SiteSetting.title) %>">
|
<meta name="twitter:title" content="<%= I18n.t("discourse_ai.share_ai.title", title: @shared_conversation.title, site_name: SiteSetting.title) %>">
|
||||||
<meta name="twitter:description" content="<%= @shared_conversation.formatted_excerpt %>">
|
<meta name="twitter:description" content="<%= @shared_conversation.formatted_excerpt %>">
|
||||||
<meta name="viewport" content="width=device-width">
|
<meta name="viewport" content="width=device-width">
|
||||||
<link rel="stylesheet" href="<%= ::UrlHelper.absolute("/plugins/discourse-ai/ai-share/share.css") %>">
|
<link rel="stylesheet" href="<%= share_asset_url("share.css") %>">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header class="site-header">
|
<header class="site-header">
|
||||||
@@ -54,7 +54,7 @@
|
|||||||
</article>
|
</article>
|
||||||
<% end %>
|
<% end %>
|
||||||
</section>
|
</section>
|
||||||
<script src="<%= share_asset_url("highlight.min.js") %>" nonce="<%= csp_nonce_placeholder %>" ></script>
|
<script src="<%= share_asset_url("highlight.js") %>" nonce="<%= csp_nonce_placeholder %>" ></script>
|
||||||
<script nonce="<%= csp_nonce_placeholder %>">
|
<script nonce="<%= csp_nonce_placeholder %>">
|
||||||
document.querySelectorAll('pre code').forEach((el) => {
|
document.querySelectorAll('pre code').forEach((el) => {
|
||||||
hljs.highlightElement(el);
|
hljs.highlightElement(el);
|
||||||
|
|||||||
@@ -18,5 +18,6 @@ export default {
|
|||||||
this.route("new");
|
this.route("new");
|
||||||
this.route("show", { path: "/:id" });
|
this.route("show", { path: "/:id" });
|
||||||
});
|
});
|
||||||
|
this.route("discourse-ai-usage", { path: "ai-usage" });
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user