Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3ce8d943a7 |
@@ -0,0 +1 @@
|
|||||||
|
module.exports = require("@discourse/lint-configs/eslint");
|
||||||
+7
-7
@@ -1,18 +1,18 @@
|
|||||||
import DiscourseRoute from "discourse/routes/discourse";
|
import DiscourseRoute from "discourse/routes/discourse";
|
||||||
|
|
||||||
export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
|
export default DiscourseRoute.extend({
|
||||||
queryParams = {
|
queryParams: {
|
||||||
llmTemplate: { refreshModel: true },
|
llmTemplate: { refreshModel: true },
|
||||||
};
|
},
|
||||||
|
|
||||||
async model() {
|
async model() {
|
||||||
const record = this.store.createRecord("ai-llm");
|
const record = this.store.createRecord("ai-llm");
|
||||||
record.provider_params = {};
|
record.provider_params = {};
|
||||||
return record;
|
return record;
|
||||||
}
|
},
|
||||||
|
|
||||||
setupController(controller, model) {
|
setupController(controller, model) {
|
||||||
super.setupController(controller, model);
|
this._super(controller, model);
|
||||||
controller.set(
|
controller.set(
|
||||||
"allLlms",
|
"allLlms",
|
||||||
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
||||||
@@ -21,5 +21,5 @@ export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
|
|||||||
"llmTemplate",
|
"llmTemplate",
|
||||||
this.paramsFor(this.routeName).llmTemplate || null
|
this.paramsFor(this.routeName).llmTemplate || null
|
||||||
);
|
);
|
||||||
}
|
},
|
||||||
}
|
});
|
||||||
|
|||||||
+5
-5
@@ -1,19 +1,19 @@
|
|||||||
import DiscourseRoute from "discourse/routes/discourse";
|
import DiscourseRoute from "discourse/routes/discourse";
|
||||||
|
|
||||||
export default class AdminPluginsShowDiscourseAiLlmsShow extends DiscourseRoute {
|
export default DiscourseRoute.extend({
|
||||||
async model(params) {
|
async model(params) {
|
||||||
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
|
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
|
||||||
const id = parseInt(params.id, 10);
|
const id = parseInt(params.id, 10);
|
||||||
const record = allLlms.findBy("id", id);
|
const record = allLlms.findBy("id", id);
|
||||||
record.provider_params = record.provider_params || {};
|
record.provider_params = record.provider_params || {};
|
||||||
return record;
|
return record;
|
||||||
}
|
},
|
||||||
|
|
||||||
setupController(controller, model) {
|
setupController(controller, model) {
|
||||||
super.setupController(controller, model);
|
this._super(controller, model);
|
||||||
controller.set(
|
controller.set(
|
||||||
"allLlms",
|
"allLlms",
|
||||||
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
this.modelFor("adminPlugins.show.discourse-ai-llms")
|
||||||
);
|
);
|
||||||
}
|
},
|
||||||
}
|
});
|
||||||
|
|||||||
+5
-5
@@ -1,7 +1,7 @@
|
|||||||
import { AUTO_GROUPS } from "discourse/lib/constants";
|
import { AUTO_GROUPS } from "discourse/lib/constants";
|
||||||
import DiscourseRoute from "discourse/routes/discourse";
|
import DiscourseRoute from "discourse/routes/discourse";
|
||||||
|
|
||||||
export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
|
export default DiscourseRoute.extend({
|
||||||
async model() {
|
async model() {
|
||||||
const record = this.store.createRecord("ai-persona");
|
const record = this.store.createRecord("ai-persona");
|
||||||
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
|
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
|
||||||
@@ -13,13 +13,13 @@ export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRou
|
|||||||
record.set("allow_personal_messages", true);
|
record.set("allow_personal_messages", true);
|
||||||
record.set("tool_details", false);
|
record.set("tool_details", false);
|
||||||
return record;
|
return record;
|
||||||
}
|
},
|
||||||
|
|
||||||
setupController(controller, model) {
|
setupController(controller, model) {
|
||||||
super.setupController(controller, model);
|
this._super(controller, model);
|
||||||
controller.set(
|
controller.set(
|
||||||
"allPersonas",
|
"allPersonas",
|
||||||
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
||||||
);
|
);
|
||||||
}
|
},
|
||||||
}
|
});
|
||||||
|
|||||||
+5
-5
@@ -1,19 +1,19 @@
|
|||||||
import DiscourseRoute from "discourse/routes/discourse";
|
import DiscourseRoute from "discourse/routes/discourse";
|
||||||
|
|
||||||
export default class AdminPluginsShowDiscourseAiPersonasShow extends DiscourseRoute {
|
export default DiscourseRoute.extend({
|
||||||
async model(params) {
|
async model(params) {
|
||||||
const allPersonas = this.modelFor(
|
const allPersonas = this.modelFor(
|
||||||
"adminPlugins.show.discourse-ai-personas"
|
"adminPlugins.show.discourse-ai-personas"
|
||||||
);
|
);
|
||||||
const id = parseInt(params.id, 10);
|
const id = parseInt(params.id, 10);
|
||||||
return allPersonas.findBy("id", id);
|
return allPersonas.findBy("id", id);
|
||||||
}
|
},
|
||||||
|
|
||||||
setupController(controller, model) {
|
setupController(controller, model) {
|
||||||
super.setupController(controller, model);
|
this._super(controller, model);
|
||||||
controller.set(
|
controller.set(
|
||||||
"allPersonas",
|
"allPersonas",
|
||||||
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
this.modelFor("adminPlugins.show.discourse-ai-personas")
|
||||||
);
|
);
|
||||||
}
|
},
|
||||||
}
|
});
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
import { service } from "@ember/service";
|
|
||||||
import { ajax } from "discourse/lib/ajax";
|
|
||||||
import DiscourseRoute from "discourse/routes/discourse";
|
|
||||||
|
|
||||||
export default class DiscourseAiUsageRoute extends DiscourseRoute {
|
|
||||||
@service store;
|
|
||||||
|
|
||||||
model() {
|
|
||||||
return ajax("/admin/plugins/discourse-ai/ai-usage.json");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+5
-7
@@ -1,7 +1,5 @@
|
|||||||
<section class="ai-persona-tool-editor__current admin-detail pull-left">
|
<AiToolEditor
|
||||||
<AiToolEditor
|
@tools={{this.allTools}}
|
||||||
@tools={{this.allTools}}
|
@model={{this.model}}
|
||||||
@model={{this.model}}
|
@presets={{this.presets}}
|
||||||
@presets={{this.presets}}
|
/>
|
||||||
/>
|
|
||||||
</section>
|
|
||||||
+5
-7
@@ -1,7 +1,5 @@
|
|||||||
<section class="ai-persona-tool-editor__current admin-detail pull-left">
|
<AiToolEditor
|
||||||
<AiToolEditor
|
@tools={{this.allTools}}
|
||||||
@tools={{this.allTools}}
|
@model={{this.model}}
|
||||||
@model={{this.model}}
|
@presets={{this.presets}}
|
||||||
@presets={{this.presets}}
|
/>
|
||||||
/>
|
|
||||||
</section>
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
<AiUsage @model={{this.model}} />
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
# frozen_string_literal: true
|
|
||||||
|
|
||||||
module DiscourseAi
|
|
||||||
module Admin
|
|
||||||
class AiUsageController < ::Admin::AdminController
|
|
||||||
requires_plugin "discourse-ai"
|
|
||||||
|
|
||||||
def show
|
|
||||||
render json: AiUsageSerializer.new(create_report, root: false)
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
|
|
||||||
def create_report
|
|
||||||
report =
|
|
||||||
DiscourseAi::Completions::Report.new(
|
|
||||||
start_date: params[:start_date]&.to_date || 30.days.ago,
|
|
||||||
end_date: params[:end_date]&.to_date || Time.current,
|
|
||||||
)
|
|
||||||
|
|
||||||
report = report.filter_by_feature(params[:feature]) if params[:feature].present?
|
|
||||||
report = report.filter_by_model(params[:model]) if params[:model].present?
|
|
||||||
report
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
# frozen_string_literal: true
|
|
||||||
|
|
||||||
module DiscourseAi
|
|
||||||
module AiBot
|
|
||||||
class ArtifactsController < ApplicationController
|
|
||||||
requires_plugin DiscourseAi::PLUGIN_NAME
|
|
||||||
before_action :require_site_settings!
|
|
||||||
|
|
||||||
skip_before_action :preload_json, :check_xhr, only: %i[show]
|
|
||||||
|
|
||||||
def show
|
|
||||||
artifact = AiArtifact.find(params[:id])
|
|
||||||
|
|
||||||
post = Post.find_by(id: artifact.post_id)
|
|
||||||
if artifact.metadata&.dig("public")
|
|
||||||
# no guardian needed
|
|
||||||
else
|
|
||||||
raise Discourse::NotFound if !post&.topic&.private_message?
|
|
||||||
raise Discourse::NotFound if !guardian.can_see?(post)
|
|
||||||
end
|
|
||||||
|
|
||||||
# Prepare the inner (untrusted) HTML document
|
|
||||||
untrusted_html = <<~HTML
|
|
||||||
<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<title>#{ERB::Util.html_escape(artifact.name)}</title>
|
|
||||||
<style>
|
|
||||||
#{artifact.css}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
#{artifact.html}
|
|
||||||
<script>
|
|
||||||
#{artifact.js}
|
|
||||||
</script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
HTML
|
|
||||||
|
|
||||||
# Prepare the outer (trusted) HTML document
|
|
||||||
trusted_html = <<~HTML
|
|
||||||
<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<title>#{ERB::Util.html_escape(artifact.name)}</title>
|
|
||||||
<style>
|
|
||||||
html, body, iframe {
|
|
||||||
margin: 0;
|
|
||||||
padding: 0;
|
|
||||||
width: 100%;
|
|
||||||
height: 100%;
|
|
||||||
border: 0;
|
|
||||||
overflow: hidden;
|
|
||||||
}
|
|
||||||
iframe {
|
|
||||||
overflow: auto;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<iframe sandbox="allow-scripts allow-forms" height="100%" width="100%" srcdoc="#{ERB::Util.html_escape(untrusted_html)}" frameborder="0"></iframe>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
HTML
|
|
||||||
|
|
||||||
response.headers.delete("X-Frame-Options")
|
|
||||||
response.headers["Content-Security-Policy"] = "script-src 'unsafe-inline';"
|
|
||||||
response.headers["X-Robots-Tag"] = "noindex"
|
|
||||||
|
|
||||||
# Render the content
|
|
||||||
render html: trusted_html.html_safe, layout: false, content_type: "text/html"
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
|
|
||||||
def require_site_settings!
|
|
||||||
if !SiteSetting.discourse_ai_enabled ||
|
|
||||||
!SiteSetting.ai_artifact_security.in?(%w[lax strict])
|
|
||||||
raise Discourse::NotFound
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
@@ -7,8 +7,7 @@ module DiscourseAi
|
|||||||
requires_login only: %i[create update destroy]
|
requires_login only: %i[create update destroy]
|
||||||
before_action :require_site_settings!
|
before_action :require_site_settings!
|
||||||
|
|
||||||
skip_before_action :preload_json, :check_xhr, only: %i[show asset]
|
skip_before_action :preload_json, :check_xhr, only: %i[show]
|
||||||
skip_before_action :verify_authenticity_token, only: ["asset"]
|
|
||||||
|
|
||||||
def create
|
def create
|
||||||
ensure_allowed_create!
|
ensure_allowed_create!
|
||||||
@@ -51,30 +50,6 @@ module DiscourseAi
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def asset
|
|
||||||
no_cookies
|
|
||||||
|
|
||||||
name = params[:name]
|
|
||||||
path, content_type =
|
|
||||||
if name == "share"
|
|
||||||
%w[share.css text/css]
|
|
||||||
elsif name == "highlight"
|
|
||||||
%w[highlight.min.js application/javascript]
|
|
||||||
else
|
|
||||||
raise Discourse::NotFound
|
|
||||||
end
|
|
||||||
|
|
||||||
content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
|
|
||||||
|
|
||||||
# note, path contains a ":version" which automatically busts the cache
|
|
||||||
# based on file content, so this is safe
|
|
||||||
response.headers["Last-Modified"] = 10.years.ago.httpdate
|
|
||||||
response.headers["Content-Length"] = content.bytesize.to_s
|
|
||||||
immutable_for 1.year
|
|
||||||
|
|
||||||
render plain: content, disposition: :nil, content_type: content_type
|
|
||||||
end
|
|
||||||
|
|
||||||
def preview
|
def preview
|
||||||
ensure_allowed_preview!
|
ensure_allowed_preview!
|
||||||
data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
|
data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ module DiscourseAi
|
|||||||
prompt.custom_instruction = params[:custom_prompt]
|
prompt.custom_instruction = params[:custom_prompt]
|
||||||
end
|
end
|
||||||
|
|
||||||
return suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
|
suggest_thumbnails(input) if prompt.id == CompletionPrompt::ILLUSTRATE_POST
|
||||||
|
|
||||||
hijack do
|
hijack do
|
||||||
render json:
|
render json:
|
||||||
|
|||||||
@@ -2,29 +2,13 @@
|
|||||||
module DiscourseAi
|
module DiscourseAi
|
||||||
module AiBot
|
module AiBot
|
||||||
module SharedAiConversationsHelper
|
module SharedAiConversationsHelper
|
||||||
# keeping it here for caching
|
# bump up version when assets change
|
||||||
def self.share_asset_url(asset_name)
|
# long term we may want to change this cause it is hard to remember
|
||||||
if !%w[share.css highlight.js].include?(asset_name)
|
# to bump versions, but for now this does the job
|
||||||
raise StandardError, "unknown asset type #{asset_name}"
|
VERSION = "1"
|
||||||
end
|
|
||||||
|
|
||||||
@urls ||= {}
|
def share_asset_url(short_path)
|
||||||
url = @urls[asset_name]
|
::UrlHelper.absolute("/plugins/discourse-ai/ai-share/#{short_path}?#{VERSION}")
|
||||||
return url if url
|
|
||||||
|
|
||||||
path = asset_name
|
|
||||||
path = "highlight.min.js" if asset_name == "highlight.js"
|
|
||||||
|
|
||||||
content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
|
|
||||||
sha1 = Digest::SHA1.hexdigest(content)
|
|
||||||
|
|
||||||
url = "/discourse-ai/ai-bot/shared-ai-conversations/asset/#{sha1}/#{asset_name}"
|
|
||||||
|
|
||||||
@urls[asset_name] = GlobalPath.cdn_path(url)
|
|
||||||
end
|
|
||||||
|
|
||||||
def share_asset_url(asset_name)
|
|
||||||
DiscourseAi::AiBot::SharedAiConversationsHelper.share_asset_url(asset_name)
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
module ::Jobs
|
||||||
|
class HotTopicsGistBatch < ::Jobs::Base
|
||||||
|
def execute(_args)
|
||||||
|
return if !SiteSetting.discourse_ai_enabled
|
||||||
|
return if !SiteSetting.ai_summarization_enabled
|
||||||
|
return if SiteSetting.ai_summarize_max_hot_topics_gists_per_batch.zero?
|
||||||
|
|
||||||
|
Topic
|
||||||
|
.joins("JOIN topic_hot_scores on topics.id = topic_hot_scores.topic_id")
|
||||||
|
.order("topic_hot_scores.score DESC")
|
||||||
|
.limit(SiteSetting.ai_summarize_max_hot_topics_gists_per_batch)
|
||||||
|
.each do |topic|
|
||||||
|
summarizer = DiscourseAi::Summarization.topic_gist(topic)
|
||||||
|
gist = summarizer.existing_summary
|
||||||
|
|
||||||
|
summarizer.force_summarize(Discourse.system_user) if gist.blank? || gist.outdated
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -9,7 +9,9 @@ module ::Jobs
|
|||||||
post = Post.find_by(id: post_id, post_type: Post.types[:regular])
|
post = Post.find_by(id: post_id, post_type: Post.types[:regular])
|
||||||
return if post&.raw.blank?
|
return if post&.raw.blank?
|
||||||
|
|
||||||
DiscourseAi::Sentiment::PostClassification.new.classify!(post)
|
DiscourseAi::PostClassificator.new(
|
||||||
|
DiscourseAi::Sentiment::SentimentClassification.new,
|
||||||
|
).classify!(post)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
+6
-3
@@ -1,20 +1,23 @@
|
|||||||
# frozen_string_literal: true
|
# frozen_string_literal: true
|
||||||
|
|
||||||
module ::Jobs
|
module ::Jobs
|
||||||
class FastTrackTopicGist < ::Jobs::Base
|
class UpdateHotTopicGist < ::Jobs::Base
|
||||||
sidekiq_options retry: false
|
sidekiq_options retry: false
|
||||||
|
|
||||||
def execute(args)
|
def execute(args)
|
||||||
return if !SiteSetting.discourse_ai_enabled
|
return if !SiteSetting.discourse_ai_enabled
|
||||||
return if !SiteSetting.ai_summarization_enabled
|
return if !SiteSetting.ai_summarization_enabled
|
||||||
return if !SiteSetting.ai_summary_gists_enabled
|
return if SiteSetting.ai_summarize_max_hot_topics_gists_per_batch.zero?
|
||||||
|
|
||||||
topic = Topic.find_by(id: args[:topic_id])
|
topic = Topic.find_by(id: args[:topic_id])
|
||||||
return if topic.blank?
|
return if topic.blank?
|
||||||
|
|
||||||
|
return if !TopicHotScore.where(topic: topic).exists?
|
||||||
|
|
||||||
summarizer = DiscourseAi::Summarization.topic_gist(topic)
|
summarizer = DiscourseAi::Summarization.topic_gist(topic)
|
||||||
gist = summarizer.existing_summary
|
gist = summarizer.existing_summary
|
||||||
return if gist.present? && !gist.outdated
|
return if gist.blank?
|
||||||
|
return if !gist.outdated
|
||||||
|
|
||||||
summarizer.force_summarize(Discourse.system_user)
|
summarizer.force_summarize(Discourse.system_user)
|
||||||
end
|
end
|
||||||
@@ -70,15 +70,14 @@ module Jobs
|
|||||||
Post
|
Post
|
||||||
.joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
|
.joins("LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id")
|
||||||
.where(deleted_at: nil)
|
.where(deleted_at: nil)
|
||||||
.where(post_type: Post.types[:regular])
|
|
||||||
.limit(limit - rebaked)
|
.limit(limit - rebaked)
|
||||||
|
|
||||||
# First, we'll try to backfill embeddings for posts that have none
|
# First, we'll try to backfill embeddings for posts that have none
|
||||||
posts
|
posts
|
||||||
.where("#{table_name}.post_id IS NULL")
|
.where("#{table_name}.post_id IS NULL")
|
||||||
.find_in_batches do |batch|
|
.find_each do |t|
|
||||||
vector_rep.gen_bulk_reprensentations(batch)
|
vector_rep.generate_representation_from(t)
|
||||||
rebaked += batch.size
|
rebaked += 1
|
||||||
end
|
end
|
||||||
|
|
||||||
return if rebaked >= limit
|
return if rebaked >= limit
|
||||||
@@ -91,28 +90,24 @@ module Jobs
|
|||||||
OR
|
OR
|
||||||
#{table_name}.strategy_version < #{strategy.version}
|
#{table_name}.strategy_version < #{strategy.version}
|
||||||
SQL
|
SQL
|
||||||
.find_in_batches do |batch|
|
.find_each do |t|
|
||||||
vector_rep.gen_bulk_reprensentations(batch)
|
vector_rep.generate_representation_from(t)
|
||||||
rebaked += batch.size
|
rebaked += 1
|
||||||
end
|
end
|
||||||
|
|
||||||
return if rebaked >= limit
|
return if rebaked >= limit
|
||||||
|
|
||||||
# Finally, we'll try to backfill embeddings for posts that have outdated
|
# Finally, we'll try to backfill embeddings for posts that have outdated
|
||||||
# embeddings due to edits. Here we only do 10% of the limit
|
# embeddings due to edits. Here we only do 10% of the limit
|
||||||
posts_batch_size = 1000
|
posts
|
||||||
|
.where("#{table_name}.updated_at < ?", 7.days.ago)
|
||||||
outdated_post_ids =
|
.order("random()")
|
||||||
posts
|
.limit((limit - rebaked) / 10)
|
||||||
.where("#{table_name}.updated_at < ?", 7.days.ago)
|
.pluck(:id)
|
||||||
.order("random()")
|
.each do |id|
|
||||||
.limit((limit - rebaked) / 10)
|
vector_rep.generate_representation_from(Post.find_by(id: id))
|
||||||
.pluck(:id)
|
rebaked += 1
|
||||||
|
end
|
||||||
outdated_post_ids.each_slice(posts_batch_size) do |batch|
|
|
||||||
vector_rep.gen_bulk_reprensentations(Post.where(id: batch))
|
|
||||||
rebaked += batch.length
|
|
||||||
end
|
|
||||||
|
|
||||||
rebaked
|
rebaked
|
||||||
end
|
end
|
||||||
@@ -125,13 +120,14 @@ module Jobs
|
|||||||
topics = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL") if !force
|
topics = topics.where("#{vector_rep.topic_table_name}.topic_id IS NULL") if !force
|
||||||
|
|
||||||
ids = topics.pluck("topics.id")
|
ids = topics.pluck("topics.id")
|
||||||
batch_size = 1000
|
|
||||||
|
|
||||||
ids.each_slice(batch_size) do |batch|
|
ids.each do |id|
|
||||||
vector_rep.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
|
topic = Topic.find_by(id: id)
|
||||||
done += batch.length
|
if topic
|
||||||
|
vector_rep.generate_representation_from(topic)
|
||||||
|
done += 1
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
done
|
done
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -19,8 +19,6 @@ module ::Jobs
|
|||||||
DiscourseAi::Summarization.topic_summary(topic).force_summarize(system_user)
|
DiscourseAi::Summarization.topic_summary(topic).force_summarize(system_user)
|
||||||
end
|
end
|
||||||
|
|
||||||
return unless SiteSetting.ai_summary_gists_enabled
|
|
||||||
|
|
||||||
gist_t = AiSummary.summary_types[:gist]
|
gist_t = AiSummary.summary_types[:gist]
|
||||||
backfill_candidates(gist_t)
|
backfill_candidates(gist_t)
|
||||||
.limit(current_budget(gist_t))
|
.limit(current_budget(gist_t))
|
||||||
@@ -31,9 +29,9 @@ module ::Jobs
|
|||||||
Topic
|
Topic
|
||||||
.where("topics.word_count >= ?", SiteSetting.ai_summary_backfill_minimum_word_count)
|
.where("topics.word_count >= ?", SiteSetting.ai_summary_backfill_minimum_word_count)
|
||||||
.joins(<<~SQL)
|
.joins(<<~SQL)
|
||||||
LEFT OUTER JOIN ai_summaries ais ON
|
LEFT OUTER JOIN ai_summaries ais ON
|
||||||
topics.id = ais.target_id AND
|
topics.id = ais.target_id AND
|
||||||
ais.target_type = 'Topic' AND
|
ais.target_type = 'Topic' AND
|
||||||
ais.summary_type = '#{summary_type}'
|
ais.summary_type = '#{summary_type}'
|
||||||
SQL
|
SQL
|
||||||
.where(
|
.where(
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
class AiApiAuditLog < ActiveRecord::Base
|
class AiApiAuditLog < ActiveRecord::Base
|
||||||
belongs_to :post
|
belongs_to :post
|
||||||
belongs_to :topic
|
belongs_to :topic
|
||||||
belongs_to :user
|
|
||||||
|
|
||||||
module Provider
|
module Provider
|
||||||
OpenAI = 1
|
OpenAI = 1
|
||||||
@@ -14,7 +13,6 @@ class AiApiAuditLog < ActiveRecord::Base
|
|||||||
Cohere = 6
|
Cohere = 6
|
||||||
Ollama = 7
|
Ollama = 7
|
||||||
SambaNova = 8
|
SambaNova = 8
|
||||||
Mistral = 9
|
|
||||||
end
|
end
|
||||||
|
|
||||||
def next_log_id
|
def next_log_id
|
||||||
@@ -44,10 +42,3 @@ end
|
|||||||
# feature_name :string(255)
|
# feature_name :string(255)
|
||||||
# language_model :string(255)
|
# language_model :string(255)
|
||||||
# feature_context :jsonb
|
# feature_context :jsonb
|
||||||
# cached_tokens :integer
|
|
||||||
#
|
|
||||||
# Indexes
|
|
||||||
#
|
|
||||||
# index_ai_api_audit_logs_on_created_at_and_feature_name (created_at,feature_name)
|
|
||||||
# index_ai_api_audit_logs_on_created_at_and_language_model (created_at,language_model)
|
|
||||||
#
|
|
||||||
|
|||||||
@@ -1,52 +0,0 @@
|
|||||||
# frozen_string_literal: true
|
|
||||||
|
|
||||||
class AiArtifact < ActiveRecord::Base
|
|
||||||
belongs_to :user
|
|
||||||
belongs_to :post
|
|
||||||
validates :html, length: { maximum: 65_535 }
|
|
||||||
validates :css, length: { maximum: 65_535 }
|
|
||||||
validates :js, length: { maximum: 65_535 }
|
|
||||||
|
|
||||||
def self.iframe_for(id)
|
|
||||||
<<~HTML
|
|
||||||
<div class='ai-artifact'>
|
|
||||||
<iframe src='#{url(id)}' frameborder="0" height="100%" width="100%"></iframe>
|
|
||||||
<a href='#{url(id)}' target='_blank'>#{I18n.t("discourse_ai.ai_artifact.link")}</a>
|
|
||||||
</div>
|
|
||||||
HTML
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.url(id)
|
|
||||||
Discourse.base_url + "/discourse-ai/ai-bot/artifacts/#{id}"
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.share_publicly(id:, post:)
|
|
||||||
artifact = AiArtifact.find_by(id: id)
|
|
||||||
artifact.update!(metadata: { public: true }) if artifact&.post&.topic&.id == post.topic.id
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.unshare_publicly(id:)
|
|
||||||
artifact = AiArtifact.find_by(id: id)
|
|
||||||
artifact&.update!(metadata: { public: false })
|
|
||||||
end
|
|
||||||
|
|
||||||
def url
|
|
||||||
self.class.url(id)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# == Schema Information
|
|
||||||
#
|
|
||||||
# Table name: ai_artifacts
|
|
||||||
#
|
|
||||||
# id :bigint not null, primary key
|
|
||||||
# user_id :integer not null
|
|
||||||
# post_id :integer not null
|
|
||||||
# name :string(255) not null
|
|
||||||
# html :string(65535)
|
|
||||||
# css :string(65535)
|
|
||||||
# js :string(65535)
|
|
||||||
# metadata :jsonb
|
|
||||||
# created_at :datetime not null
|
|
||||||
# updated_at :datetime not null
|
|
||||||
#
|
|
||||||
+10
-19
@@ -9,23 +9,15 @@ class AiSummary < ActiveRecord::Base
|
|||||||
def self.store!(strategy, llm_model, summary, og_content, human:)
|
def self.store!(strategy, llm_model, summary, og_content, human:)
|
||||||
content_ids = og_content.map { |c| c[:id] }
|
content_ids = og_content.map { |c| c[:id] }
|
||||||
|
|
||||||
AiSummary
|
AiSummary.create!(
|
||||||
.upsert(
|
target: strategy.target,
|
||||||
{
|
algorithm: llm_model.name,
|
||||||
target_id: strategy.target.id,
|
content_range: (content_ids.first..content_ids.last),
|
||||||
target_type: strategy.target.class.name,
|
summarized_text: summary,
|
||||||
algorithm: llm_model.name,
|
original_content_sha: build_sha(content_ids.join),
|
||||||
content_range: (content_ids.first..content_ids.last),
|
summary_type: strategy.type,
|
||||||
summarized_text: summary,
|
origin: !!human ? origins[:human] : origins[:system],
|
||||||
original_content_sha: build_sha(content_ids.join),
|
)
|
||||||
summary_type: strategy.type,
|
|
||||||
origin: !!human ? origins[:human] : origins[:system],
|
|
||||||
},
|
|
||||||
unique_by: %i[target_id target_type summary_type],
|
|
||||||
update_only: %i[summarized_text original_content_sha algorithm origin content_range],
|
|
||||||
)
|
|
||||||
.first
|
|
||||||
.then { AiSummary.find_by(id: _1["id"]) }
|
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.build_sha(joined_ids)
|
def self.build_sha(joined_ids)
|
||||||
@@ -59,6 +51,5 @@ end
|
|||||||
#
|
#
|
||||||
# Indexes
|
# Indexes
|
||||||
#
|
#
|
||||||
# idx_on_target_id_target_type_summary_type_3355609fbb (target_id,target_type,summary_type) UNIQUE
|
# index_ai_summaries_on_target_type_and_target_id (target_type,target_id)
|
||||||
# index_ai_summaries_on_target_type_and_target_id (target_type,target_id)
|
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ class CompletionPrompt < ActiveRecord::Base
|
|||||||
CUSTOM_PROMPT = -305
|
CUSTOM_PROMPT = -305
|
||||||
EXPLAIN = -306
|
EXPLAIN = -306
|
||||||
ILLUSTRATE_POST = -308
|
ILLUSTRATE_POST = -308
|
||||||
DETECT_TEXT_LOCALE = -309
|
|
||||||
|
|
||||||
enum :prompt_type, { text: 0, list: 1, diff: 2 }
|
enum :prompt_type, { text: 0, list: 1, diff: 2 }
|
||||||
|
|
||||||
|
|||||||
@@ -26,16 +26,6 @@ class LlmModel < ActiveRecord::Base
|
|||||||
},
|
},
|
||||||
open_ai: {
|
open_ai: {
|
||||||
organization: :text,
|
organization: :text,
|
||||||
disable_native_tools: :checkbox,
|
|
||||||
},
|
|
||||||
mistral: {
|
|
||||||
disable_native_tools: :checkbox,
|
|
||||||
},
|
|
||||||
google: {
|
|
||||||
disable_native_tools: :checkbox,
|
|
||||||
},
|
|
||||||
azure: {
|
|
||||||
disable_native_tools: :checkbox,
|
|
||||||
},
|
},
|
||||||
hugging_face: {
|
hugging_face: {
|
||||||
disable_system_prompt: :checkbox,
|
disable_system_prompt: :checkbox,
|
||||||
|
|||||||
@@ -172,7 +172,6 @@ end
|
|||||||
# updated_at :datetime not null
|
# updated_at :datetime not null
|
||||||
# force_review :boolean default(FALSE), not null
|
# force_review :boolean default(FALSE), not null
|
||||||
# reject_reason :text
|
# reject_reason :text
|
||||||
# potentially_illegal :boolean default(FALSE)
|
|
||||||
#
|
#
|
||||||
# Indexes
|
# Indexes
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -229,7 +229,6 @@ end
|
|||||||
# updated_at :datetime not null
|
# updated_at :datetime not null
|
||||||
# force_review :boolean default(FALSE), not null
|
# force_review :boolean default(FALSE), not null
|
||||||
# reject_reason :text
|
# reject_reason :text
|
||||||
# potentially_illegal :boolean default(FALSE)
|
|
||||||
#
|
#
|
||||||
# Indexes
|
# Indexes
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -34,12 +34,6 @@ class SharedAiConversation < ActiveRecord::Base
|
|||||||
|
|
||||||
def self.destroy_conversation(conversation)
|
def self.destroy_conversation(conversation)
|
||||||
conversation.destroy
|
conversation.destroy
|
||||||
|
|
||||||
maybe_topic = conversation.target
|
|
||||||
if maybe_topic.is_a?(Topic)
|
|
||||||
AiArtifact.where(post: maybe_topic.posts).update_all(metadata: { public: false })
|
|
||||||
end
|
|
||||||
|
|
||||||
::Jobs.enqueue(
|
::Jobs.enqueue(
|
||||||
:shared_conversation_adjust_upload_security,
|
:shared_conversation_adjust_upload_security,
|
||||||
target_id: conversation.target_id,
|
target_id: conversation.target_id,
|
||||||
@@ -171,7 +165,7 @@ class SharedAiConversation < ActiveRecord::Base
|
|||||||
id: post.id,
|
id: post.id,
|
||||||
user_id: post.user_id,
|
user_id: post.user_id,
|
||||||
created_at: post.created_at,
|
created_at: post.created_at,
|
||||||
cooked: cook_artifacts(post),
|
cooked: post.cooked,
|
||||||
}
|
}
|
||||||
|
|
||||||
mapped[:persona] = persona if ai_bot_participant&.id == post.user_id
|
mapped[:persona] = persona if ai_bot_participant&.id == post.user_id
|
||||||
@@ -181,24 +175,6 @@ class SharedAiConversation < ActiveRecord::Base
|
|||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.cook_artifacts(post)
|
|
||||||
html = post.cooked
|
|
||||||
return html if !%w[lax strict].include?(SiteSetting.ai_artifact_security)
|
|
||||||
|
|
||||||
doc = Nokogiri::HTML5.fragment(html)
|
|
||||||
doc
|
|
||||||
.css("div.ai-artifact")
|
|
||||||
.each do |node|
|
|
||||||
id = node["data-ai-artifact-id"].to_i
|
|
||||||
if id > 0
|
|
||||||
AiArtifact.share_publicly(id: id, post: post)
|
|
||||||
node.replace(AiArtifact.iframe_for(id))
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
doc.to_s
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def populate_user_info!(posts)
|
def populate_user_info!(posts)
|
||||||
|
|||||||
@@ -1,69 +0,0 @@
|
|||||||
# frozen_string_literal: true
|
|
||||||
|
|
||||||
class AiUsageSerializer < ApplicationSerializer
|
|
||||||
attributes :data, :features, :models, :users, :summary, :period
|
|
||||||
|
|
||||||
def data
|
|
||||||
object.tokens_by_period.as_json(
|
|
||||||
only: %i[period total_tokens total_cached_tokens total_request_tokens total_response_tokens],
|
|
||||||
)
|
|
||||||
end
|
|
||||||
|
|
||||||
def period
|
|
||||||
object.guess_period
|
|
||||||
end
|
|
||||||
|
|
||||||
def features
|
|
||||||
object.feature_breakdown.as_json(
|
|
||||||
only: %i[
|
|
||||||
feature_name
|
|
||||||
usage_count
|
|
||||||
total_tokens
|
|
||||||
total_cached_tokens
|
|
||||||
total_request_tokens
|
|
||||||
total_response_tokens
|
|
||||||
],
|
|
||||||
)
|
|
||||||
end
|
|
||||||
|
|
||||||
def models
|
|
||||||
object.model_breakdown.as_json(
|
|
||||||
only: %i[
|
|
||||||
llm
|
|
||||||
usage_count
|
|
||||||
total_tokens
|
|
||||||
total_cached_tokens
|
|
||||||
total_request_tokens
|
|
||||||
total_response_tokens
|
|
||||||
],
|
|
||||||
)
|
|
||||||
end
|
|
||||||
|
|
||||||
def users
|
|
||||||
object.user_breakdown.map do |user|
|
|
||||||
{
|
|
||||||
avatar_template: User.avatar_template(user.username, user.uploaded_avatar_id),
|
|
||||||
username: user.username,
|
|
||||||
usage_count: user.usage_count,
|
|
||||||
total_tokens: user.total_tokens,
|
|
||||||
total_cached_tokens: user.total_cached_tokens,
|
|
||||||
total_request_tokens: user.total_request_tokens,
|
|
||||||
total_response_tokens: user.total_response_tokens,
|
|
||||||
}
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def summary
|
|
||||||
{
|
|
||||||
total_tokens: object.total_tokens,
|
|
||||||
total_cached_tokens: object.total_cached_tokens,
|
|
||||||
total_request_tokens: object.total_request_tokens,
|
|
||||||
total_response_tokens: object.total_response_tokens,
|
|
||||||
total_requests: object.total_requests,
|
|
||||||
date_range: {
|
|
||||||
start: object.start_date,
|
|
||||||
end: object.end_date,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
end
|
|
||||||
end
|
|
||||||
@@ -10,7 +10,7 @@
|
|||||||
<meta name="twitter:title" content="<%= I18n.t("discourse_ai.share_ai.title", title: @shared_conversation.title, site_name: SiteSetting.title) %>">
|
<meta name="twitter:title" content="<%= I18n.t("discourse_ai.share_ai.title", title: @shared_conversation.title, site_name: SiteSetting.title) %>">
|
||||||
<meta name="twitter:description" content="<%= @shared_conversation.formatted_excerpt %>">
|
<meta name="twitter:description" content="<%= @shared_conversation.formatted_excerpt %>">
|
||||||
<meta name="viewport" content="width=device-width">
|
<meta name="viewport" content="width=device-width">
|
||||||
<link rel="stylesheet" href="<%= share_asset_url("share.css") %>">
|
<link rel="stylesheet" href="<%= ::UrlHelper.absolute("/plugins/discourse-ai/ai-share/share.css") %>">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header class="site-header">
|
<header class="site-header">
|
||||||
@@ -54,7 +54,7 @@
|
|||||||
</article>
|
</article>
|
||||||
<% end %>
|
<% end %>
|
||||||
</section>
|
</section>
|
||||||
<script src="<%= share_asset_url("highlight.js") %>" nonce="<%= csp_nonce_placeholder %>" ></script>
|
<script src="<%= share_asset_url("highlight.min.js") %>" nonce="<%= csp_nonce_placeholder %>" ></script>
|
||||||
<script nonce="<%= csp_nonce_placeholder %>">
|
<script nonce="<%= csp_nonce_placeholder %>">
|
||||||
document.querySelectorAll('pre code').forEach((el) => {
|
document.querySelectorAll('pre code').forEach((el) => {
|
||||||
hljs.highlightElement(el);
|
hljs.highlightElement(el);
|
||||||
|
|||||||
@@ -18,6 +18,5 @@ export default {
|
|||||||
this.route("new");
|
this.route("new");
|
||||||
this.route("show", { path: "/:id" });
|
this.route("show", { path: "/:id" });
|
||||||
});
|
});
|
||||||
this.route("discourse-ai-usage", { path: "ai-usage" });
|
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user