Discourse/discourse-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove all files (#1516)

Browse Source
This commit is contained in:
Sam
2026-08-28 08:24:24 +10:00
committed by GitHub
parent c312b3184a
commit 2c4e5e998e
1110 changed files with 2 additions and 133260 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Gemfile
-8
View File
@@ -1,8 +0,0 @@
# frozen_string_literal: true
source "https://rubygems.org"
group :development do
gem "rubocop-discourse"
gem "syntax_tree"
end
Gemfile.lock
-100
View File
@@ -1,100 +0,0 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (8.0.2)
base64
benchmark (>= 0.3)
bigdecimal
concurrent-ruby (~> 1.0, >= 1.3.1)
connection_pool (>= 2.2.5)
drb
i18n (>= 1.6, < 2)
logger (>= 1.4.2)
minitest (>= 5.1)
securerandom (>= 0.3)
tzinfo (~> 2.0, >= 2.0.5)
uri (>= 0.13.1)
ast (2.4.2)
base64 (0.2.0)
benchmark (0.4.0)
bigdecimal (3.1.9)
concurrent-ruby (1.3.5)
connection_pool (2.5.0)
drb (2.2.1)
i18n (1.14.7)
concurrent-ruby (~> 1.0)
json (2.10.2)
language_server-protocol (3.17.0.4)
lint_roller (1.1.0)
logger (1.6.6)
minitest (5.25.5)
parallel (1.26.3)
parser (3.3.7.1)
ast (~> 2.4.1)
racc
prettier_print (1.2.1)
racc (1.8.1)
rack (3.1.16)
rainbow (3.1.1)
regexp_parser (2.10.0)
rubocop (1.74.0)
json (~> 2.3)
language_server-protocol (~> 3.17.0.2)
lint_roller (~> 1.1.0)
parallel (~> 1.10)
parser (>= 3.3.0.2)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 2.9.3, < 3.0)
rubocop-ast (>= 1.38.0, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 4.0)
rubocop-ast (1.38.1)
parser (>= 3.3.1.0)
rubocop-capybara (2.22.1)
lint_roller (~> 1.1)
rubocop (~> 1.72, >= 1.72.1)
rubocop-discourse (3.12.1)
activesupport (>= 6.1)
lint_roller (>= 1.1.0)
rubocop (>= 1.73.2)
rubocop-capybara (>= 2.22.0)
rubocop-factory_bot (>= 2.27.0)
rubocop-rails (>= 2.30.3)
rubocop-rspec (>= 3.0.1)
rubocop-rspec_rails (>= 2.31.0)
rubocop-factory_bot (2.27.1)
lint_roller (~> 1.1)
rubocop (~> 1.72, >= 1.72.1)
rubocop-rails (2.30.3)
activesupport (>= 4.2.0)
lint_roller (~> 1.1)
rack (>= 1.1)
rubocop (>= 1.72.1, < 2.0)
rubocop-ast (>= 1.38.0, < 2.0)
rubocop-rspec (3.5.0)
lint_roller (~> 1.1)
rubocop (~> 1.72, >= 1.72.1)
rubocop-rspec_rails (2.31.0)
lint_roller (~> 1.1)
rubocop (~> 1.72, >= 1.72.1)
rubocop-rspec (~> 3.5)
ruby-progressbar (1.13.0)
securerandom (0.4.1)
syntax_tree (6.2.0)
prettier_print (>= 1.2.0)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
unicode-display_width (3.1.4)
unicode-emoji (~> 4.0, >= 4.0.4)
unicode-emoji (4.0.4)
uri (1.0.3)
PLATFORMS
ruby
DEPENDENCIES
rubocop-discourse
syntax_tree
BUNDLED WITH
2.6.6
LICENSE
-21
View File
@@ -1,21 +0,0 @@
The MIT License (MIT)
Copyright (c) Civilized Discourse Construction Kit, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
README.md
+2 -31
View File
@@ -1,32 +1,3 @@
# **Discourse AI** Plugin
# Discourse AI is no shipped with Discourse core
> [!IMPORTANT]
> This plugin has now been bundled into Discourse core. See: https://meta.discourse.org/t/bundling-more-popular-plugins-with-discourse-core/373574
**Plugin Summary**
For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco
### Evals
The directory `evals` contains AI evals for the Discourse AI plugin.
You may create a local config by copying `config/eval-llms.yml` to `config/eval-llms.local.yml` and modifying the values.
To run them use:
cd evals
./run --help
```
Usage: evals/run [options]
-e, --eval NAME Name of the evaluation to run
--list-models List models
-m, --model NAME Model to evaluate (will eval all models if not specified)
-l, --list List evals
```
To run evals you will need to configure API keys in your environment:
OPENAI_API_KEY=your_openai_api_key
ANTHROPIC_API_KEY=your_anthropic_api_key
GEMINI_API_KEY=your_gemini_api_key
see: https://github.com/discourse/discourse/tree/main/plugins/discourse-ai
about.json
-8
View File
@@ -1,8 +0,0 @@
{
"tests": {
"requiredPlugins": [
"https://github.com/discourse/discourse-prometheus",
"https://github.com/discourse/discourse-solved.git"
]
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings-edit.js
-21
View File
@@ -1,21 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiEmbeddingsEdit extends DiscourseRoute {
async model(params) {
const allEmbeddings = this.modelFor(
"adminPlugins.show.discourse-ai-embeddings"
);
const id = parseInt(params.id, 10);
const record = allEmbeddings.findBy("id", id);
record.provider_params = record.provider_params || {};
return record;
}
setupController(controller, model) {
super.setupController(controller, model);
controller.set(
"allEmbeddings",
this.modelFor("adminPlugins.show.discourse-ai-embeddings")
);
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings-new.js
-17
View File
@@ -1,17 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiEmbeddingsNew extends DiscourseRoute {
async model() {
const record = this.store.createRecord("ai-embedding");
record.provider_params = {};
return record;
}
setupController(controller, model) {
super.setupController(controller, model);
controller.set(
"allEmbeddings",
this.modelFor("adminPlugins.show.discourse-ai-embeddings")
);
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-embeddings.js
-7
View File
@@ -1,7 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiAiEmbeddingsRoute extends DiscourseRoute {
model() {
return this.store.findAll("ai-embedding");
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-features-edit.js
-35
View File
@@ -1,35 +0,0 @@
import { action } from "@ember/object";
import { ajax } from "discourse/lib/ajax";
import DiscourseRoute from "discourse/routes/discourse";
import SiteSetting from "admin/models/site-setting";
export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRoute {
async model(params) {
const allFeatures = this.modelFor(
"adminPlugins.show.discourse-ai-features"
);
const id = parseInt(params.id, 10);
const currentFeature = allFeatures.find((feature) => feature.id === id);
const { site_settings } = await ajax("/admin/config/site_settings.json", {
data: {
filter_area: `ai-features/${currentFeature.module_name}`,
plugin: "discourse-ai",
category: "discourse_ai",
},
});
currentFeature.feature_settings = site_settings.map((setting) =>
SiteSetting.create(setting)
);
return currentFeature;
}
@action
willTransition() {
// site settings may amend if a feature is enabled or disabled, so refresh the model
// even on back button
this.router.refresh("adminPlugins.show.discourse-ai-features");
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-features.js
-10
View File
@@ -1,10 +0,0 @@
import { service } from "@ember/service";
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiFeatures extends DiscourseRoute {
@service store;
async model() {
return this.store.findAll("ai-feature");
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-edit.js
-28
View File
@@ -1,28 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
async model(params) {
const id = parseInt(params.id, 10);
if (id < 0) {
// You shouldn't be able to access the edit page
// if the model is seeded
return this.router.transitionTo(
"adminPlugins.show.discourse-ai-llms.index"
);
}
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const record = allLlms.findBy("id", id);
record.provider_params = record.provider_params || {};
return record;
}
setupController(controller, model) {
super.setupController(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
);
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-new.js
-25
View File
@@ -1,25 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiLlmsNew extends DiscourseRoute {
queryParams = {
llmTemplate: { refreshModel: true },
};
async model() {
const record = this.store.createRecord("ai-llm");
record.provider_params = {};
return record;
}
setupController(controller, model) {
super.setupController(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
);
controller.set(
"llmTemplate",
this.paramsFor(this.routeName).llmTemplate || null
);
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms.js
-7
View File
@@ -1,7 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiAiLlmsRoute extends DiscourseRoute {
model() {
return this.store.findAll("ai-llm");
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-personas-edit.js
-19
View File
@@ -1,19 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
async model(params) {
const allPersonas = this.modelFor(
"adminPlugins.show.discourse-ai-personas"
);
const id = parseInt(params.id, 10);
return allPersonas.findBy("id", id);
}
setupController(controller, model) {
super.setupController(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
);
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-personas-new.js
-25
View File
@@ -1,25 +0,0 @@
import { AUTO_GROUPS } from "discourse/lib/constants";
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiPersonasNew extends DiscourseRoute {
async model() {
const record = this.store.createRecord("ai-persona");
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
record.set("rag_uploads", []);
// these match the defaults on the table
record.set("rag_chunk_tokens", 374);
record.set("rag_chunk_overlap_tokens", 10);
record.set("rag_conversation_chunks", 10);
record.set("allow_personal_messages", true);
record.set("tool_details", false);
return record;
}
setupController(controller, model) {
super.setupController(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
);
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-personas.js
-7
View File
@@ -1,7 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiAiPersonasRoute extends DiscourseRoute {
model() {
return this.store.findAll("ai-persona");
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-spam.js
-11
View File
@@ -1,11 +0,0 @@
import { service } from "@ember/service";
import { ajax } from "discourse/lib/ajax";
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiSpamRoute extends DiscourseRoute {
@service store;
model() {
return ajax("/admin/plugins/discourse-ai/ai-spam.json");
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-tools-edit.js
-20
View File
@@ -1,20 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
async model(params) {
const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
const id = parseInt(params.id, 10);
return allTools.find((tool) => tool.id === id);
}
setupController(controller) {
super.setupController(...arguments);
const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
controller.set("allTools", toolsModel);
controller.set("presets", toolsModel.resultSetMeta.presets);
controller.set("llms", toolsModel.resultSetMeta.llms);
controller.set("settings", toolsModel.resultSetMeta.settings);
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-tools-new.js
-21
View File
@@ -1,21 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiToolsNewRoute extends DiscourseRoute {
beforeModel(transition) {
this.preset = transition.to.queryParams.presetId || "empty_tool";
}
async model() {
return this.store.createRecord("ai-tool");
}
setupController(controller) {
super.setupController(...arguments);
const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
controller.set("allTools", toolsModel);
controller.set("presets", toolsModel.resultSetMeta.presets);
controller.set("llms", toolsModel.resultSetMeta.llms);
controller.set("settings", toolsModel.resultSetMeta.settings);
controller.set("selectedPreset", this.preset);
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-tools.js
-10
View File
@@ -1,10 +0,0 @@
import { service } from "@ember/service";
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiToolsRoute extends DiscourseRoute {
@service store;
model() {
return this.store.findAll("ai-tool");
}
}
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-usage.js
-11
View File
@@ -1,11 +0,0 @@
import { service } from "@ember/service";
import { ajax } from "discourse/lib/ajax";
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiUsageRoute extends DiscourseRoute {
@service store;
model() {
return ajax("/admin/plugins/discourse-ai/ai-usage.json");
}
}
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/edit.hbs
-4
View File
@@ -1,4 +0,0 @@
<AiEmbeddingsListEditor
@embeddings={{this.allEmbeddings}}
@currentEmbedding={{this.model}}
/>
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/index.hbs
-1
View File
@@ -1 +0,0 @@
<AiEmbeddingsListEditor @embeddings={{this.model}} />
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-embeddings/new.hbs
-4
View File
@@ -1,4 +0,0 @@
<AiEmbeddingsListEditor
@embeddings={{this.allEmbeddings}}
@currentEmbedding={{this.model}}
/>
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/edit.gjs
-24
View File
@@ -1,24 +0,0 @@
import RouteTemplate from "ember-route-template";
import BackButton from "discourse/components/back-button";
import SiteSettingComponent from "admin/components/site-setting";
export default RouteTemplate(
<template>
<BackButton
@route="adminPlugins.show.discourse-ai-features"
@label="discourse_ai.features.back"
/>
<section class="ai-feature-editor__header">
<h2>{{@model.name}}</h2>
<p>{{@model.description}}</p>
</section>
<section class="ai-feature-editor">
{{#each @model.feature_settings as |setting|}}
<div>
<SiteSettingComponent @setting={{setting}} />
</div>
{{/each}}
</section>
</template>
);
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-features/index.hbs
-1
View File
@@ -1 +0,0 @@
<AiFeatures @features={{this.model}} />
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-llms/edit.hbs
-1
View File
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-llms/index.hbs
-1
View File
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.model}} />
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-llms/new.hbs
-5
View File
@@ -1,5 +0,0 @@
<AiLlmsListEditor
@llms={{this.allLlms}}
@currentLlm={{this.model}}
@llmTemplate={{this.llmTemplate}}
/>
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-personas/edit.hbs
-4
View File
@@ -1,4 +0,0 @@
<AiPersonaListEditor
@personas={{this.allPersonas}}
@currentPersona={{this.model}}
/>
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-personas/index.hbs
-1
View File
@@ -1 +0,0 @@
<AiPersonaListEditor @personas={{this.model}} />
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-personas/new.hbs
-4
View File
@@ -1,4 +0,0 @@
<AiPersonaListEditor
@personas={{this.allPersonas}}
@currentPersona={{this.model}}
/>
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-spam.hbs
-1
View File
@@ -1 +0,0 @@
<AiSpam @model={{this.model}} />
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-tools/edit.hbs
-9
View File
@@ -1,9 +0,0 @@
<section class="ai-persona-tool-editor__current admin-detail pull-left">
<AiToolEditor
@tools={{this.allTools}}
@model={{this.model}}
@presets={{this.presets}}
@llms={{this.llms}}
@settings={{this.settings}}
/>
</section>
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-tools/index.hbs
-1
View File
@@ -1 +0,0 @@
<AiToolListEditor @tools={{this.model}} />
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-tools/new.hbs
-10
View File
@@ -1,10 +0,0 @@
<section class="ai-persona-tool-editor__current admin-detail pull-left">
<AiToolEditor
@tools={{this.allTools}}
@model={{this.model}}
@presets={{this.presets}}
@llms={{this.llms}}
@settings={{this.settings}}
@selectedPreset={{this.selectedPreset}}
/>
</section>
admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-usage.hbs
-1
View File
@@ -1 +0,0 @@
<AiUsage @model={{this.model}} />
app/controllers/discourse_ai/admin/ai_embeddings_controller.rb
-200
View File
@@ -1,200 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiEmbeddingsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
embedding_defs = EmbeddingDefinition.all.order(:display_name)
render json: {
ai_embeddings:
ActiveModel::ArraySerializer.new(
embedding_defs,
each_serializer: AiEmbeddingDefinitionSerializer,
root: false,
).as_json,
meta: {
provider_params: EmbeddingDefinition.provider_params,
providers: EmbeddingDefinition.provider_names,
distance_functions: EmbeddingDefinition.distance_functions,
tokenizers:
EmbeddingDefinition.tokenizer_names.map { |tn|
{ id: tn, name: tn.split("::").last }
},
presets: EmbeddingDefinition.presets,
},
}
end
def new
end
def edit
embedding_def = EmbeddingDefinition.find(params[:id])
render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
end
def create
embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
if embedding_def.save
log_ai_embedding_creation(embedding_def)
render json: AiEmbeddingDefinitionSerializer.new(embedding_def), status: :created
else
render_json_error embedding_def
end
end
def update
embedding_def = EmbeddingDefinition.find(params[:id])
if embedding_def.seeded?
return(
render_json_error(I18n.t("discourse_ai.embeddings.cannot_edit_builtin"), status: 403)
)
end
initial_attributes = embedding_def.attributes.dup
if embedding_def.update(ai_embeddings_params.except(:dimensions))
log_ai_embedding_update(embedding_def, initial_attributes)
render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
else
render_json_error embedding_def
end
end
def destroy
embedding_def = EmbeddingDefinition.find(params[:id])
if embedding_def.seeded?
return(
render_json_error(I18n.t("discourse_ai.embeddings.cannot_edit_builtin"), status: 403)
)
end
if embedding_def.id == SiteSetting.ai_embeddings_selected_model.to_i
return render_json_error(I18n.t("discourse_ai.embeddings.delete_failed"), status: 409)
end
embedding_details = {
embedding_id: embedding_def.id,
display_name: embedding_def.display_name,
provider: embedding_def.provider,
dimensions: embedding_def.dimensions,
subject: embedding_def.display_name,
}
if embedding_def.destroy
log_ai_embedding_deletion(embedding_details)
head :no_content
else
render_json_error embedding_def
end
end
def test
RateLimiter.new(
current_user,
"ai_embeddings_test_#{current_user.id}",
3,
1.minute,
).performed!
embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
DiscourseAi::Embeddings::Vector.new(embedding_def).vector_from("this is a test")
render json: { success: true }
rescue Net::HTTPBadResponse => e
render json: { success: false, error: e.message }
end
private
def ai_embeddings_params
permitted =
params.require(:ai_embedding).permit(
:display_name,
:dimensions,
:max_sequence_length,
:pg_function,
:provider,
:url,
:api_key,
:tokenizer_class,
:embed_prompt,
:search_prompt,
:matryoshka_dimensions,
)
extra_field_names = EmbeddingDefinition.provider_params.dig(permitted[:provider]&.to_sym)
if extra_field_names.present?
received_prov_params =
params.dig(:ai_embedding, :provider_params)&.slice(*extra_field_names.keys)
if received_prov_params.present?
permitted[:provider_params] = received_prov_params.permit!
end
end
permitted
end
def ai_embeddings_logger_fields
{
display_name: {
},
provider: {
},
dimensions: {
},
url: {
},
tokenizer_class: {
},
max_sequence_length: {
},
embed_prompt: {
type: :large_text,
},
search_prompt: {
type: :large_text,
},
matryoshka_dimensions: {
},
api_key: {
type: :sensitive,
},
# JSON fields should be tracked as simple changes
json_fields: [:provider_params],
}
end
def log_ai_embedding_creation(embedding_def)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
logger.log_creation("embedding", embedding_def, ai_embeddings_logger_fields, entity_details)
end
def log_ai_embedding_update(embedding_def, initial_attributes)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
logger.log_update(
"embedding",
embedding_def,
initial_attributes,
ai_embeddings_logger_fields,
entity_details,
)
end
def log_ai_embedding_deletion(embedding_details)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
logger.log_deletion("embedding", embedding_details)
end
end
end
end
app/controllers/discourse_ai/admin/ai_features_controller.rb
-66
View File
@@ -1,66 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiFeaturesController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
render json: serialize_modules(DiscourseAi::Configuration::Module.all)
end
def edit
raise Discourse::InvalidParameters.new(:id) if params[:id].blank?
a_module = DiscourseAi::Configuration::Module.find_by(id: params[:id].to_i)
render json: serialize_module(a_module)
end
private
def serialize_modules(modules)
modules.map { |a_module| serialize_module(a_module) }
end
def serialize_module(a_module)
return nil if a_module.blank?
{
id: a_module.id,
module_name: a_module.name,
module_enabled: a_module.enabled?,
features: a_module.features.map { |f| serialize_feature(f) },
}
end
def serialize_feature(feature)
{
name: feature.name,
personas: feature.persona_ids.map { |id| serialize_persona(persona_id_obj_hash[id]) },
llm_models:
feature.llm_models.map do |llm_model|
{ id: llm_model.id, name: llm_model.display_name }
end,
enabled: feature.enabled?,
}
end
def serialize_persona(persona)
return nil if persona.blank?
serialize_data(persona, AiFeaturesPersonaSerializer, root: false)
end
private
def persona_id_obj_hash
@persona_id_obj_hash ||=
begin
ids = DiscourseAi::Configuration::Feature.all.map(&:persona_ids).flatten.uniq
AiPersona.where(id: ids).index_by(&:id)
end
end
end
end
end
app/controllers/discourse_ai/admin/ai_llm_quotas_controller.rb
-59
View File
@@ -1,59 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiLlmQuotasController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
quotas = LlmQuota.includes(:group)
render json: {
quotas:
ActiveModel::ArraySerializer.new(quotas, each_serializer: LlmQuotaSerializer),
}
end
def create
quota = LlmQuota.new(quota_params)
if quota.save
render json: LlmQuotaSerializer.new(quota), status: :created
else
render_json_error quota
end
end
def update
quota = LlmQuota.find(params[:id])
if quota.update(quota_params)
render json: LlmQuotaSerializer.new(quota)
else
render_json_error quota
end
end
def destroy
quota = LlmQuota.find(params[:id])
quota.destroy!
head :no_content
rescue ActiveRecord::RecordNotFound
render json: { error: I18n.t("not_found") }, status: 404
end
private
def quota_params
params.require(:quota).permit(
:group_id,
:llm_model_id,
:max_tokens,
:max_usages,
:duration_seconds,
)
end
end
end
end
app/controllers/discourse_ai/admin/ai_llms_controller.rb
-293
View File
@@ -1,293 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiLlmsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
llms = LlmModel.all.includes(:llm_quotas).order(:display_name)
render json: {
ai_llms:
ActiveModel::ArraySerializer.new(
llms,
each_serializer: LlmModelSerializer,
root: false,
scope: {
llm_usage: DiscourseAi::Configuration::LlmEnumerator.global_usage,
},
).as_json,
meta: {
provider_params: LlmModel.provider_params,
presets: DiscourseAi::Completions::Llm.presets,
providers: DiscourseAi::Completions::Llm.provider_names,
tokenizers:
DiscourseAi::Completions::Llm.tokenizer_names.map { |tn|
{ id: tn, name: tn.split("::").last }
},
},
}
end
def new
end
def edit
llm_model = LlmModel.find(params[:id])
render json: LlmModelSerializer.new(llm_model)
end
def create
llm_model = LlmModel.new(ai_llm_params)
# we could do nested attributes but the mechanics are not ideal leading
# to lots of complex debugging, this is simpler
quota_params.each { |quota| llm_model.llm_quotas.build(quota) } if quota_params
if llm_model.save
llm_model.toggle_companion_user
log_llm_model_creation(llm_model)
render json: LlmModelSerializer.new(llm_model), status: :created
else
render_json_error llm_model
end
end
def update
llm_model = LlmModel.find(params[:id])
# Capture initial state for logging
initial_attributes = llm_model.attributes.dup
initial_quotas = llm_model.llm_quotas.map(&:attributes)
if params[:ai_llm].key?(:llm_quotas)
if quota_params
existing_quota_group_ids = llm_model.llm_quotas.pluck(:group_id)
new_quota_group_ids = quota_params.map { |q| q[:group_id] }
llm_model
.llm_quotas
.where(group_id: existing_quota_group_ids - new_quota_group_ids)
.destroy_all
quota_params.each do |quota_param|
quota = llm_model.llm_quotas.find_or_initialize_by(group_id: quota_param[:group_id])
quota.update!(quota_param)
end
else
llm_model.llm_quotas.destroy_all
end
end
if llm_model.seeded?
return render_json_error(I18n.t("discourse_ai.llm.cannot_edit_builtin"), status: 403)
end
if llm_model.update(ai_llm_params(updating: llm_model))
llm_model.toggle_companion_user
log_llm_model_update(llm_model, initial_attributes, initial_quotas)
render json: LlmModelSerializer.new(llm_model)
else
render_json_error llm_model
end
end
def destroy
llm_model = LlmModel.find(params[:id])
if llm_model.seeded?
return render_json_error(I18n.t("discourse_ai.llm.cannot_delete_builtin"), status: 403)
end
in_use_by = DiscourseAi::Configuration::LlmValidator.new.modules_using(llm_model)
if !in_use_by.empty?
return(
render_json_error(
I18n.t(
"discourse_ai.llm.delete_failed",
settings: in_use_by.join(", "),
count: in_use_by.length,
),
status: 409,
)
)
end
# Capture model details for logging before destruction
model_details = {
model_id: llm_model.id,
display_name: llm_model.display_name,
name: llm_model.name,
provider: llm_model.provider,
}
# Clean up companion users
llm_model.enabled_chat_bot = false
llm_model.toggle_companion_user
if llm_model.destroy
log_llm_model_deletion(model_details)
head :no_content
else
render_json_error llm_model
end
end
def test
RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!
llm_model = LlmModel.new(ai_llm_params)
DiscourseAi::Configuration::LlmValidator.new.run_test(llm_model)
render json: { success: true }
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
render json: { success: false, error: e.message }
end
private
def quota_params
if params[:ai_llm][:llm_quotas].present?
params[:ai_llm][:llm_quotas].map do |quota|
mapped = {}
mapped[:group_id] = quota[:group_id].to_i
mapped[:max_tokens] = quota[:max_tokens].to_i if quota[:max_tokens].present?
mapped[:max_usages] = quota[:max_usages].to_i if quota[:max_usages].present?
mapped[:duration_seconds] = quota[:duration_seconds].to_i
mapped
end
end
end
def ai_llm_params(updating: nil)
return {} if params[:ai_llm].blank?
permitted =
params.require(:ai_llm).permit(
:display_name,
:name,
:provider,
:tokenizer,
:max_prompt_tokens,
:max_output_tokens,
:api_key,
:enabled_chat_bot,
:vision_enabled,
:input_cost,
:cached_input_cost,
:output_cost,
)
provider = updating ? updating.provider : permitted[:provider]
permit_url = provider != LlmModel::BEDROCK_PROVIDER_NAME
new_url = params.dig(:ai_llm, :url)
permitted[:url] = new_url if permit_url && new_url
extra_field_names = LlmModel.provider_params.dig(provider&.to_sym)
if extra_field_names.present?
received_prov_params =
params.dig(:ai_llm, :provider_params)&.slice(*extra_field_names.keys)
if received_prov_params.present?
received_prov_params.each do |pname, value|
if extra_field_names[pname.to_sym] == :checkbox
received_prov_params[pname] = ActiveModel::Type::Boolean.new.cast(value)
end
end
permitted[:provider_params] = received_prov_params.permit!
end
end
permitted
end
def ai_llm_logger_fields
{
display_name: {
},
name: {
},
provider: {
},
tokenizer: {
},
url: {
},
max_prompt_tokens: {
},
max_output_tokens: {
},
enabled_chat_bot: {
},
vision_enabled: {
},
api_key: {
type: :sensitive,
},
input_cost: {
},
output_cost: {
},
# JSON fields should be tracked as simple changes
json_fields: [:provider_params],
}
end
def log_llm_model_creation(llm_model)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
# Add quota information as a special case
if llm_model.llm_quotas.any?
entity_details[:quotas] = llm_model
.llm_quotas
.map do |quota|
"Group #{quota.group_id}: #{quota.max_tokens} tokens, #{quota.max_usages} usages, #{quota.duration_seconds}s"
end
.join("; ")
end
logger.log_creation("llm_model", llm_model, ai_llm_logger_fields, entity_details)
end
def log_llm_model_update(llm_model, initial_attributes, initial_quotas)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { model_id: llm_model.id, subject: llm_model.display_name }
# Track quota changes separately as they're a special case
current_quotas = llm_model.llm_quotas.reload.map(&:attributes)
if initial_quotas != current_quotas
initial_quota_summary =
initial_quotas
.map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
.join("; ")
current_quota_summary =
current_quotas
.map { |q| "Group #{q["group_id"]}: #{q["max_tokens"]} tokens" }
.join("; ")
entity_details[:quotas_changed] = true
entity_details[:quotas] = "#{initial_quota_summary} → #{current_quota_summary}"
end
logger.log_update(
"llm_model",
llm_model,
initial_attributes,
ai_llm_logger_fields,
entity_details,
)
end
def log_llm_model_deletion(model_details)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
model_details[:subject] = model_details[:display_name]
logger.log_deletion("llm_model", model_details)
end
end
end
end
app/controllers/discourse_ai/admin/ai_personas_controller.rb
-403
View File
@@ -1,403 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiPersonasController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
before_action :find_ai_persona, only: %i[edit update destroy create_user export]
def index
ai_personas =
AiPersona
.ordered
.includes(:user, :uploads)
.map { |persona| LocalizedAiPersonaSerializer.new(persona, root: false) }
tools =
DiscourseAi::Personas::Persona.all_available_tools.map do |tool|
AiToolSerializer.new(tool, root: false)
end
AiTool
.where(enabled: true)
.each do |tool|
tools << {
id: "custom-#{tool.id}",
name:
I18n.t(
"discourse_ai.tools.custom_name",
name: tool.name.capitalize,
tool_name: tool.tool_name,
),
}
end
llms =
DiscourseAi::Configuration::LlmEnumerator.values_for_serialization(
allowed_seeded_llm_ids: SiteSetting.ai_bot_allowed_seeded_models_map,
)
render json: {
ai_personas: ai_personas,
meta: {
tools: tools,
llms: llms,
settings: {
rag_images_enabled: SiteSetting.ai_rag_images_enabled,
},
},
}
end
def new
end
def edit
render json: LocalizedAiPersonaSerializer.new(@ai_persona)
end
def create
ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
if ai_persona.save
RagDocumentFragment.link_target_and_uploads(ai_persona, attached_upload_ids)
log_ai_persona_creation(ai_persona)
render json: {
ai_persona: LocalizedAiPersonaSerializer.new(ai_persona, root: false),
},
status: :created
else
render_json_error ai_persona
end
end
def create_user
user = @ai_persona.create_user!
render json: BasicUserSerializer.new(user, root: "user")
end
def update
initial_attributes = @ai_persona.attributes.dup
if @ai_persona.update(ai_persona_params.except(:rag_uploads))
RagDocumentFragment.update_target_uploads(@ai_persona, attached_upload_ids)
log_ai_persona_update(@ai_persona, initial_attributes)
render json: LocalizedAiPersonaSerializer.new(@ai_persona, root: false)
else
render_json_error @ai_persona
end
end
def destroy
persona_details = {
persona_id: @ai_persona.id,
name: @ai_persona.name,
description: @ai_persona.description,
}
if @ai_persona.destroy
log_ai_persona_deletion(persona_details)
head :no_content
else
render_json_error @ai_persona
end
end
def export
persona = AiPersona.find(params[:id])
exporter = DiscourseAi::PersonaExporter.new(persona: persona)
response.headers[
"Content-Disposition"
] = "attachment; filename=\"#{persona.name.parameterize}.json\""
render json: exporter.export
end
def import
name = params.dig(:persona, :name)
existing_persona = AiPersona.find_by(name: name)
force_update = params[:force].present? && params[:force].to_s.downcase == "true"
begin
importer = DiscourseAi::PersonaImporter.new(json: params.to_unsafe_h)
if existing_persona && force_update
initial_attributes = existing_persona.attributes.dup
persona = importer.import!(overwrite: true)
log_ai_persona_update(persona, initial_attributes)
render json: LocalizedAiPersonaSerializer.new(persona, root: false)
else
persona = importer.import!
log_ai_persona_creation(persona)
render json: LocalizedAiPersonaSerializer.new(persona, root: false), status: :created
end
rescue DiscourseAi::PersonaImporter::ImportError => e
render_json_error e.message, status: :unprocessable_entity
rescue StandardError => e
Rails.logger.error("AI Persona import failed: #{e.message}")
render_json_error "Import failed: #{e.message}", status: :unprocessable_entity
end
end
def stream_reply
persona =
AiPersona.find_by(name: params[:persona_name]) ||
AiPersona.find_by(id: params[:persona_id])
return render_json_error(I18n.t("discourse_ai.errors.persona_not_found")) if persona.nil?
return render_json_error(I18n.t("discourse_ai.errors.persona_disabled")) if !persona.enabled
if persona.default_llm.blank?
return render_json_error(I18n.t("discourse_ai.errors.no_default_llm"))
end
if params[:query].blank?
return render_json_error(I18n.t("discourse_ai.errors.no_query_specified"))
end
if !persona.user_id
return render_json_error(I18n.t("discourse_ai.errors.no_user_for_persona"))
end
if !params[:username] && !params[:user_unique_id]
return render_json_error(I18n.t("discourse_ai.errors.no_user_specified"))
end
user = nil
if params[:username]
user = User.find_by_username(params[:username])
return render_json_error(I18n.t("discourse_ai.errors.user_not_found")) if user.nil?
elsif params[:user_unique_id]
user = stage_user
end
raise Discourse::NotFound if user.nil?
topic_id = params[:topic_id].to_i
topic = nil
if topic_id > 0
topic = Topic.find(topic_id)
if topic.topic_allowed_users.where(user_id: user.id).empty?
return render_json_error(I18n.t("discourse_ai.errors.user_not_allowed"))
end
end
hijack = request.env["rack.hijack"]
io = hijack.call
DiscourseAi::AiBot::ResponseHttpStreamer.queue_streamed_reply(
io: io,
persona: persona,
user: user,
topic: topic,
query: params[:query].to_s,
custom_instructions: params[:custom_instructions].to_s,
current_user: current_user,
)
end
private
AI_STREAM_CONVERSATION_UNIQUE_ID = "ai-stream-conversation-unique-id"
def stage_user
unique_id = params[:user_unique_id].to_s
field = UserCustomField.find_by(name: AI_STREAM_CONVERSATION_UNIQUE_ID, value: unique_id)
if field
field.user
else
preferred_username = params[:preferred_username]
username = UserNameSuggester.suggest(preferred_username || unique_id)
user =
User.new(
username: username,
email: "#{SecureRandom.hex}@invalid.com",
staged: true,
active: false,
)
user.custom_fields[AI_STREAM_CONVERSATION_UNIQUE_ID] = unique_id
user.save!
user
end
end
def find_ai_persona
@ai_persona = AiPersona.find(params[:id])
end
def attached_upload_ids
ai_persona_params[:rag_uploads].to_a.map { |h| h[:id] }
end
def ai_persona_params
permitted =
params.require(:ai_persona).permit(
:name,
:description,
:enabled,
:system_prompt,
:priority,
:top_p,
:temperature,
:default_llm_id,
:user_id,
:max_context_posts,
:vision_enabled,
:vision_max_pixels,
:rag_chunk_tokens,
:rag_chunk_overlap_tokens,
:rag_conversation_chunks,
:rag_llm_model_id,
:question_consolidator_llm_id,
:allow_chat_channel_mentions,
:allow_chat_direct_messages,
:allow_topic_mentions,
:allow_personal_messages,
:tool_details,
:forced_tool_count,
:force_default_llm,
allowed_group_ids: [],
rag_uploads: [:id],
)
if tools = params.dig(:ai_persona, :tools)
permitted[:tools] = permit_tools(tools)
end
if response_format = params.dig(:ai_persona, :response_format)
permitted[:response_format] = permit_response_format(response_format)
end
if examples = params.dig(:ai_persona, :examples)
permitted[:examples] = permit_examples(examples)
end
permitted
end
def permit_tools(tools)
return [] if !tools.is_a?(Array)
tools.filter_map do |tool, options, force_tool|
break nil if !tool.is_a?(String)
options&.permit! if options && options.is_a?(ActionController::Parameters)
# this is simpler from a storage perspective, 1 way to store tools
[tool, options, !!force_tool]
end
end
def permit_response_format(response_format)
return [] if !response_format.is_a?(Array)
response_format.map do |element|
if element && element.is_a?(ActionController::Parameters)
element.permit!
else
false
end
end
end
def permit_examples(examples)
return [] if !examples.is_a?(Array)
examples.map { |example_arr| example_arr.take(2).map(&:to_s) }
end
def ai_persona_logger_fields
{
name: {
},
description: {
},
enabled: {
},
priority: {
},
system_prompt: {
type: :large_text,
},
default_llm_id: {
},
temperature: {
},
top_p: {
},
user_id: {
},
max_context_posts: {
},
vision_enabled: {
},
vision_max_pixels: {
},
rag_chunk_tokens: {
},
rag_chunk_overlap_tokens: {
},
rag_conversation_chunks: {
},
rag_llm_model_id: {
},
question_consolidator_llm_id: {
},
allow_chat_channel_mentions: {
},
allow_chat_direct_messages: {
},
allow_topic_mentions: {
},
allow_personal_messages: {
},
tool_details: {
type: :large_text,
},
forced_tool_count: {
},
force_default_llm: {
},
# JSON fields
json_fields: %i[tools response_format examples allowed_group_ids],
}
end
def log_ai_persona_creation(ai_persona)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
entity_details[:tools_count] = (ai_persona.tools || []).size
logger.log_creation("persona", ai_persona, ai_persona_logger_fields, entity_details)
end
def log_ai_persona_update(ai_persona, initial_attributes)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { persona_id: ai_persona.id, subject: ai_persona.name }
entity_details[:tools_count] = ai_persona.tools.size if ai_persona.tools.present?
logger.log_update(
"persona",
ai_persona,
initial_attributes,
ai_persona_logger_fields,
entity_details,
)
end
def log_ai_persona_deletion(persona_details)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
persona_details[:subject] = persona_details[:name]
logger.log_deletion("persona", persona_details)
end
end
end
end
app/controllers/discourse_ai/admin/ai_spam_controller.rb
-202
View File
@@ -1,202 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiSpamController < ::Admin::AdminController
requires_plugin "discourse-ai"
def show
render json: AiSpamSerializer.new(spam_config, root: false)
end
def update
initial_settings = AiModerationSetting.spam
initial_data = {
custom_instructions: initial_settings&.data&.dig("custom_instructions"),
llm_model_id: initial_settings&.llm_model_id,
ai_persona_id: initial_settings&.ai_persona_id,
}
initial_custom_instructions = initial_settings&.data&.dig("custom_instructions")
initial_llm_model_id = initial_settings&.llm_model_id
updated_params = {}
if allowed_params.key?(:llm_model_id)
llm_model_id = updated_params[:llm_model_id] = allowed_params[:llm_model_id]
if llm_model_id.to_i < 0 &&
!SiteSetting.ai_spam_detection_model_allowed_seeded_models_map.include?(
llm_model_id.to_s,
)
return(
render_json_error(
I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"),
status: 422,
)
)
end
end
if allowed_params.key?(:ai_persona_id)
updated_params[:ai_persona_id] = allowed_params[:ai_persona_id]
persona = AiPersona.find_by(id: allowed_params[:ai_persona_id])
if persona.nil? ||
persona.response_format.to_a.none? { |rf|
rf["key"] == "spam" && rf["type"] == "boolean"
}
return(
render_json_error(
I18n.t("discourse_ai.llm.configuration.invalid_persona_response_format"),
status: 422,
)
)
end
end
updated_params[:data] = {
custom_instructions: allowed_params[:custom_instructions],
} if allowed_params.key?(:custom_instructions)
if updated_params.present?
# not using upsert cause we will not get the correct validation errors
if AiModerationSetting.spam
AiModerationSetting.spam.update!(updated_params)
else
AiModerationSetting.create!(updated_params.merge(setting_type: :spam))
end
log_ai_spam_update(initial_data, allowed_params)
end
is_enabled = ActiveModel::Type::Boolean.new.cast(allowed_params[:is_enabled])
if allowed_params.key?(:is_enabled)
if is_enabled && !AiModerationSetting.spam&.llm_model_id
return(
render_json_error(
I18n.t("discourse_ai.llm.configuration.must_select_model"),
status: 422,
)
)
end
SiteSetting.set_and_log("ai_spam_detection_enabled", is_enabled, current_user)
end
render json: AiSpamSerializer.new(spam_config, root: false)
end
def test
url = params[:post_url].to_s
post = nil
if url.match?(/^\d+$/)
post_id = url.to_i
post = Post.find_by(id: post_id)
end
route = UrlHelper.rails_route_from_url(url) if !post
if route
if route[:controller] == "topics"
post_number = route[:post_number] || 1
post = Post.with_deleted.find_by(post_number: post_number, topic_id: route[:topic_id])
end
end
raise Discourse::NotFound if !post
result =
DiscourseAi::AiModeration::SpamScanner.test_post(
post,
custom_instructions: params[:custom_instructions],
llm_id: params[:llm_id],
)
render json: result
end
def fix_errors
case params[:error]
when "spam_scanner_not_admin"
begin
DiscourseAi::AiModeration::SpamScanner.fix_spam_scanner_not_admin
render json: success_json
rescue ActiveRecord::RecordInvalid
render_json_error(
I18n.t("discourse_ai.spam_detection.bot_user_update_failed"),
status: :unprocessable_entity,
)
rescue StandardError
render_json_error(
I18n.t("discourse_ai.spam_detection.unexpected"),
status: :internal_server_error,
)
end
else
render_json_error(
I18n.t("discourse_ai.spam_detection.invalid_error_type"),
status: :bad_request,
)
end
end
private
def log_ai_spam_update(initial_data, params)
changes_to_log = {}
initial_llm_model_id = initial_data[:llm_model_id]
if params.key?(:llm_model_id) && initial_llm_model_id.to_s != params[:llm_model_id].to_s
old_model_name =
LlmModel.find_by(id: initial_llm_model_id)&.display_name || initial_llm_model_id
new_model_name =
LlmModel.find_by(id: params[:llm_model_id])&.display_name || params[:llm_model_id]
changes_to_log[:llm_model_id] = "#{old_model_name} → #{new_model_name}"
end
initial_custom_instructions = initial_data[:custom_instructions]
if params.key?(:custom_instructions) &&
initial_custom_instructions != params[:custom_instructions]
changes_to_log[:custom_instructions] = params[:custom_instructions]
end
initial_ai_persona_id = initial_data[:ai_persona_id]
if params.key?(:ai_persona_id) && initial_ai_persona_id.to_s != params[:ai_persona_id].to_s
old_persona_name =
AiPersona.find_by(id: initial_ai_persona_id)&.name || initial_ai_persona_id
new_persona_name =
AiPersona.find_by(id: params[:ai_persona_id])&.name || params[:ai_persona_id]
changes_to_log[:ai_persona_id] = "#{old_persona_name} → #{new_persona_name}"
end
if changes_to_log.present?
changes_to_log[:subject] = I18n.t("discourse_ai.spam_detection.logging_subject")
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
logger.log_custom("update_ai_spam_settings", changes_to_log)
end
end
def allowed_params
params.permit(:is_enabled, :llm_model_id, :custom_instructions, :ai_persona_id)
end
def spam_config
spam_config = {
enabled: SiteSetting.ai_spam_detection_enabled,
settings: AiModerationSetting.spam,
}
spam_config[:stats] = DiscourseAi::AiModeration::SpamReport.generate(min_date: 1.week.ago)
if spam_config[:stats].scanned_count > 0
spam_config[
:flagging_username
] = DiscourseAi::AiModeration::SpamScanner.flagging_user&.username
end
spam_config
end
end
end
end
app/controllers/discourse_ai/admin/ai_tools_controller.rb
-206
View File
@@ -1,206 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiToolsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
before_action :find_ai_tool, only: %i[test edit update destroy export]
def index
ai_tools = AiTool.all
render_serialized({ ai_tools: ai_tools }, AiCustomToolListSerializer, root: false)
end
def new
end
def edit
render_serialized(@ai_tool, AiCustomToolSerializer)
end
def create
ai_tool = AiTool.new(ai_tool_params)
ai_tool.created_by_id = current_user.id
if ai_tool.save
RagDocumentFragment.link_target_and_uploads(ai_tool, attached_upload_ids)
log_ai_tool_creation(ai_tool)
render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
else
render_json_error ai_tool
end
end
def export
response.headers[
"Content-Disposition"
] = "attachment; filename=\"#{@ai_tool.tool_name}.json\""
render_serialized(@ai_tool, AiCustomToolSerializer)
end
def import
existing_tool = AiTool.find_by(tool_name: ai_tool_params[:tool_name])
force_update = params[:force].present? && params[:force].to_s.downcase == "true"
if existing_tool && !force_update
return(
render_json_error "Tool with tool_name '#{ai_tool_params[:tool_name]}' already exists. Use force=true to overwrite.",
status: :conflict
)
end
if existing_tool && force_update
initial_attributes = existing_tool.attributes.dup
if existing_tool.update(ai_tool_params)
log_ai_tool_update(existing_tool, initial_attributes)
render_serialized(existing_tool, AiCustomToolSerializer)
else
render_json_error existing_tool
end
else
ai_tool = AiTool.new(ai_tool_params)
ai_tool.created_by_id = current_user.id
if ai_tool.save
log_ai_tool_creation(ai_tool)
render_serialized(ai_tool, AiCustomToolSerializer, status: :created)
else
render_json_error ai_tool
end
end
end
def update
initial_attributes = @ai_tool.attributes.dup
if @ai_tool.update(ai_tool_params)
RagDocumentFragment.update_target_uploads(@ai_tool, attached_upload_ids)
log_ai_tool_update(@ai_tool, initial_attributes)
render_serialized(@ai_tool, AiCustomToolSerializer)
else
render_json_error @ai_tool
end
end
def destroy
tool_logger_details = {
tool_id: @ai_tool.id,
name: @ai_tool.name,
tool_name: @ai_tool.tool_name,
subject: @ai_tool.name,
}
if @ai_tool.destroy
log_ai_tool_deletion(tool_logger_details)
head :no_content
else
render_json_error @ai_tool
end
end
def test
@ai_tool.assign_attributes(ai_tool_params) if params[:ai_tool]
parameters = params[:parameters].to_unsafe_h
# we need an llm so we have a tokenizer
# but will do without if none is available
llm = LlmModel.first&.to_llm
runner = @ai_tool.runner(parameters, llm: llm, bot_user: current_user)
result = runner.invoke
if result.is_a?(Hash) && result[:error]
render_json_error result[:error]
else
render json: { output: result }
end
rescue ActiveRecord::RecordNotFound => e
render_json_error e.message, status: 400
rescue => e
render_json_error "Error executing the tool: #{e.message}", status: 400
end
private
def attached_upload_ids
params[:ai_tool][:rag_uploads].to_a.map { |h| h[:id] }
end
def find_ai_tool
@ai_tool = AiTool.find(params[:id].to_i)
end
def ai_tool_params
params
.require(:ai_tool)
.permit(
:name,
:tool_name,
:description,
:script,
:summary,
:rag_chunk_tokens,
:rag_chunk_overlap_tokens,
:rag_llm_model_id,
rag_uploads: [:id],
parameters: [:name, :type, :description, :required, enum: []],
)
.except(:rag_uploads)
end
def ai_tool_logger_fields
{
name: {
},
tool_name: {
},
description: {
},
summary: {
},
enabled: {
},
rag_chunk_tokens: {
},
rag_chunk_overlap_tokens: {
},
rag_llm_model_id: {
},
script: {
type: :large_text,
},
parameters: {
type: :large_text,
},
}
end
def log_ai_tool_creation(ai_tool)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
entity_details[:parameter_count] = ai_tool.parameters.size if ai_tool.parameters.present?
logger.log_creation("tool", ai_tool, ai_tool_logger_fields, entity_details)
end
def log_ai_tool_update(ai_tool, initial_attributes)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { tool_id: ai_tool.id, subject: ai_tool.name }
logger.log_update(
"tool",
ai_tool,
initial_attributes,
ai_tool_logger_fields,
entity_details,
)
end
def log_ai_tool_deletion(tool_details)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
logger.log_deletion("tool", tool_details)
end
end
end
end
app/controllers/discourse_ai/admin/ai_usage_controller.rb
-30
View File
@@ -1,30 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiUsageController < ::Admin::AdminController
requires_plugin "discourse-ai"
def show
end
def report
render json: AiUsageSerializer.new(create_report, root: false)
end
private
def create_report
report =
DiscourseAi::Completions::Report.new(
start_date: params[:start_date]&.to_date || 30.days.ago,
end_date: params[:end_date]&.to_date || Time.current,
)
report = report.filter_by_feature(params[:feature]) if params[:feature].present?
report = report.filter_by_model(params[:model]) if params[:model].present?
report
end
end
end
end
app/controllers/discourse_ai/admin/dashboard_controller.rb
-12
View File
@@ -1,12 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class DashboardController < ::Admin::StaffController
requires_plugin DiscourseAi::PLUGIN_NAME
def sentiment
end
end
end
end
app/controllers/discourse_ai/admin/rag_document_fragments_controller.rb
-76
View File
@@ -1,76 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class RagDocumentFragmentsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def indexing_status_check
if params[:target_type] == "AiPersona"
@target = AiPersona.find(params[:target_id])
elsif params[:target_type] == "AiTool"
@target = AiTool.find(params[:target_id])
else
raise Discourse::InvalidParameters.new("Invalid target type")
end
render json: RagDocumentFragment.indexing_status(@target, @target.uploads)
end
def upload_file
file = params[:file] || params[:files].first
if !DiscourseAi::Embeddings.enabled?
raise Discourse::InvalidAccess.new("Embeddings not enabled")
end
validate_extension!(file.original_filename)
validate_file_size!(file.tempfile.size)
hijack do
upload =
UploadCreator.new(
file.tempfile,
file.original_filename,
type: "discourse_ai_rag_upload",
skip_validations: true,
).create_for(current_user.id)
if upload.persisted?
render json: UploadSerializer.new(upload)
else
render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
end
end
end
private
def validate_extension!(filename)
extension = File.extname(filename)[1..-1] || ""
authorized_extensions = %w[txt md pdf]
authorized_extensions.concat(%w[png jpg jpeg]) if SiteSetting.ai_rag_images_enabled
if !authorized_extensions.include?(extension)
raise Discourse::InvalidParameters.new(
I18n.t(
"upload.unauthorized",
authorized_extensions: authorized_extensions.join(" "),
),
)
end
end
def validate_file_size!(filesize)
max_size_bytes = 20.megabytes
if filesize > max_size_bytes
raise Discourse::InvalidParameters.new(
I18n.t(
"upload.attachments.too_large_humanized",
max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
),
)
end
end
end
end
end
app/controllers/discourse_ai/ai_bot/artifact_key_values_controller.rb
-124
View File
@@ -1,124 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module AiBot
class ArtifactKeyValuesController < ::ApplicationController
requires_plugin DiscourseAi::PLUGIN_NAME
before_action :ensure_logged_in, only: %i[set destroy]
before_action :find_artifact
PER_PAGE_MAX = 100
def index
page = index_params[:page].to_i
page = 1 if page < 1
per_page = index_params[:per_page].to_i
per_page = PER_PAGE_MAX if per_page < 1 || per_page > PER_PAGE_MAX
query = build_index_query
total_count = query.count
key_values =
query
.includes(:user)
.order(:user_id, :key, :created_at)
.offset((page - 1) * per_page)
.limit(per_page + 1)
has_more = key_values.length > per_page
key_values = key_values.first(per_page) if has_more
render json: {
key_values:
ActiveModel::ArraySerializer.new(
key_values,
each_serializer: AiArtifactKeyValueSerializer,
keys_only: params[:keys_only] == "true",
).as_json,
has_more: has_more,
total_count: total_count,
users:
key_values
.map { |kv| kv.user }
.uniq
.map { |u| BasicUserSerializer.new(u, root: nil).as_json },
}
end
def destroy
if params[:key].blank?
render json: { error: "Key parameter is required" }, status: :bad_request
return
end
key_value = @artifact.key_values.find_by(user_id: current_user.id, key: params[:key])
if key_value.nil?
render json: { error: "Key not found" }, status: :not_found
elsif key_value.destroy
head :ok
else
render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
end
end
def set
key_value =
@artifact.key_values.find_or_initialize_by(
user: current_user,
key: key_value_params[:key],
)
key_value.assign_attributes(key_value_params.except(:key))
if key_value.save
render json: AiArtifactKeyValueSerializer.new(key_value).as_json
else
render json: { errors: key_value.errors.full_messages }, status: :unprocessable_entity
end
end
private
def key_value_params
params.permit(:key, :value, :public)
end
def index_params
@index_params ||= params.permit(:page, :per_page, :key, :keys_only, :all_users)
end
def build_index_query
query = @artifact.key_values
query =
if current_user&.admin?
query
elsif current_user
query.where("user_id = ? OR public = true", current_user.id)
else
query.where(public: true)
end
query = query.where("key = ?", index_params[:key]) if index_params[:key].present?
if index_params[:all_users].to_s != "true"
if current_user
query = query.where(user_id: current_user.id)
else
query = query.where("1 = 0")
end
end
query
end
def find_artifact
@artifact = AiArtifact.find_by(id: params[:artifact_id])
raise Discourse::NotFound if !@artifact
raise Discourse::NotFound if !@artifact.public? && guardian.anonymous?
raise Discourse::NotFound if !@artifact.public? && !guardian.can_see?(@artifact.post)
end
end
end
end
app/controllers/discourse_ai/ai_bot/artifacts_controller.rb
-344
View File
@@ -1,344 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module AiBot
class ArtifactsController < ApplicationController
requires_plugin DiscourseAi::PLUGIN_NAME
before_action :require_site_settings!
skip_before_action :preload_json, :check_xhr, only: %i[show]
def show
artifact = AiArtifact.find(params[:id])
post = Post.find_by(id: artifact.post_id)
if artifact.public?
# no guardian needed
else
raise Discourse::NotFound if !post&.topic&.private_message?
raise Discourse::NotFound if !guardian.can_see?(post)
end
name = artifact.name
artifact_version = nil
if params[:version].present?
artifact_version = artifact.versions.find_by(version_number: params[:version])
raise Discourse::NotFound if !artifact_version
end
untrusted_html = build_untrusted_html(artifact_version || artifact, name)
trusted_html = build_trusted_html(artifact, artifact_version, name, untrusted_html)
set_security_headers
render html: trusted_html.html_safe, layout: false, content_type: "text/html"
end
private
def build_untrusted_html(artifact, name)
js = prepare_javascript(artifact.js)
<<~HTML
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>#{ERB::Util.html_escape(name)}</title>
<style>
#{artifact.css}
</style>
#{build_iframe_javascript}
</head>
<body>
#{artifact.html}
#{js}
</body>
</html>
HTML
end
def build_trusted_html(artifact, artifact_version, name, untrusted_html)
<<~HTML
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>#{ERB::Util.html_escape(name)}</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, user-scalable=yes, viewport-fit=cover, interactive-widget=resizes-content">
<meta name="csrf-token" content="#{form_authenticity_token}">
<style>
html, body, iframe {
margin: 0;
padding: 0;
width: 100%;
height: 100%;
border: 0;
overflow: hidden;
}
iframe {
overflow: auto;
}
</style>
</head>
<body>
<iframe sandbox="allow-scripts allow-forms" height="100%" width="100%" srcdoc="#{ERB::Util.html_escape(untrusted_html)}" frameborder="0"></iframe>
#{build_parent_javascript(artifact)}
</body>
</html>
HTML
end
def prepare_javascript(js)
return "" if js.blank?
if !js.match?(%r{\A\s*<script.*</script>}mi)
mod = ""
mod = " type=\"module\"" if js.match?(/\A\s*import.*/)
js = "<script#{mod}>\n#{js}\n</script>"
end
js
end
def user_data
{
username: current_user ? current_user.username : nil,
user_id: current_user ? current_user.id : nil,
name: current_user ? current_user.name : nil,
}
end
def build_iframe_javascript
<<~JAVASCRIPT
<script>
window._discourse_user_data = #{user_data.to_json};
window.discourseArtifactReady = new Promise(resolve => {
window._resolveArtifactData = resolve;
});
// Key-value store API
window.discourseArtifact = {
get: function(key) {
return window._postMessageRequest('get', { key: key });
},
set: function(key, value, options = {}) {
return window._postMessageRequest('set', {
key: key,
value: value,
public: options.public || false
});
},
delete: function(key) {
return window._postMessageRequest('delete', { key: key });
},
index: function(filter = {}) {
return window._postMessageRequest('index', filter);
}
};
window._postMessageRequest = function(action, data) {
return new Promise((resolve, reject) => {
const requestId = Math.random().toString(36).substr(2, 9);
const messageHandler = function(event) {
if (event.data && event.data.requestId === requestId) {
window.removeEventListener('message', messageHandler);
if (event.data.error) {
reject(new Error(event.data.error));
} else {
resolve(event.data.result);
}
}
};
window.addEventListener('message', messageHandler);
window.parent.postMessage({
type: 'discourse-artifact-kv',
action: action,
data: data,
requestId: requestId
}, '*');
});
};
window.addEventListener('message', function(event) {
if (event.data && event.data.type === 'discourse-artifact-data') {
window.discourseArtifactData = event.data.dataset || {};
Object.assign(window.discourseArtifactData, window._discourse_user_data);
window._resolveArtifactData(window.discourseArtifactData);
}
});
</script>
JAVASCRIPT
end
def build_parent_javascript(artifact)
<<~JAVASCRIPT
<script>
document.querySelector('iframe').addEventListener('load', function() {
try {
const iframeWindow = this.contentWindow;
const message = { type: 'discourse-artifact-data', dataset: {} };
if (window.frameElement && window.frameElement.dataset) {
Object.assign(message.dataset, window.frameElement.dataset);
}
iframeWindow.postMessage(message, '*');
} catch (e) {
console.error('Error passing data to artifact:', e);
}
});
// Handle key-value store requests from iframe
window.addEventListener('message', async function(event) {
if (event.data && event.data.type === 'discourse-artifact-kv') {
const { action, data, requestId } = event.data;
const artifactId = #{artifact.id};
try {
const result = await handleKeyValueRequest(action, data, artifactId);
event.source.postMessage({
requestId: requestId,
result: result
}, '*');
} catch (error) {
event.source.postMessage({
requestId: requestId,
error: error.message
}, '*');
}
}
});
async function handleKeyValueRequest(action, data, artifactId) {
const baseUrl = '/discourse-ai/ai-bot/artifact-key-values/' + artifactId + ".json";
const csrfToken = document.querySelector('meta[name="csrf-token"]')?.content || '';
switch (action) {
case 'get':
return await handleGetRequest(baseUrl, data, csrfToken);
case 'set':
return await handleSetRequest(baseUrl, data, csrfToken);
case 'index':
return await handleIndexRequest(baseUrl, data, csrfToken);
case 'delete':
return await handleDeleteRequest(baseUrl, data, csrfToken);
default:
throw new Error('Unknown action: ' + action);
}
}
async function handleGetRequest(baseUrl, data, csrfToken) {
const response = await fetch(baseUrl + '?key=' + encodeURIComponent(data.key), {
method: 'GET',
headers: {
'X-CSRF-Token': csrfToken,
'Content-Type': 'application/json'
},
credentials: 'same-origin'
});
if (!response.ok) throw new Error('Failed to get key-value');
const result = await response.json();
const keyValue = result.key_values.find(kv => kv.key === data.key);
return keyValue ? keyValue.value : null;
}
async function handleSetRequest(baseUrl, data, csrfToken) {
const response = await fetch(baseUrl, {
method: 'POST',
headers: {
'X-CSRF-Token': csrfToken,
'Content-Type': 'application/json'
},
credentials: 'same-origin',
body: JSON.stringify({
key: data.key,
value: data.value,
public: data.public
})
});
if (!response.ok) {
const errorData = await response.json();
throw new Error(errorData.errors ? errorData.errors.join(', ') : 'Failed to set key-value');
}
return await response.json();
}
async function handleDeleteRequest(baseUrl, data, csrfToken) {
const response = await fetch(baseUrl, {
method: 'DELETE',
body: JSON.stringify({ key: data.key }),
headers: {
'X-CSRF-Token': csrfToken,
'Content-Type': 'application/json'
},
credentials: 'same-origin'
});
if (!response.ok) {
if (response.status === 404) {
throw new Error('Key not found');
}
const errorData = await response.json();
throw new Error(errorData.errors ? errorData.errors.join(', ') : 'Failed to delete key-value');
}
return true;
}
async function handleIndexRequest(baseUrl, data, csrfToken) {
const params = new URLSearchParams();
if (data.key) params.append('key', data.key);
if (data.all_users) params.append('all_users', data.all_users);
if (data.keys_only) params.append('keys_only', data.keys_only);
if (data.page) params.append('page', data.page);
if (data.per_page) params.append('per_page', data.per_page);
const response = await fetch(baseUrl + '?' + params.toString(), {
method: 'GET',
headers: {
'X-CSRF-Token': csrfToken,
'Content-Type': 'application/json'
},
credentials: 'same-origin'
});
if (!response.ok) throw new Error('Failed to get key-values');
const result = await response.json();
const userMap = {};
result.users.forEach(user => {
userMap[user.id] = user;
});
result.key_values.forEach(kv => {
if (kv.user_id && userMap[kv.user_id]) {
kv.user = userMap[kv.user_id];
}
});
return result;
}
</script>
JAVASCRIPT
end
def set_security_headers
response.headers.delete("X-Frame-Options")
response.headers[
"Content-Security-Policy"
] = "script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval' #{AiArtifact::ALLOWED_CDN_SOURCES.join(" ")};"
response.headers["X-Robots-Tag"] = "noindex"
end
def require_site_settings!
if !SiteSetting.discourse_ai_enabled ||
!SiteSetting.ai_artifact_security.in?(%w[lax hybrid strict])
raise Discourse::NotFound
end
end
end
end
end
app/controllers/discourse_ai/ai_bot/bot_controller.rb
-108
View File
@@ -1,108 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module AiBot
class BotController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login
def show_debug_info_by_id
log = AiApiAuditLog.find(params[:id])
raise Discourse::NotFound if !log.topic
guardian.ensure_can_debug_ai_bot_conversation!(log.topic)
render json: AiApiAuditLogSerializer.new(log, root: false), status: 200
end
def show_debug_info
post = Post.find(params[:post_id])
guardian.ensure_can_debug_ai_bot_conversation!(post)
posts =
Post
.where("post_number <= ?", post.post_number)
.where(topic_id: post.topic_id)
.order("post_number DESC")
debug_info = AiApiAuditLog.where(post: posts).order(created_at: :desc).first
render json: AiApiAuditLogSerializer.new(debug_info, root: false), status: 200
end
def stop_streaming_response
post = Post.find(params[:post_id])
guardian.ensure_can_see!(post)
Discourse.redis.del("gpt_cancel:#{post.id}")
render json: {}, status: 200
end
def show_bot_username
bot_user = DiscourseAi::AiBot::EntryPoint.find_user_from_model(params[:username])
raise Discourse::InvalidParameters.new(:username) if !bot_user
render json: { bot_username: bot_user.username_lower }, status: 200
end
def discover
ai_persona =
AiPersona
.all_personas(enabled_only: false)
.find { |persona| persona.id == SiteSetting.ai_bot_discover_persona.to_i }
if ai_persona.nil? || !current_user.in_any_groups?(ai_persona.allowed_group_ids.to_a)
raise Discourse::InvalidAccess.new
end
if ai_persona.default_llm_id.blank?
render_json_error "Discover persona is missing a default LLM model.", status: 503
return
end
query = params[:query]
raise Discourse::InvalidParameters.new("Missing query to discover") if query.blank?
RateLimiter.new(current_user, "ai_bot_discover_#{current_user.id}", 3, 1.minute).performed!
Jobs.enqueue(:stream_discover_reply, user_id: current_user.id, query: query)
render json: {}, status: 200
end
def discover_continue_convo
raise Discourse::InvalidParameters.new("user_id") if !params[:user_id]
raise Discourse::InvalidParameters.new("query") if !params[:query]
raise Discourse::InvalidParameters.new("context") if !params[:context]
user = User.find(params[:user_id])
bot_user_id = AiPersona.find_by(id: SiteSetting.ai_bot_discover_persona).user_id
bot_username = User.find_by(id: bot_user_id).username
query = params[:query]
context = "[quote]\n#{params[:context]}\n[/quote]"
post =
PostCreator.create!(
user,
title:
I18n.t("discourse_ai.ai_bot.discoveries.continue_conversation.title", query: query),
raw:
I18n.t(
"discourse_ai.ai_bot.discoveries.continue_conversation.raw",
query: query,
context: context,
),
archetype: Archetype.private_message,
target_usernames: bot_username,
skip_validations: true,
)
render json: success_json.merge(topic_id: post.topic_id)
rescue StandardError => e
render json: failed_json.merge(errors: [e.message]), status: 422
end
end
end
end
app/controllers/discourse_ai/ai_bot/conversations_controller.rb
-39
View File
@@ -1,39 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module AiBot
class ConversationsController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login
def index
page = params[:page].to_i
per_page = params[:per_page]&.to_i || 40
base_query =
Topic
.private_messages_for_user(current_user)
.where(user: current_user) # Only show PMs where the current user is the author
.joins(
"INNER JOIN topic_custom_fields tcf ON tcf.topic_id = topics.id
AND tcf.name = '#{DiscourseAi::AiBot::TOPIC_AI_BOT_PM_FIELD}'
AND tcf.value = 't'",
)
.distinct
total = base_query.count
pms = base_query.order(last_posted_at: :desc).offset(page * per_page).limit(per_page)
render json: {
conversations: serialize_data(pms, ListableTopicSerializer),
meta: {
total: total,
page: page,
per_page: per_page,
has_more: total > (page + 1) * per_page,
},
}
end
end
end
end
app/controllers/discourse_ai/ai_bot/shared_ai_conversations_controller.rb
-136
View File
@@ -1,136 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module AiBot
class SharedAiConversationsController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login only: %i[create update destroy]
before_action :require_site_settings!
skip_before_action :preload_json, :check_xhr, only: %i[show asset]
skip_before_action :redirect_to_login_if_required, :verify_authenticity_token, only: %i[asset]
def create
ensure_allowed_create!
RateLimiter.new(current_user, "share-ai-conversation", 10, 1.minute).performed!
shared_conversation = SharedAiConversation.share_conversation(current_user, @topic)
if shared_conversation.persisted?
render json: success_json.merge(share_key: shared_conversation.share_key)
else
render json:
failed_json.merge(error: I18n.t("discourse_ai.share_ai.errors.failed_to_share")),
status: :unprocessable_entity
end
end
def destroy
ensure_allowed_destroy!
SharedAiConversation.destroy_conversation(@shared_conversation)
render json:
success_json.merge(
message: I18n.t("discourse_ai.share_ai.errors.conversation_deleted"),
)
end
def show
@shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
raise Discourse::NotFound if @shared_conversation.blank?
expires_in 1.minute, public: true
response.headers["X-Robots-Tag"] = "noindex"
if request.format.json?
render json: success_json.merge(@shared_conversation.to_json)
else
render "show", layout: false
end
end
def asset
no_cookies
name = params[:name]
path, content_type =
if name == "share"
%w[share.css text/css]
elsif name == "highlight"
%w[highlight.min.js application/javascript]
else
raise Discourse::NotFound
end
content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
# note, path contains a ":version" which automatically busts the cache
# based on file content, so this is safe
response.headers["Last-Modified"] = 10.years.ago.httpdate
response.headers["Content-Length"] = content.bytesize.to_s
immutable_for 1.year
render plain: content, disposition: :nil, content_type: content_type
end
def preview
ensure_allowed_preview!
data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
data[:error] = @error if @error
data[:share_key] = @shared_conversation.share_key if @shared_conversation
data[:topic_id] = @topic.id
render json: data
end
private
def require_site_settings!
if !SiteSetting.discourse_ai_enabled ||
!SiteSetting.ai_bot_public_sharing_allowed_groups_map.any? ||
!SiteSetting.ai_bot_enabled
raise Discourse::NotFound
end
end
def ensure_allowed_preview!
@topic = Topic.find_by(id: params[:topic_id])
raise Discourse::NotFound if !@topic
@shared_conversation = SharedAiConversation.find_by(target: @topic)
@error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
if @error == :not_allowed
raise Discourse::InvalidAccess.new(
nil,
nil,
custom_message: "discourse_ai.share_ai.errors.#{@error}",
)
end
end
def ensure_allowed_destroy!
@shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
raise Discourse::InvalidAccess if @shared_conversation.blank?
guardian.ensure_can_destroy_shared_ai_bot_conversation!(@shared_conversation)
end
def ensure_allowed_create!
@topic = Topic.find_by(id: params[:topic_id])
raise Discourse::NotFound if !@topic
error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
if error
raise Discourse::InvalidAccess.new(
nil,
nil,
custom_message: "discourse_ai.share_ai.errors.#{error}",
)
end
end
end
end
end
app/controllers/discourse_ai/ai_helper/assistant_controller.rb
-241
View File
@@ -1,241 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module AiHelper
class AssistantController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login
before_action :ensure_can_request_suggestions
before_action :rate_limiter_performed!
include SecureUploadEndpointHelpers
RATE_LIMITS = {
"default" => {
amount: 6,
interval: 3.minutes,
},
"caption_image" => {
amount: 20,
interval: 1.minute,
},
}.freeze
def suggest
input = get_text_param!
force_default_locale = params[:force_default_locale] || false
raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?
if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
end
if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
return suggest_thumbnails(input)
end
hijack do
render json:
DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
params[:mode],
input,
current_user,
force_default_locale: force_default_locale,
custom_prompt: params[:custom_prompt],
),
status: 200
end
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
status: 502
end
def suggest_title
if params[:topic_id]
topic = Topic.find_by(id: params[:topic_id])
input = DiscourseAi::Summarization::Strategies::TopicSummary.new(topic).targets_data
else
input = get_text_param!
end
hijack do
render json:
DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
DiscourseAi::AiHelper::Assistant::GENERATE_TITLES,
input,
current_user,
),
status: 200
end
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
status: 502
end
def suggest_category
if params[:topic_id]
opts = { topic_id: params[:topic_id] }
else
input = get_text_param!
opts = { text: input }
end
render json: DiscourseAi::AiHelper::SemanticCategorizer.new(current_user, opts).categories,
status: 200
end
def suggest_tags
if params[:topic_id]
opts = { topic_id: params[:topic_id] }
else
input = get_text_param!
opts = { text: input }
end
render json: DiscourseAi::AiHelper::SemanticCategorizer.new(current_user, opts).tags,
status: 200
end
def suggest_thumbnails(input)
hijack do
thumbnails = DiscourseAi::AiHelper::Painter.new.commission_thumbnails(input, current_user)
render json: { thumbnails: thumbnails }, status: 200
end
end
def stream_suggestion
text = get_text_param!
location = params[:location]
raise Discourse::InvalidParameters.new(:location) if !location
raise Discourse::InvalidParameters.new(:mode) if params[:mode].blank?
if params[:mode] == DiscourseAi::AiHelper::Assistant::ILLUSTRATE_POST
return suggest_thumbnails(input)
end
if params[:mode] == DiscourseAi::AiHelper::Assistant::CUSTOM_PROMPT
raise Discourse::InvalidParameters.new(:custom_prompt) if params[:custom_prompt].blank?
end
# to stream we must have an appropriate client_id
# otherwise we may end up streaming the data to the wrong client
raise Discourse::InvalidParameters.new(:client_id) if params[:client_id].blank?
channel_id = next_channel_id
progress_channel = "discourse_ai_helper/stream_suggestions/#{channel_id}"
if location == "composer"
Jobs.enqueue(
:stream_composer_helper,
user_id: current_user.id,
text: text,
prompt: params[:mode],
custom_prompt: params[:custom_prompt],
force_default_locale: params[:force_default_locale] || false,
client_id: params[:client_id],
progress_channel:,
)
else
post_id = get_post_param!
post = Post.includes(:topic).find_by(id: post_id)
raise Discourse::InvalidParameters.new(:post_id) unless post
Jobs.enqueue(
:stream_post_helper,
post_id: post.id,
user_id: current_user.id,
text: text,
prompt: params[:mode],
custom_prompt: params[:custom_prompt],
client_id: params[:client_id],
progress_channel:,
)
end
render json: { success: true, progress_channel: }, status: 200
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
status: 502
end
def caption_image
image_url = params[:image_url]
image_url_type = params[:image_url_type]
raise Discourse::InvalidParameters.new(:image_url) if !image_url
raise Discourse::InvalidParameters.new(:image_url) if !image_url_type
if image_url_type == "short_path"
image = Upload.find_by(sha1: Upload.sha1_from_short_path(image_url))
elsif image_url_type == "short_url"
image = Upload.find_by(sha1: Upload.sha1_from_short_url(image_url))
else
image = upload_from_full_url(image_url)
end
raise Discourse::NotFound if image.blank?
check_secure_upload_permission(image) if image.secure?
user = current_user
hijack do
caption = DiscourseAi::AiHelper::Assistant.new.generate_image_caption(image, user)
render json: {
caption:
"#{caption} (#{I18n.t("discourse_ai.ai_helper.image_caption.attribution")})",
},
status: 200
end
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed, Net::HTTPBadResponse
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
status: 502
end
private
CHANNEL_ID_KEY = "discourse_ai_helper_next_channel_id"
def next_channel_id
Discourse
.redis
.pipelined do |pipeline|
pipeline.incr(CHANNEL_ID_KEY)
pipeline.expire(CHANNEL_ID_KEY, 1.day)
end
.first
end
def get_text_param!
params[:text].tap { |t| raise Discourse::InvalidParameters.new(:text) if t.blank? }
end
def get_post_param!
params[:post_id].tap { |t| raise Discourse::InvalidParameters.new(:post_id) if t.blank? }
end
def rate_limiter_performed!
action_rate_limit = RATE_LIMITS[action_name] || RATE_LIMITS["default"]
RateLimiter.new(
current_user,
"ai_assistant",
action_rate_limit[:amount],
action_rate_limit[:interval],
).performed!
end
def ensure_can_request_suggestions
allowed_groups =
(
SiteSetting.composer_ai_helper_allowed_groups_map |
SiteSetting.post_ai_helper_allowed_groups_map
)
raise Discourse::InvalidAccess if !current_user.in_any_groups?(allowed_groups)
end
end
end
end
app/controllers/discourse_ai/discord/bot_controller.rb
-50
View File
@@ -1,50 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Discord
class BotController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
skip_before_action :verify_authenticity_token
def interactions
# Request signature verification
begin
verify_request!
rescue Ed25519::VerifyError
return head :unauthorized
end
body = request.body.read
interaction = JSON.parse(body, object_class: OpenStruct)
if interaction.type == 1
# Respond to Discord PING request
render json: { type: 1 }
else
if !SiteSetting.ai_discord_allowed_guilds_map.include?(interaction.guild_id)
return head :forbidden
end
response = { type: 5, data: { content: "Searching..." } }
hijack { render json: response }
# Respond to Discord command
Jobs.enqueue(:stream_discord_reply, interaction: body)
end
end
private
def verify_request!
signature = request.headers["X-Signature-Ed25519"]
timestamp = request.headers["X-Signature-Timestamp"]
verify_key.verify([signature].pack("H*"), "#{timestamp}#{request.raw_post}")
end
def verify_key
Ed25519::VerifyKey.new([SiteSetting.ai_discord_app_public_key].pack("H*")).freeze
end
end
end
end
app/controllers/discourse_ai/embeddings/embeddings_controller.rb
-97
View File
@@ -1,97 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Embeddings
class EmbeddingsController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
SEMANTIC_SEARCH_TYPE = "semantic_search"
MAX_HYDE_SEARCHES_PER_MINUTE = 4
MAX_SEARCHES_PER_MINUTE = 100
def search
query = params[:q].to_s
skip_hyde = params[:hyde].to_s.downcase == "false" || params[:hyde].to_s == "0"
if query.length < SiteSetting.min_search_term_length
raise Discourse::InvalidParameters.new(:q)
end
grouped_results =
Search::GroupedSearchResults.new(
type_filter: SEMANTIC_SEARCH_TYPE,
term: query,
search_context: guardian,
use_pg_headlines_for_excerpt: false,
can_lazy_load_categories: guardian.can_lazy_load_categories?,
)
semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
if !skip_hyde && !semantic_search.cached_query?(query)
RateLimiter.new(
current_user,
"semantic-search",
MAX_HYDE_SEARCHES_PER_MINUTE,
1.minutes,
).performed!
else
RateLimiter.new(
current_user,
"semantic-search-non-hyde",
MAX_SEARCHES_PER_MINUTE,
1.minutes,
).performed!
end
hijack do
begin
semantic_search
.search_for_topics(query, _page = 1, hyde: !skip_hyde)
.each { |topic_post| grouped_results.add(topic_post) }
render_serialized(
grouped_results,
GroupedSearchResultSerializer,
result: grouped_results,
)
rescue Discourse::InvalidAccess
render_json_error(I18n.t("invalid_access"), status: 403)
end
end
end
def quick_search
# this search function searches posts (vs: topics)
# it requires post embeddings and a reranker
# it will not perform a hyde expantion
query = params[:q].to_s
if query.length < SiteSetting.min_search_term_length
raise Discourse::InvalidParameters.new(:q)
end
grouped_results =
Search::GroupedSearchResults.new(
type_filter: SEMANTIC_SEARCH_TYPE,
term: query,
search_context: guardian,
use_pg_headlines_for_excerpt: false,
)
semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
if !semantic_search.cached_query?(query)
RateLimiter.new(current_user, "semantic-search", 60, 1.minutes).performed!
end
hijack do
semantic_search.quick_search(query).each { |topic_post| grouped_results.add(topic_post) }
render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
end
end
end
end
end
app/controllers/discourse_ai/sentiment/sentiment_controller.rb
-97
View File
@@ -1,97 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Sentiment
class SentimentController < ::Admin::StaffController
include Constants
requires_plugin ::DiscourseAi::PLUGIN_NAME
DEFAULT_POSTS_LIMIT = 50
MAX_POSTS_LIMIT = 100
def posts
group_by = params.required(:group_by)&.to_sym
group_value = params.required(:group_value).presence
start_date = params[:start_date].presence
end_date = params[:end_date]
threshold = SENTIMENT_THRESHOLD
raise Discourse::InvalidParameters if %i[category tag].exclude?(group_by)
limit = fetch_limit_from_params(default: DEFAULT_POSTS_LIMIT, max: MAX_POSTS_LIMIT)
offset = params[:offset].to_i || 0
case group_by
when :category
grouping_clause = "c.name"
grouping_join = "" # categories already joined
when :tag
grouping_clause = "tags.name"
grouping_join =
"INNER JOIN topic_tags tt ON tt.topic_id = p.topic_id INNER JOIN tags ON tags.id = tt.tag_id"
end
posts =
DB.query(
<<~SQL,
SELECT
p.id AS post_id,
p.topic_id,
t.fancy_title AS topic_title,
p.cooked as post_cooked,
p.user_id,
p.post_number,
p.created_at AS created_at,
u.username,
u.name,
u.uploaded_avatar_id,
c.id AS category_id,
(CASE
WHEN (cr.classification::jsonb->'positive')::float > :threshold THEN 'positive'
WHEN (cr.classification::jsonb->'negative')::float > :threshold THEN 'negative'
ELSE 'neutral'
END) AS sentiment
FROM posts p
INNER JOIN topics t ON t.id = p.topic_id
INNER JOIN classification_results cr ON cr.target_id = p.id AND cr.target_type = 'Post'
LEFT JOIN users u ON u.id = p.user_id
LEFT JOIN categories c ON c.id = t.category_id
#{grouping_join}
WHERE
#{grouping_clause} = :group_value AND
t.archetype = 'regular' AND
p.user_id > 0 AND
cr.model_used = 'cardiffnlp/twitter-roberta-base-sentiment-latest' AND
((:start_date IS NULL OR p.created_at > :start_date) AND (:end_date IS NULL OR p.created_at < :end_date))
AND p.deleted_at IS NULL
ORDER BY p.created_at DESC
LIMIT :limit OFFSET :offset
SQL
group_value: group_value,
start_date: start_date,
end_date: end_date,
threshold: threshold,
limit: limit + 1,
offset: offset,
)
has_more = posts.length > limit
posts.pop if has_more
render_json_dump(
posts:
serialize_data(
posts,
AiSentimentPostSerializer,
scope: guardian,
add_raw: true,
add_excerpt: true,
add_title: true,
),
has_more: has_more,
next_offset: has_more ? offset + limit : nil,
)
end
end
end
end
app/controllers/discourse_ai/summarization/chat_summary_controller.rb
-40
View File
@@ -1,40 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Summarization
class ChatSummaryController < ::Chat::ApiController
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_plugin ::Chat::PLUGIN_NAME
VALID_SINCE_VALUES = [1, 3, 6, 12, 24, 72, 168]
def show
since = params[:since].to_i
raise Discourse::InvalidParameters.new(:since) if !VALID_SINCE_VALUES.include?(since)
channel = ::Chat::Channel.find(params[:channel_id])
guardian.ensure_can_join_chat_channel!(channel)
summarizer = DiscourseAi::Summarization.chat_channel_summary(channel, since)
raise Discourse::NotFound.new unless summarizer
guardian.ensure_can_request_summary!
RateLimiter.new(current_user, "channel_summary", 6, 5.minutes).performed!
hijack do
strategy = DiscourseAi::Summarization::Strategies::ChatMessages.new(channel, since)
summarized_text =
if strategy.targets_data.empty?
I18n.t("discourse_ai.summarization.chat.no_targets")
else
summarizer.summarize(current_user)&.summarized_text
end
render json: { summary: summarized_text }
end
end
end
end
end
app/controllers/discourse_ai/summarization/summary_controller.rb
-46
View File
@@ -1,46 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Summarization
class SummaryController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def show
topic = Topic.find(params[:topic_id])
guardian.ensure_can_see!(topic)
raise Discourse::NotFound if !guardian.can_see_summary?(topic)
RateLimiter.new(current_user, "summary", 6, 5.minutes).performed! if current_user
opts = params.permit(:skip_age_check)
skip_age_check = opts[:skip_age_check] == "true"
summarization_service = DiscourseAi::TopicSummarization.for(topic, current_user)
if params[:stream] && current_user
cached_summary = summarization_service.cached_summary
if cached_summary && !skip_age_check
render_serialized(cached_summary, AiTopicSummarySerializer)
return
end
Jobs.enqueue(
:stream_topic_ai_summary,
topic_id: topic.id,
user_id: current_user.id,
skip_age_check: skip_age_check,
)
render json: success_json
else
hijack do
summary = summarization_service.summarize(skip_age_check: skip_age_check)
render_serialized(summary, AiTopicSummarySerializer)
end
end
end
end
end
end
app/helpers/discourse_ai/ai_bot/shared_ai_conversations_helper.rb
-31
View File
@@ -1,31 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module AiBot
module SharedAiConversationsHelper
# keeping it here for caching
def self.share_asset_url(asset_name)
if !%w[share.css highlight.js].include?(asset_name)
raise StandardError, "unknown asset type #{asset_name}"
end
@urls ||= {}
url = @urls[asset_name]
return url if url
path = asset_name
path = "highlight.min.js" if asset_name == "highlight.js"
content = File.read(DiscourseAi.public_asset_path("ai-share/#{path}"))
sha1 = Digest::SHA1.hexdigest(content)
url = "/discourse-ai/ai-bot/shared-ai-conversations/asset/#{sha1}/#{asset_name}"
@urls[asset_name] = GlobalPath.cdn_path(url)
end
def share_asset_url(asset_name)
DiscourseAi::AiBot::SharedAiConversationsHelper.share_asset_url(asset_name)
end
end
end
end
app/jobs/regular/ai_spam_scan.rb
-13
View File
@@ -1,13 +0,0 @@
# frozen_string_literal: true
module Jobs
class AiSpamScan < ::Jobs::Base
def execute(args)
return if !args[:post_id]
post = Post.find_by(id: args[:post_id])
return if !post
DiscourseAi::AiModeration::SpamScanner.perform_scan(post)
end
end
end
app/jobs/regular/create_ai_chat_reply.rb
-28
View File
@@ -1,28 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class CreateAiChatReply < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
channel = ::Chat::Channel.find_by(id: args[:channel_id])
return if channel.blank?
message = ::Chat::Message.find_by(id: args[:message_id])
return if message.blank?
personaClass =
DiscourseAi::Personas::Persona.find_by(id: args[:persona_id], user: message.user)
return if personaClass.blank?
user = User.find_by(id: personaClass.user_id)
bot = DiscourseAi::Personas::Bot.as(user, persona: personaClass.new)
DiscourseAi::AiBot::Playground.new(bot).reply_to_chat_message(
message,
channel,
args[:context_post_ids],
)
end
end
end
app/jobs/regular/create_ai_reply.rb
-26
View File
@@ -1,26 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class CreateAiReply < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
return unless bot_user = User.find_by(id: args[:bot_user_id])
return unless post = Post.includes(:topic).find_by(id: args[:post_id])
persona_id = args[:persona_id]
begin
persona = DiscourseAi::Personas::Persona.find_by(user: post.user, id: persona_id)
raise DiscourseAi::Personas::Bot::BOT_NOT_FOUND if persona.nil?
bot = DiscourseAi::Personas::Bot.as(bot_user, persona: persona.new)
DiscourseAi::AiBot::Playground.new(bot).reply_to(post, feature_name: "bot")
rescue DiscourseAi::Personas::Bot::BOT_NOT_FOUND
Rails.logger.warn(
"Bot not found for post #{post.id} - perhaps persona was deleted or bot was disabled",
)
end
end
end
end
app/jobs/regular/detect_translate_post.rb
-61
View File
@@ -1,61 +0,0 @@
# frozen_string_literal: true
module Jobs
class DetectTranslatePost < ::Jobs::Base
cluster_concurrency 1
sidekiq_options retry: false
def execute(args)
return if !DiscourseAi::Translation.enabled?
return if args[:post_id].blank?
post = Post.find_by(id: args[:post_id])
return if post.blank? || post.raw.blank? || post.deleted_at.present? || post.user_id <= 0
topic = post.topic
return if topic.blank?
if SiteSetting.ai_translation_backfill_limit_to_public_content
return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
else
if topic.archetype == Archetype.private_message &&
!TopicAllowedGroup.exists?(topic_id: topic.id)
return
end
end
# the user may fill locale in manually
if (detected_locale = post.locale).blank?
begin
detected_locale = DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
rescue FinalDestination::SSRFDetector::LookupFailedError
# this job is non-critical
# the backfill job will handle failures
return
end
end
return if detected_locale.blank?
locales = SiteSetting.content_localization_supported_locales.split("|")
return if locales.blank?
locales.each do |locale|
next if LocaleNormalizer.is_same?(locale, detected_locale)
regionless_locale = locale.split("_").first
next if post.post_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
begin
DiscourseAi::Translation::PostLocalizer.localize(post, locale)
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
"Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
end
end
MessageBus.publish("/topic/#{post.topic_id}", type: :localized, id: post.id)
end
end
end
app/jobs/regular/detect_translate_topic.rb
-59
View File
@@ -1,59 +0,0 @@
# frozen_string_literal: true
module Jobs
class DetectTranslateTopic < ::Jobs::Base
cluster_concurrency 1
sidekiq_options retry: false
def execute(args)
return if !DiscourseAi::Translation.enabled?
return if args[:topic_id].blank?
topic = Topic.find_by(id: args[:topic_id])
if topic.blank? || topic.title.blank? || topic.deleted_at.present? || topic.user_id <= 0
return
end
if SiteSetting.ai_translation_backfill_limit_to_public_content
return if topic.category&.read_restricted? || topic.archetype == Archetype.private_message
else
if topic.archetype == Archetype.private_message &&
!TopicAllowedGroup.exists?(topic_id: topic.id)
return
end
end
if (detected_locale = topic.locale).blank?
begin
detected_locale = DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
rescue FinalDestination::SSRFDetector::LookupFailedError
# this job is non-critical
# the backfill job will handle failures
return
end
end
return if detected_locale.blank?
locales = SiteSetting.content_localization_supported_locales.split("|")
return if locales.blank?
locales.each do |locale|
next if LocaleNormalizer.is_same?(locale, detected_locale)
regionless_locale = locale.split("_").first
next if topic.topic_localizations.where("locale LIKE ?", "#{regionless_locale}%").exists?
begin
DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
"Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
end
end
MessageBus.publish("/topic/#{topic.id}", type: :localized, id: 1)
end
end
end
app/jobs/regular/digest_rag_upload.rb
-204
View File
@@ -1,204 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class DigestRagUpload < ::Jobs::Base
CHUNK_SIZE = 1024
CHUNK_OVERLAP = 64
MAX_FRAGMENTS = 100_000
# TODO(roman): Add a way to automatically recover from errors, resulting in unindexed uploads.
def execute(args)
return if (upload = Upload.find_by(id: args[:upload_id])).nil?
target_type = args[:target_type]
target_id = args[:target_id]
return if !target_type || !target_id
target = target_type.constantize.find_by(id: target_id)
return if !target
vector_rep = DiscourseAi::Embeddings::Vector.instance
tokenizer = vector_rep.tokenizer
chunk_tokens = target.rag_chunk_tokens
overlap_tokens = target.rag_chunk_overlap_tokens
fragment_ids = RagDocumentFragment.where(target: target, upload: upload).pluck(:id)
# Check if this is the first time we process this upload.
if fragment_ids.empty?
document = get_uploaded_file(upload: upload, target: target)
return if document.nil?
RagDocumentFragment.publish_status(upload, { total: 0, indexed: 0, left: 0 })
fragment_ids = []
idx = 0
ActiveRecord::Base.transaction do
chunk_document(
file: document,
tokenizer: tokenizer,
chunk_tokens: chunk_tokens,
overlap_tokens: overlap_tokens,
) do |chunk, metadata|
fragment_ids << RagDocumentFragment.create!(
target: target,
fragment: chunk,
fragment_number: idx + 1,
upload: upload,
metadata: metadata,
).id
idx += 1
if idx > MAX_FRAGMENTS
Rails.logger.warn("Upload #{upload.id} has too many fragments, truncating.")
break
end
end
end
end
fragment_ids.each_slice(50) do |slice|
Jobs.enqueue(:generate_rag_embeddings, fragment_ids: slice)
end
end
private
def chunk_document(file:, tokenizer:, chunk_tokens:, overlap_tokens:)
buffer = +""
current_metadata = nil
done = false
overlap = ""
# generally this will be plenty
read_size = chunk_tokens * 10
while buffer.present? || !done
if buffer.length < read_size
read = file.read(read_size)
done = true if read.nil?
read = Encodings.to_utf8(read) if read
buffer << (read || "")
end
# at this point we unconditionally have 2x CHUNK_SIZE worth of data in the buffer
metadata_regex = /\[\[metadata (.*?)\]\]/m
before_metadata, new_metadata, after_metadata = buffer.split(metadata_regex)
to_chunk = nil
if before_metadata.present?
to_chunk = before_metadata
elsif after_metadata.present?
current_metadata = new_metadata
to_chunk = after_metadata
buffer = buffer.split(metadata_regex, 2).last
overlap = ""
else
current_metadata = new_metadata
buffer = buffer.split(metadata_regex, 2).last
overlap = ""
next
end
chunk, split_char = first_chunk(to_chunk, tokenizer: tokenizer, chunk_tokens: chunk_tokens)
buffer = buffer[chunk.length..-1]
processed_chunk = overlap + chunk
processed_chunk.strip!
processed_chunk.gsub!(/\n[\n]+/, "\n\n")
yield processed_chunk, current_metadata
current_chunk_tokens = tokenizer.encode(chunk)
overlap_token_ids = current_chunk_tokens[-overlap_tokens..-1] || current_chunk_tokens
overlap = ""
while overlap_token_ids.present?
begin
padding = split_char
padding = " " if padding.empty?
overlap = tokenizer.decode(overlap_token_ids) + padding
break if overlap.encoding == Encoding::UTF_8
rescue StandardError
# it is possible that we truncated mid char
end
overlap_token_ids.shift
end
# remove first word it is probably truncated
overlap = overlap.split(/\s/, 2).last.to_s.lstrip
end
end
def first_chunk(text, chunk_tokens:, tokenizer:, splitters: ["\n\n", "\n", ".", ""])
return text, " " if tokenizer.tokenize(text).length <= chunk_tokens
splitters = splitters.find_all { |s| text.include?(s) }.compact
buffer = +""
split_char = nil
splitters.each do |splitter|
split_char = splitter
text
.split(split_char)
.each do |part|
break if tokenizer.tokenize(buffer + split_char + part).length > chunk_tokens
buffer << split_char
buffer << part
end
break if buffer.length > 0
end
[buffer, split_char]
end
def get_uploaded_file(upload:, target:)
if %w[png jpg jpeg].include?(upload.extension) && !SiteSetting.ai_rag_images_enabled
raise Discourse::InvalidAccess.new(
"The setting ai_rag_images_enabled is false, can not index images",
)
end
if upload.extension == "pdf"
return(
DiscourseAi::Utils::PdfToText.as_fake_file(
upload: upload,
llm_model: SiteSetting.ai_rag_images_enabled ? target.rag_llm_model : nil,
user: Discourse.system_user,
)
)
end
if %w[png jpg jpeg].include?(upload.extension)
return(
DiscourseAi::Utils::ImageToText.as_fake_file(
uploads: [upload],
llm_model: target.rag_llm_model,
user: Discourse.system_user,
)
)
end
store = Discourse.store
@file ||=
if store.external?
# Upload#filesize could be approximate.
# add two extra Mbs to make sure that we'll be able to download the upload.
max_filesize = upload.filesize + 2.megabytes
store.download(upload, max_file_size_kb: max_filesize)
else
File.open(store.path_for(upload))
end
end
end
end
app/jobs/regular/fast_track_topic_gist.rb
-22
View File
@@ -1,22 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class FastTrackTopicGist < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
return if !SiteSetting.discourse_ai_enabled
return if !SiteSetting.ai_summarization_enabled
return if !SiteSetting.ai_summary_gists_enabled
topic = Topic.find_by(id: args[:topic_id])
return if topic.blank?
summarizer = DiscourseAi::Summarization.topic_gist(topic)
gist = summarizer.existing_summary
return if gist.present? && (!gist.outdated || gist.created_at >= 5.minutes.ago)
summarizer.summarize(Discourse.system_user)
end
end
end
app/jobs/regular/generate_chat_thread_title.rb
-21
View File
@@ -1,21 +0,0 @@
# frozen_string_literal: true
module Jobs
class GenerateChatThreadTitle < ::Jobs::Base
sidekiq_options queue: "low"
def execute(args)
return unless SiteSetting.ai_helper_automatic_chat_thread_title
return if (thread_id = args[:thread_id]).blank?
thread = ::Chat::Thread.find_by_id(thread_id)
return if thread.nil? || thread.title.present?
title = DiscourseAi::AiHelper::ChatThreadTitler.new(thread).suggested_title
return if title.blank?
# TODO use a proper API that will make the new title update live
thread.update!(title: title)
end
end
end
app/jobs/regular/generate_embeddings.rb
-22
View File
@@ -1,22 +0,0 @@
# frozen_string_literal: true
module Jobs
class GenerateEmbeddings < ::Jobs::Base
sidekiq_options queue: "low"
def execute(args)
return unless DiscourseAi::Embeddings.enabled?
return if args[:target_type].blank? || args[:target_id].blank?
target = args[:target_type].constantize.find_by_id(args[:target_id])
return if target.nil? || target.deleted_at.present?
topic = target.is_a?(Topic) ? target : target.topic
post = target.is_a?(Post) ? target : target.first_post
return if topic.blank? || post.blank?
return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
return if post.raw.blank?
DiscourseAi::Embeddings::Vector.instance.generate_representation_from(target)
end
end
end
app/jobs/regular/generate_inferred_concepts.rb
-70
View File
@@ -1,70 +0,0 @@
# frozen_string_literal: true
module Jobs
class GenerateInferredConcepts < ::Jobs::Base
sidekiq_options queue: "low"
# Process items to generate new concepts
#
# @param args [Hash] Contains job arguments
# @option args [String] :item_type Required - Type of items to process ('topics' or 'posts')
# @option args [Array<Integer>] :item_ids Required - List of item IDs to process
# @option args [Integer] :batch_size (100) Number of items to process in each batch
# @option args [Boolean] :match_only (false) Only match against existing concepts without generating new ones
def execute(args = {})
return if args[:item_ids].blank? || args[:item_type].blank?
if %w[topics posts].exclude?(args[:item_type])
Rails.logger.error("Invalid item_type for GenerateInferredConcepts: #{args[:item_type]}")
return
end
# Process items in smaller batches to avoid memory issues
batch_size = args[:batch_size] || 100
# Get the list of item IDs
item_ids = args[:item_ids]
match_only = args[:match_only] || false
# Process items in batches
item_ids.each_slice(batch_size) do |batch_item_ids|
process_batch(batch_item_ids, args[:item_type], match_only)
end
end
private
def process_batch(item_ids, item_type, match_only)
klass = item_type.singularize.classify.constantize
items = klass.where(id: item_ids)
manager = DiscourseAi::InferredConcepts::Manager.new
items.each do |item|
begin
process_item(item, item_type, match_only, manager)
rescue => e
Rails.logger.error(
"Error generating concepts from #{item_type.singularize} #{item.id}: #{e.message}\n#{e.backtrace.join("\n")}",
)
end
end
end
def process_item(item, item_type, match_only, manager)
# Use the Manager method that handles both identifying and creating concepts
if match_only
if item_type == "topics"
manager.match_topic_to_concepts(item)
else # posts
manager.match_post_to_concepts(item)
end
else
if item_type == "topics"
manager.generate_concepts_from_topic(item)
else # posts
manager.generate_concepts_from_post(item)
end
end
end
end
end
app/jobs/regular/generate_rag_embeddings.rb
-25
View File
@@ -1,25 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class GenerateRagEmbeddings < ::Jobs::Base
sidekiq_options queue: "ultra_low"
# we could also restrict concurrency but this takes so long if it is not concurrent
def execute(args)
return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
vector = DiscourseAi::Embeddings::Vector.instance
# generate_representation_from checks compares the digest value to make sure
# the embedding is only generated once per fragment unless something changes.
fragments.map { |fragment| vector.generate_representation_from(fragment) }
last_fragment = fragments.last
target = last_fragment.target
upload = last_fragment.upload
indexing_status = RagDocumentFragment.indexing_status(target, [upload])[upload.id]
RagDocumentFragment.publish_status(upload, indexing_status)
end
end
end
app/jobs/regular/localize_categories.rb
-53
View File
@@ -1,53 +0,0 @@
# frozen_string_literal: true
module Jobs
class LocalizeCategories < ::Jobs::Base
cluster_concurrency 1
sidekiq_options retry: false
def execute(args)
return if !DiscourseAi::Translation.enabled?
limit = args[:limit]
raise Discourse::InvalidParameters.new(:limit) if limit.nil?
return if limit <= 0
locales = SiteSetting.content_localization_supported_locales.split("|")
categories = Category.where("locale IS NOT NULL")
if SiteSetting.ai_translation_backfill_limit_to_public_content
categories = categories.where(read_restricted: false)
end
categories = categories.order(:id).limit(limit)
return if categories.empty?
remaining_limit = limit
categories.each do |category|
break if remaining_limit <= 0
existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)
missing_locales = locales - existing_locales - [category.locale]
missing_locales.each do |locale|
break if remaining_limit <= 0
next if LocaleNormalizer.is_same?(locale, category.locale)
begin
DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
"Failed to translate category #{category.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
ensure
remaining_limit -= 1
end
end
if existing_locales.include?(category.locale)
CategoryLocalization.find_by(category_id: category.id, locale: category.locale).destroy
end
end
end
end
end
app/jobs/regular/localize_posts.rb
-71
View File
@@ -1,71 +0,0 @@
# frozen_string_literal: true
module Jobs
class LocalizePosts < ::Jobs::Base
cluster_concurrency 1
sidekiq_options retry: false
def execute(args)
limit = args[:limit]
raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
return if !DiscourseAi::Translation.backfill_enabled?
locales = SiteSetting.content_localization_supported_locales.split("|")
locales.each do |locale|
base_locale = locale.split("_").first
posts =
Post
.joins(
"LEFT JOIN post_localizations pl ON pl.post_id = posts.id AND pl.locale LIKE '#{base_locale}%'",
)
.where(
"posts.created_at > ?",
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
)
.where(deleted_at: nil)
.where("posts.user_id > 0")
.where.not(raw: [nil, ""])
.where.not(locale: nil)
.where("posts.locale NOT LIKE '#{base_locale}%'")
.where("pl.id IS NULL")
posts = posts.joins(:topic)
if SiteSetting.ai_translation_backfill_limit_to_public_content
# exclude all PMs
# and only include posts from public categories
posts =
posts
.where.not(topics: { archetype: Archetype.private_message })
.where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
else
# all regular topics, and group PMs
posts =
posts.where(
"topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
Archetype.private_message,
)
end
posts = posts.order(updated_at: :desc).limit(limit)
next if posts.empty?
posts.each do |post|
begin
DiscourseAi::Translation::PostLocalizer.localize(post, locale)
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
"Failed to translate post #{post.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
end
end
DiscourseAi::Translation::VerboseLogger.log("Translated #{posts.size} posts to #{locale}")
end
end
end
end
app/jobs/regular/localize_topics.rb
-68
View File
@@ -1,68 +0,0 @@
# frozen_string_literal: true
module Jobs
class LocalizeTopics < ::Jobs::Base
cluster_concurrency 1
sidekiq_options retry: false
def execute(args)
limit = args[:limit]
raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
return if !DiscourseAi::Translation.backfill_enabled?
locales = SiteSetting.content_localization_supported_locales.split("|")
locales.each do |locale|
base_locale = locale.split("_").first
topics =
Topic
.joins(
"LEFT JOIN topic_localizations tl ON tl.topic_id = topics.id AND tl.locale LIKE '#{base_locale}%'",
)
.where(
"topics.created_at > ?",
SiteSetting.ai_translation_backfill_max_age_days.days.ago,
)
.where(deleted_at: nil)
.where("topics.user_id > 0")
.where.not(locale: nil)
.where("topics.locale NOT LIKE '#{base_locale}%'")
.where("tl.id IS NULL")
if SiteSetting.ai_translation_backfill_limit_to_public_content
# exclude all PMs
# and only include posts from public categories
topics =
topics
.where.not(archetype: Archetype.private_message)
.where(category_id: Category.where(read_restricted: false).select(:id))
else
# all regular topics, and group PMs
topics =
topics.where(
"topics.archetype != ? OR topics.id IN (SELECT topic_id FROM topic_allowed_groups)",
Archetype.private_message,
)
end
topics = topics.order(updated_at: :desc).limit(limit)
next if topics.empty?
topics.each do |topic|
begin
DiscourseAi::Translation::TopicLocalizer.localize(topic, locale)
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
"Failed to translate topic #{topic.id} to #{locale}: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
end
end
DiscourseAi::Translation::VerboseLogger.log("Translated #{topics.size} topics to #{locale}")
end
end
end
end
app/jobs/regular/manage_embedding_def_search_index.rb
-13
View File
@@ -1,13 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class ManageEmbeddingDefSearchIndex < ::Jobs::Base
def execute(args)
embedding_def = EmbeddingDefinition.find_by(id: args[:id])
return if embedding_def.nil?
return if DiscourseAi::Embeddings::Schema.correctly_indexed?(embedding_def)
DiscourseAi::Embeddings::Schema.prepare_search_indexes(embedding_def)
end
end
end
app/jobs/regular/post_sentiment_analysis.rb
-17
View File
@@ -1,17 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class PostSentimentAnalysis < ::Jobs::Base
sidekiq_options queue: "low"
def execute(args)
return unless SiteSetting.ai_sentiment_enabled
return if (post_id = args[:post_id]).blank?
post = Post.find_by(id: post_id, post_type: Post.types[:regular])
return if post&.raw.blank?
DiscourseAi::Sentiment::PostClassification.new.classify!(post)
end
end
end
app/jobs/regular/shared_conversation_adjust_upload_security.rb
-51
View File
@@ -1,51 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class SharedConversationAdjustUploadSecurity < ::Jobs::Base
def execute(args)
if args[:conversation_id].present?
# The conversation context includes post cooked content so this
# must be updated when target uploads security changes.
update_conversation(args[:conversation_id])
elsif args[:target_id].present? && args[:target_type].present?
# If we deleted the conversation then we just need to update the target's
# uploads security, no need to update the conversation.
update_target(args[:target_id], args[:target_type])
end
end
private
def update_conversation(conversation_id)
conversation = SharedAiConversation.find_by(id: conversation_id)
return if conversation.blank?
# NOTE: Only Topics are supported for now, in future we will need a more flexible
# way of doing this.
if conversation.target_type == "Topic"
rebaked_posts = TopicUploadSecurityManager.new(conversation.target).run
if rebaked_posts.any?
new_context =
conversation.context.map do |context_post|
rebaked_post = rebaked_posts.find { |p| p.id == context_post["id"] }
context_post["cooked"] = rebaked_post.cooked if rebaked_post
context_post
end
conversation.update(context: new_context)
end
end
end
def update_target(target_id, target_type)
# NOTE: Only Topics are supported for now, in future we will need a more flexible
# way of doing this.
if target_type == "Topic"
topic = target_type.constantize.find_by(id: target_id)
return if topic.blank?
TopicUploadSecurityManager.new(topic).run
end
end
end
end
app/jobs/regular/stream_composer_helper.rb
-27
View File
@@ -1,27 +0,0 @@
# frozen_string_literal: true
module Jobs
class StreamComposerHelper < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
return unless args[:prompt]
return unless user = User.find_by(id: args[:user_id])
return unless args[:text]
return unless args[:client_id]
return unless args[:progress_channel]
helper_mode = args[:prompt]
DiscourseAi::AiHelper::Assistant.new.stream_prompt(
helper_mode,
args[:text],
user,
args[:progress_channel],
force_default_locale: args[:force_default_locale],
client_id: args[:client_id],
custom_prompt: args[:custom_prompt],
)
end
end
end
app/jobs/regular/stream_discord_reply.rb
-19
View File
@@ -1,19 +0,0 @@
# frozen_string_literal: true
module Jobs
class StreamDiscordReply < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
interaction = args[:interaction]
return unless SiteSetting.ai_discord_search_enabled
if SiteSetting.ai_discord_search_mode == "persona"
DiscourseAi::Discord::Bot::PersonaReplier.new(interaction).handle_interaction!
else
DiscourseAi::Discord::Bot::Search.new(interaction).handle_interaction!
end
end
end
end
app/jobs/regular/stream_discover_reply.rb
-57
View File
@@ -1,57 +0,0 @@
# frozen_string_literal: true
module Jobs
class StreamDiscoverReply < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
return if (user = User.find_by(id: args[:user_id])).nil?
return if (query = args[:query]).blank?
ai_persona_klass =
AiPersona
.all_personas(enabled_only: false)
.find { |persona| persona.id == SiteSetting.ai_bot_discover_persona.to_i }
if ai_persona_klass.nil? || !user.in_any_groups?(ai_persona_klass.allowed_group_ids.to_a)
return
end
return if (llm_model = LlmModel.find_by(id: ai_persona_klass.default_llm_id)).nil?
bot =
DiscourseAi::Personas::Bot.as(
Discourse.system_user,
persona: ai_persona_klass.new,
model: llm_model,
)
streamed_reply = +""
start = Time.now
base = { query: query, model_used: llm_model.display_name }
context =
DiscourseAi::Personas::BotContext.new(
messages: [{ type: :user, content: query }],
skip_tool_details: true,
)
bot.reply(context) do |partial|
streamed_reply << partial
# Throttle updates.
if (Time.now - start > 0.3) || Rails.env.test?
payload = base.merge(done: false, ai_discover_reply: streamed_reply)
publish_update(user, payload)
start = Time.now
end
end
publish_update(user, base.merge(done: true, ai_discover_reply: streamed_reply))
end
def publish_update(user, payload)
MessageBus.publish("/discourse-ai/ai-bot/discover", payload, user_ids: [user.id])
end
end
end
app/jobs/regular/stream_post_helper.rb
-42
View File
@@ -1,42 +0,0 @@
# frozen_string_literal: true
module Jobs
class StreamPostHelper < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
return unless post = Post.includes(:topic).find_by(id: args[:post_id])
return unless user = User.find_by(id: args[:user_id])
return unless args[:text]
return unless args[:progress_channel]
return unless args[:client_id]
topic = post.topic
reply_to = post.reply_to_post
return unless user.guardian.can_see?(post)
helper_mode = args[:prompt]
if helper_mode == DiscourseAi::AiHelper::Assistant::EXPLAIN
input = <<~TEXT.strip
<term>#{args[:text]}</term>
<context>#{post.raw}</context>
<topic>#{topic.title}</topic>
#{reply_to ? "<replyTo>#{reply_to.raw}</replyTo>" : nil}
TEXT
else
input = args[:text]
end
DiscourseAi::AiHelper::Assistant.new.stream_prompt(
helper_mode,
input,
user,
args[:progress_channel],
custom_prompt: args[:custom_prompt],
client_id: args[:client_id],
)
end
end
end
app/jobs/regular/stream_topic_ai_summary.rb
-50
View File
@@ -1,50 +0,0 @@
# frozen_string_literal: true
module Jobs
class StreamTopicAiSummary < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
return unless topic = Topic.find_by(id: args[:topic_id])
return unless user = User.find_by(id: args[:user_id])
strategy = DiscourseAi::Summarization.topic_summary(topic)
return if strategy.nil? || !Guardian.new(user).can_see_summary?(topic)
guardian = Guardian.new(user)
return unless guardian.can_see?(topic)
skip_age_check = !!args[:skip_age_check]
streamed_summary = +""
start = Time.now
summary =
DiscourseAi::TopicSummarization
.new(strategy, user)
.summarize(skip_age_check: skip_age_check) do |partial_summary|
streamed_summary << partial_summary
# Throttle updates.
if (Time.now - start > 0.3) || Rails.env.test?
payload = { done: false, ai_topic_summary: { summarized_text: streamed_summary } }
publish_update(topic, user, payload)
start = Time.now
end
end
publish_update(
topic,
user,
AiTopicSummarySerializer.new(summary, { scope: guardian }).as_json.merge(done: true),
)
end
private
def publish_update(topic, user, payload)
MessageBus.publish("/discourse-ai/summaries/topic/#{topic.id}", payload, user_ids: [user.id])
end
end
end
app/jobs/scheduled/categories_locale_detection_backfill.rb
-36
View File
@@ -1,36 +0,0 @@
# frozen_string_literal: true
module Jobs
class CategoriesLocaleDetectionBackfill < ::Jobs::Scheduled
every 1.hour
sidekiq_options retry: false
cluster_concurrency 1
def execute(args)
return if !DiscourseAi::Translation.backfill_enabled?
categories = Category.where(locale: nil)
if SiteSetting.ai_translation_backfill_limit_to_public_content
categories = categories.where(read_restricted: false)
end
limit = SiteSetting.ai_translation_backfill_hourly_rate
categories = categories.limit(limit)
return if categories.empty?
categories.each do |category|
begin
DiscourseAi::Translation::CategoryLocaleDetector.detect_locale(category)
rescue FinalDestination::SSRFDetector::LookupFailedError
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
"Failed to detect category #{category.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
end
end
DiscourseAi::Translation::VerboseLogger.log("Detected #{categories.size} category locales")
end
end
end
app/jobs/scheduled/category_localization_backfill.rb
-15
View File
@@ -1,15 +0,0 @@
# frozen_string_literal: true
module Jobs
class CategoryLocalizationBackfill < ::Jobs::Scheduled
every 1.hour
cluster_concurrency 1
def execute(args)
return if !DiscourseAi::Translation.backfill_enabled?
limit = SiteSetting.ai_translation_backfill_hourly_rate
Jobs.enqueue(:localize_categories, limit:)
end
end
end
app/jobs/scheduled/embeddings_backfill.rb
-151
View File
@@ -1,151 +0,0 @@
# frozen_string_literal: true
module Jobs
class EmbeddingsBackfill < ::Jobs::Scheduled
every 5.minutes
sidekiq_options queue: "low"
cluster_concurrency 1
def execute(args)
return unless DiscourseAi::Embeddings.enabled?
limit = SiteSetting.ai_embeddings_backfill_batch_size
if limit > 50_000
limit = 50_000
Rails.logger.warn(
"Limiting backfill batch size to 50,000 to avoid OOM errors, reduce ai_embeddings_backfill_batch_size to avoid this warning",
)
end
production_vector = DiscourseAi::Embeddings::Vector.instance
if SiteSetting.ai_embeddings_backfill_model.present? &&
SiteSetting.ai_embeddings_backfill_model != SiteSetting.ai_embeddings_selected_model
backfill_vector =
DiscourseAi::Embeddings::Vector.new(
EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_backfill_model),
)
end
topic_work_list = []
topic_work_list << production_vector
topic_work_list << backfill_vector if backfill_vector
topic_work_list.each do |vector|
rebaked = 0
table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
vector_def = vector.vdef
topics =
Topic
.joins(
"LEFT JOIN #{table_name} ON #{table_name}.topic_id = topics.id AND #{table_name}.model_id = #{vector_def.id}",
)
.where(archetype: Archetype.default)
.where(deleted_at: nil)
.order("topics.bumped_at DESC")
rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
next if rebaked >= limit
# Then, we'll try to backfill embeddings for topics that have outdated
# embeddings, be it model or strategy version
relation = topics.where(<<~SQL).limit(limit - rebaked)
#{table_name}.model_version < #{vector_def.version}
OR
#{table_name}.strategy_version < #{vector_def.strategy_version}
SQL
rebaked += populate_topic_embeddings(vector, relation, force: true)
next if rebaked >= limit
# Finally, we'll try to backfill embeddings for topics that have outdated
# embeddings due to edits or new replies. Here we only do 10% of the limit
relation =
topics
.where("#{table_name}.updated_at < ?", 6.hours.ago)
.where("#{table_name}.updated_at < topics.updated_at")
.limit((limit - rebaked) / 10)
populate_topic_embeddings(vector, relation, force: true)
next unless SiteSetting.ai_embeddings_per_post_enabled
# Now for posts
table_name = DiscourseAi::Embeddings::Schema::POSTS_TABLE
posts_batch_size = 1000
posts =
Post
.joins(
"LEFT JOIN #{table_name} ON #{table_name}.post_id = posts.id AND #{table_name}.model_id = #{vector_def.id}",
)
.where(deleted_at: nil)
.where(post_type: Post.types[:regular])
# First, we'll try to backfill embeddings for posts that have none
posts
.where("#{table_name}.post_id IS NULL")
.limit(limit - rebaked)
.pluck(:id)
.each_slice(posts_batch_size) do |batch|
vector.gen_bulk_reprensentations(Post.where(id: batch))
rebaked += batch.length
end
next if rebaked >= limit
# Then, we'll try to backfill embeddings for posts that have outdated
# embeddings, be it model or strategy version
posts
.where(<<~SQL)
#{table_name}.model_version < #{vector_def.version}
OR
#{table_name}.strategy_version < #{vector_def.strategy_version}
SQL
.limit(limit - rebaked)
.pluck(:id)
.each_slice(posts_batch_size) do |batch|
vector.gen_bulk_reprensentations(Post.where(id: batch))
rebaked += batch.length
end
next if rebaked >= limit
# Finally, we'll try to backfill embeddings for posts that have outdated
# embeddings due to edits. Here we only do 10% of the limit
posts
.where("#{table_name}.updated_at < ?", 7.days.ago)
.order("random()")
.limit((limit - rebaked) / 10)
.pluck(:id)
.each_slice(posts_batch_size) do |batch|
vector.gen_bulk_reprensentations(Post.where(id: batch))
rebaked += batch.length
end
end
end
private
def populate_topic_embeddings(vector, topics, force: false)
done = 0
topics =
topics.where("#{DiscourseAi::Embeddings::Schema::TOPICS_TABLE}.topic_id IS NULL") if !force
ids = topics.pluck("topics.id")
batch_size = 1000
ids.each_slice(batch_size) do |batch|
vector.gen_bulk_reprensentations(Topic.where(id: batch).order("topics.bumped_at DESC"))
done += batch.length
end
done
end
end
end
app/jobs/scheduled/generate_concepts_from_popular_items.rb
-87
View File
@@ -1,87 +0,0 @@
# frozen_string_literal: true
module Jobs
class GenerateConceptsFromPopularItems < ::Jobs::Scheduled
every 1.day
# This job runs daily and generates new concepts from popular topics and posts
# It selects items based on engagement metrics and generates concepts from their content
def execute(_args)
return unless SiteSetting.inferred_concepts_enabled
process_popular_topics
process_popular_posts
end
private
def process_popular_topics
# Find candidate topics that are popular and don't have concepts yet
manager = DiscourseAi::InferredConcepts::Manager.new
candidates =
manager.find_candidate_topics(
limit: SiteSetting.inferred_concepts_daily_topics_limit || 20,
min_posts: SiteSetting.inferred_concepts_min_posts || 5,
min_likes: SiteSetting.inferred_concepts_min_likes || 10,
min_views: SiteSetting.inferred_concepts_min_views || 100,
created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
)
return if candidates.blank?
# Process candidate topics - first generate concepts, then match
Jobs.enqueue(
:generate_inferred_concepts,
item_type: "topics",
item_ids: candidates.map(&:id),
batch_size: 10,
)
if SiteSetting.inferred_concepts_background_match
# Schedule a follow-up job to match existing concepts
Jobs.enqueue_in(
1.hour,
:generate_inferred_concepts,
item_type: "topics",
item_ids: candidates.map(&:id),
batch_size: 10,
match_only: true,
)
end
end
def process_popular_posts
# Find candidate posts that are popular and don't have concepts yet
manager = DiscourseAi::InferredConcepts::Manager.new
candidates =
manager.find_candidate_posts(
limit: SiteSetting.inferred_concepts_daily_posts_limit || 30,
min_likes: SiteSetting.inferred_concepts_post_min_likes || 5,
exclude_first_posts: true,
created_after: SiteSetting.inferred_concepts_lookback_days.days.ago,
)
return if candidates.blank?
# Process candidate posts - first generate concepts, then match
Jobs.enqueue(
:generate_inferred_concepts,
item_type: "posts",
item_ids: candidates.map(&:id),
batch_size: 10,
)
if SiteSetting.inferred_concepts_background_match
# Schedule a follow-up job to match against existing concepts
Jobs.enqueue_in(
1.hour,
:generate_inferred_concepts,
item_type: "posts",
item_ids: candidates.map(&:id),
batch_size: 10,
match_only: true,
)
end
end
end
end
app/jobs/scheduled/post_localization_backfill.rb
-17
View File
@@ -1,17 +0,0 @@
# frozen_string_literal: true
module Jobs
class PostLocalizationBackfill < ::Jobs::Scheduled
every 5.minutes
cluster_concurrency 1
def execute(args)
return if !DiscourseAi::Translation.backfill_enabled?
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
return if limit == 0
Jobs.enqueue(:localize_posts, limit:)
end
end
end
app/jobs/scheduled/posts_locale_detection_backfill.rb
-54
View File
@@ -1,54 +0,0 @@
# frozen_string_literal: true
module Jobs
class PostsLocaleDetectionBackfill < ::Jobs::Scheduled
every 5.minutes
sidekiq_options retry: false
cluster_concurrency 1
def execute(args)
return if !DiscourseAi::Translation.backfill_enabled?
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
posts =
Post
.where(locale: nil)
.where(deleted_at: nil)
.where("posts.user_id > 0")
.where("posts.created_at > ?", SiteSetting.ai_translation_backfill_max_age_days.days.ago)
.where.not(raw: [nil, ""])
if SiteSetting.ai_translation_backfill_limit_to_public_content
posts =
posts
.joins(:topic)
.where(topics: { category_id: Category.where(read_restricted: false).select(:id) })
.where("archetype != ?", Archetype.private_message)
else
posts =
posts.joins(:topic).where(
"topics.archetype != ? OR EXISTS (SELECT 1 FROM topic_allowed_groups WHERE topic_id = topics.id)",
Archetype.private_message,
)
end
posts = posts.order(updated_at: :desc).limit(limit)
return if posts.empty?
posts.each do |post|
begin
DiscourseAi::Translation::PostLocaleDetector.detect_locale(post)
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
"Failed to detect post #{post.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
end
end
DiscourseAi::Translation::VerboseLogger.log("Detected #{posts.size} post locales")
end
end
end
app/jobs/scheduled/remove_orphaned_embeddings.rb
-11
View File
@@ -1,11 +0,0 @@
# frozen_string_literal: true
module Jobs
class RemoveOrphanedEmbeddings < ::Jobs::Scheduled
every 1.week
def execute(_args)
DiscourseAi::Embeddings::Schema.remove_orphaned_data
end
end
end
app/jobs/scheduled/sentiment_backfill.rb
-30
View File
@@ -1,30 +0,0 @@
# frozen_string_literal: true
module Jobs
class SentimentBackfill < ::Jobs::Scheduled
every 5.minutes
cluster_concurrency 1
def execute(_args)
return if !SiteSetting.ai_sentiment_enabled
base_budget = SiteSetting.ai_sentiment_backfill_maximum_posts_per_hour
return if base_budget.zero?
# Split budget in 12 intervals, but make sure is at least one.
#
# This is not exact as we don't have a way of tracking how many
# posts we classified in the current hour, but it's a good enough approximation.
limit_per_job = [base_budget, 12].max / 12
classificator = DiscourseAi::Sentiment::PostClassification.new
return if !classificator.has_classifiers?
posts =
DiscourseAi::Sentiment::PostClassification.backfill_query(
max_age_days: SiteSetting.ai_sentiment_backfill_post_max_age_days,
).limit(limit_per_job)
classificator.bulk_classify!(posts)
end
end
end
app/jobs/scheduled/summaries_backfill.rb
-91
View File
@@ -1,91 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class SummariesBackfill < ::Jobs::Scheduled
every 5.minutes
cluster_concurrency 1
def execute(_args)
return if !SiteSetting.discourse_ai_enabled
return if !SiteSetting.ai_summarization_enabled
return if SiteSetting.ai_summary_backfill_maximum_topics_per_hour.zero?
system_user = Discourse.system_user
if SiteSetting.ai_summary_gists_enabled
gist_t = AiSummary.summary_types[:gist]
backfill_candidates(gist_t)
.limit(current_budget(gist_t))
.each do |topic|
strategy = DiscourseAi::Summarization.topic_gist(topic)
try_summarize(strategy, system_user, topic)
end
end
complete_t = AiSummary.summary_types[:complete]
backfill_candidates(complete_t)
.limit(current_budget(complete_t))
.each do |topic|
strategy = DiscourseAi::Summarization.topic_summary(topic)
try_summarize(strategy, system_user, topic)
end
end
def try_summarize(strategy, user, topic)
existing_summary = strategy.existing_summary
if existing_summary.blank? || existing_summary.outdated
begin
strategy.summarize(user)
rescue => e
Rails.logger.error("Error summarizing topic #{topic.id}: #{e.class.name} - #{e.message}")
raise e
end
else
# Hiding or deleting a post, and creating a small action alters the Topic#highest_post_number.
# We use this as a quick way to select potential backfill candidates without relying on original_content_sha.
# At this point, we are confident the summary doesn't need to be regenerated so something other than a regular reply
# caused the number to change in the topic.
existing_summary.update!(highest_target_number: topic.highest_post_number)
end
end
def backfill_candidates(summary_type)
max_age_days = SiteSetting.ai_summary_backfill_topic_max_age_days
Topic
.where("topics.word_count >= ?", SiteSetting.ai_summary_backfill_minimum_word_count)
.joins(<<~SQL)
LEFT OUTER JOIN ai_summaries ais ON
topics.id = ais.target_id AND
ais.target_type = 'Topic' AND
ais.summary_type = '#{summary_type}'
SQL
.where("topics.last_posted_at > current_timestamp - INTERVAL '#{max_age_days.to_i} DAY'")
.where(
<<~SQL, # (1..1) gets stored ad (1..2).
ais.id IS NULL OR (
ais.highest_target_number < topics.highest_post_number
AND ais.updated_at < (current_timestamp - INTERVAL '5 minutes')
)
SQL
)
.order("ais.updated_at DESC NULLS FIRST, topics.last_posted_at DESC")
end
def current_budget(type)
# Split budget in 12 intervals, but make sure is at least one.
base_budget = SiteSetting.ai_summary_backfill_maximum_topics_per_hour
limit_per_job = [base_budget, 12].max / 12
used_budget =
AiSummary.system.where("created_at > ?", 1.hour.ago).where(summary_type: type).count
current_budget = [(base_budget - used_budget), limit_per_job].min
return 0 if current_budget < 0
current_budget
end
end
end
app/jobs/scheduled/topic_localization_backfill.rb
-15
View File
@@ -1,15 +0,0 @@
# frozen_string_literal: true
module Jobs
class TopicLocalizationBackfill < ::Jobs::Scheduled
every 5.minutes
cluster_concurrency 1
def execute(args)
return if !DiscourseAi::Translation.backfill_enabled?
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
Jobs.enqueue(:localize_topics, limit:)
end
end
end
app/jobs/scheduled/topics_locale_detection_backfill.rb
-52
View File
@@ -1,52 +0,0 @@
# frozen_string_literal: true
module Jobs
class TopicsLocaleDetectionBackfill < ::Jobs::Scheduled
every 5.minutes
sidekiq_options retry: false
cluster_concurrency 1
def execute(args)
return if !DiscourseAi::Translation.backfill_enabled?
limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
topics =
Topic
.where(locale: nil, deleted_at: nil)
.where("topics.user_id > 0")
.where("topics.created_at > ?", SiteSetting.ai_translation_backfill_max_age_days.days.ago)
if SiteSetting.ai_translation_backfill_limit_to_public_content
topics =
topics.where(category_id: Category.where(read_restricted: false).select(:id)).where(
"archetype != ?",
Archetype.private_message,
)
else
topics =
topics.where(
"archetype != ? OR EXISTS (SELECT 1 FROM topic_allowed_groups WHERE topic_id = topics.id)",
Archetype.private_message,
)
end
topics = topics.order(updated_at: :desc).limit(limit)
return if topics.empty?
topics.each do |topic|
begin
DiscourseAi::Translation::TopicLocaleDetector.detect_locale(topic)
rescue FinalDestination::SSRFDetector::LookupFailedError
# do nothing, there are too many sporadic lookup failures
rescue => e
DiscourseAi::Translation::VerboseLogger.log(
"Failed to detect topic #{topic.id}'s locale: #{e.message}\n\n#{e.backtrace[0..3].join("\n")}",
)
end
end
DiscourseAi::Translation::VerboseLogger.log("Detected #{topics.size} topic locales")
end
end
end
app/mailers/ai_report_mailer.rb
-9
View File
@@ -1,9 +0,0 @@
# frozen_string_literal: true
class AiReportMailer < ActionMailer::Base
include Email::BuildEmailHelper
def send_report(to_address, opts = {})
build_email(to_address, **opts)
end
end
app/models/ai_api_audit_log.rb
-55
View File
@@ -1,55 +0,0 @@
# frozen_string_literal: true
class AiApiAuditLog < ActiveRecord::Base
belongs_to :post
belongs_to :topic
belongs_to :user
module Provider
OpenAI = 1
Anthropic = 2
HuggingFaceTextGeneration = 3
Gemini = 4
Vllm = 5
Cohere = 6
Ollama = 7
SambaNova = 8
Mistral = 9
OpenRouter = 10
end
def next_log_id
self.class.where("id > ?", id).where(topic_id: topic_id).order(id: :asc).pluck(:id).first
end
def prev_log_id
self.class.where("id < ?", id).where(topic_id: topic_id).order(id: :desc).pluck(:id).first
end
end
# == Schema Information
#
# Table name: ai_api_audit_logs
#
# id :bigint not null, primary key
# provider_id :integer not null
# user_id :integer
# request_tokens :integer
# response_tokens :integer
# raw_request_payload :string
# raw_response_payload :string
# created_at :datetime not null
# updated_at :datetime not null
# topic_id :integer
# post_id :integer
# feature_name :string(255)
# language_model :string(255)
# feature_context :jsonb
# cached_tokens :integer
# duration_msecs :integer
#
# Indexes
#
# index_ai_api_audit_logs_on_created_at_and_feature_name (created_at,feature_name)
# index_ai_api_audit_logs_on_created_at_and_language_model (created_at,language_model)
#
app/models/ai_artifact.rb
-119
View File
@@ -1,119 +0,0 @@
# frozen_string_literal: true
class AiArtifact < ActiveRecord::Base
has_many :versions, class_name: "AiArtifactVersion", dependent: :destroy
has_many :key_values, class_name: "AiArtifactKeyValue", dependent: :destroy
belongs_to :user
belongs_to :post
validates :html, length: { maximum: 65_535 }
validates :css, length: { maximum: 65_535 }
validates :js, length: { maximum: 65_535 }
ALLOWED_CDN_SOURCES = %w[
https://cdn.jsdelivr.net
https://cdnjs.cloudflare.com
https://unpkg.com
https://ajax.googleapis.com
https://d3js.org
https://code.jquery.com
https://esm.sh
]
def self.artifact_version_attribute(version)
if version
"data-artifact-version='#{version}'"
else
""
end
end
def self.iframe_for(id, version = nil)
<<~HTML
<div class='ai-artifact'>
<iframe src='#{url(id, version)}' frameborder="0" height="100%" width="100%"></iframe>
<div class='ai-artifact-controls'>
<a href='#{url(id, version)}' class='link-artifact' target='_blank'>#{I18n.t("discourse_ai.ai_artifact.link")}</a>
<a href class='copy-embed' data-artifact-id="#{id}" #{artifact_version_attribute(version)} data-url="#{url(id, version)}">#{I18n.t("discourse_ai.ai_artifact.copy_embed")}</a>
</div>
</div>
HTML
end
def self.url(id, version = nil)
url = Discourse.base_url + "/discourse-ai/ai-bot/artifacts/#{id}"
if version
"#{url}/#{version}"
else
url
end
end
def self.share_publicly(id:, post:)
artifact = AiArtifact.find_by(id: id)
if artifact&.post&.topic&.id == post.topic.id
artifact.metadata ||= {}
artifact.metadata[:public] = true
artifact.save!
end
end
def self.unshare_publicly(id:)
artifact = AiArtifact.find_by(id: id)
artifact&.update!(metadata: { public: false })
end
def url
self.class.url(id)
end
def apply_diff(html_diff: nil, css_diff: nil, js_diff: nil, change_description: nil)
differ = DiscourseAi::Utils::DiffUtils
html = html_diff ? differ.apply_hunk(self.html, html_diff) : self.html
css = css_diff ? differ.apply_hunk(self.css, css_diff) : self.css
js = js_diff ? differ.apply_hunk(self.js, js_diff) : self.js
create_new_version(html: html, css: css, js: js, change_description: change_description)
end
def create_new_version(html: nil, css: nil, js: nil, change_description: nil)
latest_version = versions.order(version_number: :desc).first
new_version_number = latest_version ? latest_version.version_number + 1 : 1
version = nil
transaction do
# Create the version record
version =
versions.create!(
version_number: new_version_number,
html: html || self.html,
css: css || self.css,
js: js || self.js,
change_description: change_description,
)
save!
end
version
end
def public?
!!metadata&.dig("public")
end
end
# == Schema Information
#
# Table name: ai_artifacts
#
# id :bigint not null, primary key
# user_id :integer not null
# post_id :integer not null
# name :string(255) not null
# html :string(65535)
# css :string(65535)
# js :string(65535)
# metadata :jsonb
# created_at :datetime not null
# updated_at :datetime not null
#
app/models/ai_artifact_key_value.rb
-56
View File
@@ -1,56 +0,0 @@
# frozen_string_literal: true
class AiArtifactKeyValue < ActiveRecord::Base
belongs_to :ai_artifact
belongs_to :user
validates :key, presence: true, length: { maximum: 50 }
validates :value,
presence: true,
length: {
maximum: ->(_) { SiteSetting.ai_artifact_kv_value_max_length },
}
attribute :public, :boolean, default: false
validates :ai_artifact, presence: true
validates :user, presence: true
validates :key, uniqueness: { scope: %i[ai_artifact_id user_id] }
validate :validate_max_keys_per_user_per_artifact
private
def validate_max_keys_per_user_per_artifact
return unless ai_artifact_id && user_id
max_keys = SiteSetting.ai_artifact_max_keys_per_user_per_artifact
existing_count = self.class.where(ai_artifact_id: ai_artifact_id, user_id: user_id).count
# Don't count the current record if it's being updated
existing_count -= 1 if persisted?
if existing_count >= max_keys
errors.add(
:base,
I18n.t("discourse_ai.ai_artifact.errors.max_keys_exceeded", count: max_keys),
)
end
end
end
# == Schema Information
#
# Table name: ai_artifact_key_values
#
# id :bigint not null, primary key
# ai_artifact_id :bigint not null
# user_id :integer not null
# key :string(50) not null
# value :string(20000) not null
# public :boolean default(FALSE), not null
# created_at :datetime not null
# updated_at :datetime not null
#
# Indexes
#
# index_ai_artifact_kv_unique (ai_artifact_id,user_id,key) UNIQUE
#
app/models/ai_artifact_version.rb
-40
View File
@@ -1,40 +0,0 @@
# frozen_string_literal: true
class AiArtifactVersion < ActiveRecord::Base
belongs_to :ai_artifact
validates :html, length: { maximum: 65_535 }
validates :css, length: { maximum: 65_535 }
validates :js, length: { maximum: 65_535 }
# used when generating test cases
def write_to(path)
css_path = "#{path}/main.css"
html_path = "#{path}/main.html"
js_path = "#{path}/main.js"
instructions_path = "#{path}/instructions.txt"
File.write(css_path, css)
File.write(html_path, html)
File.write(js_path, js)
File.write(instructions_path, change_description)
end
end
# == Schema Information
#
# Table name: ai_artifact_versions
#
# id :bigint not null, primary key
# ai_artifact_id :bigint not null
# version_number :integer not null
# html :string(65535)
# css :string(65535)
# js :string(65535)
# metadata :jsonb
# change_description :string
# created_at :datetime not null
# updated_at :datetime not null
#
# Indexes
#
# index_ai_artifact_versions_on_ai_artifact_id_and_version_number (ai_artifact_id,version_number) UNIQUE
#
app/models/ai_moderation_setting.rb
-34
View File
@@ -1,34 +0,0 @@
# frozen_string_literal: true
class AiModerationSetting < ActiveRecord::Base
belongs_to :llm_model
belongs_to :ai_persona
validates :llm_model_id, presence: true
validates :setting_type, presence: true
validates :setting_type, uniqueness: true
def self.spam
find_by(setting_type: :spam)
end
def custom_instructions
data["custom_instructions"]
end
end
# == Schema Information
#
# Table name: ai_moderation_settings
#
# id :bigint not null, primary key
# setting_type :enum not null
# data :jsonb
# llm_model_id :bigint not null
# created_at :datetime not null
# updated_at :datetime not null
# ai_persona_id :bigint default(-31), not null
#
# Indexes
#
# index_ai_moderation_settings_on_setting_type (setting_type) UNIQUE
#
app/models/ai_persona.rb
-432
View File
@@ -1,432 +0,0 @@
# frozen_string_literal: true
class AiPersona < ActiveRecord::Base
# TODO remove this line 01-10-2025
self.ignored_columns = %i[default_llm question_consolidator_llm]
# places a hard limit, so per site we cache a maximum of 500 classes
MAX_PERSONAS_PER_SITE = 500
validates :name, presence: true, uniqueness: true, length: { maximum: 100 }
validates :description, presence: true, length: { maximum: 2000 }
validates :system_prompt, presence: true, length: { maximum: 10_000_000 }
validate :system_persona_unchangeable, on: :update, if: :system
validate :chat_preconditions
validate :allowed_seeded_model, if: :default_llm_id
validate :well_formated_examples
validates :max_context_posts, numericality: { greater_than: 0 }, allow_nil: true
# leaves some room for growth but sets a maximum to avoid memory issues
# we may want to revisit this in the future
validates :vision_max_pixels, numericality: { greater_than: 0, maximum: 4_000_000 }
validates :rag_chunk_tokens, numericality: { greater_than: 0, maximum: 50_000 }
validates :rag_chunk_overlap_tokens, numericality: { greater_than: -1, maximum: 200 }
validates :rag_conversation_chunks, numericality: { greater_than: 0, maximum: 1000 }
validates :forced_tool_count, numericality: { greater_than: -2, maximum: 100_000 }
validate :tools_can_not_be_duplicated
has_many :rag_document_fragments, dependent: :destroy, as: :target
belongs_to :created_by, class_name: "User"
belongs_to :user
belongs_to :default_llm, class_name: "LlmModel"
belongs_to :question_consolidator_llm, class_name: "LlmModel"
belongs_to :rag_llm_model, class_name: "LlmModel"
has_many :upload_references, as: :target, dependent: :destroy
has_many :uploads, through: :upload_references
before_destroy :ensure_not_system
before_update :regenerate_rag_fragments
def self.persona_cache
@persona_cache ||= ::DiscourseAi::MultisiteHash.new("persona_cache")
end
scope :ordered, -> { order("priority DESC, lower(name) ASC") }
def self.all_personas(enabled_only: true)
persona_cache[:value] ||= AiPersona
.ordered
.all
.limit(MAX_PERSONAS_PER_SITE)
.map(&:class_instance)
if enabled_only
persona_cache[:value].select { |p| p.enabled }
else
persona_cache[:value]
end
end
def self.persona_users(user: nil)
persona_users =
persona_cache[:persona_users] ||= AiPersona
.where(enabled: true)
.joins(:user)
.map do |persona|
{
id: persona.id,
user_id: persona.user_id,
username: persona.user.username_lower,
allowed_group_ids: persona.allowed_group_ids,
default_llm_id: persona.default_llm_id,
force_default_llm: persona.force_default_llm,
allow_chat_channel_mentions: persona.allow_chat_channel_mentions,
allow_chat_direct_messages: persona.allow_chat_direct_messages,
allow_topic_mentions: persona.allow_topic_mentions,
allow_personal_messages: persona.allow_personal_messages,
}
end
if user
persona_users.select { |persona_user| user.in_any_groups?(persona_user[:allowed_group_ids]) }
else
persona_users
end
end
def self.allowed_modalities(
user: nil,
allow_chat_channel_mentions: false,
allow_chat_direct_messages: false,
allow_topic_mentions: false,
allow_personal_messages: false
)
index =
"modality-#{allow_chat_channel_mentions}-#{allow_chat_direct_messages}-#{allow_topic_mentions}-#{allow_personal_messages}"
personas =
persona_cache[index.to_sym] ||= persona_users.select do |persona|
next true if allow_chat_channel_mentions && persona[:allow_chat_channel_mentions]
next true if allow_chat_direct_messages && persona[:allow_chat_direct_messages]
next true if allow_topic_mentions && persona[:allow_topic_mentions]
next true if allow_personal_messages && persona[:allow_personal_messages]
false
end
if user
personas.select { |u| user.in_any_groups?(u[:allowed_group_ids]) }
else
personas
end
end
after_commit :bump_cache
def bump_cache
self.class.persona_cache.flush!
end
def tools_can_not_be_duplicated
return unless tools.is_a?(Array)
seen_tools = Set.new
custom_tool_ids = Set.new
builtin_tool_names = Set.new
tools.each do |tool|
inner_name, _, _ = tool.is_a?(Array) ? tool : [tool, nil]
if inner_name.start_with?("custom-")
custom_tool_ids.add(inner_name.split("-", 2).last.to_i)
else
builtin_tool_names.add(inner_name.downcase)
end
if seen_tools.include?(inner_name)
errors.add(:tools, I18n.t("discourse_ai.ai_bot.personas.cannot_have_duplicate_tools"))
break
else
seen_tools.add(inner_name)
end
end
return if errors.any?
# Checking if there are any duplicate tool_names between custom and builtin tools
if builtin_tool_names.present? && custom_tool_ids.present?
AiTool
.where(id: custom_tool_ids)
.pluck(:tool_name)
.each do |tool_name|
if builtin_tool_names.include?(tool_name.downcase)
errors.add(:tools, I18n.t("discourse_ai.ai_bot.personas.cannot_have_duplicate_tools"))
break
end
end
end
end
def class_instance
attributes = %i[
id
user_id
system
mentionable
default_llm_id
max_context_posts
vision_enabled
vision_max_pixels
rag_conversation_chunks
question_consolidator_llm_id
allow_chat_channel_mentions
allow_chat_direct_messages
allow_topic_mentions
allow_personal_messages
force_default_llm
name
description
allowed_group_ids
tool_details
enabled
]
instance_attributes = {}
attributes.each do |attr|
value = self.read_attribute(attr)
instance_attributes[attr] = value
end
instance_attributes[:username] = user&.username_lower
options = {}
force_tool_use = []
tools =
self.tools.filter_map do |element|
klass = nil
element = [element] if element.is_a?(String)
inner_name, current_options, should_force_tool_use =
element.is_a?(Array) ? element : [element, nil]
if inner_name.start_with?("custom-")
custom_tool_id = inner_name.split("-", 2).last.to_i
if AiTool.exists?(id: custom_tool_id, enabled: true)
klass = DiscourseAi::Personas::Tools::Custom.class_instance(custom_tool_id)
end
else
inner_name = inner_name.gsub("Tool", "")
inner_name = "List#{inner_name}" if %w[Categories Tags].include?(inner_name)
begin
klass = "DiscourseAi::Personas::Tools::#{inner_name}".constantize
options[klass] = current_options if current_options
rescue StandardError
end
end
force_tool_use << klass if should_force_tool_use
klass
end
persona_class = DiscourseAi::Personas::Persona.system_personas_by_id[self.id]
if persona_class
return(
# we need a new copy so we don't leak information
# across sites
Class.new(persona_class) do
# required for localization
define_singleton_method(:to_s) { persona_class.to_s }
instance_attributes.each do |key, value|
# description/name are localized
define_singleton_method(key) { value } if key != :description && key != :name
end
define_method(:options) { options }
end
)
end
ai_persona_id = self.id
Class.new(DiscourseAi::Personas::Persona) do
instance_attributes.each { |key, value| define_singleton_method(key) { value } }
define_singleton_method(:to_s) do
"#<#{self.class.name} @name=#{name} @allowed_group_ids=#{allowed_group_ids.join(",")}>"
end
define_singleton_method(:inspect) { to_s }
define_method(:initialize) do |*args, **kwargs|
@ai_persona = AiPersona.find_by(id: ai_persona_id)
super(*args, **kwargs)
end
define_method(:tools) { tools }
define_method(:force_tool_use) { force_tool_use }
define_method(:forced_tool_count) { @ai_persona&.forced_tool_count }
define_method(:options) { options }
define_method(:temperature) { @ai_persona&.temperature }
define_method(:top_p) { @ai_persona&.top_p }
define_method(:system_prompt) { @ai_persona&.system_prompt || "You are a helpful bot." }
define_method(:uploads) { @ai_persona&.uploads }
define_method(:response_format) { @ai_persona&.response_format }
define_method(:examples) { @ai_persona&.examples }
end
end
FIRST_PERSONA_USER_ID = -1200
def create_user!
raise "User already exists" if user_id && User.exists?(user_id)
# find the first id smaller than FIRST_USER_ID that is not taken
id = nil
id = DB.query_single(<<~SQL, FIRST_PERSONA_USER_ID, FIRST_PERSONA_USER_ID - 200).first
WITH seq AS (
SELECT generate_series(?, ?, -1) AS id
)
SELECT seq.id FROM seq
LEFT JOIN users ON users.id = seq.id
WHERE users.id IS NULL
ORDER BY seq.id DESC
SQL
id = DB.query_single(<<~SQL).first if id.nil?
SELECT min(id) - 1 FROM users
SQL
# note .invalid is a reserved TLD which will route nowhere
user =
User.new(
email: "#{SecureRandom.hex}@does-not-exist.invalid",
name: name.titleize,
username: UserNameSuggester.suggest(name + "_bot"),
active: true,
approved: true,
trust_level: TrustLevel[4],
id: id,
)
user.save!(validate: false)
update!(user_id: user.id)
user
end
def regenerate_rag_fragments
if rag_chunk_tokens_changed? || rag_chunk_overlap_tokens_changed?
RagDocumentFragment.where(target: self).delete_all
end
end
def features
DiscourseAi::Configuration::Feature.find_features_using(persona_id: id)
end
private
def chat_preconditions
if (
allow_chat_channel_mentions || allow_chat_direct_messages || allow_topic_mentions ||
force_default_llm
) && !default_llm_id
errors.add(:default_llm, I18n.t("discourse_ai.ai_bot.personas.default_llm_required"))
end
end
def system_persona_unchangeable
error_msg = I18n.t("discourse_ai.ai_bot.personas.cannot_edit_system_persona")
if top_p_changed? || temperature_changed? || system_prompt_changed? || name_changed? ||
description_changed?
errors.add(:base, error_msg)
elsif tools_changed?
old_tools = tools_change[0]
new_tools = tools_change[1]
old_tool_names = old_tools.map { |t| t.is_a?(Array) ? t[0] : t }.to_set
new_tool_names = new_tools.map { |t| t.is_a?(Array) ? t[0] : t }.to_set
errors.add(:base, error_msg) if old_tool_names != new_tool_names
elsif response_format_changed?
old_format = response_format_change[0].map { |f| f["key"] }.to_set
new_format = response_format_change[1].map { |f| f["key"] }.to_set
errors.add(:base, error_msg) if old_format != new_format
elsif examples_changed?
old_examples = examples_change[0].flatten.to_set
new_examples = examples_change[1].flatten.to_set
errors.add(:base, error_msg) if old_examples != new_examples
end
end
def ensure_not_system
if system
errors.add(:base, I18n.t("discourse_ai.ai_bot.personas.cannot_delete_system_persona"))
throw :abort
end
end
def allowed_seeded_model
return if default_llm_id.blank?
return if default_llm.nil?
return if !default_llm.seeded?
return if SiteSetting.ai_bot_allowed_seeded_models_map.include?(default_llm.id.to_s)
errors.add(:default_llm, I18n.t("discourse_ai.llm.configuration.invalid_seeded_model"))
end
def well_formated_examples
return if examples.blank?
if examples.is_a?(Array) &&
examples.all? { |e| e.is_a?(Array) && e.length == 2 && e.all?(&:present?) }
return
end
errors.add(:examples, I18n.t("discourse_ai.personas.malformed_examples"))
end
end
# == Schema Information
#
# Table name: ai_personas
#
# id :bigint not null, primary key
# name :string(100) not null
# description :string(2000) not null
# system_prompt :string(10000000) not null
# allowed_group_ids :integer default([]), not null, is an Array
# created_by_id :integer
# enabled :boolean default(TRUE), not null
# created_at :datetime not null
# updated_at :datetime not null
# system :boolean default(FALSE), not null
# priority :boolean default(FALSE), not null
# temperature :float
# top_p :float
# user_id :integer
# max_context_posts :integer
# vision_enabled :boolean default(FALSE), not null
# vision_max_pixels :integer default(1048576), not null
# rag_chunk_tokens :integer default(374), not null
# rag_chunk_overlap_tokens :integer default(10), not null
# rag_conversation_chunks :integer default(10), not null
# tool_details :boolean default(TRUE), not null
# tools :json not null
# forced_tool_count :integer default(-1), not null
# allow_chat_channel_mentions :boolean default(FALSE), not null
# allow_chat_direct_messages :boolean default(FALSE), not null
# allow_topic_mentions :boolean default(FALSE), not null
# allow_personal_messages :boolean default(TRUE), not null
# force_default_llm :boolean default(FALSE), not null
# rag_llm_model_id :bigint
# default_llm_id :bigint
# question_consolidator_llm_id :bigint
# response_format :jsonb
# examples :jsonb
#
# Indexes
#
# index_ai_personas_on_name (name) UNIQUE
#
app/models/ai_spam_log.rb
-27
View File
@@ -1,27 +0,0 @@
# frozen_string_literal: true
class AiSpamLog < ActiveRecord::Base
belongs_to :post
belongs_to :llm_model
belongs_to :ai_api_audit_log
belongs_to :reviewable
end
# == Schema Information
#
# Table name: ai_spam_logs
#
# id :bigint not null, primary key
# post_id :bigint not null
# llm_model_id :bigint not null
# ai_api_audit_log_id :bigint
# reviewable_id :bigint
# is_spam :boolean not null
# payload :string(20000) default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
# error :string(3000)
#
# Indexes
#
# index_ai_spam_logs_on_post_id (post_id)
#

Some files were not shown because too many files have changed in this diff Show More