Compare commits

..
Author SHA1 Message Date
David Taylor 593e8dc2a5 DEV: Update modifyClass calls
- Update `controller:topic` modification to native class syntax
- Remove search-result-entry modification in favour of plugin-outlet-based solution
2024-12-02 11:49:22 +00:00
980 changed files with 2548225 additions and 69152 deletions
-7
View File
@@ -1,10 +1,3 @@
< 3.5.0.beta7-dev: cd14b0c0bee0cf63c59b64b6f7213e31a37f11a7
< 3.5.0.beta6-dev: 3e74eea1e5e3143888d67a8d8a11206df214dc24
< 3.5.0.beta3-dev: 09a68414804a1447f52e5d60691ba59742cda9ec
< 3.5.0.beta2-dev: de8624416a15b3d8e7ad350b083cc1420451ccec
< 3.5.0.beta1-dev: bdef136080074a993e7c4f5ca562edc31a8ba756
< 3.4.0.beta4-dev: a53719ab8eb071459f215227421b3ea4987e5f87
< 3.4.0.beta4-dev: 20612fde52d3f740cad64823ef8aadb0748b567f
< 3.4.0.beta3-dev: decf1bb49d737ea15308400f22f89d1d1e71d13d
< 3.4.0.beta1-dev: 9d887ad4ace8e33c3fe7dbb39237e882c08b4f0b
< 3.3.0.beta5-dev: 4d8090002f6dcd8e34d41033606bf131fa221475
-5
View File
@@ -2,8 +2,3 @@ node_modules
/gems
/auto_generated
.env
evals/log
evals/cases
config/eval-llms.local.yml
# this gets rid of search results from ag, ripgrep, etc
public/ai-share/highlight.min.js
-3
View File
@@ -2,6 +2,3 @@ inherit_gem:
rubocop-discourse: stree-compat.yml
RSpec/NamedSubject:
Enabled: false
Style/GlobalVars:
AllowedVariables: [$prometheus_client]
+45 -59
View File
@@ -1,93 +1,79 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (8.0.2)
activesupport (7.1.3.4)
base64
benchmark (>= 0.3)
bigdecimal
concurrent-ruby (~> 1.0, >= 1.3.1)
concurrent-ruby (~> 1.0, >= 1.0.2)
connection_pool (>= 2.2.5)
drb
i18n (>= 1.6, < 2)
logger (>= 1.4.2)
minitest (>= 5.1)
securerandom (>= 0.3)
tzinfo (~> 2.0, >= 2.0.5)
uri (>= 0.13.1)
mutex_m
tzinfo (~> 2.0)
ast (2.4.2)
base64 (0.2.0)
benchmark (0.4.0)
bigdecimal (3.1.9)
concurrent-ruby (1.3.5)
connection_pool (2.5.0)
bigdecimal (3.1.8)
concurrent-ruby (1.3.3)
connection_pool (2.4.1)
drb (2.2.1)
i18n (1.14.7)
i18n (1.14.5)
concurrent-ruby (~> 1.0)
json (2.10.2)
language_server-protocol (3.17.0.4)
lint_roller (1.1.0)
logger (1.6.6)
minitest (5.25.5)
parallel (1.26.3)
parser (3.3.7.1)
json (2.7.2)
language_server-protocol (3.17.0.3)
minitest (5.24.1)
mutex_m (0.2.0)
parallel (1.25.1)
parser (3.3.3.0)
ast (~> 2.4.1)
racc
prettier_print (1.2.1)
racc (1.8.1)
rack (3.1.16)
racc (1.8.0)
rack (3.1.6)
rainbow (3.1.1)
regexp_parser (2.10.0)
rubocop (1.74.0)
regexp_parser (2.9.2)
rexml (3.3.9)
rubocop (1.64.1)
json (~> 2.3)
language_server-protocol (~> 3.17.0.2)
lint_roller (~> 1.1.0)
language_server-protocol (>= 3.17.0)
parallel (~> 1.10)
parser (>= 3.3.0.2)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 2.9.3, < 3.0)
rubocop-ast (>= 1.38.0, < 2.0)
regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0)
rubocop-ast (>= 1.31.1, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 4.0)
rubocop-ast (1.38.1)
unicode-display_width (>= 2.4.0, < 3.0)
rubocop-ast (1.31.3)
parser (>= 3.3.1.0)
rubocop-capybara (2.22.1)
lint_roller (~> 1.1)
rubocop (~> 1.72, >= 1.72.1)
rubocop-discourse (3.12.1)
rubocop-capybara (2.21.0)
rubocop (~> 1.41)
rubocop-discourse (3.8.1)
activesupport (>= 6.1)
lint_roller (>= 1.1.0)
rubocop (>= 1.73.2)
rubocop-capybara (>= 2.22.0)
rubocop-factory_bot (>= 2.27.0)
rubocop-rails (>= 2.30.3)
rubocop (>= 1.59.0)
rubocop-capybara (>= 2.0.0)
rubocop-factory_bot (>= 2.0.0)
rubocop-rails (>= 2.25.0)
rubocop-rspec (>= 3.0.1)
rubocop-rspec_rails (>= 2.31.0)
rubocop-factory_bot (2.27.1)
lint_roller (~> 1.1)
rubocop (~> 1.72, >= 1.72.1)
rubocop-rails (2.30.3)
rubocop-rspec_rails (>= 2.30.0)
rubocop-factory_bot (2.26.1)
rubocop (~> 1.61)
rubocop-rails (2.25.1)
activesupport (>= 4.2.0)
lint_roller (~> 1.1)
rack (>= 1.1)
rubocop (>= 1.72.1, < 2.0)
rubocop-ast (>= 1.38.0, < 2.0)
rubocop-rspec (3.5.0)
lint_roller (~> 1.1)
rubocop (~> 1.72, >= 1.72.1)
rubocop-rspec_rails (2.31.0)
lint_roller (~> 1.1)
rubocop (~> 1.72, >= 1.72.1)
rubocop-rspec (~> 3.5)
rubocop (>= 1.33.0, < 2.0)
rubocop-ast (>= 1.31.1, < 2.0)
rubocop-rspec (3.0.2)
rubocop (~> 1.61)
rubocop-rspec_rails (2.30.0)
rubocop (~> 1.61)
rubocop-rspec (~> 3, >= 3.0.1)
ruby-progressbar (1.13.0)
securerandom (0.4.1)
syntax_tree (6.2.0)
prettier_print (>= 1.2.0)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
unicode-display_width (3.1.4)
unicode-emoji (~> 4.0, >= 4.0.4)
unicode-emoji (4.0.4)
uri (1.0.3)
unicode-display_width (2.5.0)
PLATFORMS
ruby
@@ -97,4 +83,4 @@ DEPENDENCIES
syntax_tree
BUNDLED WITH
2.6.6
2.5.10
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) Civilized Discourse Construction Kit, Inc.
Copyright (c) 2021 Civilized Discourse Construction Kit
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
-24
View File
@@ -3,27 +3,3 @@
**Plugin Summary**
For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco
### Evals
The directory `evals` contains AI evals for the Discourse AI plugin.
You may create a local config by copying `config/eval-llms.yml` to `config/eval-llms.local.yml` and modifying the values.
To run them use:
cd evals
./run --help
```
Usage: evals/run [options]
-e, --eval NAME Name of the evaluation to run
--list-models List models
-m, --model NAME Model to evaluate (will eval all models if not specified)
-l, --list List evals
```
To run evals you will need to configure API keys in your environment:
OPENAI_API_KEY=your_openai_api_key
ANTHROPIC_API_KEY=your_anthropic_api_key
GEMINI_API_KEY=your_gemini_api_key
-8
View File
@@ -1,8 +0,0 @@
{
"tests": {
"requiredPlugins": [
"https://github.com/discourse/discourse-prometheus",
"https://github.com/discourse/discourse-solved.git"
]
}
}
@@ -1,21 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiEmbeddingsEdit extends DiscourseRoute {
async model(params) {
const allEmbeddings = this.modelFor(
"adminPlugins.show.discourse-ai-embeddings"
);
const id = parseInt(params.id, 10);
const record = allEmbeddings.findBy("id", id);
record.provider_params = record.provider_params || {};
return record;
}
setupController(controller, model) {
super.setupController(controller, model);
controller.set(
"allEmbeddings",
this.modelFor("adminPlugins.show.discourse-ai-embeddings")
);
}
}
@@ -1,17 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiEmbeddingsNew extends DiscourseRoute {
async model() {
const record = this.store.createRecord("ai-embedding");
record.provider_params = {};
return record;
}
setupController(controller, model) {
super.setupController(controller, model);
controller.set(
"allEmbeddings",
this.modelFor("adminPlugins.show.discourse-ai-embeddings")
);
}
}
@@ -1,7 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiAiEmbeddingsRoute extends DiscourseRoute {
model() {
return this.store.findAll("ai-embedding");
}
}
@@ -1,35 +0,0 @@
import { action } from "@ember/object";
import { ajax } from "discourse/lib/ajax";
import DiscourseRoute from "discourse/routes/discourse";
import SiteSetting from "admin/models/site-setting";
export default class AdminPluginsShowDiscourseAiFeaturesEdit extends DiscourseRoute {
async model(params) {
const allFeatures = this.modelFor(
"adminPlugins.show.discourse-ai-features"
);
const id = parseInt(params.id, 10);
const currentFeature = allFeatures.find((feature) => feature.id === id);
const { site_settings } = await ajax("/admin/config/site_settings.json", {
data: {
filter_area: `ai-features/${currentFeature.module_name}`,
plugin: "discourse-ai",
category: "discourse_ai",
},
});
currentFeature.feature_settings = site_settings.map((setting) =>
SiteSetting.create(setting)
);
return currentFeature;
}
@action
willTransition() {
// site settings may amend if a feature is enabled or disabled, so refresh the model
// even on back button
this.router.refresh("adminPlugins.show.discourse-ai-features");
}
}
@@ -1,10 +0,0 @@
import { service } from "@ember/service";
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiFeatures extends DiscourseRoute {
@service store;
async model() {
return this.store.findAll("ai-feature");
}
}
@@ -1,18 +1,9 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
export default class AdminPluginsShowDiscourseAiLlmsShow extends DiscourseRoute {
async model(params) {
const id = parseInt(params.id, 10);
if (id < 0) {
// You shouldn't be able to access the edit page
// if the model is seeded
return this.router.transitionTo(
"adminPlugins.show.discourse-ai-llms.index"
);
}
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const id = parseInt(params.id, 10);
const record = allLlms.findBy("id", id);
record.provider_params = record.provider_params || {};
return record;
@@ -1,6 +1,6 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiPersonasEdit extends DiscourseRoute {
export default class AdminPluginsShowDiscourseAiPersonasShow extends DiscourseRoute {
async model(params) {
const allPersonas = this.modelFor(
"adminPlugins.show.discourse-ai-personas"
@@ -1,11 +0,0 @@
import { service } from "@ember/service";
import { ajax } from "discourse/lib/ajax";
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiSpamRoute extends DiscourseRoute {
@service store;
model() {
return ajax("/admin/plugins/discourse-ai/ai-spam.json");
}
}
@@ -1,10 +1,6 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiToolsNewRoute extends DiscourseRoute {
beforeModel(transition) {
this.preset = transition.to.queryParams.presetId || "empty_tool";
}
async model() {
return this.store.createRecord("ai-tool");
}
@@ -12,10 +8,8 @@ export default class DiscourseAiToolsNewRoute extends DiscourseRoute {
setupController(controller) {
super.setupController(...arguments);
const toolsModel = this.modelFor("adminPlugins.show.discourse-ai-tools");
controller.set("allTools", toolsModel);
controller.set("presets", toolsModel.resultSetMeta.presets);
controller.set("llms", toolsModel.resultSetMeta.llms);
controller.set("settings", toolsModel.resultSetMeta.settings);
controller.set("selectedPreset", this.preset);
}
}
@@ -1,6 +1,6 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
export default class DiscourseAiToolsShowRoute extends DiscourseRoute {
async model(params) {
const allTools = this.modelFor("adminPlugins.show.discourse-ai-tools");
const id = parseInt(params.id, 10);
@@ -14,7 +14,5 @@ export default class DiscourseAiToolsEditRoute extends DiscourseRoute {
controller.set("allTools", toolsModel);
controller.set("presets", toolsModel.resultSetMeta.presets);
controller.set("llms", toolsModel.resultSetMeta.llms);
controller.set("settings", toolsModel.resultSetMeta.settings);
}
}
@@ -1,4 +0,0 @@
<AiEmbeddingsListEditor
@embeddings={{this.allEmbeddings}}
@currentEmbedding={{this.model}}
/>
@@ -1 +0,0 @@
<AiEmbeddingsListEditor @embeddings={{this.model}} />
@@ -1,4 +0,0 @@
<AiEmbeddingsListEditor
@embeddings={{this.allEmbeddings}}
@currentEmbedding={{this.model}}
/>
@@ -1,24 +0,0 @@
import RouteTemplate from "ember-route-template";
import BackButton from "discourse/components/back-button";
import SiteSettingComponent from "admin/components/site-setting";
export default RouteTemplate(
<template>
<BackButton
@route="adminPlugins.show.discourse-ai-features"
@label="discourse_ai.features.back"
/>
<section class="ai-feature-editor__header">
<h2>{{@model.name}}</h2>
<p>{{@model.description}}</p>
</section>
<section class="ai-feature-editor">
{{#each @model.feature_settings as |setting|}}
<div>
<SiteSettingComponent @setting={{setting}} />
</div>
{{/each}}
</section>
</template>
);
@@ -1 +0,0 @@
<AiFeatures @features={{this.model}} />
@@ -1 +0,0 @@
<AiSpam @model={{this.model}} />
@@ -3,8 +3,5 @@
@tools={{this.allTools}}
@model={{this.model}}
@presets={{this.presets}}
@llms={{this.llms}}
@settings={{this.settings}}
@selectedPreset={{this.selectedPreset}}
/>
</section>
@@ -3,7 +3,5 @@
@tools={{this.allTools}}
@model={{this.model}}
@presets={{this.presets}}
@llms={{this.llms}}
@settings={{this.settings}}
/>
</section>
@@ -1,200 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiEmbeddingsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
embedding_defs = EmbeddingDefinition.all.order(:display_name)
render json: {
ai_embeddings:
ActiveModel::ArraySerializer.new(
embedding_defs,
each_serializer: AiEmbeddingDefinitionSerializer,
root: false,
).as_json,
meta: {
provider_params: EmbeddingDefinition.provider_params,
providers: EmbeddingDefinition.provider_names,
distance_functions: EmbeddingDefinition.distance_functions,
tokenizers:
EmbeddingDefinition.tokenizer_names.map { |tn|
{ id: tn, name: tn.split("::").last }
},
presets: EmbeddingDefinition.presets,
},
}
end
def new
end
def edit
embedding_def = EmbeddingDefinition.find(params[:id])
render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
end
def create
embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
if embedding_def.save
log_ai_embedding_creation(embedding_def)
render json: AiEmbeddingDefinitionSerializer.new(embedding_def), status: :created
else
render_json_error embedding_def
end
end
def update
embedding_def = EmbeddingDefinition.find(params[:id])
if embedding_def.seeded?
return(
render_json_error(I18n.t("discourse_ai.embeddings.cannot_edit_builtin"), status: 403)
)
end
initial_attributes = embedding_def.attributes.dup
if embedding_def.update(ai_embeddings_params.except(:dimensions))
log_ai_embedding_update(embedding_def, initial_attributes)
render json: AiEmbeddingDefinitionSerializer.new(embedding_def)
else
render_json_error embedding_def
end
end
def destroy
embedding_def = EmbeddingDefinition.find(params[:id])
if embedding_def.seeded?
return(
render_json_error(I18n.t("discourse_ai.embeddings.cannot_edit_builtin"), status: 403)
)
end
if embedding_def.id == SiteSetting.ai_embeddings_selected_model.to_i
return render_json_error(I18n.t("discourse_ai.embeddings.delete_failed"), status: 409)
end
embedding_details = {
embedding_id: embedding_def.id,
display_name: embedding_def.display_name,
provider: embedding_def.provider,
dimensions: embedding_def.dimensions,
subject: embedding_def.display_name,
}
if embedding_def.destroy
log_ai_embedding_deletion(embedding_details)
head :no_content
else
render_json_error embedding_def
end
end
def test
RateLimiter.new(
current_user,
"ai_embeddings_test_#{current_user.id}",
3,
1.minute,
).performed!
embedding_def = EmbeddingDefinition.new(ai_embeddings_params)
DiscourseAi::Embeddings::Vector.new(embedding_def).vector_from("this is a test")
render json: { success: true }
rescue Net::HTTPBadResponse => e
render json: { success: false, error: e.message }
end
private
def ai_embeddings_params
permitted =
params.require(:ai_embedding).permit(
:display_name,
:dimensions,
:max_sequence_length,
:pg_function,
:provider,
:url,
:api_key,
:tokenizer_class,
:embed_prompt,
:search_prompt,
:matryoshka_dimensions,
)
extra_field_names = EmbeddingDefinition.provider_params.dig(permitted[:provider]&.to_sym)
if extra_field_names.present?
received_prov_params =
params.dig(:ai_embedding, :provider_params)&.slice(*extra_field_names.keys)
if received_prov_params.present?
permitted[:provider_params] = received_prov_params.permit!
end
end
permitted
end
def ai_embeddings_logger_fields
{
display_name: {
},
provider: {
},
dimensions: {
},
url: {
},
tokenizer_class: {
},
max_sequence_length: {
},
embed_prompt: {
type: :large_text,
},
search_prompt: {
type: :large_text,
},
matryoshka_dimensions: {
},
api_key: {
type: :sensitive,
},
# JSON fields should be tracked as simple changes
json_fields: [:provider_params],
}
end
def log_ai_embedding_creation(embedding_def)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
logger.log_creation("embedding", embedding_def, ai_embeddings_logger_fields, entity_details)
end
def log_ai_embedding_update(embedding_def, initial_attributes)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
entity_details = { embedding_id: embedding_def.id, subject: embedding_def.display_name }
logger.log_update(
"embedding",
embedding_def,
initial_attributes,
ai_embeddings_logger_fields,
entity_details,
)
end
def log_ai_embedding_deletion(embedding_details)
logger = DiscourseAi::Utils::AiStaffActionLogger.new(current_user)
logger.log_deletion("embedding", embedding_details)
end
end
end
end
@@ -1,66 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiFeaturesController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
render json: serialize_modules(DiscourseAi::Configuration::Module.all)
end
def edit
raise Discourse::InvalidParameters.new(:id) if params[:id].blank?
a_module = DiscourseAi::Configuration::Module.find_by(id: params[:id].to_i)
render json: serialize_module(a_module)
end
private
def serialize_modules(modules)
modules.map { |a_module| serialize_module(a_module) }
end
def serialize_module(a_module)
return nil if a_module.blank?
{
id: a_module.id,
module_name: a_module.name,
module_enabled: a_module.enabled?,
features: a_module.features.map { |f| serialize_feature(f) },
}
end
def serialize_feature(feature)
{
name: feature.name,
personas: feature.persona_ids.map { |id| serialize_persona(persona_id_obj_hash[id]) },
llm_models:
feature.llm_models.map do |llm_model|
{ id: llm_model.id, name: llm_model.display_name }
end,
enabled: feature.enabled?,
}
end
def serialize_persona(persona)
return nil if persona.blank?
serialize_data(persona, AiFeaturesPersonaSerializer, root: false)
end
private
def persona_id_obj_hash
@persona_id_obj_hash ||=
begin
ids = DiscourseAi::Configuration::Feature.all.map(&:persona_ids).flatten.uniq
AiPersona.where(id: ids).index_by(&:id)
end
end
end
end
end
@@ -1,59 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiLlmQuotasController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
quotas = LlmQuota.includes(:group)
render json: {
quotas:
ActiveModel::ArraySerializer.new(quotas, each_serializer: LlmQuotaSerializer),
}
end
def create
quota = LlmQuota.new(quota_params)
if quota.save
render json: LlmQuotaSerializer.new(quota), status: :created
else
render_json_error quota
end
end
def update
quota = LlmQuota.find(params[:id])
if quota.update(quota_params)
render json: LlmQuotaSerializer.new(quota)
else
render_json_error quota
end
end
def destroy
quota = LlmQuota.find(params[:id])
quota.destroy!
head :no_content
rescue ActiveRecord::RecordNotFound
render json: { error: I18n.t("not_found") }, status: 404
end
private
def quota_params
params.require(:quota).permit(
:group_id,
:llm_model_id,
:max_tokens,
:max_usages,
:duration_seconds,
)
end
end
end
end

Some files were not shown because too many files have changed in this diff Show More