Compare commits

..
Author SHA1 Message Date
Sam Saffron 6e6ced4554 Work in progress... post embeddings and version
Feel free to continue any work here, will pick it up again next week
2023-05-12 15:28:12 +10:00
634 changed files with 4209 additions and 2581694 deletions
-7
View File
@@ -1,8 +1 @@
< 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
< 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
< 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
< 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
3.1.0.beta3: 057fbe1ce6
+3
View File
@@ -0,0 +1,3 @@
{
"extends": "eslint-config-discourse"
}
-1
View File
@@ -1 +0,0 @@
module.exports = require("@discourse/lint-configs/eslint");
-21
View File
@@ -1,21 +0,0 @@
app/assets/stylesheets/vendor/
documentation/
package.json
config/locales/**/*.yml
!config/locales/**/*.en*.yml
script/import_scripts/**/*.yml
plugins/**/lib/javascripts/locale
public/
!/app/assets/javascripts/discourse/public
vendor/
app/assets/javascripts/discourse/tests/fixtures
spec/
node_modules/
dist/
tmp/
**/*.rb
**/*.html
**/*.json
**/*.md
+1
View File
@@ -0,0 +1 @@
{}
-1
View File
@@ -1 +0,0 @@
module.exports = require("@discourse/lint-configs/prettier");
-2
View File
@@ -1,4 +1,2 @@
inherit_gem:
rubocop-discourse: stree-compat.yml
RSpec/NamedSubject:
Enabled: false
+1 -1
View File
@@ -1,2 +1,2 @@
--print-width=100
--plugins=plugin/trailing_comma,plugin/disable_auto_ternary
--plugins=plugin/trailing_comma,disable_ternary
-1
View File
@@ -1 +0,0 @@
module.exports = require("@discourse/lint-configs/template-lint");
+4
View File
@@ -0,0 +1,4 @@
module.exports = {
plugins: ["ember-template-lint-plugin-discourse"],
extends: "discourse:recommended",
};
+1
View File
@@ -5,4 +5,5 @@ source "https://rubygems.org"
group :development do
gem "rubocop-discourse"
gem "syntax_tree"
gem "syntax_tree-disable_ternary"
end
+28 -57
View File
@@ -1,79 +1,50 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (7.1.3.2)
base64
bigdecimal
concurrent-ruby (~> 1.0, >= 1.0.2)
connection_pool (>= 2.2.5)
drb
i18n (>= 1.6, < 2)
minitest (>= 5.1)
mutex_m
tzinfo (~> 2.0)
ast (2.4.2)
base64 (0.2.0)
bigdecimal (3.1.6)
concurrent-ruby (1.2.3)
connection_pool (2.4.1)
drb (2.2.1)
i18n (1.14.1)
concurrent-ruby (~> 1.0)
json (2.7.1)
language_server-protocol (3.17.0.3)
minitest (5.22.2)
mutex_m (0.2.0)
parallel (1.24.0)
parser (3.3.0.5)
json (2.6.2)
parallel (1.22.1)
parser (3.1.2.1)
ast (~> 2.4.1)
racc
prettier_print (1.2.1)
racc (1.7.3)
prettier_print (1.2.0)
rainbow (3.1.1)
regexp_parser (2.9.0)
rexml (3.2.8)
strscan (>= 3.0.9)
rubocop (1.61.0)
regexp_parser (2.6.0)
rexml (3.2.5)
rubocop (1.36.0)
json (~> 2.3)
language_server-protocol (>= 3.17.0)
parallel (~> 1.10)
parser (>= 3.3.0.2)
parser (>= 3.1.2.1)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0)
rubocop-ast (>= 1.30.0, < 2.0)
rubocop-ast (>= 1.20.1, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 3.0)
rubocop-ast (1.31.1)
parser (>= 3.3.0.4)
rubocop-capybara (2.20.0)
rubocop (~> 1.41)
rubocop-discourse (3.7.1)
activesupport (>= 6.1)
rubocop (>= 1.59.0)
rubocop-capybara (>= 2.0.0)
rubocop-factory_bot (>= 2.0.0)
rubocop-rspec (>= 2.25.0)
rubocop-factory_bot (2.25.1)
rubocop (~> 1.41)
rubocop-rspec (2.27.1)
rubocop (~> 1.40)
rubocop-capybara (~> 2.17)
rubocop-factory_bot (~> 2.22)
ruby-progressbar (1.13.0)
strscan (3.1.0)
syntax_tree (6.2.0)
unicode-display_width (>= 1.4.0, < 3.0)
rubocop-ast (1.21.0)
parser (>= 3.1.1.0)
rubocop-discourse (3.0)
rubocop (>= 1.1.0)
rubocop-rspec (>= 2.0.0)
rubocop-rspec (2.13.2)
rubocop (~> 1.33)
ruby-progressbar (1.11.0)
syntax_tree (5.1.0)
prettier_print (>= 1.2.0)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
unicode-display_width (2.5.0)
syntax_tree-disable_ternary (1.0.0)
unicode-display_width (2.3.0)
PLATFORMS
arm64-darwin-20
ruby
x86_64-darwin-18
x86_64-darwin-19
x86_64-darwin-20
x86_64-linux
DEPENDENCIES
rubocop-discourse
syntax_tree
syntax_tree-disable_ternary
BUNDLED WITH
2.5.4
2.3.10
+2 -2
View File
@@ -1,5 +1,5 @@
# **Discourse AI** Plugin
# **Plugin Name** Plugin
**Plugin Summary**
For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco
For more information, please see: **url to meta topic**
@@ -1,16 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model() {
const record = this.store.createRecord("ai-llm");
return record;
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
);
},
});
@@ -1,17 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model(params) {
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const id = parseInt(params.id, 10);
return allLlms.findBy("id", id);
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
);
},
});
@@ -1,7 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiAiLlmsRoute extends DiscourseRoute {
model() {
return this.store.findAll("ai-llm");
}
}
@@ -1,23 +0,0 @@
import { AUTO_GROUPS } from "discourse/lib/constants";
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model() {
const record = this.store.createRecord("ai-persona");
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
record.set("rag_uploads", []);
// these match the defaults on the table
record.set("rag_chunk_tokens", 374);
record.set("rag_chunk_overlap_tokens", 10);
record.set("rag_conversation_chunks", 10);
return record;
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
);
},
});
@@ -1,19 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model(params) {
const allPersonas = this.modelFor(
"adminPlugins.show.discourse-ai-personas"
);
const id = parseInt(params.id, 10);
return allPersonas.findBy("id", id);
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
);
},
});
@@ -1,7 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiAiPersonasRoute extends DiscourseRoute {
model() {
return this.store.findAll("ai-persona");
}
}
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.model}} />
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
<AiPersonaListEditor @personas={{this.model}} />
@@ -1,10 +0,0 @@
<DBreadcrumbsItem as |linkClass|>
<LinkTo @route="adminPlugins.show.discourse-ai-personas" class={{linkClass}}>
{{@model.plugin.nameTitleized}}
</LinkTo>
</DBreadcrumbsItem>
<AiPersonaListEditor
@personas={{this.allPersonas}}
@currentPersona={{this.model}}
/>
@@ -1,4 +0,0 @@
<AiPersonaListEditor
@personas={{this.allPersonas}}
@currentPersona={{this.model}}
/>
@@ -1,83 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiLlmsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
llms = LlmModel.all
render json: {
ai_llms:
ActiveModel::ArraySerializer.new(
llms,
each_serializer: LlmModelSerializer,
root: false,
).as_json,
meta: {
providers: DiscourseAi::Completions::Llm.provider_names,
tokenizers:
DiscourseAi::Completions::Llm.tokenizer_names.map { |tn|
{ id: tn, name: tn.split("::").last }
},
},
}
end
def show
llm_model = LlmModel.find(params[:id])
render json: LlmModelSerializer.new(llm_model)
end
def create
llm_model = LlmModel.new(ai_llm_params)
if llm_model.save
render json: { ai_persona: llm_model }, status: :created
else
render_json_error llm_model
end
end
def update
llm_model = LlmModel.find(params[:id])
if llm_model.update(ai_llm_params)
render json: llm_model
else
render_json_error llm_model
end
end
def test
RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!
llm_model = LlmModel.new(ai_llm_params)
DiscourseAi::Completions::Llm.proxy_from_obj(llm_model).generate(
"How much is 1 + 1?",
user: current_user,
feature_name: "llm_validator",
)
render json: { success: true }
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
render json: { success: false, error: e.message }
end
private
def ai_llm_params
params.require(:ai_llm).permit(
:display_name,
:name,
:provider,
:tokenizer,
:max_prompt_tokens,
:url,
:api_key,
)
end
end
end
end
@@ -1,181 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiPersonasController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
before_action :find_ai_persona,
only: %i[show update destroy create_user indexing_status_check]
def index
ai_personas =
AiPersona.ordered.map do |persona|
# we use a special serializer here cause names and descriptions are
# localized for system personas
LocalizedAiPersonaSerializer.new(persona, root: false)
end
tools =
DiscourseAi::AiBot::Personas::Persona.all_available_tools.map do |tool|
AiToolSerializer.new(tool, root: false)
end
llms =
DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
{ id: hash[:value], name: hash[:name] }
end
render json: { ai_personas: ai_personas, meta: { commands: tools, llms: llms } }
end
def show
render json: LocalizedAiPersonaSerializer.new(@ai_persona)
end
def create
ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
if ai_persona.save
RagDocumentFragment.link_persona_and_uploads(ai_persona, attached_upload_ids)
render json: { ai_persona: ai_persona }, status: :created
else
render_json_error ai_persona
end
end
def create_user
user = @ai_persona.create_user!
render json: BasicUserSerializer.new(user, root: "user")
end
def update
if @ai_persona.update(ai_persona_params.except(:rag_uploads))
RagDocumentFragment.update_persona_uploads(@ai_persona, attached_upload_ids)
render json: @ai_persona
else
render_json_error @ai_persona
end
end
def destroy
if @ai_persona.destroy
head :no_content
else
render_json_error @ai_persona
end
end
def upload_file
file = params[:file] || params[:files].first
if !SiteSetting.ai_embeddings_enabled?
raise Discourse::InvalidAccess.new("Embeddings not enabled")
end
validate_extension!(file.original_filename)
validate_file_size!(file.tempfile.size)
hijack do
upload =
UploadCreator.new(
file.tempfile,
file.original_filename,
type: "discourse_ai_rag_upload",
skip_validations: true,
).create_for(current_user.id)
if upload.persisted?
render json: UploadSerializer.new(upload)
else
render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
end
end
end
def indexing_status_check
render json: RagDocumentFragment.indexing_status(@ai_persona, @ai_persona.uploads)
end
private
def find_ai_persona
@ai_persona = AiPersona.find(params[:id])
end
def attached_upload_ids
ai_persona_params[:rag_uploads].to_a.map { |h| h[:id] }
end
def ai_persona_params
permitted =
params.require(:ai_persona).permit(
:name,
:description,
:enabled,
:system_prompt,
:priority,
:top_p,
:temperature,
:default_llm,
:user_id,
:mentionable,
:max_context_posts,
:vision_enabled,
:vision_max_pixels,
:rag_chunk_tokens,
:rag_chunk_overlap_tokens,
:rag_conversation_chunks,
:question_consolidator_llm,
:allow_chat,
allowed_group_ids: [],
rag_uploads: [:id],
)
if commands = params.dig(:ai_persona, :commands)
permitted[:commands] = permit_commands(commands)
end
permitted
end
def permit_commands(commands)
return [] if !commands.is_a?(Array)
commands.filter_map do |command, options|
break nil if !command.is_a?(String)
options&.permit! if options && options.is_a?(ActionController::Parameters)
if options
[command, options]
else
command
end
end
end
def validate_extension!(filename)
extension = File.extname(filename)[1..-1] || ""
authorized_extensions = %w[txt md]
if !authorized_extensions.include?(extension)
raise Discourse::InvalidParameters.new(
I18n.t(
"upload.unauthorized",
authorized_extensions: authorized_extensions.join(" "),
),
)
end
end
def validate_file_size!(filesize)
max_size_bytes = 20.megabytes
if filesize > max_size_bytes
raise Discourse::InvalidParameters.new(
I18n.t(
"upload.attachments.too_large_humanized",
max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
),
)
end
end
end
end
end
@@ -1,12 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class DashboardController < ::Admin::StaffController
requires_plugin DiscourseAi::PLUGIN_NAME
def sentiment
end
end
end
end
@@ -6,21 +6,6 @@ module DiscourseAi
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login
def show_debug_info
post = Post.find(params[:post_id])
guardian.ensure_can_debug_ai_bot_conversation!(post)
posts =
Post
.where("post_number <= ?", post.post_number)
.where(topic_id: post.topic_id)
.order("post_number DESC")
debug_info = AiApiAuditLog.where(post: posts).order(created_at: :desc).first
render json: debug_info, status: 200
end
def stop_streaming_response
post = Post.find(params[:post_id])
guardian.ensure_can_see!(post)
@@ -29,15 +14,6 @@ module DiscourseAi
render json: {}, status: 200
end
def show_bot_username
bot_user_id = DiscourseAi::AiBot::EntryPoint.map_bot_model_to_user_id(params[:username])
raise Discourse::InvalidParameters.new(:username) if !bot_user_id
bot_username_lower = User.find(bot_user_id).username_lower
render json: { bot_username: bot_username_lower }, status: 200
end
end
end
end
@@ -1,111 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module AiBot
class SharedAiConversationsController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login only: %i[create update destroy]
before_action :require_site_settings!
skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required, only: %i[show]
def create
ensure_allowed_create!
RateLimiter.new(current_user, "share-ai-conversation", 10, 1.minute).performed!
shared_conversation = SharedAiConversation.share_conversation(current_user, @topic)
if shared_conversation.persisted?
render json: success_json.merge(share_key: shared_conversation.share_key)
else
render json:
failed_json.merge(error: I18n.t("discourse_ai.share_ai.errors.failed_to_share")),
status: :unprocessable_entity
end
end
def destroy
ensure_allowed_destroy!
SharedAiConversation.destroy_conversation(@shared_conversation)
render json:
success_json.merge(
message: I18n.t("discourse_ai.share_ai.errors.conversation_deleted"),
)
end
def show
@shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
raise Discourse::NotFound if @shared_conversation.blank?
expires_in 1.minute, public: true
response.headers["X-Robots-Tag"] = "noindex"
if request.format.json?
render json: success_json.merge(@shared_conversation.to_json)
else
render "show", layout: false
end
end
def preview
ensure_allowed_preview!
data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
data[:error] = @error if @error
data[:share_key] = @shared_conversation.share_key if @shared_conversation
data[:topic_id] = @topic.id
render json: data
end
private
def require_site_settings!
if !SiteSetting.discourse_ai_enabled ||
!SiteSetting.ai_bot_public_sharing_allowed_groups_map.any? ||
!SiteSetting.ai_bot_enabled
raise Discourse::NotFound
end
end
def ensure_allowed_preview!
@topic = Topic.find_by(id: params[:topic_id])
raise Discourse::NotFound if !@topic
@shared_conversation = SharedAiConversation.find_by(target: @topic)
@error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
if @error == :not_allowed
raise Discourse::InvalidAccess.new(
nil,
nil,
custom_message: "discourse_ai.share_ai.errors.#{@error}",
)
end
end
def ensure_allowed_destroy!
@shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
raise Discourse::InvalidAccess if @shared_conversation.blank?
guardian.ensure_can_destroy_shared_ai_bot_conversation!(@shared_conversation)
end
def ensure_allowed_create!
@topic = Topic.find_by(id: params[:topic_id])
raise Discourse::NotFound if !@topic
error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
if error
raise Discourse::InvalidAccess.new(
nil,
nil,
custom_message: "discourse_ai.share_ai.errors.#{error}",
)
end
end
end
end
end

Some files were not shown because too many files have changed in this diff Show More