Compare commits

..
Author SHA1 Message Date
Sam Saffron a8f54e2709 DEV: adding a concept of prompt builder
This is half baked and not working, but starts demonstrating the idea
2023-08-25 15:54:16 +10:00
653 changed files with 100551 additions and 1576166 deletions
-5
View File
@@ -1,8 +1,3 @@
< 3.3.0.beta2-dev: 61890b667c06299841ae88946f84a112f00060e1
< 3.3.0.beta1-dev: a65c4076a63104b61d213534db7f89aa3c8acd9e
< 3.2.0.beta5-dev: c70f43f130fb98e4a1adb1afd9b621aaa49088e3
< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
< 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
< 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
3.1.0.beta3: 057fbe1ce6
+3
View File
@@ -0,0 +1,3 @@
{
"extends": "eslint-config-discourse"
}
-1
View File
@@ -1 +0,0 @@
module.exports = require("@discourse/lint-configs/eslint");
-21
View File
@@ -1,21 +0,0 @@
app/assets/stylesheets/vendor/
documentation/
package.json
config/locales/**/*.yml
!config/locales/**/*.en*.yml
script/import_scripts/**/*.yml
plugins/**/lib/javascripts/locale
public/
!/app/assets/javascripts/discourse/public
vendor/
app/assets/javascripts/discourse/tests/fixtures
spec/
node_modules/
dist/
tmp/
**/*.rb
**/*.html
**/*.json
**/*.md
+1
View File
@@ -0,0 +1 @@
{}
-1
View File
@@ -1 +0,0 @@
module.exports = require("@discourse/lint-configs/prettier");
-2
View File
@@ -1,4 +1,2 @@
inherit_gem:
rubocop-discourse: stree-compat.yml
RSpec/NamedSubject:
Enabled: false
+1 -1
View File
@@ -1,2 +1,2 @@
--print-width=100
--plugins=plugin/trailing_comma,plugin/disable_auto_ternary
--plugins=plugin/trailing_comma,disable_ternary
-1
View File
@@ -1 +0,0 @@
module.exports = require("@discourse/lint-configs/template-lint");
+4
View File
@@ -0,0 +1,4 @@
module.exports = {
plugins: ["ember-template-lint-plugin-discourse"],
extends: "discourse:recommended",
};
+1
View File
@@ -5,4 +5,5 @@ source "https://rubygems.org"
group :development do
gem "rubocop-discourse"
gem "syntax_tree"
gem "syntax_tree-disable_ternary"
end
+28 -57
View File
@@ -1,79 +1,50 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (7.1.3.2)
base64
bigdecimal
concurrent-ruby (~> 1.0, >= 1.0.2)
connection_pool (>= 2.2.5)
drb
i18n (>= 1.6, < 2)
minitest (>= 5.1)
mutex_m
tzinfo (~> 2.0)
ast (2.4.2)
base64 (0.2.0)
bigdecimal (3.1.6)
concurrent-ruby (1.2.3)
connection_pool (2.4.1)
drb (2.2.1)
i18n (1.14.1)
concurrent-ruby (~> 1.0)
json (2.7.1)
language_server-protocol (3.17.0.3)
minitest (5.22.2)
mutex_m (0.2.0)
parallel (1.24.0)
parser (3.3.0.5)
json (2.6.2)
parallel (1.22.1)
parser (3.1.2.1)
ast (~> 2.4.1)
racc
prettier_print (1.2.1)
racc (1.7.3)
prettier_print (1.2.0)
rainbow (3.1.1)
regexp_parser (2.9.0)
rexml (3.2.8)
strscan (>= 3.0.9)
rubocop (1.61.0)
regexp_parser (2.6.0)
rexml (3.2.5)
rubocop (1.36.0)
json (~> 2.3)
language_server-protocol (>= 3.17.0)
parallel (~> 1.10)
parser (>= 3.3.0.2)
parser (>= 3.1.2.1)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0)
rubocop-ast (>= 1.30.0, < 2.0)
rubocop-ast (>= 1.20.1, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 3.0)
rubocop-ast (1.31.1)
parser (>= 3.3.0.4)
rubocop-capybara (2.20.0)
rubocop (~> 1.41)
rubocop-discourse (3.7.1)
activesupport (>= 6.1)
rubocop (>= 1.59.0)
rubocop-capybara (>= 2.0.0)
rubocop-factory_bot (>= 2.0.0)
rubocop-rspec (>= 2.25.0)
rubocop-factory_bot (2.25.1)
rubocop (~> 1.41)
rubocop-rspec (2.27.1)
rubocop (~> 1.40)
rubocop-capybara (~> 2.17)
rubocop-factory_bot (~> 2.22)
ruby-progressbar (1.13.0)
strscan (3.1.0)
syntax_tree (6.2.0)
unicode-display_width (>= 1.4.0, < 3.0)
rubocop-ast (1.21.0)
parser (>= 3.1.1.0)
rubocop-discourse (3.0)
rubocop (>= 1.1.0)
rubocop-rspec (>= 2.0.0)
rubocop-rspec (2.13.2)
rubocop (~> 1.33)
ruby-progressbar (1.11.0)
syntax_tree (5.1.0)
prettier_print (>= 1.2.0)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
unicode-display_width (2.5.0)
syntax_tree-disable_ternary (1.0.0)
unicode-display_width (2.3.0)
PLATFORMS
arm64-darwin-20
ruby
x86_64-darwin-18
x86_64-darwin-19
x86_64-darwin-20
x86_64-linux
DEPENDENCIES
rubocop-discourse
syntax_tree
syntax_tree-disable_ternary
BUNDLED WITH
2.5.4
2.3.10
+2 -2
View File
@@ -1,5 +1,5 @@
# **Discourse AI** Plugin
# **Plugin Name** Plugin
**Plugin Summary**
For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco
For more information, please see: **url to meta topic**
@@ -1,16 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model() {
const record = this.store.createRecord("ai-llm");
return record;
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
);
},
});
@@ -1,17 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model(params) {
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const id = parseInt(params.id, 10);
return allLlms.findBy("id", id);
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allLlms",
this.modelFor("adminPlugins.show.discourse-ai-llms")
);
},
});
@@ -1,7 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiAiLlmsRoute extends DiscourseRoute {
model() {
return this.store.findAll("ai-llm");
}
}
@@ -1,23 +0,0 @@
import { AUTO_GROUPS } from "discourse/lib/constants";
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model() {
const record = this.store.createRecord("ai-persona");
record.set("allowed_group_ids", [AUTO_GROUPS.trust_level_0.id]);
record.set("rag_uploads", []);
// these match the defaults on the table
record.set("rag_chunk_tokens", 374);
record.set("rag_chunk_overlap_tokens", 10);
record.set("rag_conversation_chunks", 10);
return record;
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
);
},
});
@@ -1,19 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default DiscourseRoute.extend({
async model(params) {
const allPersonas = this.modelFor(
"adminPlugins.show.discourse-ai-personas"
);
const id = parseInt(params.id, 10);
return allPersonas.findBy("id", id);
},
setupController(controller, model) {
this._super(controller, model);
controller.set(
"allPersonas",
this.modelFor("adminPlugins.show.discourse-ai-personas")
);
},
});
@@ -1,7 +0,0 @@
import DiscourseRoute from "discourse/routes/discourse";
export default class DiscourseAiAiPersonasRoute extends DiscourseRoute {
model() {
return this.store.findAll("ai-persona");
}
}
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.model}} />
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
<AiLlmsListEditor @llms={{this.allLlms}} @currentLlm={{this.model}} />
@@ -1 +0,0 @@
<AiPersonaListEditor @personas={{this.model}} />
@@ -1,10 +0,0 @@
<DBreadcrumbsItem as |linkClass|>
<LinkTo @route="adminPlugins.show.discourse-ai-personas" class={{linkClass}}>
{{@model.plugin.nameTitleized}}
</LinkTo>
</DBreadcrumbsItem>
<AiPersonaListEditor
@personas={{this.allPersonas}}
@currentPersona={{this.model}}
/>
@@ -1,4 +0,0 @@
<AiPersonaListEditor
@personas={{this.allPersonas}}
@currentPersona={{this.model}}
/>
@@ -1,83 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiLlmsController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
def index
llms = LlmModel.all
render json: {
ai_llms:
ActiveModel::ArraySerializer.new(
llms,
each_serializer: LlmModelSerializer,
root: false,
).as_json,
meta: {
providers: DiscourseAi::Completions::Llm.provider_names,
tokenizers:
DiscourseAi::Completions::Llm.tokenizer_names.map { |tn|
{ id: tn, name: tn.split("::").last }
},
},
}
end
def show
llm_model = LlmModel.find(params[:id])
render json: LlmModelSerializer.new(llm_model)
end
def create
llm_model = LlmModel.new(ai_llm_params)
if llm_model.save
render json: { ai_persona: llm_model }, status: :created
else
render_json_error llm_model
end
end
def update
llm_model = LlmModel.find(params[:id])
if llm_model.update(ai_llm_params)
render json: llm_model
else
render_json_error llm_model
end
end
def test
RateLimiter.new(current_user, "llm_test_#{current_user.id}", 3, 1.minute).performed!
llm_model = LlmModel.new(ai_llm_params)
DiscourseAi::Completions::Llm.proxy_from_obj(llm_model).generate(
"How much is 1 + 1?",
user: current_user,
feature_name: "llm_validator",
)
render json: { success: true }
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
render json: { success: false, error: e.message }
end
private
def ai_llm_params
params.require(:ai_llm).permit(
:display_name,
:name,
:provider,
:tokenizer,
:max_prompt_tokens,
:url,
:api_key,
)
end
end
end
end
@@ -1,181 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiPersonasController < ::Admin::AdminController
requires_plugin ::DiscourseAi::PLUGIN_NAME
before_action :find_ai_persona,
only: %i[show update destroy create_user indexing_status_check]
def index
ai_personas =
AiPersona.ordered.map do |persona|
# we use a special serializer here cause names and descriptions are
# localized for system personas
LocalizedAiPersonaSerializer.new(persona, root: false)
end
tools =
DiscourseAi::AiBot::Personas::Persona.all_available_tools.map do |tool|
AiToolSerializer.new(tool, root: false)
end
llms =
DiscourseAi::Configuration::LlmEnumerator.values.map do |hash|
{ id: hash[:value], name: hash[:name] }
end
render json: { ai_personas: ai_personas, meta: { commands: tools, llms: llms } }
end
def show
render json: LocalizedAiPersonaSerializer.new(@ai_persona)
end
def create
ai_persona = AiPersona.new(ai_persona_params.except(:rag_uploads))
if ai_persona.save
RagDocumentFragment.link_persona_and_uploads(ai_persona, attached_upload_ids)
render json: { ai_persona: ai_persona }, status: :created
else
render_json_error ai_persona
end
end
def create_user
user = @ai_persona.create_user!
render json: BasicUserSerializer.new(user, root: "user")
end
def update
if @ai_persona.update(ai_persona_params.except(:rag_uploads))
RagDocumentFragment.update_persona_uploads(@ai_persona, attached_upload_ids)
render json: @ai_persona
else
render_json_error @ai_persona
end
end
def destroy
if @ai_persona.destroy
head :no_content
else
render_json_error @ai_persona
end
end
def upload_file
file = params[:file] || params[:files].first
if !SiteSetting.ai_embeddings_enabled?
raise Discourse::InvalidAccess.new("Embeddings not enabled")
end
validate_extension!(file.original_filename)
validate_file_size!(file.tempfile.size)
hijack do
upload =
UploadCreator.new(
file.tempfile,
file.original_filename,
type: "discourse_ai_rag_upload",
skip_validations: true,
).create_for(current_user.id)
if upload.persisted?
render json: UploadSerializer.new(upload)
else
render json: failed_json.merge(errors: upload.errors.full_messages), status: 422
end
end
end
def indexing_status_check
render json: RagDocumentFragment.indexing_status(@ai_persona, @ai_persona.uploads)
end
private
def find_ai_persona
@ai_persona = AiPersona.find(params[:id])
end
def attached_upload_ids
ai_persona_params[:rag_uploads].to_a.map { |h| h[:id] }
end
def ai_persona_params
permitted =
params.require(:ai_persona).permit(
:name,
:description,
:enabled,
:system_prompt,
:priority,
:top_p,
:temperature,
:default_llm,
:user_id,
:mentionable,
:max_context_posts,
:vision_enabled,
:vision_max_pixels,
:rag_chunk_tokens,
:rag_chunk_overlap_tokens,
:rag_conversation_chunks,
:question_consolidator_llm,
:allow_chat,
allowed_group_ids: [],
rag_uploads: [:id],
)
if commands = params.dig(:ai_persona, :commands)
permitted[:commands] = permit_commands(commands)
end
permitted
end
def permit_commands(commands)
return [] if !commands.is_a?(Array)
commands.filter_map do |command, options|
break nil if !command.is_a?(String)
options&.permit! if options && options.is_a?(ActionController::Parameters)
if options
[command, options]
else
command
end
end
end
def validate_extension!(filename)
extension = File.extname(filename)[1..-1] || ""
authorized_extensions = %w[txt md]
if !authorized_extensions.include?(extension)
raise Discourse::InvalidParameters.new(
I18n.t(
"upload.unauthorized",
authorized_extensions: authorized_extensions.join(" "),
),
)
end
end
def validate_file_size!(filesize)
max_size_bytes = 20.megabytes
if filesize > max_size_bytes
raise Discourse::InvalidParameters.new(
I18n.t(
"upload.attachments.too_large_humanized",
max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes),
),
)
end
end
end
end
end
@@ -1,12 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class DashboardController < ::Admin::StaffController
requires_plugin DiscourseAi::PLUGIN_NAME
def sentiment
end
end
end
end
@@ -6,21 +6,6 @@ module DiscourseAi
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login
def show_debug_info
post = Post.find(params[:post_id])
guardian.ensure_can_debug_ai_bot_conversation!(post)
posts =
Post
.where("post_number <= ?", post.post_number)
.where(topic_id: post.topic_id)
.order("post_number DESC")
debug_info = AiApiAuditLog.where(post: posts).order(created_at: :desc).first
render json: debug_info, status: 200
end
def stop_streaming_response
post = Post.find(params[:post_id])
guardian.ensure_can_see!(post)
@@ -1,111 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module AiBot
class SharedAiConversationsController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login only: %i[create update destroy]
before_action :require_site_settings!
skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required, only: %i[show]
def create
ensure_allowed_create!
RateLimiter.new(current_user, "share-ai-conversation", 10, 1.minute).performed!
shared_conversation = SharedAiConversation.share_conversation(current_user, @topic)
if shared_conversation.persisted?
render json: success_json.merge(share_key: shared_conversation.share_key)
else
render json:
failed_json.merge(error: I18n.t("discourse_ai.share_ai.errors.failed_to_share")),
status: :unprocessable_entity
end
end
def destroy
ensure_allowed_destroy!
SharedAiConversation.destroy_conversation(@shared_conversation)
render json:
success_json.merge(
message: I18n.t("discourse_ai.share_ai.errors.conversation_deleted"),
)
end
def show
@shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
raise Discourse::NotFound if @shared_conversation.blank?
expires_in 1.minute, public: true
response.headers["X-Robots-Tag"] = "noindex"
if request.format.json?
render json: success_json.merge(@shared_conversation.to_json)
else
render "show", layout: false
end
end
def preview
ensure_allowed_preview!
data = SharedAiConversation.build_conversation_data(@topic, include_usernames: true)
data[:error] = @error if @error
data[:share_key] = @shared_conversation.share_key if @shared_conversation
data[:topic_id] = @topic.id
render json: data
end
private
def require_site_settings!
if !SiteSetting.discourse_ai_enabled ||
!SiteSetting.ai_bot_public_sharing_allowed_groups_map.any? ||
!SiteSetting.ai_bot_enabled
raise Discourse::NotFound
end
end
def ensure_allowed_preview!
@topic = Topic.find_by(id: params[:topic_id])
raise Discourse::NotFound if !@topic
@shared_conversation = SharedAiConversation.find_by(target: @topic)
@error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
if @error == :not_allowed
raise Discourse::InvalidAccess.new(
nil,
nil,
custom_message: "discourse_ai.share_ai.errors.#{@error}",
)
end
end
def ensure_allowed_destroy!
@shared_conversation = SharedAiConversation.find_by(share_key: params[:share_key])
raise Discourse::InvalidAccess if @shared_conversation.blank?
guardian.ensure_can_destroy_shared_ai_bot_conversation!(@shared_conversation)
end
def ensure_allowed_create!
@topic = Topic.find_by(id: params[:topic_id])
raise Discourse::NotFound if !@topic
error = DiscourseAi::AiBot::EntryPoint.ai_share_error(@topic, guardian)
if error
raise Discourse::InvalidAccess.new(
nil,
nil,
custom_message: "discourse_ai.share_ai.errors.#{error}",
)
end
end
end
end
end

Some files were not shown because too many files have changed in this diff Show More