Compare commits

..
Author SHA1 Message Date
Sam Saffron 6e6ced4554 Work in progress... post embeddings and version
Feel free to continue any work here, will pick it up again next week
2023-05-12 15:28:12 +10:00
389 changed files with 2997 additions and 1145105 deletions
-4
View File
@@ -1,5 +1 @@
< 3.2.0.beta4-dev: 493b48477a9288f2f1b6f021954088705a6673d6
< 3.2.0.beta2-dev: 3bced1c6f525553fe2b5b79d29bd66c1cf2a47d5
3.1.999: c0415bb7eb878b1b7abf112d65cba981030df8df
< 3.2.0.beta1-dev: 3e7c99de894b03e9161d9481da118fe893991a1d
3.1.0.beta3: 057fbe1ce6
+3
View File
@@ -0,0 +1,3 @@
{
"extends": "eslint-config-discourse"
}
-1
View File
@@ -1 +0,0 @@
module.exports = require("@discourse/lint-configs/eslint");
-21
View File
@@ -1,21 +0,0 @@
app/assets/stylesheets/vendor/
documentation/
package.json
config/locales/**/*.yml
!config/locales/**/*.en*.yml
script/import_scripts/**/*.yml
plugins/**/lib/javascripts/locale
public/
!/app/assets/javascripts/discourse/public
vendor/
app/assets/javascripts/discourse/tests/fixtures
spec/
node_modules/
dist/
tmp/
**/*.rb
**/*.html
**/*.json
**/*.md
+1
View File
@@ -0,0 +1 @@
{}
-1
View File
@@ -1 +0,0 @@
module.exports = require("@discourse/lint-configs/prettier");
-2
View File
@@ -1,4 +1,2 @@
inherit_gem:
rubocop-discourse: stree-compat.yml
RSpec/NamedSubject:
Enabled: false
+1 -1
View File
@@ -1,2 +1,2 @@
--print-width=100
--plugins=plugin/trailing_comma,plugin/disable_auto_ternary
--plugins=plugin/trailing_comma,disable_ternary
+4 -1
View File
@@ -1 +1,4 @@
module.exports = require("@discourse/lint-configs/template-lint");
module.exports = {
plugins: ["ember-template-lint-plugin-discourse"],
extends: "discourse:recommended",
};
+1
View File
@@ -5,4 +5,5 @@ source "https://rubygems.org"
group :development do
gem "rubocop-discourse"
gem "syntax_tree"
gem "syntax_tree-disable_ternary"
end
+20 -28
View File
@@ -2,45 +2,36 @@ GEM
remote: https://rubygems.org/
specs:
ast (2.4.2)
json (2.6.3)
language_server-protocol (3.17.0.3)
parallel (1.23.0)
parser (3.2.2.4)
json (2.6.2)
parallel (1.22.1)
parser (3.1.2.1)
ast (~> 2.4.1)
racc
prettier_print (1.2.1)
racc (1.7.2)
prettier_print (1.2.0)
rainbow (3.1.1)
regexp_parser (2.8.2)
rexml (3.2.6)
rubocop (1.57.2)
regexp_parser (2.6.0)
rexml (3.2.5)
rubocop (1.36.0)
json (~> 2.3)
language_server-protocol (>= 3.17.0)
parallel (~> 1.10)
parser (>= 3.2.2.4)
parser (>= 3.1.2.1)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0)
rubocop-ast (>= 1.28.1, < 2.0)
rubocop-ast (>= 1.20.1, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 3.0)
rubocop-ast (1.30.0)
parser (>= 3.2.1.0)
rubocop-capybara (2.19.0)
rubocop (~> 1.41)
rubocop-discourse (3.4.0)
unicode-display_width (>= 1.4.0, < 3.0)
rubocop-ast (1.21.0)
parser (>= 3.1.1.0)
rubocop-discourse (3.0)
rubocop (>= 1.1.0)
rubocop-rspec (>= 2.0.0)
rubocop-factory_bot (2.24.0)
rubocop-rspec (2.13.2)
rubocop (~> 1.33)
rubocop-rspec (2.25.0)
rubocop (~> 1.40)
rubocop-capybara (~> 2.17)
rubocop-factory_bot (~> 2.22)
ruby-progressbar (1.13.0)
syntax_tree (6.2.0)
ruby-progressbar (1.11.0)
syntax_tree (5.1.0)
prettier_print (>= 1.2.0)
unicode-display_width (2.5.0)
syntax_tree-disable_ternary (1.0.0)
unicode-display_width (2.3.0)
PLATFORMS
arm64-darwin-20
@@ -53,6 +44,7 @@ PLATFORMS
DEPENDENCIES
rubocop-discourse
syntax_tree
syntax_tree-disable_ternary
BUNDLED WITH
2.4.21
2.3.10
+2 -2
View File
@@ -1,5 +1,5 @@
# **Discourse AI** Plugin
# **Plugin Name** Plugin
**Plugin Summary**
For more information, please see: https://meta.discourse.org/t/discourse-ai/259214?u=falco
For more information, please see: **url to meta topic**
@@ -1,71 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class AiPersonasController < ::Admin::AdminController
before_action :find_ai_persona, only: %i[show update destroy]
def index
ai_personas =
AiPersona.ordered.map do |persona|
# we use a special serializer here cause names and descriptions are
# localized for system personas
LocalizedAiPersonaSerializer.new(persona, root: false)
end
commands =
DiscourseAi::AiBot::Personas::Persona.all_available_commands.map do |command|
{ id: command.to_s.split("::").last, name: command.name.humanize.titleize }
end
render json: { ai_personas: ai_personas, meta: { commands: commands } }
end
def show
render json: LocalizedAiPersonaSerializer.new(@ai_persona)
end
def create
ai_persona = AiPersona.new(ai_persona_params)
if ai_persona.save
render json: { ai_persona: ai_persona }, status: :created
else
render_json_error ai_persona
end
end
def update
if @ai_persona.update(ai_persona_params)
render json: @ai_persona
else
render_json_error @ai_persona
end
end
def destroy
if @ai_persona.destroy
head :no_content
else
render_json_error @ai_persona
end
end
private
def find_ai_persona
@ai_persona = AiPersona.find(params[:id])
end
def ai_persona_params
params.require(:ai_persona).permit(
:name,
:description,
:enabled,
:system_prompt,
:enabled,
:priority,
allowed_group_ids: [],
commands: [],
)
end
end
end
end
@@ -1,12 +0,0 @@
# frozen_string_literal: true
module DiscourseAi
module Admin
class DashboardController < ::Admin::StaffController
requires_plugin DiscourseAi::PLUGIN_NAME
def sentiment
end
end
end
end
@@ -14,15 +14,6 @@ module DiscourseAi
render json: {}, status: 200
end
def show_bot_username
bot_user_id = DiscourseAi::AiBot::EntryPoint.map_bot_model_to_user_id(params[:username])
raise Discourse::InvalidParameters.new(:username) if !bot_user_id
bot_username_lower = User.find(bot_user_id).username_lower
render json: { bot_username: bot_username_lower }, status: 200
end
end
end
end
@@ -6,123 +6,40 @@ module DiscourseAi
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login
before_action :ensure_can_request_suggestions
before_action :rate_limiter_performed!, except: %i[prompts]
def prompts
render json:
ActiveModel::ArraySerializer.new(
DiscourseAi::AiHelper::Assistant.new.available_prompts,
DiscourseAi::AiHelper::LlmPrompt.new.available_prompts,
root: false,
),
status: 200
end
def suggest
input = get_text_param!
raise Discourse::InvalidParameters.new(:text) if params[:text].blank?
prompt = CompletionPrompt.find_by(id: params[:mode])
raise Discourse::InvalidParameters.new(:mode) if !prompt || !prompt.enabled?
if prompt.prompt_type == "custom_prompt" && params[:custom_prompt].blank?
raise Discourse::InvalidParameters.new(:custom_prompt)
end
RateLimiter.new(current_user, "ai_assistant", 6, 3.minutes).performed!
hijack do
render json:
DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
DiscourseAi::AiHelper::LlmPrompt.new.generate_and_send_prompt(
prompt,
input,
current_user,
params[:text],
),
status: 200
end
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
status: 502
end
def suggest_title
input = get_text_param!
prompt = CompletionPrompt.enabled_by_name("generate_titles")
raise Discourse::InvalidParameters.new(:mode) if !prompt
hijack do
render json:
DiscourseAi::AiHelper::Assistant.new.generate_and_send_prompt(
prompt,
input,
current_user,
),
status: 200
end
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
status: 502
end
def suggest_category
input = get_text_param!
input_hash = { text: input }
render json:
DiscourseAi::AiHelper::SemanticCategorizer.new(
input_hash,
current_user,
).categories,
status: 200
end
def suggest_tags
input = get_text_param!
input_hash = { text: input }
render json: DiscourseAi::AiHelper::SemanticCategorizer.new(input_hash, current_user).tags,
status: 200
end
def suggest_thumbnails
input = get_text_param!
hijack do
thumbnails = DiscourseAi::AiHelper::Painter.new.commission_thumbnails(input, current_user)
render json: { thumbnails: thumbnails }, status: 200
end
end
def explain
post_id = get_post_param!
term_to_explain = get_text_param!
post = Post.includes(:topic).find_by(id: post_id)
raise Discourse::InvalidParameters.new(:post_id) unless post
render json:
DiscourseAi::AiHelper::TopicHelper.new(current_user).explain(
term_to_explain,
post,
),
status: 200
rescue DiscourseAi::Completions::Endpoints::Base::CompletionFailed => e
rescue ::DiscourseAi::Inference::OpenAiCompletions::CompletionFailed,
::DiscourseAi::Inference::AnthropicCompletions::CompletionFailed => e
render_json_error I18n.t("discourse_ai.ai_helper.errors.completion_request_failed"),
status: 502
end
private
def get_text_param!
params[:text].tap { |t| raise Discourse::InvalidParameters.new(:text) if t.blank? }
end
def get_post_param!
params[:post_id].tap { |t| raise Discourse::InvalidParameters.new(:post_id) if t.blank? }
end
def rate_limiter_performed!
RateLimiter.new(current_user, "ai_assistant", 6, 3.minutes).performed!
end
def ensure_can_request_suggestions
user_group_ids = current_user.group_ids
@@ -8,11 +8,8 @@ module DiscourseAi
SEMANTIC_SEARCH_TYPE = "semantic_search"
def search
query = params[:q].to_s
if query.length < SiteSetting.min_search_term_length
raise Discourse::InvalidParameters.new(:q)
end
query = params[:q]
page = (params[:page] || 1).to_i
grouped_results =
Search::GroupedSearchResults.new(
@@ -22,19 +19,17 @@ module DiscourseAi
use_pg_headlines_for_excerpt: false,
)
semantic_search = DiscourseAi::Embeddings::SemanticSearch.new(guardian)
model =
DiscourseAi::Embeddings::Model.instantiate(
SiteSetting.ai_embeddings_semantic_search_model,
)
if !semantic_search.cached_query?(query)
RateLimiter.new(current_user, "semantic-search", 4, 1.minutes).performed!
end
DiscourseAi::Embeddings::SemanticSearch
.new(guardian, model)
.search_for_topics(query, page)
.each { |topic_post| grouped_results.add(topic_post) }
hijack do
semantic_search
.search_for_topics(query)
.each { |topic_post| grouped_results.add(topic_post) }
render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
end
render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
end
end
end
@@ -0,0 +1,48 @@
# frozen_string_literal: true
module DiscourseAi
module Summarization
class SummaryController < ::ApplicationController
requires_plugin ::DiscourseAi::PLUGIN_NAME
requires_login
VALID_SINCE_VALUES = [1, 3, 6, 12, 24, 72, 168]
VALID_TARGETS = %w[chat_channel topic]
def show
raise PluginDisabled unless SiteSetting.ai_summarization_enabled
target_type = params[:target_type]
raise Discourse::InvalidParameters.new(:target_type) if !VALID_TARGETS.include?(target_type)
since = nil
if target_type == "chat_channel"
since = params[:since].to_i
raise Discourse::InvalidParameters.new(:since) if !VALID_SINCE_VALUES.include?(since)
target = Chat::Channel.find_by(id: params[:target_id])
raise Discourse::NotFound.new(:chat_channel) if !target
raise Discourse::InvalidAccess if !guardian.can_join_chat_channel?(target)
else
target = Topic.find_by(id: params[:target_id])
raise Discourse::NotFound.new(:topic) if !target
raise Discourse::InvalidAccess if !guardian.can_see_topic?(target)
end
RateLimiter.new(
current_user,
"ai_summarization",
6,
SiteSetting.ai_summarization_rate_limit_minutes.minutes,
).performed!
hijack do
summary =
DiscourseAi::Summarization::SummaryGenerator.new(target, current_user).summarize!(since)
render json: { summary: summary }, status: 200
end
end
end
end
end
-3
View File
@@ -4,7 +4,6 @@ class AiApiAuditLog < ActiveRecord::Base
module Provider
OpenAI = 1
Anthropic = 2
HuggingFaceTextGeneration = 3
end
end
@@ -15,8 +14,6 @@ end
# id :bigint not null, primary key
# provider_id :integer not null
# user_id :integer
# topic_id :integer
# post_id :integer
# request_tokens :integer
# response_tokens :integer
# raw_request_payload :string
-179
View File
@@ -1,179 +0,0 @@
# frozen_string_literal: true
class AiPersona < ActiveRecord::Base
# places a hard limit, so per site we cache a maximum of 500 classes
MAX_PERSONAS_PER_SITE = 500
validates :name, presence: true, uniqueness: true, length: { maximum: 100 }
validates :description, presence: true, length: { maximum: 2000 }
validates :system_prompt, presence: true, length: { maximum: 10_000_000 }
validate :system_persona_unchangeable, on: :update, if: :system
before_destroy :ensure_not_system
class MultisiteHash
def initialize(id)
@hash = Hash.new { |h, k| h[k] = {} }
@id = id
MessageBus.subscribe(channel_name) { |message| @hash[message.data] = {} }
end
def channel_name
"/multisite-hash-#{@id}"
end
def current_db
RailsMultisite::ConnectionManagement.current_db
end
def [](key)
@hash.dig(current_db, key)
end
def []=(key, val)
@hash[current_db][key] = val
end
def flush!
@hash[current_db] = {}
MessageBus.publish(channel_name, current_db)
end
end
def self.persona_cache
@persona_cache ||= MultisiteHash.new("persona_cache")
end
scope :ordered, -> { order("priority DESC, lower(name) ASC") }
def self.all_personas
persona_cache[:value] ||= AiPersona
.ordered
.where(enabled: true)
.all
.limit(MAX_PERSONAS_PER_SITE)
.map(&:class_instance)
end
after_commit :bump_cache
def bump_cache
self.class.persona_cache.flush!
end
def class_instance
allowed_group_ids = self.allowed_group_ids
id = self.id
system = self.system
persona_class = DiscourseAi::AiBot::Personas.system_personas_by_id[self.id]
if persona_class
persona_class.define_singleton_method :allowed_group_ids do
allowed_group_ids
end
persona_class.define_singleton_method :id do
id
end
persona_class.define_singleton_method :system do
system
end
return persona_class
end
name = self.name
description = self.description
ai_persona_id = self.id
commands =
self.commands.filter_map do |inner_name|
begin
("DiscourseAi::AiBot::Commands::#{inner_name}").constantize
rescue StandardError
nil
end
end
Class.new(DiscourseAi::AiBot::Personas::Persona) do
define_singleton_method :id do
id
end
define_singleton_method :name do
name
end
define_singleton_method :description do
description
end
define_singleton_method :system do
system
end
define_singleton_method :allowed_group_ids do
allowed_group_ids
end
define_singleton_method :to_s do
"#<DiscourseAi::AiBot::Personas::Persona::Custom @name=#{self.name} @allowed_group_ids=#{self.allowed_group_ids.join(",")}>"
end
define_singleton_method :inspect do
"#<DiscourseAi::AiBot::Personas::Persona::Custom @name=#{self.name} @allowed_group_ids=#{self.allowed_group_ids.join(",")}>"
end
define_method :initialize do |*args, **kwargs|
@ai_persona = AiPersona.find_by(id: ai_persona_id)
super(*args, **kwargs)
end
define_method :commands do
commands
end
define_method :system_prompt do
@ai_persona&.system_prompt || "You are a helpful bot."
end
end
end
private
def system_persona_unchangeable
if system_prompt_changed? || commands_changed? || name_changed? || description_changed?
errors.add(:base, I18n.t("discourse_ai.ai_bot.personas.cannot_edit_system_persona"))
end
end
def ensure_not_system
if system
errors.add(:base, I18n.t("discourse_ai.ai_bot.personas.cannot_delete_system_persona"))
throw :abort
end
end
end
# == Schema Information
#
# Table name: ai_personas
#
# id :bigint not null, primary key
# name :string(100) not null
# description :string(2000) not null
# commands :string default([]), not null, is an Array
# system_prompt :string(10000000) not null
# allowed_group_ids :integer default([]), not null, is an Array
# created_by_id :integer
# enabled :boolean default(TRUE), not null
# created_at :datetime not null
# updated_at :datetime not null
# system :boolean default(FALSE), not null
# priority :integer default(0), not null
#
# Indexes
#
# index_ai_personas_on_name (name) UNIQUE
#
+9 -21
View File
@@ -1,37 +1,24 @@
# frozen_string_literal: true
class CompletionPrompt < ActiveRecord::Base
# TODO(roman): Remove may 2024.
self.ignored_columns = ["provider"]
TRANSLATE = -301
GENERATE_TITLES = -302
PROOFREAD = -303
MARKDOWN_TABLE = -304
CUSTOM_PROMPT = -305
EXPLAIN = -306
# TODO(roman): Remove sept 2023.
self.ignored_columns = ["value"]
enum :prompt_type, { text: 0, list: 1, diff: 2 }
validates :messages, length: { maximum: 20 }
validate :each_message_length
def self.enabled_by_name(name)
where(enabled: true).find_by(name: name)
end
def messages_with_input(input)
return unless input
messages_hash.merge(input: "<input>#{input}</input")
def messages_with_user_input(user_input)
if ::DiscourseAi::AiHelper::LlmPrompt.new.enabled_provider == "openai"
self.messages << { role: "user", content: user_input }
else
self.messages << { "role" => "Input", "content" => "<input>#{user_input}</input>" }
end
end
private
def messages_hash
@messages_hash ||= messages.symbolize_keys!
end
def each_message_length
messages.each_with_index do |msg, idx|
next if msg["content"].length <= 1000
@@ -53,6 +40,7 @@ end
# created_at :datetime not null
# updated_at :datetime not null
# messages :jsonb
# provider :text
#
# Indexes
#
-9
View File
@@ -1,9 +0,0 @@
# frozen_string_literal: true
class PostCustomPrompt < ActiveRecord::Base
belongs_to :post
end
class ::Post
has_one :post_custom_prompt, dependent: :destroy
end
@@ -1,11 +1,5 @@
# frozen_string_literal: true
class AiChatChannelSerializer < ApplicationSerializer
attributes :id, :chatable, :chatable_type, :chatable_url, :slug
def title
# Display all participants for a DM.
# For category channels, the argument is ignored.
object.title(nil)
end
attributes :id, :chatable, :chatable_type, :chatable_url, :title, :slug
end
@@ -1,23 +0,0 @@
# frozen_string_literal: true
class LocalizedAiPersonaSerializer < ApplicationSerializer
root "ai_persona"
attributes :id,
:name,
:description,
:enabled,
:system,
:priority,
:commands,
:system_prompt,
:allowed_group_ids
def name
object.class_instance.name
end
def description
object.class_instance.description
end
end
@@ -1,14 +0,0 @@
export default {
resource: "admin.adminPlugins",
path: "/plugins",
map() {
this.route("discourse-ai", { path: "discourse-ai" }, function () {
this.route("ai-personas", { path: "ai_personas" }, function () {
this.route("new", { path: "/new" });
this.route("show", { path: "/:id" });
});
});
},
};
@@ -1,10 +0,0 @@
export default {
resource: "admin.dashboard",
path: "/dashboard",
map() {
this.route("admin.dashboardSentiment", {
path: "/dashboard/sentiment",
resetNamespace: true,
});
},
};
@@ -1,17 +0,0 @@
import RestAdapter from "discourse/adapters/rest";
export default class Adapter extends RestAdapter {
jsonMode = true;
basePath() {
return "/admin/plugins/discourse-ai/";
}
pathFor() {
return super.pathFor(...arguments) + ".json";
}
apiNameFor() {
return "ai-persona";
}
}
@@ -1,28 +0,0 @@
import RestModel from "discourse/models/rest";
const ATTRIBUTES = [
"name",
"description",
"commands",
"system_prompt",
"allowed_group_ids",
"enabled",
"system",
"priority",
];
export default class AiPersona extends RestModel {
updateProperties() {
let attrs = this.getProperties(ATTRIBUTES);
attrs.id = this.id;
return attrs;
}
createProperties() {
return this.getProperties(ATTRIBUTES);
}
workingCopy() {
return AiPersona.create(this.createProperties());
}
}
@@ -1,18 +0,0 @@
<div class="bot-panel ai-bot-available-bot-options">
<div class="menu-panel drop-down">
<div class="panel-body">
<div class="panel-body-contents">
<div class="sidebar-hamburger-dropdown">
{{#each this.botNames as |bot|}}
<DButton
@class="btn-flat ai-bot-available-bot-content"
@translatedTitle={{bot.humanized}}
@translatedLabel={{bot.humanized}}
@action={{action "composeMessageWithTargetBot" bot.modelName}}
/>
{{/each}}
</div>
</div>
</div>
</div>
</div>
@@ -1,33 +0,0 @@
import Component from "@glimmer/component";
import { action } from "@ember/object";
import { inject as service } from "@ember/service";
import I18n from "I18n";
import { composeAiBotMessage } from "discourse/plugins/discourse-ai/discourse/lib/ai-bot-helper";
export default class AiBotHeaderPanel extends Component {
@service siteSettings;
@service composer;
@action
composeMessageWithTargetBot(target) {
this.#composeAiBotMessage(target);
}
get botNames() {
return this.enabledBotOptions.map((bot) => {
return {
humanized: I18n.t(`discourse_ai.ai_bot.bot_names.${bot}`),
modelName: bot,
};
});
}
get enabledBotOptions() {
return this.siteSettings.ai_bot_enabled_chat_bots.split("|");
}
#composeAiBotMessage(targetBot) {
this.args.closePanel();
composeAiBotMessage(targetBot, this.composer);
}
}

Some files were not shown because too many files have changed in this diff Show More