Discourse/discourse-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove all files (#1516)

Browse Source
This commit is contained in:
Sam
2026-08-28 08:24:24 +10:00
committed by GitHub
parent c312b3184a
commit 2c4e5e998e
1110 changed files with 2 additions and 133260 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/fixtures/ai_bot/602_bot_users.rb
-3
View File
@@ -1,3 +0,0 @@
# frozen_string_literal: true
DiscourseAi::AiBot::SiteSettingsExtension.enable_or_disable_ai_bots
db/fixtures/personas/603_ai_personas.rb
-91
View File
@@ -1,91 +0,0 @@
# frozen_string_literal: true
summarization_personas = [DiscourseAi::Personas::Summarizer, DiscourseAi::Personas::ShortSummarizer]
def from_setting(setting_name)
DB
.query_single(
"SELECT value FROM site_settings WHERE name = :setting_name",
setting_name: setting_name,
)
&.first
&.split("|")
end
DiscourseAi::Personas::Persona.system_personas.each do |persona_class, id|
persona = AiPersona.find_by(id: id)
if !persona
persona = AiPersona.new
persona.id = id
if persona_class == DiscourseAi::Personas::WebArtifactCreator
# this is somewhat sensitive, so we default it to staff
persona.allowed_group_ids = [Group::AUTO_GROUPS[:staff]]
elsif summarization_personas.include?(persona_class)
# Copy group permissions from site settings.
default_groups = [Group::AUTO_GROUPS[:staff], Group::AUTO_GROUPS[:trust_level_3]]
setting_name = "ai_custom_summarization_allowed_groups"
if persona_class == DiscourseAi::Personas::ShortSummarizer
setting_name = "ai_summary_gists_allowed_groups"
default_groups = [Group::AUTO_GROUPS[:everyone]]
end
persona.allowed_group_ids = from_setting(setting_name) || default_groups
elsif persona_class == DiscourseAi::Personas::CustomPrompt
setting_name = "ai_helper_custom_prompts_allowed_groups"
default_groups = [Group::AUTO_GROUPS[:staff]]
persona.allowed_group_ids = from_setting(setting_name) || default_groups
elsif persona_class == DiscourseAi::Personas::ContentCreator
persona.allowed_group_ids = [Group::AUTO_GROUPS[:everyone]]
else
persona.allowed_group_ids = [Group::AUTO_GROUPS[:trust_level_0]]
end
persona.enabled = persona_class.default_enabled
persona.priority = true if persona_class == DiscourseAi::Personas::General
end
names = [
persona_class.name,
persona_class.name + " 1",
persona_class.name + " 2",
persona_class.name + SecureRandom.hex,
]
persona.name = DB.query_single(<<~SQL, names, id).first
SELECT guess_name
FROM (
SELECT unnest(Array[?]) AS guess_name
FROM (SELECT 1) as t
) x
LEFT JOIN ai_personas ON ai_personas.name = x.guess_name AND ai_personas.id <> ?
WHERE ai_personas.id IS NULL
ORDER BY x.guess_name ASC
LIMIT 1
SQL
persona.description = persona_class.description
persona.system = true
instance = persona_class.new
tools = {}
instance.tools.map { |tool| tool.to_s.split("::").last }.each { |name| tools[name] = nil }
existing_tools = persona.tools || []
existing_tools.each do |tool|
if tool.is_a?(Array)
name, value = tool
tools[name] = value if tools.key?(name)
end
end
persona.tools = tools.map { |name, value| [name, value] }
persona.response_format = instance.response_format
persona.examples = instance.examples
persona.system_prompt = instance.system_prompt
persona.top_p = instance.top_p
persona.temperature = instance.temperature
persona.save!(validate: false)
end
db/migrate/20230224165056_create_classification_results_table.rb
-19
View File
@@ -1,19 +0,0 @@
# frozen_string_literal: true
class CreateClassificationResultsTable < ActiveRecord::Migration[7.0]
def change
create_table :classification_results do |t|
t.string :model_used, null: true
t.string :classification_type, null: true
t.integer :target_id, null: true
t.string :target_type, null: true
t.jsonb :classification, null: true
t.timestamps
end
add_index :classification_results,
%i[target_id target_type model_used],
unique: true,
name: "unique_classification_target_per_type"
end
end
db/migrate/20230307125342_created_model_accuracy_table.rb
-16
View File
@@ -1,16 +0,0 @@
# frozen_string_literal: true
class CreatedModelAccuracyTable < ActiveRecord::Migration[7.0]
def change
create_table :model_accuracies do |t|
t.string :model, null: false
t.string :classification_type, null: false
t.integer :flags_agreed, null: false, default: 0
t.integer :flags_disagreed, null: false, default: 0
t.timestamps
end
add_index :model_accuracies, %i[model], unique: true
end
end
db/migrate/20230314184514_migrate_discourse_ai_reviewables.rb
-13
View File
@@ -1,13 +0,0 @@
# frozen_string_literal: true
class MigrateDiscourseAiReviewables < ActiveRecord::Migration[7.0]
def up
DB.exec("UPDATE reviewables SET type='ReviewableAiPost' WHERE type='ReviewableAIPost'")
DB.exec(
"UPDATE reviewables SET type='ReviewableAiChatMessage' WHERE type='ReviewableAIChatMessage'",
)
end
def down
raise ActiveRecord::IrreversibleMigration
end
end
db/migrate/20230316160714_create_completion_prompt_table.rb
-15
View File
@@ -1,15 +0,0 @@
# frozen_string_literal: true
class CreateCompletionPromptTable < ActiveRecord::Migration[7.0]
def change
create_table :completion_prompts do |t|
t.string :name, null: false
t.string :translated_name
t.integer :prompt_type, null: false, default: 0
t.text :value, null: false
t.boolean :enabled, null: false, default: true
t.timestamps
end
add_index :completion_prompts, %i[name], unique: true
end
end
db/migrate/20230320122645_delete_duplicated_seeded_prompts.rb
-18
View File
@@ -1,18 +0,0 @@
# frozen_string_literal: true
class DeleteDuplicatedSeededPrompts < ActiveRecord::Migration[7.0]
def up
DB.exec <<~SQL
DELETE FROM completion_prompts
WHERE (
(id = 1 AND name = 'translate') OR
(id = 2 AND name = 'generate_titles') OR
(id = 3 AND name = 'proofread')
)
SQL
end
def down
raise ActiveRecord::IrreversibleMigration
end
end
db/migrate/20230320185619_multi_message_completion_prompts.rb
-7
View File
@@ -1,7 +0,0 @@
# frozen_string_literal: true
class MultiMessageCompletionPrompts < ActiveRecord::Migration[7.0]
def change
add_column :completion_prompts, :messages, :jsonb
end
end
db/migrate/20230320191928_drop_completion_prompt_value.rb
-7
View File
@@ -1,7 +0,0 @@
# frozen_string_literal: true
class DropCompletionPromptValue < ActiveRecord::Migration[7.0]
def change
remove_column :completion_prompts, :value, :text
end
end
db/migrate/20230322142028_make_dropped_value_column_nullable.rb
-14
View File
@@ -1,14 +0,0 @@
# frozen_string_literal: true
class MakeDroppedValueColumnNullable < ActiveRecord::Migration[7.0]
def up
if column_exists?(:completion_prompts, :value)
Migration::SafeMigrate.disable!
change_column_null :completion_prompts, :value, true
Migration::SafeMigrate.enable!
end
end
def down
raise ActiveRecord::IrreversibleMigration
end
end
db/migrate/20230406135943_add_provider_to_completion_prompts.rb
-22
View File
@@ -1,22 +0,0 @@
# frozen_string_literal: true
class AddProviderToCompletionPrompts < ActiveRecord::Migration[7.0]
def up
remove_index :completion_prompts, name: "index_completion_prompts_on_name"
add_column :completion_prompts, :provider, :text
add_index :completion_prompts, %i[name], unique: false
# set provider for existing prompts
DB.exec <<~SQL
UPDATE completion_prompts
SET provider = 'openai'
WHERE provider IS NULL;
SQL
end
def down
remove_column :completion_prompts, :provider
remove_index :completion_prompts, name: "index_completion_prompts_on_name"
add_index :completion_prompts, %i[name], unique: true
end
end
db/migrate/20230424055354_create_ai_api_audit_logs.rb
-15
View File
@@ -1,15 +0,0 @@
# frozen_string_literal: true
class CreateAiApiAuditLogs < ActiveRecord::Migration[7.0]
def change
create_table :ai_api_audit_logs do |t|
t.integer :provider_id, null: false
t.integer :user_id
t.integer :request_tokens
t.integer :response_tokens
t.string :raw_request_payload
t.string :raw_response_payload
t.timestamps
end
end
end
db/migrate/20230519003106_post_custom_prompts.rb
-13
View File
@@ -1,13 +0,0 @@
# frozen_string_literal: true
class PostCustomPrompts < ActiveRecord::Migration[7.0]
def change
create_table :post_custom_prompts do |t|
t.integer :post_id, null: false
t.json :custom_prompt, null: false
t.timestamps
end
add_index :post_custom_prompts, :post_id, unique: true
end
end
db/migrate/20230710171141_enable_pg_vector_extension.rb
-18
View File
@@ -1,18 +0,0 @@
# frozen_string_literal: true
class EnablePgVectorExtension < ActiveRecord::Migration[7.0]
def change
begin
enable_extension :vector
rescue Exception => e
if DB.query_single("SELECT 1 FROM pg_available_extensions WHERE name = 'vector';").empty?
STDERR.puts "------------------------------DISCOURSE AI ERROR----------------------------------"
STDERR.puts " Discourse AI requires the pgvector extension on the PostgreSQL database."
STDERR.puts " Run a `./launcher rebuild app` to fix it on a standard install."
STDERR.puts " Alternatively, you can remove Discourse AI to rebuild."
STDERR.puts "------------------------------DISCOURSE AI ERROR----------------------------------"
end
raise e
end
end
end
db/migrate/20230710171142_create_ai_topic_embeddings_table.rb
-27
View File
@@ -1,27 +0,0 @@
# frozen_string_literal: true
class CreateAiTopicEmbeddingsTable < ActiveRecord::Migration[7.0]
def change
create_table :ai_topic_embeddings_1_1, id: false do |t|
t.integer :topic_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(768)", null: false
t.timestamps
t.index :topic_id, unique: true
end
create_table :ai_topic_embeddings_2_1, id: false do |t|
t.integer :topic_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(1536)", null: false
t.timestamps
t.index :topic_id, unique: true
end
end
end
db/migrate/20230710171143_migrate_embeddings_from_dedicated_database.rb
-63
View File
@@ -1,63 +0,0 @@
# frozen_string_literal: true
class MigrateEmbeddingsFromDedicatedDatabase < ActiveRecord::Migration[7.0]
def up
return unless SiteSetting.ai_embeddings_enabled
return if SiteSetting.ai_embeddings_pg_connection_string.blank?
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
vector_reps =
[
DiscourseAi::Embeddings::VectorRepresentations::AllMpnetBaseV2,
DiscourseAi::Embeddings::VectorRepresentations::TextEmbeddingAda002,
].map { |k| k.new(truncation) }
vector_reps.each do |vector_rep|
new_table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
old_table_name = "topic_embeddings_#{vector_rep.name.underscore}"
begin
row_count =
DiscourseAi::Database::Connection
.db
.query_single("SELECT COUNT(*) FROM #{old_table_name}")
.first
if row_count > 0
puts "Migrating #{row_count} embeddings from #{old_table_name} to #{new_table_name}"
last_topic_id = 0
loop do
batch = DiscourseAi::Database::Connection.db.query(<<-SQL)
SELECT topic_id, embedding
FROM #{old_table_name}
WHERE topic_id > #{last_topic_id}
ORDER BY topic_id ASC
LIMIT 50
SQL
break if batch.empty?
DB.exec(<<-SQL)
INSERT INTO #{new_table_name} (topic_id, model_version, strategy_version, digest, embeddings, created_at, updated_at)
VALUES #{batch.map { |r| "(#{r.topic_id}, 0, 0, '', '#{r.embedding}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" }.join(", ")}
ON CONFLICT (topic_id)
DO NOTHING
SQL
last_topic_id = batch.last.topic_id
end
end
rescue PG::Error => e
Rails.logger.error(
"Error #{e} migrating embeddings from #{old_table_name} to #{new_table_name}",
)
end
end
end
def down
# no-op
end
end
db/migrate/20230727170222_create_multilingual_topic_embeddings_table.rb
-16
View File
@@ -1,16 +0,0 @@
# frozen_string_literal: true
class CreateMultilingualTopicEmbeddingsTable < ActiveRecord::Migration[7.0]
def change
create_table :ai_topic_embeddings_3_1, id: false do |t|
t.integer :topic_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(1024)", null: false
t.timestamps
t.index :topic_id, unique: true
end
end
end
db/migrate/20230831033812_rename_ai_helper_add_ai_pm_to_header_setting.rb
-11
View File
@@ -1,11 +0,0 @@
# frozen_string_literal: true
class RenameAiHelperAddAiPmToHeaderSetting < ActiveRecord::Migration[7.0]
def up
execute "UPDATE site_settings SET name = 'ai_bot_add_to_header' WHERE name = 'ai_helper_add_ai_pm_to_header'"
end
def down
execute "UPDATE site_settings SET name = 'ai_helper_add_ai_pm_to_header' WHERE name = 'ai_bot_add_to_header'"
end
end
db/migrate/20231003155701_create_bge_topic_embeddings_table.rb
-16
View File
@@ -1,16 +0,0 @@
# frozen_string_literal: true
class CreateBgeTopicEmbeddingsTable < ActiveRecord::Migration[7.0]
def change
create_table :ai_topic_embeddings_4_1, id: false do |t|
t.integer :topic_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(1024)", null: false
t.timestamps
t.index :topic_id, unique: true
end
end
end
db/migrate/20231031050538_add_topic_id_post_id_to_ai_audit_log.rb
-8
View File
@@ -1,8 +0,0 @@
# frozen_string_literal: true
class AddTopicIdPostIdToAiAuditLog < ActiveRecord::Migration[7.0]
def change
add_column :ai_api_audit_logs, :topic_id, :integer
add_column :ai_api_audit_logs, :post_id, :integer
end
end
db/migrate/20231109011155_create_ai_personas.rb
-18
View File
@@ -1,18 +0,0 @@
# frozen_string_literal: true
#
class CreateAiPersonas < ActiveRecord::Migration[7.0]
def change
create_table :ai_personas do |t|
t.string :name, null: false, unique: true, limit: 100
t.string :description, null: false, limit: 2000
t.string :commands, array: true, default: [], null: false
t.string :system_prompt, null: false, limit: 10_000_000
t.integer :allowed_group_ids, array: true, default: [], null: false
t.integer :created_by_id
t.boolean :enabled, default: true, null: false
t.timestamps
end
add_index :ai_personas, :name, unique: true
end
end
db/migrate/20231117050928_add_system_and_priority_to_ai_personas.rb
-8
View File
@@ -1,8 +0,0 @@
# frozen_string_literal: true
class AddSystemAndPriorityToAiPersonas < ActiveRecord::Migration[7.0]
def change
add_column :ai_personas, :system, :boolean, null: false, default: false
add_column :ai_personas, :priority, :boolean, null: false, default: false
end
end
db/migrate/20231120033747_remove_site_settings.rb
-12
View File
@@ -1,12 +0,0 @@
# frozen_string_literal: true
class RemoveSiteSettings < ActiveRecord::Migration[7.0]
def up
DB.exec(<<~SQL, %w[ai_bot_enabled_chat_commands ai_bot_enabled_personas])
DELETE FROM site_settings WHERE name IN (?)
SQL
end
def down
raise ActiveRecord::IrreversibleMigration
end
end
db/migrate/20231123224203_switch_to_generic_completion_prompts.rb
-9
View File
@@ -1,9 +0,0 @@
# frozen_string_literal: true
class SwitchToGenericCompletionPrompts < ActiveRecord::Migration[7.0]
def change
remove_column :completion_prompts, :provider, :text
DB.exec("DELETE FROM completion_prompts WHERE (id < 0 AND id > -300)")
end
end
db/migrate/20231128151234_recreate_generate_titles_prompt.rb
-11
View File
@@ -1,11 +0,0 @@
# frozen_string_literal: true
class RecreateGenerateTitlesPrompt < ActiveRecord::Migration[7.0]
def up
DB.exec("DELETE FROM completion_prompts WHERE id = -302")
end
def down
raise ActiveRecord::IrreversibleMigration
end
end
db/migrate/20231202013850_convert_ai_personas_commands_to_json.rb
-31
View File
@@ -1,31 +0,0 @@
# frozen_string_literal: true
class ConvertAiPersonasCommandsToJson < ActiveRecord::Migration[7.0]
def up
# this all may be a bit surprising, but interestingly this makes all our backend code
# cross compatible
# upgrading ["a", "b", "c"] to json simply works cause in both cases
# rails will cast to a string array and all code simply expects a string array
#
# this change was made so we can also start storing parameters with the commands
execute <<~SQL
ALTER TABLE ai_personas
ALTER COLUMN commands DROP DEFAULT
SQL
execute <<~SQL
ALTER TABLE ai_personas
ALTER COLUMN commands
TYPE json USING array_to_json(commands)
SQL
execute <<~SQL
ALTER TABLE ai_personas
ALTER COLUMN commands
SET DEFAULT '[]'::json
SQL
end
def down
raise ActiveRecord::IrreversibleMigration
end
end
db/migrate/20231227223301_create_gemini_topic_embeddings_table.rb
-16
View File
@@ -1,16 +0,0 @@
# frozen_string_literal: true
class CreateGeminiTopicEmbeddingsTable < ActiveRecord::Migration[7.0]
def change
create_table :ai_topic_embeddings_5_1, id: false do |t|
t.integer :topic_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(768)", null: false
t.timestamps
t.index :topic_id, unique: true
end
end
end
db/migrate/20231228213036_create_ai_post_embeddings_tables.rb
-60
View File
@@ -1,60 +0,0 @@
# frozen_string_literal: true
class CreateAiPostEmbeddingsTables < ActiveRecord::Migration[7.0]
def change
create_table :ai_post_embeddings_1_1, id: false do |t|
t.integer :post_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(768)", null: false
t.timestamps
t.index :post_id, unique: true
end
create_table :ai_post_embeddings_2_1, id: false do |t|
t.integer :post_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(1536)", null: false
t.timestamps
t.index :post_id, unique: true
end
create_table :ai_post_embeddings_3_1, id: false do |t|
t.integer :post_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(1024)", null: false
t.timestamps
t.index :post_id, unique: true
end
create_table :ai_post_embeddings_4_1, id: false do |t|
t.integer :post_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(1024)", null: false
t.timestamps
t.index :post_id, unique: true
end
create_table :ai_post_embeddings_5_1, id: false do |t|
t.integer :post_id, null: false
t.integer :model_version, null: false
t.integer :strategy_version, null: false
t.text :digest, null: false
t.column :embeddings, "vector(768)", null: false
t.timestamps
t.index :post_id, unique: true
end
end
end
db/migrate/20240104013944_add_params_to_completion_prompt.rb
-8
View File
@@ -1,8 +0,0 @@
# frozen_string_literal: true
class AddParamsToCompletionPrompt < ActiveRecord::Migration[7.0]
def change
add_column :completion_prompts, :temperature, :integer
add_column :completion_prompts, :stop_sequences, :string, array: true
end
end
db/migrate/20240119152348_explicit_provider_backwards_compat.rb
-91
View File
@@ -1,91 +0,0 @@
# frozen_string_literal: true
class ExplicitProviderBackwardsCompat < ActiveRecord::Migration[7.0]
def up
backfill_settings("composer_ai_helper_enabled", "ai_helper_model")
backfill_settings(
"ai_embeddings_semantic_search_enabled",
"ai_embeddings_semantic_search_hyde_model",
)
end
def down
raise ActiveRecord::IrreversibleMigration
end
def backfill_settings(feature_setting_name, llm_setting_name)
feature_enabled =
DB.query_single(
"SELECT value FROM site_settings WHERE name = :setting_name",
setting_name: feature_setting_name,
).first == "t"
setting_value =
DB
.query_single(
"SELECT value FROM site_settings WHERE name = :llm_setting",
llm_setting: llm_setting_name,
)
.first
.to_s
providers = %w[aws_bedrock anthropic open_ai hugging_face vllm google]
# Sanity check to make sure we won't add provider twice.
return if providers.include?(setting_value.split(":").first)
if !setting_value && feature_enabled
# Enabled and using old default (gpt-3.5-turbo)
DB.exec(
"UPDATE site_settings SET value='open_ai:gpt-3.5-turbo' WHERE name=:llm_setting",
llm_setting: llm_setting_name,
)
elsif setting_value && !feature_enabled
# They'll have to choose an LLM model again before enabling the feature
DB.exec("DELETE FROM site_settings WHERE name=:llm_setting", llm_setting: llm_setting_name)
elsif setting_value && feature_enabled
DB.exec(
"UPDATE site_settings SET value=:new_value WHERE name=:llm_setting",
llm_setting: llm_setting_name,
new_value: append_provider(setting_value),
)
end
end
def append_provider(value)
open_ai_models = %w[gpt-3.5-turbo gpt-4 gpt-3.5-turbo-16k gpt-4-32k gpt-4-turbo gpt-4o]
return "open_ai:#{value}" if open_ai_models.include?(value)
return "google:#{value}" if value == "gemini-pro"
hf_models = %w[StableBeluga2 Upstage-Llama-2-*-instruct-v2 Llama2-*-chat-hf Llama2-chat-hf]
return "hugging_face:#{value}" if hf_models.include?(value)
# Models available through multiple providers
claude_models = %w[claude-instant-1 claude-2]
if claude_models.include?(value)
has_bedrock_creds =
DB.query_single(
"SELECT value FROM site_settings WHERE name = 'ai_bedrock_secret_access_key' OR name = 'ai_bedrock_access_key_id' ",
).length > 0
if has_bedrock_creds
return "aws_bedrock:#{value}"
else
return "anthropic:#{value}"
end
end
mixtral_models = %w[mistralai/Mixtral-8x7B-Instruct-v0.1 mistralai/Mistral-7B-Instruct-v0.2]
if mixtral_models.include?(value)
vllm_configured =
DB.query_single(
"SELECT value FROM site_settings WHERE name = 'ai_vllm_endpoint_srv' OR name = 'ai_vllm_endpoint' ",
).length > 0
if vllm_configured
"vllm:#{value}"
else
"hugging_face:#{value}"
end
end
end
end

Some files were not shown because too many files have changed in this diff Show More