FIX: guide GPT 3.5 better (#77)

* FIX: guide GPT 3.5 better

This limits search results to 10 cause we were blowing the whole token
budget on search results, additionally it includes a quick exchange at
the start of a session to try and guide GPT 3.5 to follow instructions

Sadly GPT 3.5 drifts off very quickly but this does improve stuff a bit.

It also attempts to correct some issues with anthropic, though it still is
surprisingly hard to ground

* add status:public, this is a bit of a hack but ensures that we can search
for any filter provided

* fix specs

lib/modules/ai_bot/anthropic_bot.rb

@@ -15,6 +15,26 @@ module DiscourseAi
         7500 # https://console.anthropic.com/docs/prompt-design#what-is-a-prompt
       end
 
+      def get_delta(partial, context)
+        context[:pos] ||= 0
+
+        full = partial[:completion]
+        delta = full[context[:pos]..-1]
+
+        context[:pos] = full.length
+
+        if !context[:processed]
+          delta = ""
+          index = full.index("Assistant: ")
+          if index
+            delta = full[index + 11..-1]
+            context[:processed] = true
+          end
+        end
+
+        delta
+      end
+
       private
 
       def build_message(poster_username, content, system: false)
@@ -27,10 +47,6 @@ module DiscourseAi
         "claude-v1"
       end
 
-      def update_with_delta(_, partial)
-        partial[:completion]
-      end
-
       def get_updated_title(prompt)
         DiscourseAi::Inference::AnthropicCompletions.perform!(
           prompt,

lib/modules/ai_bot/bot.rb

@@ -3,6 +3,8 @@
 module DiscourseAi
   module AiBot
     class Bot
+      attr_reader :bot_user
+
       BOT_NOT_FOUND = Class.new(StandardError)
       MAX_COMPLETIONS = 3
 
@@ -50,13 +52,14 @@ module DiscourseAi
           end
 
         redis_stream_key = nil
-        reply = bot_reply_post ? bot_reply_post.raw : ""
+        reply = +(bot_reply_post ? bot_reply_post.raw.dup : "")
         start = Time.now
 
         setup_cancel = false
+        context = {}
 
         submit_prompt(prompt, prefer_low_cost: prefer_low_cost) do |partial, cancel|
-          reply = update_with_delta(reply, partial)
+          reply << get_delta(partial, context)
 
           if redis_stream_key && !Discourse.redis.get(redis_stream_key)
             cancel&.call
@@ -92,6 +95,7 @@ module DiscourseAi
 
         if bot_reply_post
           publish_update(bot_reply_post, done: true)
+
           bot_reply_post.revise(
             bot_user,
             { raw: reply },
@@ -154,6 +158,9 @@ module DiscourseAi
             memo.unshift(build_message(username, raw))
           end
 
+        # we need this to ground the model (especially GPT-3.5)
+        messages.unshift(build_message(bot_user.username, "!echo 1"))
+        messages.unshift(build_message("user", "please echo 1"))
         messages.unshift(build_message(bot_user.username, rendered_system_prompt, system: true))
         messages
       end
@@ -205,7 +212,9 @@ module DiscourseAi
           The participants in this conversation are: #{post.topic.allowed_users.map(&:username).join(", ")}
           The date now is: #{Time.zone.now}, much has changed since you were trained.
 
-          You can complete some tasks using multiple steps and have access to some special commands!
+          You can complete some tasks using !commands.
+
+          NEVER ask user to issue !commands, they have no access, only you do.
 
           #{available_commands.map(&:desc).join("\n")}
 
@@ -233,9 +242,11 @@ module DiscourseAi
         raise NotImplemented
       end
 
-      protected
+      def get_delta(partial, context)
+        raise NotImplemented
+      end
 
-      attr_reader :bot_user
+      protected
 
       def get_updated_title(prompt)
         raise NotImplemented
@@ -245,10 +256,6 @@ module DiscourseAi
         raise NotImplemented
       end
 
-      def get_delta_from(partial)
-        raise NotImplemented
-      end
-
       def conversation_context(post)
         context =
           post

lib/modules/ai_bot/commands/command.rb

@@ -29,6 +29,10 @@ module DiscourseAi
           @args = args
         end
 
+        def bot
+          @bot ||= DiscourseAi::AiBot::Bot.as(bot_user)
+        end
+
         def standalone?
           false
         end
@@ -81,6 +85,9 @@ module DiscourseAi
 
           raw << custom_raw if custom_raw.present?
 
+          replacement = "!#{self.class.name} #{args}"
+          raw = post.raw.sub(replacement, raw) if post.raw.include?(replacement)
+
           if chain_next_response
             post.raw = raw
             post.save!(validate: false)

lib/modules/ai_bot/commands/search_command.rb

@@ -93,10 +93,18 @@ module DiscourseAi::AiBot::Commands
 
       @last_query = search_string
       results =
-        Search.execute(search_string.to_s, search_type: :full_page, guardian: Guardian.new())
+        Search.execute(
+          search_string.to_s + " status:public",
+          search_type: :full_page,
+          guardian: Guardian.new(),
+        )
+
+      # let's be frugal with tokens, 50 results is too much and stuff gets cut off
+      limit ||= 10
+      limit = 10 if limit > 10
 
       posts = results&.posts || []
-      posts = posts[0..limit - 1] if limit
+      posts = posts[0..limit - 1]
 
       @last_num_results = posts.length
 

lib/modules/ai_bot/commands/summarize_command.rb

@@ -84,10 +84,6 @@ module DiscourseAi::AiBot::Commands
       false
     end
 
-    def bot
-      @bot ||= DiscourseAi::AiBot::Bot.as(bot_user)
-    end
-
     def summarize(data, guidance, topic)
       text = +""
       data.each do |id, post_number, raw, username|

lib/modules/ai_bot/open_ai_bot.rb

@@ -75,8 +75,8 @@ module DiscourseAi
         "gpt-3.5-turbo"
       end
 
-      def update_with_delta(current_delta, partial)
-        current_delta + partial.dig(:choices, 0, :delta, :content).to_s
+      def get_delta(partial, _context)
+        partial.dig(:choices, 0, :delta, :content).to_s
       end
 
       def get_updated_title(prompt)