Support for OpenBuddy with finetune

It's a bit of a hack but works for now
Batched mode fails, will be fixed later

Author: John

examples/falcon/falcon_main.cpp

@@ -340,6 +340,20 @@ int main(int argc, char ** argv) {
                         inp_system_baseline = ::falcon_tokenize(ctx, ">>INTRODUCTION<<"+params.system_baseline_prompt+"\n", false);
                 }
                 break;
+            case FINETUNE_OPENBUDDY:
+                inp_pfx = ::falcon_tokenize(ctx, "User: ", false);
+                inp_sfx = ::falcon_tokenize(ctx, "\nAssistant:", false);
+                if (!params.system_prompt.size())
+                {
+                    inp_system = ::falcon_tokenize(ctx, "Consider a conversation between User (a human) and Assistant (named Buddy).\nBuddy is an INTP-T, a friendly, intelligent and multilingual AI assistant, by OpenBuddy team, running on https://github.com/cmp-nct/ggllm.cpp\nBuddy cannot access the Internet.\nBuddy can fluently speak the user's language (e.g. English, Chinese).\nBuddy can generate poems, stories, code, essays, songs, and more.\nBuddy possesses knowledge about the world, history, and culture, but not everything. Knowledge cutoff: 2021-09.\nBuddy's responses are always positive, unharmful, safe, creative, high-quality, human-like, and interesting.\n\nUser: Hi.\nAssistant: Hi, I'm Buddy, your AI assistant. How can I help you today?\n", false);
+                } else
+                if (params.system_prompt.size() &&!params.sys_prompt_is_raw)
+                {
+                    inp_system = ::falcon_tokenize(ctx, ">>INTRODUCTION<<"+params.system_prompt+"\n", false);
+                    if(!params.sys_prompt_simple)
+                        inp_system_baseline = ::falcon_tokenize(ctx, ">>INTRODUCTION<<"+params.system_baseline_prompt+"\n", false);
+                }
+                break;
             case FINETUNE_FALCONINSTRUCT:
                 inp_pfx = ::falcon_tokenize(ctx, "User: ", false);
                 inp_sfx = ::falcon_tokenize(ctx, "\nAssistant:", false); // must not include space

falcon_convert.py

@@ -108,7 +108,15 @@ def bytes_to_unicode():
 
 for i in range(hparams["vocab_size"]):
     if i in reverse_vocab:
-        text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+        try:
+            text = bytearray([byte_decoder[c] for c in reverse_vocab[i]])
+        except KeyError:
+            text = bytearray()
+            for c in reverse_vocab[i]:
+                if ord(c) < 256:  # single byte character
+                    text.append(byte_decoder[ord(c)])
+                else:  # multibyte special token character
+                    text.extend(c.encode('utf-8'))
     else:
         print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
         padding_token = f"[PAD{i}]".encode("utf8")

libfalcon.cpp

@@ -1497,6 +1497,10 @@ t_finetune_type falcon_detect_finetune(falcon_context * ctx, std::string model_p
             return FINETUNE_OPENASSISTANT;
         }
     }
+    if (ctx->vocab.id_to_token.size() == 70144)
+    {
+        return FINETUNE_OPENBUDDY;
+    }
     if (model_lower.find("wizard") != std::string::npos) {
         return FINETUNE_WIZARD;
     }

libfalcon.h

@@ -268,8 +268,8 @@ extern "C" {
 
     // Token Id -> String. Uses the vocabulary in the provided context
     LLAMA_API const char * falcon_token_to_str(const struct falcon_context * ctx, falcon_token token);
-    typedef enum { FINETUNE_UNSPECIFIED, FINETUNE_NONE, FINETUNE_ALPACA, FINETUNE_OPENASSISTANT, FINETUNE_OPENASSIST_V1, FINETUNE_WIZARD, FINETUNE_FALCONINSTRUCT } t_finetune_type;
-    static const char *FINETUNE_NAME[7] = { "UNSPECIFIED", "NONE", "ALPACA", "OPENASSISTANT", "OPENASSIST_V1", "WIZARD", "FALCONINSTRUCT" };
+    typedef enum { FINETUNE_UNSPECIFIED, FINETUNE_NONE, FINETUNE_ALPACA, FINETUNE_OPENASSISTANT, FINETUNE_OPENASSIST_V1, FINETUNE_WIZARD, FINETUNE_FALCONINSTRUCT, FINETUNE_OPENBUDDY } t_finetune_type;
+    static const char *FINETUNE_NAME[8] = { "UNSPECIFIED", "NONE", "ALPACA", "OPENASSISTANT", "OPENASSIST_V1", "WIZARD", "FALCONINSTRUCT", "OPENBUDDY"};