llama grammar

dev-llama-3-small.yaml

@@ -11,10 +11,18 @@ config_file: |
   parameters:
     model: downloads/llama_backend/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf
   
+  stopwords:
+  - <|im_end|>
+  - <dummy32000>
+  - <|eot_id|>
+  - <|end_of_text|>
   template:
+    chat: |
+      <|begin_of_text|>{{.Input }}
+      <|start_header_id|>assistant<|end_header_id|>
     chat_message: |
       <|start_header_id|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "tool"}}tool{{else if eq .RoleName "user"}}user{{end}}<|end_header_id|>
-
+  
       {{ if .FunctionCall -}}
       Function call:
       {{ else if eq .RoleName "tool" -}}
@@ -26,29 +34,43 @@ config_file: |
       {{ toJson .FunctionCall -}}
       {{ end -}}
       <|eot_id|>
+    completion: |
+      {{.Input}}
     function: |
       <|start_header_id|>system<|end_header_id|>
-
-      You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools:
-      <tools>
+  
+      You have access to the following functions:
+  
       {{range .Functions}}
-      {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }}
+      Use the function '{{.Name}}' to '{{.Description}}'
+      {{toJson .Parameters}}
       {{end}}
-      </tools>
-      Use the following pydantic model json schema for each tool call you will make:
-      {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-      Function call:
-    chat: |
-      <|begin_of_text|>{{.Input }}
+  
+      Think very carefully before calling functions.
+      If a you choose to call a function ONLY reply in the following format with no prefix or suffix:
+  
+      <function=example_function_name>{{`{{"example_name": "example_value"}}`}}</function>
+  
+      Reminder:
+      - If looking for real time information use relevant functions before falling back to searching on internet
+      - Function calls MUST follow the specified format, start with <function= and end with </function>
+      - Required parameters MUST be specified
+      - Only call one function at a time
+      - Put the entire function call reply on one line
+      <|eot_id|>
+      {{.Input }}
       <|start_header_id|>assistant<|end_header_id|>
-    completion: |
-      {{.Input}}
+  function:
+    disable_no_action: true
+    grammar:
+      #disable: true
+      no_mixed_free_string: true
+      mixed_mode: true
+      schema_type: llama3.1 # or JSON is supported too (json)
+    response_regex:
+    - <function=(?P<name>\w+)>(?P<arguments>.*)</function>
   context_size: 8192
-  stopwords:
-    - <|im_end|>
-    - <dummy32000>
-    - "<|eot_id|>"
-    - <|end_of_text|>
+
   usage: |
         curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
             "model": "gpt-4",