From a6343018040b33110eb5822d7013be3c4849d1db Mon Sep 17 00:00:00 2001
From: Martin Forell <martin.forell@kit.edu>
Date: Tue, 30 Jul 2024 08:50:01 +0000
Subject: [PATCH] llama grammar
---
dev-llama-3-small.yaml | 58 +++++++++++++++++++++++++++++-------------
1 file changed, 40 insertions(+), 18 deletions(-)
diff --git a/dev-llama-3-small.yaml b/dev-llama-3-small.yaml
index 5134df0..45cec93 100644
--- a/dev-llama-3-small.yaml
+++ b/dev-llama-3-small.yaml
@@ -11,10 +11,18 @@ config_file: |
parameters:
model: downloads/llama_backend/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf
+ stopwords:
+ - <|im_end|>
+ - <dummy32000>
+ - <|eot_id|>
+ - <|end_of_text|>
template:
+ chat: |
+ <|begin_of_text|>{{.Input }}
+ <|start_header_id|>assistant<|end_header_id|>
chat_message: |
<|start_header_id|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "tool"}}tool{{else if eq .RoleName "user"}}user{{end}}<|end_header_id|>
-
+
{{ if .FunctionCall -}}
Function call:
{{ else if eq .RoleName "tool" -}}
@@ -26,29 +34,43 @@ config_file: |
{{ toJson .FunctionCall -}}
{{ end -}}
<|eot_id|>
+ completion: |
+ {{.Input}}
function: |
<|start_header_id|>system<|end_header_id|>
-
- You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools:
- <tools>
+
+ You have access to the following functions:
+
{{range .Functions}}
- {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }}
+ Use the function '{{.Name}}' to '{{.Description}}'
+ {{toJson .Parameters}}
{{end}}
- </tools>
- Use the following pydantic model json schema for each tool call you will make:
- {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
- Function call:
- chat: |
- <|begin_of_text|>{{.Input }}
+
+ Think very carefully before calling functions.
+ If a you choose to call a function ONLY reply in the following format with no prefix or suffix:
+
+ <function=example_function_name>{{`{{"example_name": "example_value"}}`}}</function>
+
+ Reminder:
+ - If looking for real time information use relevant functions before falling back to searching on internet
+ - Function calls MUST follow the specified format, start with <function= and end with </function>
+ - Required parameters MUST be specified
+ - Only call one function at a time
+ - Put the entire function call reply on one line
+ <|eot_id|>
+ {{.Input }}
<|start_header_id|>assistant<|end_header_id|>
- completion: |
- {{.Input}}
+ function:
+ disable_no_action: true
+ grammar:
+ #disable: true
+ no_mixed_free_string: true
+ mixed_mode: true
+ schema_type: llama3.1 # or JSON is supported too (json)
+ response_regex:
+ - <function=(?P<name>\w+)>(?P<arguments>.*)</function>
context_size: 8192
- stopwords:
- - <|im_end|>
- - <dummy32000>
- - "<|eot_id|>"
- - <|end_of_text|>
+
usage: |
curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
"model": "gpt-4",
--
GitLab