name: "mixtral"

description: |
  This is a mixtral expert model

license: "https://ai.meta.com/llama/license/"
urls:
- https://huggingface.co/TheBloke/dolphin-2.5-mixtral-8x7b-GGUF

config_file: |
  name: mixtral
  context_size: 8000
  backend: llama
  f16: true
  mmap: true
  threads: 4
  parameters:
    model: model_data/text_generation/TheBloke/mixtral/dolphin-2.5-mixtral-8x7b.Q5_K_M.gguf
    temperature: 0.2
    top_k: 40
    top_p: 0.95
  template:
    chat_message: mixtral-chatml
    chat: mixtral-chatml-block
    completion: completion
  gpu_layers: 12  
  stopwords:
  - <|im_end|>

prompt_templates:
- name: "mixtral-chatml"
  content: |
    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
    {{if .Content}}{{.Content}}{{end}}
    <|im_end|>
- name: "mixtral-chatml-block"
  content: |
    {{.Input}}
    <|im_start|>assistant
- name: "completion"
  content: |
    {{.Input}}
    
files:
- filename: "model_data/text_generation/TheBloke/mixtral/dolphin-2.5-mixtral-8x7b.Q5_K_M.gguf"
  sha256: "b0a57cd79a0a1b056e964984fcfb94f6802d4e7487efe463b18fba6d6147734e"
  uri: "https://huggingface.co/TheBloke/dolphin-2.5-mixtral-8x7b-GGUF/resolve/main/dolphin-2.5-mixtral-8x7b.Q5_K_M.gguf"