sauerkrautlm

index.yaml

@@ -5,6 +5,9 @@
 - url: https://gitlab.kit.edu/kit/aifb/BIS/infrastruktur/localai/localai-model-gallery/-/raw/main/mixtral.yaml
   name: mixtral
   license: other
+- url: https://gitlab.kit.edu/kit/aifb/BIS/infrastruktur/localai/localai-model-gallery/-/raw/main/sauerkrautlm.yaml
+  name: sauerkrautlm
+  license: other
 
 - url: https://gitlab.kit.edu/kit/aifb/BIS/infrastruktur/localai/localai-model-gallery/-/raw/main/neural.yaml
   name: neural

sauerkrautlm.yaml

+name: "sauerkrautlm"
+
+description: |
+  This is a mistral model
+
+license: "https://ai.meta.com/llama/license/"
+urls:
+- https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-GGUF
+
+config_file: |
+  name: sauerkrautlm
+  context_size: 4096
+  f16: true
+  mmap: true
+  threads: 6
+  parameters:
+    model: model_data/text_generation/TheBloke/sauerkraut/sauerkrautlm-7b-v1-mistral.Q5_K_M.gguf
+    temperature: 0.2
+    top_k: 40
+    top_p: 0.95
+  template:
+    chat_message: chatml
+    chat: chatml-block
+    completion: completion
+  gpu_layers: 12  
+  stopwords:
+  - <|im_end|>
+
+prompt_templates:
+- name: "chatml"
+  content: |
+    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
+    {{if .Content}}{{.Content}}{{end}}
+    <|im_end|>
+- name: "chatml-block"
+  content: |
+    {{.Input}}
+    <|im_start|>assistant
+- name: "completion"
+  content: |
+    {{.Input}}
+    
+files:
+- filename: "model_data/text_generation/TheBloke/sauerkraut/sauerkrautlm-7b-v1-mistral.Q5_K_M.gguf"
+  sha256: "b1779f1d2a8aaa6720da6e16aa603a319be7b343afc7445f38c87f58890b7a2c"
+  uri: "https://huggingface.co/TheBloke/SauerkrautLM-7B-v1-mistral-GGUF/resolve/main/sauerkrautlm-7b-v1-mistral.Q5_K_M.gguf"
+  
\ No newline at end of file