From 9475900a1085b8277808b0a0b1555c59f7eb6d36 Mon Sep 17 00:00:00 2001
From: Pherkel
Date: Mon, 18 Sep 2023 12:44:34 +0200
Subject: small fixes

---
 data/tokenizers/tokens_german.txt | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 data/tokenizers/tokens_german.txt

(limited to 'data')

diff --git a/data/tokenizers/tokens_german.txt b/data/tokenizers/tokens_german.txt
new file mode 100644
index 0000000..57f2c3a
--- /dev/null
+++ b/data/tokenizers/tokens_german.txt
@@ -0,0 +1,38 @@
+_
+<BLANK>
+<UNK>
+<SPACE>
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+é
+à
+ä
+ö
+ß
+ü
+-
+'
-- 
cgit v1.2.3