1 files changed, 38 insertions, 0 deletions
diff --git a/data/tokenizers/tokens_german.txt b/data/tokenizers/tokens_german.txt
new file mode 100644
index 0000000..57f2c3a
--- /dev/null
+++ b/data/tokenizers/tokens_german.txt
@@ -0,0 +1,38 @@
+_
+<BLANK>
+<UNK>
+<SPACE>
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+é
+à
+ä
+ö
+ß
+ü
+-
+'