aboutsummaryrefslogtreecommitdiff
path: root/swr2_asr/tokenizer.py
diff options
context:
space:
mode:
authorPherkel2023-09-11 14:33:18 +0200
committerPherkel2023-09-11 14:33:18 +0200
commit01fae2b5e395e84db6a7e9819b6f98777c46e845 (patch)
treee07dec6ccfc9c994af8f5d292be38ea216adfaa1 /swr2_asr/tokenizer.py
parent9aba8d447ea727afb4208e41dcdcff9157446162 (diff)
readd space token, idk why its there but it could break things
Diffstat (limited to 'swr2_asr/tokenizer.py')
-rw-r--r--swr2_asr/tokenizer.py1
1 files changed, 1 insertions, 0 deletions
diff --git a/swr2_asr/tokenizer.py b/swr2_asr/tokenizer.py
index 69ced81..d92465a 100644
--- a/swr2_asr/tokenizer.py
+++ b/swr2_asr/tokenizer.py
@@ -44,6 +44,7 @@ class CharTokenizer:
ü
-
'
+
"""
self.char_map = {}