diff options
author | Pherkel | 2023-09-11 14:33:18 +0200 |
---|---|---|
committer | Pherkel | 2023-09-11 14:33:18 +0200 |
commit | 01fae2b5e395e84db6a7e9819b6f98777c46e845 (patch) | |
tree | e07dec6ccfc9c994af8f5d292be38ea216adfaa1 /swr2_asr/tokenizer.py | |
parent | 9aba8d447ea727afb4208e41dcdcff9157446162 (diff) |
readd space token, idk why its there but it could break things
Diffstat (limited to 'swr2_asr/tokenizer.py')
-rw-r--r-- | swr2_asr/tokenizer.py | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/swr2_asr/tokenizer.py b/swr2_asr/tokenizer.py index 69ced81..d92465a 100644 --- a/swr2_asr/tokenizer.py +++ b/swr2_asr/tokenizer.py @@ -44,6 +44,7 @@ class CharTokenizer: ü - ' + """ self.char_map = {} |