diff options
Diffstat (limited to 'swr2_asr/utils/tokenizer.py')
| -rw-r--r-- | swr2_asr/utils/tokenizer.py | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/swr2_asr/utils/tokenizer.py b/swr2_asr/utils/tokenizer.py index ee89cdb..9abf57d 100644 --- a/swr2_asr/utils/tokenizer.py +++ b/swr2_asr/utils/tokenizer.py @@ -121,6 +121,8 @@ class CharTokenizer: load_tokenizer.index_map[int(index)] = char return load_tokenizer + + #TO DO check about the weird unknown tokens etc. def create_txt(self,path:str): with open(path, 'w',encoding="utf-8") as file: for key,value in self.char_map(): |