diff options
| author | JoJoBarthold2 | 2023-09-16 15:25:19 +0200 |
|---|---|---|
| committer | JoJoBarthold2 | 2023-09-16 15:25:19 +0200 |
| commit | ea42dd50f167307d52fb128823904fe46f1118ec (patch) | |
| tree | 2111b367f671ff614a1bd860028ed5797b665391 /swr2_asr/utils | |
| parent | 80544737bc4338bd0cde305b8bccb7c5209e1bdc (diff) | |
added a todo
Diffstat (limited to 'swr2_asr/utils')
| -rw-r--r-- | swr2_asr/utils/tokenizer.py | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/swr2_asr/utils/tokenizer.py b/swr2_asr/utils/tokenizer.py index ee89cdb..9abf57d 100644 --- a/swr2_asr/utils/tokenizer.py +++ b/swr2_asr/utils/tokenizer.py @@ -121,6 +121,8 @@ class CharTokenizer: load_tokenizer.index_map[int(index)] = char return load_tokenizer + + #TO DO check about the weird unknown tokens etc. def create_txt(self,path:str): with open(path, 'w',encoding="utf-8") as file: for key,value in self.char_map(): |