From c09ff76ba6f4c5dd5de64a401efcd27449150aec Mon Sep 17 00:00:00 2001 From: Pherkel Date: Mon, 18 Sep 2023 15:05:21 +0200 Subject: added support for lm decoder during training --- swr2_asr/utils/tokenizer.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'swr2_asr/utils/tokenizer.py') diff --git a/swr2_asr/utils/tokenizer.py b/swr2_asr/utils/tokenizer.py index b1de83e..4e3fddd 100644 --- a/swr2_asr/utils/tokenizer.py +++ b/swr2_asr/utils/tokenizer.py @@ -29,9 +29,17 @@ class CharTokenizer: """Use a character map and convert integer labels to an text sequence""" string = [] for i in labels: + i = int(i) string.append(self.index_map[i]) return "".join(string).replace("", " ") + def decode_batch(self, labels: list[list[int]]) -> list[str]: + """Use a character map and convert integer labels to an text sequence""" + string = [] + for label in labels: + string.append(self.decode(label)) + return string + def get_vocab_size(self) -> int: """Get the number of unique characters in the dataset""" return len(self.char_map) -- cgit v1.2.3