From 58b30927bd870604a4077a8af9ec3cad7b0be21c Mon Sep 17 00:00:00 2001
From: Pherkel
Date: Mon, 11 Sep 2023 21:52:42 +0200
Subject: changed config to yaml!

---
 config.philipp.yaml | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 config.philipp.yaml

(limited to 'config.philipp.yaml')

diff --git a/config.philipp.yaml b/config.philipp.yaml
new file mode 100644
index 0000000..638b5ef
--- /dev/null
+++ b/config.philipp.yaml
@@ -0,0 +1,29 @@
+model: 
+  n_cnn_layers: 3
+  n_rnn_layers: 5
+  rnn_dim: 512
+  n_feats: 128 # number of mel features
+  stride: 2
+  dropout: 0.25 # recommended to be around 0.4 for smaller datasets, 0.1 for really large datasets
+
+training:
+  learning_rate: 0.0005
+  batch_size: 2 # recommended to maximum number that fits on the GPU (batch size of 32 fits on a 12GB GPU)
+  epochs: 3 
+  eval_every_n: 1 # evaluate every n epochs
+  num_workers: 4 # number of workers for dataloader
+
+dataset:
+  download: True
+  dataset_root_path: "/Volumes/pherkel 1/SWR2-ASR" # files will be downloaded into this dir
+  language_name: "mls_german_opus"
+  limited_supervision: True # set to True if you want to use limited supervision
+  dataset_percentage: 0.01 # percentage of dataset to use (1.0 = 100%)
+  shuffle: True
+
+tokenizer:
+  tokenizer_path: "data/tokenizers/char_tokenizer_german.json"
+
+checkpoints:
+  model_load_path: ~ # path to load model from
+  model_save_path: ~ # path to save model to
\ No newline at end of file
-- 
cgit v1.2.3


From 64dbb9d32a51b1bce6c9de67069dc8f5943a5399 Mon Sep 17 00:00:00 2001
From: Pherkel
Date: Mon, 11 Sep 2023 22:16:26 +0200
Subject: added n_feats from config

---
 config.philipp.yaml    | 2 +-
 swr2_asr/train.py      | 4 ++--
 swr2_asr/utils/data.py | 7 ++++---
 3 files changed, 7 insertions(+), 6 deletions(-)

(limited to 'config.philipp.yaml')

diff --git a/config.philipp.yaml b/config.philipp.yaml
index 638b5ef..6b905cd 100644
--- a/config.philipp.yaml
+++ b/config.philipp.yaml
@@ -4,7 +4,7 @@ model:
   rnn_dim: 512
   n_feats: 128 # number of mel features
   stride: 2
-  dropout: 0.25 # recommended to be around 0.4 for smaller datasets, 0.1 for really large datasets
+  dropout: 0.25 # recommended to be around 0.4-0.6 for smaller datasets, 0.1 for really large datasets
 
 training:
   learning_rate: 0.0005
diff --git a/swr2_asr/train.py b/swr2_asr/train.py
index eb79ee2..ca70d21 100644
--- a/swr2_asr/train.py
+++ b/swr2_asr/train.py
@@ -223,8 +223,8 @@ def main(config_path: str):
         )
     tokenizer = CharTokenizer.from_file(tokenizer_config["tokenizer_path"])
 
-    train_data_processing = DataProcessing("train", tokenizer)
-    valid_data_processing = DataProcessing("valid", tokenizer)
+    train_data_processing = DataProcessing("train", tokenizer, {"n_feats": model_config["n_feats"]})
+    valid_data_processing = DataProcessing("valid", tokenizer, {"n_feats": model_config["n_feats"]})
 
     train_loader = DataLoader(
         dataset=train_dataset,
diff --git a/swr2_asr/utils/data.py b/swr2_asr/utils/data.py
index 0e06eec..10f0ea8 100644
--- a/swr2_asr/utils/data.py
+++ b/swr2_asr/utils/data.py
@@ -15,18 +15,19 @@ from swr2_asr.utils.tokenizer import CharTokenizer
 class DataProcessing:
     """Data processing class for the dataloader"""
 
-    def __init__(self, data_type: str, tokenizer: CharTokenizer):
+    def __init__(self, data_type: str, tokenizer: CharTokenizer, hparams: dict):
         self.data_type = data_type
         self.tokenizer = tokenizer
+        n_features = hparams["n_feats"]
 
         if data_type == "train":
             self.audio_transform = torch.nn.Sequential(
-                torchaudio.transforms.MelSpectrogram(sample_rate=16000, n_mels=128),
+                torchaudio.transforms.MelSpectrogram(sample_rate=16000, n_mels=n_features),
                 torchaudio.transforms.FrequencyMasking(freq_mask_param=30),
                 torchaudio.transforms.TimeMasking(time_mask_param=100),
             )
         elif data_type == "valid":
-            self.audio_transform = torchaudio.transforms.MelSpectrogram()
+            self.audio_transform = torchaudio.transforms.MelSpectrogram(n_mels=n_features)
 
     def __call__(self, data) -> tuple[Tensor, Tensor, list, list]:
         spectrograms = []
-- 
cgit v1.2.3


From 6f5513140f153206cfa91df3077e67ce58043d35 Mon Sep 17 00:00:00 2001
From: Pherkel
Date: Mon, 11 Sep 2023 22:58:19 +0200
Subject: model loading is broken :(

---
 config.philipp.yaml   |   9 +++-
 config.train.yaml     |  28 ----------
 config.yaml           |  34 ++++++++++++
 swr2_asr/inference.py | 140 ++++++++++++++++++++++----------------------------
 swr2_asr/train.py     |   2 +-
 5 files changed, 103 insertions(+), 110 deletions(-)
 delete mode 100644 config.train.yaml
 create mode 100644 config.yaml

(limited to 'config.philipp.yaml')

diff --git a/config.philipp.yaml b/config.philipp.yaml
index 6b905cd..4a723c6 100644
--- a/config.philipp.yaml
+++ b/config.philipp.yaml
@@ -12,6 +12,7 @@ training:
   epochs: 3 
   eval_every_n: 1 # evaluate every n epochs
   num_workers: 4 # number of workers for dataloader
+  device: "cuda" # device to run inference on if gpu is available, else "cpu" will be set automatically
 
 dataset:
   download: True
@@ -25,5 +26,9 @@ tokenizer:
   tokenizer_path: "data/tokenizers/char_tokenizer_german.json"
 
 checkpoints:
-  model_load_path: ~ # path to load model from
-  model_save_path: ~ # path to save model to
\ No newline at end of file
+  model_load_path: "data/runs/epoch30" # path to load model from
+  model_save_path: ~ # path to save model to
+
+inference:
+  model_load_path: "data/runs/epoch30" # path to load model from
+  device: "cuda" # device to run inference on if gpu is available, else "cpu" will be set automatically
\ No newline at end of file
diff --git a/config.train.yaml b/config.train.yaml
deleted file mode 100644
index c82439d..0000000
--- a/config.train.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-model: 
-  n_cnn_layers: 3
-  n_rnn_layers: 5
-  rnn_dim: 512
-  n_feats: 128 # number of mel features
-  stride: 2
-  dropout: 0.25 # recommended to be around 0.4 for smaller datasets, 0.1 for really large datasets
-
-training:
-  learning_rate: 5e-4
-  batch_size: 8 # recommended to maximum number that fits on the GPU (batch size of 32 fits on a 12GB GPU)
-  epochs: 3 
-  eval_every_n: 3 # evaluate every n epochs
-  num_workers: 8 # number of workers for dataloader
-
-dataset:
-  download: True
-  dataset_root_path: "YOUR/PATH" # files will be downloaded into this dir
-  language_name: "mls_german_opus"
-  limited_supervision: False # set to True if you want to use limited supervision
-  dataset_percentage: 1.0 # percentage of dataset to use (1.0 = 100%)
-
-tokenizer:
-  tokenizer_path: "data/tokenizers/char_tokenizer_german.yaml"
-
-checkpoints:
-  model_load_path: "YOUR/PATH" # path to load model from
-  model_save_path: "YOUR/PATH" # path to save model to
\ No newline at end of file
diff --git a/config.yaml b/config.yaml
new file mode 100644
index 0000000..e5ff43a
--- /dev/null
+++ b/config.yaml
@@ -0,0 +1,34 @@
+model: 
+  n_cnn_layers: 3
+  n_rnn_layers: 5
+  rnn_dim: 512
+  n_feats: 128 # number of mel features
+  stride: 2
+  dropout: 0.3 # recommended to be around 0.4 for smaller datasets, 0.1 for really large datasets
+
+training:
+  learning_rate: 5e-4
+  batch_size: 8 # recommended to maximum number that fits on the GPU (batch size of 32 fits on a 12GB GPU)
+  epochs: 3 
+  eval_every_n: 3 # evaluate every n epochs
+  num_workers: 8 # number of workers for dataloader
+
+dataset:
+  download: True
+  dataset_root_path: "YOUR/PATH" # files will be downloaded into this dir
+  language_name: "mls_german_opus"
+  limited_supervision: False # set to True if you want to use limited supervision
+  dataset_percentage: 1.0 # percentage of dataset to use (1.0 = 100%)
+  shuffle: True
+
+tokenizer:
+  tokenizer_path: "data/tokenizers/char_tokenizer_german.yaml"
+
+checkpoints:
+  model_load_path: "YOUR/PATH" # path to load model from
+  model_save_path: "YOUR/PATH" # path to save model to
+
+inference:
+  model_load_path: "YOUR/PATH" # path to load model from
+  beam_width: 10 # beam width for beam search
+  device: "cuda" # device to run inference on if gpu is available, else "cpu" will be set automatically
\ No newline at end of file
diff --git a/swr2_asr/inference.py b/swr2_asr/inference.py
index f8342f7..6495a9a 100644
--- a/swr2_asr/inference.py
+++ b/swr2_asr/inference.py
@@ -1,35 +1,20 @@
 """Training script for the ASR model."""
-from typing import TypedDict
-
+import click
 import torch
 import torch.nn.functional as F
 import torchaudio
+import yaml
 
 from swr2_asr.model_deep_speech import SpeechRecognitionModel
 from swr2_asr.utils.tokenizer import CharTokenizer
 
 
-class HParams(TypedDict):
-    """Type for the hyperparameters of the model."""
-
-    n_cnn_layers: int
-    n_rnn_layers: int
-    rnn_dim: int
-    n_class: int
-    n_feats: int
-    stride: int
-    dropout: float
-    learning_rate: float
-    batch_size: int
-    epochs: int
-
-
-def greedy_decoder(output, tokenizer, collapse_repeated=True):
+def greedy_decoder(output, tokenizer: CharTokenizer, collapse_repeated=True):
     """Greedily decode a sequence."""
     arg_maxes = torch.argmax(output, dim=2)  # pylint: disable=no-member
-    blank_label = tokenizer.encode(" ").ids[0]
+    blank_label = tokenizer.get_blank_token()
     decodes = []
-    for _i, args in enumerate(arg_maxes):
+    for args in arg_maxes:
         decode = []
         for j, index in enumerate(args):
             if index != blank_label:
@@ -40,75 +25,72 @@ def greedy_decoder(output, tokenizer, collapse_repeated=True):
     return decodes
 
 
-def main() -> None:
+@click.command()
+@click.option(
+    "--config_path",
+    default="config.yaml",
+    help="Path to yaml config file",
+    type=click.Path(exists=True),
+)
+@click.option(
+    "--file_path",
+    help="Path to audio file",
+    type=click.Path(exists=True),
+)
+def main(config_path: str, file_path: str) -> None:
     """inference function."""
-
-    device = "cuda" if torch.cuda.is_available() else "cpu"
+    with open(config_path, "r", encoding="utf-8") as yaml_file:
+        config_dict = yaml.safe_load(yaml_file)
+
+    # Create separate dictionaries for each top-level key
+    model_config = config_dict.get("model", {})
+    tokenizer_config = config_dict.get("tokenizer", {})
+    inference_config = config_dict.get("inference", {})
+
+    if inference_config["device"] == "cpu":
+        device = "cpu"
+    elif inference_config["device"] == "cuda":
+        device = "cuda" if torch.cuda.is_available() else "cpu"
     device = torch.device(device)  # pylint: disable=no-member
 
-    tokenizer = CharTokenizer.from_file("char_tokenizer_german.json")
-
-    spectrogram_hparams = {
-        "sample_rate": 16000,
-        "n_fft": 400,
-        "win_length": 400,
-        "hop_length": 160,
-        "n_mels": 128,
-        "f_min": 0,
-        "f_max": 8000,
-        "power": 2.0,
-    }
-
-    hparams = HParams(
-        n_cnn_layers=3,
-        n_rnn_layers=5,
-        rnn_dim=512,
-        n_class=tokenizer.get_vocab_size(),
-        n_feats=128,
-        stride=2,
-        dropout=0.1,
-        learning_rate=0.1,
-        batch_size=30,
-        epochs=100,
-    )
+    tokenizer = CharTokenizer.from_file(tokenizer_config["tokenizer_path"])
 
     model = SpeechRecognitionModel(
-        hparams["n_cnn_layers"],
-        hparams["n_rnn_layers"],
-        hparams["rnn_dim"],
-        hparams["n_class"],
-        hparams["n_feats"],
-        hparams["stride"],
-        hparams["dropout"],
+        model_config["n_cnn_layers"],
+        model_config["n_rnn_layers"],
+        model_config["rnn_dim"],
+        tokenizer.get_vocab_size(),
+        model_config["n_feats"],
+        model_config["stride"],
+        model_config["dropout"],
     ).to(device)
 
-    checkpoint = torch.load("model8", map_location=device)
-    state_dict = {
-        k[len("module.") :] if k.startswith("module.") else k: v
-        for k, v in checkpoint["model_state_dict"].items()
-    }
-    model.load_state_dict(state_dict)
-
-    # waveform, sample_rate = torchaudio.load("test.opus")
-    waveform, sample_rate = torchaudio.load("marvin_rede.flac")  # pylint: disable=no-member
-    if sample_rate != spectrogram_hparams["sample_rate"]:
-        resampler = torchaudio.transforms.Resample(sample_rate, spectrogram_hparams["sample_rate"])
+    checkpoint = torch.load(inference_config["model_load_path"], map_location=device)
+    print(checkpoint["model_state_dict"].keys())
+    model.load_state_dict(checkpoint["model_state_dict"], strict=False)
+    model.eval()
+    waveform, sample_rate = torchaudio.load(file_path)  # pylint: disable=no-member
+    if waveform.shape[0] != 1:
+        waveform = waveform[1]
+        waveform = waveform.unsqueeze(0)
+    if sample_rate != 16000:
+        resampler = torchaudio.transforms.Resample(sample_rate, 16000)
         waveform = resampler(waveform)
+        sample_rate = 16000
+
+    data_processing = torchaudio.transforms.MelSpectrogram(n_mels=model_config["n_feats"])
+
+    spec = data_processing(waveform).squeeze(0).transpose(0, 1)
 
-    spec = (
-        torchaudio.transforms.MelSpectrogram(**spectrogram_hparams)(waveform)
-        .squeeze(0)
-        .transpose(0, 1)
-    )
-    specs = [spec]
-    specs = torch.nn.utils.rnn.pad_sequence(specs, batch_first=True).unsqueeze(1).transpose(2, 3)
+    spec = spec.unsqueeze(0)
+    spec = spec.transpose(1, 2)
+    spec = spec.unsqueeze(0)
+    output = model(spec)  # pylint: disable=not-callable
+    output = F.log_softmax(output, dim=2)  # (batch, time, n_class)
+    decoded_preds = greedy_decoder(output, tokenizer)
 
-    output = model(specs)  # pylint: disable=not-callable
-    output = F.log_softmax(output, dim=2)
-    output = output.transpose(0, 1)  # (time, batch, n_class)
-    decodes = greedy_decoder(output, tokenizer)
-    print(decodes)
+    print(decoded_preds)
 
 
 if __name__ == "__main__":
-    main()
+    main()  # pylint: disable=no-value-for-parameter
diff --git a/swr2_asr/train.py b/swr2_asr/train.py
index ca70d21..ec25918 100644
--- a/swr2_asr/train.py
+++ b/swr2_asr/train.py
@@ -263,7 +263,7 @@ def main(config_path: str):
     prev_epoch = 0
 
     if checkpoints_config["model_load_path"] is not None:
-        checkpoint = torch.load(checkpoints_config["model_load_path"])
+        checkpoint = torch.load(checkpoints_config["model_load_path"], map_location=device)
         model.load_state_dict(checkpoint["model_state_dict"])
         optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
         prev_epoch = checkpoint["epoch"]
-- 
cgit v1.2.3


From 4aff1fcd70cd8601541a1dd5bd820b0263ed1362 Mon Sep 17 00:00:00 2001
From: Philipp Merkel
Date: Mon, 11 Sep 2023 22:36:28 +0000
Subject: fix: switched up training and test splits in train.py

---
 config.philipp.yaml         | 22 +++++++++++-----------
 swr2_asr/train.py           |  8 +++-----
 swr2_asr/utils/data.py      | 31 -------------------------------
 swr2_asr/utils/tokenizer.py | 12 ------------
 4 files changed, 14 insertions(+), 59 deletions(-)

(limited to 'config.philipp.yaml')

diff --git a/config.philipp.yaml b/config.philipp.yaml
index 4a723c6..f72ce2e 100644
--- a/config.philipp.yaml
+++ b/config.philipp.yaml
@@ -4,30 +4,30 @@ model:
   rnn_dim: 512
   n_feats: 128 # number of mel features
   stride: 2
-  dropout: 0.25 # recommended to be around 0.4-0.6 for smaller datasets, 0.1 for really large datasets
+  dropout: 0.2 # recommended to be around 0.4-0.6 for smaller datasets, 0.1 for really large datasets
 
 training:
   learning_rate: 0.0005
-  batch_size: 2 # recommended to maximum number that fits on the GPU (batch size of 32 fits on a 12GB GPU)
-  epochs: 3 
-  eval_every_n: 1 # evaluate every n epochs
+  batch_size: 32 # recommended to maximum number that fits on the GPU (batch size of 32 fits on a 12GB GPU)
+  epochs: 150 
+  eval_every_n: 5 # evaluate every n epochs
   num_workers: 4 # number of workers for dataloader
   device: "cuda" # device to run inference on if gpu is available, else "cpu" will be set automatically
 
 dataset:
-  download: True
-  dataset_root_path: "/Volumes/pherkel 1/SWR2-ASR" # files will be downloaded into this dir
+  download: true
+  dataset_root_path: "data" # files will be downloaded into this dir
   language_name: "mls_german_opus"
-  limited_supervision: True # set to True if you want to use limited supervision
-  dataset_percentage: 0.01 # percentage of dataset to use (1.0 = 100%)
-  shuffle: True
+  limited_supervision: false # set to True if you want to use limited supervision
+  dataset_percentage: 1 # percentage of dataset to use (1.0 = 100%)
+  shuffle: true
 
 tokenizer:
   tokenizer_path: "data/tokenizers/char_tokenizer_german.json"
 
 checkpoints:
-  model_load_path: "data/runs/epoch30" # path to load model from
-  model_save_path: ~ # path to save model to
+  model_load_path: "data/runs/epoch31" # path to load model from
+  model_save_path: "data/runs/epoch" # path to save model to
 
 inference:
   model_load_path: "data/runs/epoch30" # path to load model from
diff --git a/swr2_asr/train.py b/swr2_asr/train.py
index ec25918..3ed3ac8 100644
--- a/swr2_asr/train.py
+++ b/swr2_asr/train.py
@@ -187,16 +187,14 @@ def main(config_path: str):
     dataset_config = config_dict.get("dataset", {})
     tokenizer_config = config_dict.get("tokenizer", {})
     checkpoints_config = config_dict.get("checkpoints", {})
-
-    print(training_config["learning_rate"])
-
+    
     if not os.path.isdir(dataset_config["dataset_root_path"]):
         os.makedirs(dataset_config["dataset_root_path"])
 
     train_dataset = MLSDataset(
         dataset_config["dataset_root_path"],
         dataset_config["language_name"],
-        Split.TEST,
+        Split.TRAIN,
         download=dataset_config["download"],
         limited=dataset_config["limited_supervision"],
         size=dataset_config["dataset_percentage"],
@@ -204,7 +202,7 @@ def main(config_path: str):
     valid_dataset = MLSDataset(
         dataset_config["dataset_root_path"],
         dataset_config["language_name"],
-        Split.TRAIN,
+        Split.TEST,
         download=dataset_config["download"],
         limited=dataset_config["limited_supervision"],
         size=dataset_config["dataset_percentage"],
diff --git a/swr2_asr/utils/data.py b/swr2_asr/utils/data.py
index 10f0ea8..d551c98 100644
--- a/swr2_asr/utils/data.py
+++ b/swr2_asr/utils/data.py
@@ -134,11 +134,6 @@ class MLSDataset(Dataset):
 
     def initialize_limited(self) -> None:
         """Initializes the limited supervision dataset"""
-        # get file handles
-        # get file paths
-        # get transcripts
-        # create train or validation split
-
         handles = set()
 
         train_root_path = os.path.join(self.dataset_path, self.language, "train")
@@ -348,29 +343,3 @@ class MLSDataset(Dataset):
             dataset_lookup_entry["chapterid"],
             idx,
         )  # type: ignore
-
-
-if __name__ == "__main__":
-    DATASET_PATH = "/Volumes/pherkel/SWR2-ASR"
-    LANGUAGE = "mls_german_opus"
-    split = Split.DEV
-    DOWNLOAD = False
-
-    dataset = MLSDataset(DATASET_PATH, LANGUAGE, split, download=DOWNLOAD)
-
-    dataloader = DataLoader(
-        dataset,
-        batch_size=1,
-        shuffle=True,
-        collate_fn=DataProcessing(
-            "train", CharTokenizer.from_file("data/tokenizers/char_tokenizer_german.json")
-        ),
-    )
-
-    for batch in dataloader:
-        print(batch)
-        break
-
-    print(len(dataset))
-
-    print(dataset[0])
diff --git a/swr2_asr/utils/tokenizer.py b/swr2_asr/utils/tokenizer.py
index 22569eb..1cc7b84 100644
--- a/swr2_asr/utils/tokenizer.py
+++ b/swr2_asr/utils/tokenizer.py
@@ -120,15 +120,3 @@ class CharTokenizer:
                     load_tokenizer.char_map[char] = int(index)
                     load_tokenizer.index_map[int(index)] = char
         return load_tokenizer
-
-
-if __name__ == "__main__":
-    tokenizer = CharTokenizer.train("/Volumes/pherkel 1/SWR2-ASR", "mls_german_opus")
-    print(tokenizer.char_map)
-    print(tokenizer.index_map)
-    print(tokenizer.get_vocab_size())
-    print(tokenizer.get_blank_token())
-    print(tokenizer.get_unk_token())
-    print(tokenizer.get_space_token())
-    print(tokenizer.encode("hallo welt"))
-    print(tokenizer.decode([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
-- 
cgit v1.2.3