sayitinsaami.yle.fi scraper

JSON = "http://sayitinsaami.yle.fi/js/fraasit.js?2019-02-15"

import requests

rawdata = requests.get(JSON)

to_clean = rawdata.text
to_clean = to_clean.replace(" f:", ' "f":').replace(" fi:", ' "fi":').replace(" sv:", ' "sv":').replace(" en:", ' "en":').replace(" sme:", ' "sme":').replace(" smn:", ' "smn":').replace(" sms:", ' "sms":').replace(" cats:", ' "cats":')

if to_clean.startswith("FRAASIT = "):
    to_clean = to_clean[10:]
to_clean = to_clean.strip()
if to_clean.endswith("},\n]"):
    to_clean = to_clean.replace("},\n]", "}]")

import json
data = json.loads(to_clean)

with open("sayitinsaami_output.tsv", "w") as outf:
    for item in data:
        outf.write(f'{item["sme"]}\thttp://sanosesaameksi.katrikoivula.com/audio/sme/{item["f"]}.m4a\n')

!cat output.tsv|awk -F'\t' '{print $2}'|while read i;do f=$(echo $i|awk -F/ '{print $NF}'); if [ ! -e $f ];then wget -c $i;fi;done

with open("sayitinsaami.json", "w") as cleaned:
    json.dump(data, cleaned)

!ls *.m4a|zip sayitinsaami_audio.zip -@

!for f in *.m4a;do ffmpeg -i $f  -acodec pcm_s16le -ac 1 -ar 16000 $(basename $f .m4a).wav;done

!ls -al *.wav

text = []
audio = []
for item in data:
    text.append(item["sme"])
    audio.append(f'{item["f"]}.wav')

%%capture
!pip install datasets

%%capture
!pip install torchaudio librosa

from datasets import Dataset, Audio

/home/joregan/miniconda3/envs/hf_new/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

text = []
audio = []
with open("/home/joregan/sayitinsaami/sayitinsaami_output.tsv") as tsvf:
    for line in tsvf.readlines():
        parts = line.split("\t")
        text.append(parts[0])
        urlparts = parts[1].split("/")
        audio_path = "/home/joregan/sayitinsaami/" + urlparts[-1].replace("m4a", "wav").strip()
        audio.append(audio_path)

audio_ds = Dataset.from_dict({"text": text, "audio": audio}).cast_column("audio", Audio())

audio_ds[1]

{'text': 'Buorre beaivi!',
 'audio': {'path': '/home/joregan/sayitinsaami/102.wav',
  'array': array([-3.05175781e-05, -6.10351562e-05, -6.10351562e-05, ...,
          0.00000000e+00,  0.00000000e+00,  0.00000000e+00]),
  'sampling_rate': 16000}}

%%capture
!pip install --upgrade evaluate jiwer

def clean_text(batch):
    if "(duaali)" in batch["text"]:
        batch["text"] = batch["text"].replace("(duaali)", "").strip()
    if "..." in batch["text"]:
        batch["text"] = batch["text"].replace("...", "").strip()
    if batch["text"][-1] in "?!.":
        batch["text"] = batch["text"][:-1]
    return batch

audio_ds = audio_ds.map(clean_text)

Map:   0%|          | 0/888 [00:00<?, ? examples/s]Map: 100%|██████████| 888/888 [00:00<00:00, 11158.84 examples/s]

from transformers import pipeline
import torch

MODEL = "NbAiLab/whisper-large-sme"
LANG = "fi"

pipe = pipeline(task="automatic-speech-recognition", model=MODEL, chunk_length_s=30, device=0)
pipe.model.config.forced_decoder_ids = pipe.tokenizer.get_decoder_prompt_ids(language=LANG, task="transcribe")

import evaluate

metric = evaluate.load("wer")

Downloading builder script: 100%|██████████| 4.49k/4.49k [00:00<00:00, 2.94MB/s]

from transformers.pipelines.pt_utils import KeyDataset

from tqdm.auto import tqdm

preds = []

for out in tqdm(pipe(KeyDataset(audio_ds, "audio"))):
    preds.append(out["text"])

100%|██████████| 888/888 [11:46<00:00,  1.26it/s]

from evaluate import load

wer_metric = load("wer")

wer_ortho = 100 * wer_metric.compute(
    references=audio_ds["text"], predictions=preds
)
wer_ortho

69.94623655913979

def clean_for_wer(text):
    if text[-1] in "?!.":
        text = text[:-1]
    return text.lower().strip()

preds = [clean_for_wer(x) for x in preds]
refs = [clean_for_wer(x) for x in audio_ds["text"]]

wer_ortho = 100 * wer_metric.compute(
    references=refs, predictions=preds
)
wer_ortho

51.45161290322581