Convert IPA to Waxholm
For generating a lexicon for MFA
VOWELS = {"A", "A:", "E", "E0", "E:", "I", "I:", "O", "O:", "U", "U:", "Y", "Y:",
"Ä", "Ä3", "Ä4", "Ä:", "Å", "Å:", "Ö", "Ö3", "Ö4", "Ö:"}
VOICELESS_STOPS = {"k", "t", "p", "2t"}
IPA2WAX = {
'<hes>': 'öh', '<ha>': 'ha', '<kl>': 'kl', '<pa>': 'pa', '<sm>': 'sm', '<v>': 'v',
'œ̞ː': 'Ö3', 'ɑː': 'A:', 'eː': 'E:', 'iː': 'I:', 'uː': 'O:', 'ʉː': 'U:', 'yː': 'Y:',
'æː': 'Ä3', 'ɛː': 'Ä:', 'œ̞': 'Ö4', 'øː': 'Ö:', 'oː': 'Å:',
'ɖ': '2D', 'ɭ': '2L', 'ɳ': '2N', 'ʂ': '2S', 'ʈ': '2T',
'a': 'A', 'b': 'B', 'd': 'D', 'e': 'E', 'ə': 'E0', 'f': 'F', 'ɡ': 'G', 'h': 'H',
'ɪ': 'I', 'j': 'J', 'k': 'K', 'l': 'L', 'm': 'M', 'n': 'N', 'ŋ': 'NG', 'ʊ': 'O',
'p': 'P', 'r': 'R', 's': 'S', 'ɧ': 'SJ', 't': 'T', 'ɕ': 'TJ', 'ɵ': 'U', 'v': 'V',
'ʏ': 'Y', 'ɛ': 'Ä', 'æ': 'Ä4', 'œ': 'Ö', 'ɔ': 'Å', ' ': 'SIL'
}
SPECIAL = {"<pad>", "<s>", "</s>", "<unk>", ""}
_VOCAB = sorted(IPA2WAX, key=len, reverse=True)
def ipa_to_phones(text):
phones = []
for word in text.replace("|", " ").split():
if word in SPECIAL:
continue
i = 0
while i < len(word):
for sym in _VOCAB:
if word.startswith(sym, i):
phones.append(IPA2WAX[sym])
i += len(sym)
break
else:
raise ValueError(f"unknown IPA {word[i:]!r} in {word!r}")
return phones
ipa_to_phones("ɔtɪ")
sample = '{"word": "på", "word_start_ms": 52780, "word_end_ms": 52840, "tokens": [{"text": "poː", "start_ms": 52780, "end_ms": 52860, "annotations": {}}], "has_epenthetic": false, "annotations": {"match_method": "exact_timing"}, "metadata": {"id": "CA37298"}, "ipa": "poː", "dict_match_score": 1.0}'
import json
data = json.loads(sample)
data.keys()
import json
lines = []
with open("/tmp/storspigg_word_pronunciations.jsonl") as inf:
for line in inf.readlines():
sample = json.loads(line)
if sample["dict_match_score"] != 1.0:
continue
line = f'{sample["word"]}\t{" ".join(ipa_to_phones(sample["ipa"]))}'
if not line in lines:
lines.append(line)
with open("/tmp/storspigg_waxholm.dict", "w") as outf:
for line in sorted(lines):
print(line, file=outf)
numbers = {}
with open("sp_numlist") as f:
for line in f.readlines():
line = line.strip()
if not ' ' in line:
continue
key = line.replace(" ", "")
numbers[key] = line
numbers
from pathlib import Path
for text_file in Path("/tmp/storpigg/mfa-corpus").glob("**/*.lab"):
with open(text_file) as f:
text = f.read().strip()
if not text:
continue
outwords = []
words = text.split()
for word in words:
if word in numbers:
outwords.append(numbers[word])
else:
outwords.append(word)
outtext = " ".join(outwords)
if outtext != text:
with open(text_file, "w") as f:
f.write(outtext)
with open("tmpinlist") as f, open("tmpoutlist", "w") as of:
for line in f.readlines():
line = line.strip()
if not line:
continue
parts = line.split("\t")
word = parts[0]
ipa = parts[1]
of.write(f"{word}\t{' '.join(ipa_to_phones(ipa))}\n")
ipa_to_phones("deːmɪts heliːn<v>")