Waxholm IPA -> Original labels
To play with pyshiro
IPA_TOK = """
{
"": 4,
"ɖ": 5,
"ɭ": 6,
"ɳ": 7,
"ʂ": 8,
"ʈ": 9,
"</s>": 2,
"<pad>": 0,
"<s>": 1,
"<unk>": 3,
"a": 10,
"ɑː": 11,
"b": 12,
"d": 13,
"e": 14,
"ə": 15,
"eː": 16,
"f": 17,
"ɡ": 18,
"h": 19,
"ɪ": 20,
"iː": 21,
"j": 22,
"k": 23,
"l": 24,
"m": 25,
"n": 26,
"ŋ": 27,
"ʊ": 28,
"uː": 29,
"p": 30,
"r": 31,
"s": 32,
"ɧ": 33,
"t": 34,
"ɕ": 35,
"ɵ": 36,
"ʉː": 37,
"v": 38,
"ʏ": 39,
"yː": 40,
"ɛ": 41,
"æː": 42,
"æ": 43,
"ɛː": 44,
"œ": 45,
"œ̞ː": 46,
"œ̞": 47,
"øː": 48,
"ɔ": 49,
"oː": 50,
"<ha>": 51,
"<hes>": 52,
"<kl>": 53,
"<pa>": 54,
"<sm>": 55,
"<v>": 56,
"|": 57
}
"""
WH_TOK = """
{
"": 4,
"2D": 5,
"2L": 6,
"2N": 7,
"2S": 8,
"2T": 9,
"</s>": 2,
"<pad>": 0,
"<s>": 1,
"<unk>": 3,
"A": 10,
"A:": 11,
"B": 12,
"D": 13,
"E": 14,
"E0": 15,
"E:": 16,
"F": 17,
"G": 18,
"H": 19,
"I": 20,
"I:": 21,
"J": 22,
"K": 23,
"L": 24,
"M": 25,
"N": 26,
"NG": 27,
"O": 28,
"O:": 29,
"P": 30,
"R": 31,
"S": 32,
"SJ": 33,
"T": 34,
"TJ": 35,
"U": 36,
"U:": 37,
"V": 38,
"Y": 39,
"Y:": 40,
"[": 41,
"[3": 42,
"[4": 43,
"[:": 44,
"\\": 45,
"\\3": 46,
"\\4": 47,
"\\:": 48,
"]": 49,
"]:": 50,
"ha": 51,
"hes": 52,
"kl": 53,
"pa": 54,
"sm": 55,
"v": 56,
"|": 57
}
"""
ORIG = """
{
"": 4,
"2D": 5,
"2L": 6,
"2N": 7,
"2S": 8,
"2T": 9,
"</s>": 2,
"<pad>": 0,
"<s>": 1,
"<unk>": 3,
"A": 10,
"A:": 11,
"B": 12,
"D": 13,
"E": 14,
"E0": 15,
"E:": 16,
"F": 17,
"G": 18,
"H": 19,
"I": 20,
"I:": 21,
"J": 22,
"K": 23,
"L": 24,
"M": 25,
"N": 26,
"NG": 27,
"O": 28,
"O:": 29,
"P": 30,
"R": 31,
"S": 32,
"SJ": 33,
"T": 34,
"TJ": 35,
"U": 36,
"U:": 37,
"V": 38,
"Y": 39,
"Y:": 40,
"Ä": 41,
"Ä3": 42,
"Ä4": 43,
"Ä:": 44,
"Ö": 45,
"Ö3": 46,
"Ö4": 47,
"Ö:": 48,
"Å": 49,
"Å:": 50,
"ha": 51,
"öh": 52,
"kl": 53,
"pa": 54,
"sm": 55,
"v": 56,
"|": 57
}
"""
import json
SPECIAL = {"<pad>", "<s>", "</s>", "<unk>", "", "|"}
def load_ipa_to_waxholm(ipa_string, wax_string):
ipa = json.loads(ipa_string)
wax = json.loads(wax_string)
id2wax = {v: k for k, v in wax.items()} # bridge through shared IDs
return {sym: id2wax[i] for sym, i in ipa.items()}
mapping = load_ipa_to_waxholm(IPA_TOK, ORIG)
def ipa_to_waxholm(text, mapping, sil="sil"):
vocab = sorted((s for s in mapping if s not in SPECIAL), key=len, reverse=True)
out = [sil]
for word in text.replace("|", " ").split():
if word in SPECIAL:
continue
i = 0
while i < len(word):
for sym in vocab: # longest match wins
if word.startswith(sym, i):
out.append(mapping[sym])
i += len(sym)
break
else:
raise ValueError(f"unknown IPA {word[i:]!r} in {word!r}")
out.append(sil)
return out
" ".join(ipa_to_waxholm("<hes> fœ̞sta tɑːlaʁə <pa> kaʁʊliːn ɕyːbə koːɖeː vɑːsoːɡuːɖ".replace("ʁ", "r"), mapping))
vocab = sorted((s for s in mapping if s not in SPECIAL), key=len, reverse=True)
new_mapping = {sym: mapping[sym] for sym in vocab}
new_mapping