Convert Folkets lexicon pronunciations to JSON
Ignoring everything else; incomplete
Based on this
Set up ICU
!pip install pyicu
Get data
Example entry:
<word class="pp" comment="endast vid sifferuttryck" lang="sv" value="à"><translation comment="used only with numerical expressions" value="at" />
<phonetic soundFile="à.swf" value="a" />
<see type="saldo" value="à||à..1||à..pp.1" />
<example value="två koppar kaffe à 8 kronor (styck)"><translation value="two cups of coffee at 8 kronor (each)" />
</example>
<definition value="till ett pris av"><translation value="at a price of" />
</definition>
</word>
!wget https://folkets-lexikon.csc.kth.se/folkets/folkets_sv_en_public.xml
from lxml import etree
tree = etree.parse("folkets_sv_en_public.xml")
audio = {}
phon = {}
compounds = {}
for entry in tree.iter("word"):
value = entry.get("value")
word = value
if not isinstance(word, str):
word = str(entry)
phonetic = entry.find("phonetic")
if phonetic is None:
continue
if not isinstance(phonetic, etree._Element):
print(phonetic)
continue
if "|" in value:
compound = value.split("|")
word = value.replace("|", "")
if word in compounds:
if compound == compounds[word]:
continue
else:
print("Error:", compound, compounds[word])
compounds[word] = compound
phon_value = phonetic.get("value")
if phon_value:
if not entry in phon:
phon[word] = list()
if not phon_value in phon[word]:
phon[word].append(phon_value)
sound_file = phonetic.get("soundFile")
if sound_file:
if not entry in audio:
audio[word] = set()
audio[word].add(sound_file)
phon = {k: v[0] for k, v in phon.items()}
phon
Set up transliterator
TRANSLIT_SV = """
r '+' n → ɳ ;
r '+' s → ʂ ;
r '+' l → ɭ ;
r '+' t → ʈ ;
r '+' d → ɖ ;
A \: → ˈɑː ;
a \: → ɑː ;
A → ˈa ;
a → a ;
I \: → ˈiː ;
i \: → iː ;
I → ˈɪ ;
i → ɪ ;
E \: → ˈeː ;
e \: → eː ;
E → ˈɛ ;
e → ɛ ;
Å \: → ˈoː ;
å \: → oː ;
Å → ˈɔ ;
å → ɔ ;
\@ → ŋ ;
2 → ² ;
\: → ː ;
\$ → ɧ ;
g → ɡ ;
"""
import icu
def transliterator_from_rules(name, rules):
fromrules = icu.Transliterator.createFromRules(name, rules)
icu.Transliterator.registerInstance(fromrules)
return icu.Transliterator.createInstance(name)
swelex_trans = transliterator_from_rules("swelex_trans", TRANSLIT_SV)
assert swelex_trans.transliterate('asistAn:s') == "asɪstˈanːs"
assert swelex_trans.transliterate("²Ab:år:e") == "²ˈabːɔrːɛ"
assert swelex_trans.transliterate("abånemA@:") == "abɔnɛmˈaŋː"
assert swelex_trans.transliterate("abÅr+t:") == "abˈɔʈː"
swelex_trans.transliterate("abÅr+t:")
def collapse_available_fields(data):
output = []
for i in range(1, 10):
if data[f"available_field{i}"] != "":
output.append(data[f"available_field{i}"])
del data[f"available_field{i}"]
data["available_fields"] = output
return data
def collapse_transliterations(data, transliterator):
output = []
for i in range(1, 5):
if data[f"transliteration{i}"] != "":
tmp = {}
tmp["transliteration"] = data[f"transliteration{i}"]
tmp["ipa"] = transliterator.transliterate(data[f"transliteration{i}"])
tmp["certainty"] = data[f"certainty_trans_{i}"]
tmp["status"] = data[f"status_trans_{i}"]
tmp["language_code"] = data[f"language_code_trans_{i}"]
output.append(tmp)
del data[f"transliteration{i}"]
del data[f"certainty_trans_{i}"]
del data[f"status_trans_{i}"]
del data[f"language_code_trans_{i}"]
data["transliterations"] = output
return data
import json
import io
with open("svlex.json", "w") as outf:
swelexf = io.StringIO(data["sv"])
swelex = csv.DictReader(swelexf, delimiter=';', fieldnames=field_names, quoting=csv.QUOTE_NONE)
for row in swelex:
row["decomp"] = [f for f in row["decomp"].split("+") if f != ""]
row = collapse_available_fields(row)
row = collapse_transliterations(row, swelex_trans)
jsonstr = json.dumps(row)
outf.write(jsonstr + "\n")
data["da"] = data["da"].replace('\r', '')
for lang in ["no", "da"]:
with open(f"{lang}lex.json", "w", newline='') as outf:
swelexf = io.StringIO(data[lang])
swelex = csv.DictReader(swelexf, delimiter=';', fieldnames=field_names, quoting=csv.QUOTE_NONE)
for row in swelex:
row["decomp"] = [f for f in row["decomp"].split("+") if f != ""]
row = collapse_available_fields(row)
row = collapse_transliterations(row, nstlex_trans[lang])
jsonstr = json.dumps(row)
outf.write(jsonstr + "\n")