Canuint.ie scraper
Pieces towards a scraper
LANDING = "https://www.canuint.ie/ga/"
import requests
from bs4 import BeautifulSoup
def get_regions(url):
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
regions = []
maplist = soup.find('div', class_='mapList')
for area in maplist.find_all('li', {"class": "area"}):
data = {}
data['lat'] = area.get('data-lat')
data['lon'] = area.get('data-lon')
data['name'] = area.get('data-name')
for link in area.find_all('a'):
if link.text == data['name']:
data['url'] = 'https://www.canuint.ie' + link.get('href')
regions.append(data)
return regions
areas = get_regions(LANDING)
def get_transcripts(url):
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
transcripts = []
for recording in soup.find_all('li', class_='recording'):
if recording.get('data-is-transcribed') != "1":
continue
for link in recording.find_all('a'):
if link.text.strip() == 'Léigh & éist':
transcripts.append('https://www.canuint.ie' + link.get('href'))
return transcripts
get_transcripts(areas[1]['url'])