[cy] add basic Welsh crawler

[cwd24]

Tested with the following local change, to reduce redirects when run from the UK:

diff --git a/Lib/corpuscrawler/util.py b/Lib/corpuscrawler/util.py index f22e6b9..e6b294c 100644 --- a/Lib/corpuscrawler/util.py +++ b/Lib/corpuscrawler/util.py @@ -316,7 +316,7 @@ class Crawler(object):

using the same site structure for all languages.

def crawl_bbc_news(crawler, out, urlprefix):

sitemap = {'http://www.bbc.com/burmese/world-41146701': None}

• sitemap = crawler.fetch_sitemap('http://www.bbc.com/sitemap.xml')
• sitemap = crawler.fetch_sitemap('http://www.bbc.co.uk/sitemap.xml') pubdate_regex = re.compile(r'"dateModified":\s*"([0-9T:+-]{25})"') for url in sorted(sitemap.keys()): if not urlpath(url).startswith(urlprefix):

[brawer]

Thank you!