Skip to content

Commit

Permalink
use context setter for Irish Times (requires at least TLSv1_2)
Browse files Browse the repository at this point in the history
  • Loading branch information
jimregan committed Nov 16, 2019
1 parent bc012db commit a24ed38
Showing 1 changed file with 3 additions and 0 deletions.
3 changes: 3 additions & 0 deletions Lib/corpuscrawler/crawl_ga.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from __future__ import absolute_import, print_function, unicode_literals
import re
import sys
import ssl
from corpuscrawler.util import (
crawl_udhr, clean_paragraphs, cleantext, extract, striptags, urlpath
)
Expand Down Expand Up @@ -117,6 +118,7 @@ def _irishtimes_section_list(crawler, out, url):


def crawl_irishtimes(crawler, out):
crawler.set_context(ssl.SSLContext(ssl.PROTOCOL_TLSv1_2))
start = 'https://www.irishtimes.com/culture/treibh'
pubdatere1 = re.compile(
r'<meta itemprop="datePublished" content="([^"]*)"/>')
Expand Down Expand Up @@ -148,6 +150,7 @@ def crawl_irishtimes(crawler, out):
html.split('<div class="article_bodycopy">')[1]):
cleaned = cleantext(paragraph)
out.write(cleaned + '\n')
crawler.set_context(ssl.SSLContext(ssl.PROTOCOL_TLSv1))


def crawl_chg(crawler, out):
Expand Down

0 comments on commit a24ed38

Please sign in to comment.