Add skipping HTTPS check

This commit is contained in:
Ivaylo Ivanov 2018-12-19 10:05:37 +01:00
parent d279e4531a
commit c6800ae591
1 changed files with 6 additions and 1 deletions

View File

@ -1,5 +1,7 @@
from requests import get
from requests.exceptions import RequestException
from requests.packages import urllib3
from urllib3.exceptions import InsecureRequestWarning
from contextlib import closing
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
@ -37,10 +39,13 @@ class Scanner:
chrome_options.add_argument("--window-size=1920x1080")
driver = webdriver.Chrome(chrome_options=chrome_options)
# Disable HTTPS request warnings
urllib3.disable_warnings(category=InsecureRequestWarning)
# Loop through all pages
for url in self.urls:
try:
with closing(get(url, stream=True)) as resp:
with closing(get(url, stream=True, verify=False)) as resp:
if self.response_is_correct(resp):
print("################################")
print("INFO: Scraping {0}".format(url))