Commit c6800ae5 authored by Ivaylo Ivanov's avatar Ivaylo Ivanov

Add skipping HTTPS check

parent d279e453
from requests import get
from requests.exceptions import RequestException
from requests.packages import urllib3
from urllib3.exceptions import InsecureRequestWarning
from contextlib import closing
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
......@@ -37,10 +39,13 @@ class Scanner:
chrome_options.add_argument("--window-size=1920x1080")
driver = webdriver.Chrome(chrome_options=chrome_options)
# Disable HTTPS request warnings
urllib3.disable_warnings(category=InsecureRequestWarning)
# Loop through all pages
for url in self.urls:
try:
with closing(get(url, stream=True)) as resp:
with closing(get(url, stream=True, verify=False)) as resp:
if self.response_is_correct(resp):
print("################################")
print("INFO: Scraping {0}".format(url))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment