|
from selenium import webdriver |
|
import time |
|
import requests |
|
import cookielib |
|
import urllib2 |
|
import logging |
|
import sys |
|
import random |
|
|
|
profile = webdriver.firefox.firefox_profile.FirefoxProfile() |
|
profile.set_preference( |
|
"general.useragent.override", |
|
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:26.0)" |
|
" Gecko/20100101 Firefox/26.0") |
|
driver = webdriver.Firefox(firefox_profile=profile) |
|
driver.get('http://www.morguefile.com/archive') |
|
|
|
|
|
|
|
cookie_jar = cookielib.LWPCookieJar() |
|
handlers = [ |
|
urllib2.HTTPHandler(), |
|
urllib2.HTTPSHandler(), |
|
] |
|
[h.set_http_debuglevel(1) for h in handlers] |
|
handlers.append(urllib2.HTTPCookieProcessor(cookie_jar)) |
|
#using selenium driver cookies, returns a list of dictionaries |
|
cookies = driver.get_cookies() |
|
opener = urllib2.build_opener(*handlers) |
|
opener.addheaders = [( |
|
'User-agent', |
|
'Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:26.0) ' |
|
'Gecko/20100101 Firefox/26.0' |
|
)] |
|
logger = logging.getLogger("cookielib") |
|
logger.addHandler(logging.StreamHandler(sys.stdout)) |
|
logger.setLevel(logging.DEBUG) |
|
for item in cookies: |
|
opener.addheaders.append(('Cookie', '{}={}'.format( |
|
item['name'], item['value'] |
|
))) |
|
logger.info('{}={}'.format(item['name'], item['value'])) |
|
|
|
|
|
|
|
for i in range(10): |
|
driver.execute_script("window.scrollTo(0,document.body.scrollHeight);") |
|
time.sleep(1) |
|
|
|
ALL_IMAGES_ELE = driver.find_elements_by_xpath('//div[@id="mfile_browser"]//div[@id="img_thumbs"]//span[@class="mthumb_brdr"]//a') |
|
|
|
for index,img in enumerate(ALL_IMAGES_ELE): |
|
print index |
|
src = img.get_attribute('href') |
|
src = src.replace("display", "download") |
|
print src |
|
resp = opener.open(src) |
|
content = resp.read() |
|
with open(str(index*random.random())+"_ext.jpg", "wb") as code: |
|
code.write(content) |
|
print "."*80 |
|
|
|
driver.close() |
