You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

176 lines
5.1 KiB
Python

import urllib.request as urlreq
from bs4 import BeautifulSoup
import zipfile
import time
import os
import sys
blacklist = ['b.thumbs.redditmedia.com', 'reddit.com']
dl_dir = './.cache/'
img_ext = ['jpg', 'jpeg', 'png'] # define the urls we are searching for
hdr = { # request header
'User-Agent': """Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko)
Chrome/23.0.1271.64 Safari/537.11""",
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 'Accept-Encoding': 'none', 'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'}
def print_progress(iteration, total, prefix='', suffix='', decimals=1, length=100, fill=''):
percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
filled_length = int(length * iteration // total)
bar = fill * filled_length + '-' * (length - filled_length)
print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end='\r')
# Print New Line on Complete
if iteration == total:
print()
def spinning_cursor():
while True:
for cursor in '|/-\\':
yield cursor
def get_extension(fstring):
return fstring.split('.')[-1].lower()
def get_soup4url(url):
""" Returns a soup for the url with 10 retrys """
req = urlreq.Request(url, headers=hdr)
html = None
for x in range(0, 10):
try:
html = urlreq.urlopen(req).read()
break
except Exception as e:
print('[-]', e)
if html:
soup = BeautifulSoup(html, "lxml")
return soup
return False
def has_source(tag):
if tag.has_attr('src'):
try:
return get_extension(tag['src']) in img_ext
except IndexError or KeyError:
return False
elif tag.has_attr('data-url'):
try:
tag['src'] = tag['data-url']
return get_extension(tag['src']) in img_ext
except IndexError or KeyError:
return False
else:
return False
def get_next_url(baseurl, url):
ids = []
soup = get_soup4url(url)
if not soup:
return False
for t in soup.find_all(has_source):
if 'redditmedia' not in t['src']:
try:
fname = t['data-fullname']
ids.append(fname)
except KeyError:
pass
ids = [_id for _id in ids if _id]
if len(ids) == 0:
return False
_id = ids[-1]
next_url = '{}/?after={}'.format(baseurl, _id)
return next_url
def get_img4site(url):
soup = get_soup4url(url)
if not soup:
return False
ret = []
sys.stdout.write('.')
sys.stdout.flush()
for t in soup.find_all(has_source):
try:
if 'redditmedia' not in t['src']:
img = t['src']
if 'http' not in img.split('/')[0] and '//' not in img.split('.')[0]:
img = url + img
if 'http' not in img.split('/')[0]:
img = 'http:' + img
if img.strip('http://').strip('https://').split('/')[0] in blacklist:
img = None
if img:
ret.append(img)
except KeyError:
pass
return ret
def get_img4sub(url, length=-1):
baseurl = url
imgs = []
print('[ ] 1/2 Getting images...')
if length >= 0:
for x in range(length):
time.sleep(0.1) # we don't want to flood with requests
imgs.extend(get_img4site(url))
url = get_next_url(baseurl, url)
if not url:
break
sys.stdout.write('\b')
else:
while url:
time.sleep(0.1) # we don't want to flood with requests
imgs.extend(get_img4site(url))
url = get_next_url(baseurl, url)
return imgs
def download_images(imgs, zfile):
count = 0
imgcount = len(imgs)
print('[ ] Downloading %s images' % imgcount)
if not os.path.isdir(dl_dir):
os.mkdir(dl_dir)
print_progress(count, imgcount, prefix="2/2 Downloading: ", suffix="Complete")
for img in imgs:
print_progress(count+1, imgcount, prefix="2/2 Downloading: ", suffix="Complete")
imgname = img.split('/')[-1]
name = dl_dir + imgname
if os.path.isfile(name):
continue
f = open(name, "wb")
req = urlreq.Request(img, headers=hdr)
image = urlreq.urlopen(req)
f.write(image.read())
f.close()
zfile.write(name, imgname, zipfile.ZIP_DEFLATED)
try:
os.remove(name)
except FileNotFoundError or PermissionError:
pass
time.sleep(0.1) # no don't penetrate
count += 1
def download_subreddit(sub):
mode = 'w'
if os.path.isfile(sub + '.zip'):
mode = 'a'
url = 'https://old.reddit.com/r/%s/' % sub
imgs = get_img4sub(url)
zfile = zipfile.ZipFile('%s.zip' % sub, mode)
download_images(imgs, zfile)
zfile.close()
if __name__ == '__main__':
download_subreddit('Animewallpaper')