Trivernis
/
reddit-riddle
mirror of https://github.com/Trivernis/reddit-riddle.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added pipfile and shebag line

Browse Source 
- added pipfile for pipenv
- added shebag line to execute the file directly
- (converted to unix line endings)
pull/6/head
Trivernis 5 years ago
parent 77468223e2
commit 9fcfe625d4
5 changed files with 418 additions and 293 deletions
Show Stats Download Patch File Download Diff File

13
Pipfile
Unescape Escape View File

@ -0,0 +1,13 @@
[[source]]
url = "https://pypi.python.org/simple"
verify_ssl = true
name = "pypi"
[packages]
PyYAML = "*"
praw = "*"
[dev-packages]
[requires]
python_version = "3.7"

111
Pipfile.lock generated
Unescape Escape View File

@ -0,0 +1,111 @@
{
"_meta": {
"hash": {
"sha256": "e030a28963c27bc726b49ad8bc68cf9648c19fde4e1a5a76d1fc8a5955b06cd1"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.7"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.python.org/simple",
"verify_ssl": true
}
]
},
"default": {
"certifi": {
"hashes": [
"sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
"sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
],
"version": "==2019.9.11"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"idna": {
"hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
],
"version": "==2.8"
},
"praw": {
"hashes": [
"sha256:2e5c98e49fe60e5308255ed147b670d350f98281f84f582df30f87de727b6de2",
"sha256:cb8f85541ad4c6b10214ef9639acccfb5fed7ffee977be169b85357d2d2ea6d9"
],
"index": "pypi",
"version": "==6.4.0"
},
"prawcore": {
"hashes": [
"sha256:25dd14bf121bc0ad2ffc78e2322d9a01a516017105a5596cc21bb1e9a928b40c",
"sha256:ab5558efb438aa73fc66c4178bfc809194dea3ce2addf4dec873de7e2fd2824e"
],
"version": "==1.0.1"
},
"pyyaml": {
"hashes": [
"sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
"sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4",
"sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8",
"sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696",
"sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34",
"sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9",
"sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73",
"sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299",
"sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b",
"sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae",
"sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681",
"sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41",
"sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"
],
"index": "pypi",
"version": "==5.1.2"
},
"requests": {
"hashes": [
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
],
"version": "==2.22.0"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"version": "==1.12.0"
},
"update-checker": {
"hashes": [
"sha256:59cfad7f9a0ee99f95f1dfc60f55bf184937bcab46a7270341c2c33695572453",
"sha256:70e39446fccf77b21192cf7a8214051fa93a636dc3b5c8b602b589d100a168b8"
],
"version": "==0.16"
},
"urllib3": {
"hashes": [
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
],
"version": "==1.25.6"
},
"websocket-client": {
"hashes": [
"sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9",
"sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a"
],
"version": "==0.56.0"
}
},
"develop": {}
}

22
default-config.yaml
Unescape Escape View File

@ -1,12 +1,12 @@
# user app credentials # user app credentials
credentials: credentials:
client_id: your app-client id client_id: your app-client id
client_secret: your app-client secret client_secret: your app-client secret
# required extension of the file to be downloaded # required extension of the file to be downloaded
image-extensions: image-extensions:
- png - png
- jpg - jpg
- jpeg - jpeg
min-size: 5 # minimum size in kilobytes min-size: 5 # minimum size in kilobytes

2
requirements.txt
Unescape Escape View File

@ -1,2 +1,2 @@
PyYaml PyYaml
praw praw

563
riddle.py
Unescape Escape View File

@ -1,281 +1,282 @@
#!/usr/bin/env python3
# coding: utf-8
# author: u/Trivernis # coding: utf-8
import os # author: u/Trivernis
import shutil import os
import yaml import shutil
import praw import yaml
import optparse import praw
import zipfile import optparse
import urllib.request as urlreq import zipfile
import urllib.request as urlreq
user_agent = 'python:riddle:3.0 (by u/Trivernis)' # the reddit api user-agent
img_ext = ['jpg', 'jpeg', 'png'] # default used extensions to filter for images user_agent = 'python:riddle:3.0 (by u/Trivernis)' # the reddit api user-agent
min_size = 5 # minimum size in kilobytes. changeable in settings img_ext = ['jpg', 'jpeg', 'png'] # default used extensions to filter for images
min_size = 5 # minimum size in kilobytes. changeable in settings
def assert_dir_exist(dirpath):
""" def assert_dir_exist(dirpath):
Creates the directory if it doesn't exist """
:param dirpath: path to the directory Creates the directory if it doesn't exist
:return: None :param dirpath: path to the directory
""" :return: None
if not os.path.exists(dirpath): """
os.mkdir(dirpath) if not os.path.exists(dirpath):
os.mkdir(dirpath)
def download_file(url: str, dest: str, progressbar = None):
""" def download_file(url: str, dest: str, progressbar = None):
Downloads a url to a file """
:param url: download url Downloads a url to a file
:param dest: download destination :param url: download url
:param progressbar: The progressbar instance to clear it before writing an error message :param dest: download destination
:return: Success? :param progressbar: The progressbar instance to clear it before writing an error message
""" :return: Success?
f = open(dest, "wb") """
req = urlreq.Request(url) f = open(dest, "wb")
success = False req = urlreq.Request(url)
try: success = False
image = urlreq.urlopen(req) try:
f.write(image.read()) image = urlreq.urlopen(req)
success = True f.write(image.read())
except ConnectionError: success = True
if progressbar: except ConnectionError:
progressbar.clear() if progressbar:
print('\r[-] Connection Error') progressbar.clear()
except urlreq.HTTPError as err: print('\r[-] Connection Error')
if progressbar: except urlreq.HTTPError as err:
progressbar.clear() if progressbar:
print('\r[-] HTTPError for %s: %s' % (url, err)) progressbar.clear()
except urlreq.URLError as err: print('\r[-] HTTPError for %s: %s' % (url, err))
if progressbar: except urlreq.URLError as err:
progressbar.clear() if progressbar:
print('\r[-] URLError for %s: %s' % (url, err)) progressbar.clear()
f.close() print('\r[-] URLError for %s: %s' % (url, err))
try: f.close()
file_size = round(os.path.getsize(dest) / 1000) try:
if not success: file_size = round(os.path.getsize(dest) / 1000)
os.remove(dest) if not success:
elif file_size < min_size: os.remove(dest)
os.remove(dest) elif file_size < min_size:
success = False os.remove(dest)
if progressbar: success = False
progressbar.clear() if progressbar:
print('\r[-] Removed %s: Too small (%s kb)' % (dest, file_size)) progressbar.clear()
except IOError as err: print('\r[-] Removed %s: Too small (%s kb)' % (dest, file_size))
if progressbar: except IOError as err:
progressbar.clear() if progressbar:
print('\r[-] Error when removing file %s: %s' % (dest, err)) progressbar.clear()
return success print('\r[-] Error when removing file %s: %s' % (dest, err))
return success
class ProgressBar:
""" class ProgressBar:
A simple progressbar. """
""" A simple progressbar.
"""
def __init__(self, total=100, prefix='', suffix='', length=50, fill='█'):
self.prefix = prefix def __init__(self, total=100, prefix='', suffix='', length=50, fill='█'):
self.suffix = suffix self.prefix = prefix
self.fill = fill self.suffix = suffix
self.length = length self.fill = fill
self.total = total self.length = length
self.progress = 0 self.total = total
self.textlength = 0 self.progress = 0
self.textlength = 0
def tick(self):
""" def tick(self):
Next step of the progressbar. The stepwidth is always 1. """
:return: Next step of the progressbar. The stepwidth is always 1.
""" :return:
self.progress += 1 """
self._print_progress() self.progress += 1
self._print_progress()
def setprogress(self, progress: float):
""" def setprogress(self, progress: float):
Set the progress of the bar. """
:param progress: progress in percent Set the progress of the bar.
:return: None :param progress: progress in percent
""" :return: None
self.progress = progress """
self._print_progress() self.progress = progress
self._print_progress()
def _print_progress(self):
iteration = self.progress def _print_progress(self):
total = self.total iteration = self.progress
prefix = self.prefix total = self.total
suffix = self.suffix prefix = self.prefix
suffix = self.suffix
percent = ("{0:." + str(1) + "f}").format(100 * (iteration / float(total)))
filled_length = int(self.length * iteration // total) percent = ("{0:." + str(1) + "f}").format(100 * (iteration / float(total)))
bar = self.fill * filled_length + '-' * (self.length - filled_length) filled_length = int(self.length * iteration // total)
textout = '\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix) bar = self.fill * filled_length + '-' * (self.length - filled_length)
print(textout, end='\r') textout = '\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix)
self.textlength = len(textout) print(textout, end='\r')
# Print new line on complete self.textlength = len(textout)
if iteration == total: # Print new line on complete
print() if iteration == total:
print()
def clear(self):
""" def clear(self):
clear last progress output """
:return: clear last progress output
""" :return:
print(' '*self.textlength, end='\r') """
print(' '*self.textlength, end='\r')
def parser_init():
""" def parser_init():
Initializes and parses command line arguments """
:return: dict, list Initializes and parses command line arguments
""" :return: dict, list
parser = optparse.OptionParser(usage="usage: %prog [options] [subreddits]") """
parser.add_option('-c', '--count', dest='count', parser = optparse.OptionParser(usage="usage: %prog [options] [subreddits]")
type='int', default=None, parser.add_option('-c', '--count', dest='count',
help="""The number of images to download for each subreddit. type='int', default=None,
If not set it is the maximum fetchable number.""") help="""The number of images to download for each subreddit.
parser.add_option('-o', '--output', dest='output', If not set it is the maximum fetchable number.""")
type='str', default=None, parser.add_option('-o', '--output', dest='output',
help="""The name of the output folder. type='str', default=None,
If none is specified, it\'s the subreddits name.""") help="""The name of the output folder.
parser.add_option('-z', '--zip', dest='zip', If none is specified, it\'s the subreddits name.""")
action='store_true', default=False, parser.add_option('-z', '--zip', dest='zip',
help='Stores the images in a zip file if true') action='store_true', default=False,
parser.add_option('--nsfw', dest='nsfw', help='Stores the images in a zip file if true')
action='store_true', default=False, parser.add_option('--nsfw', dest='nsfw',
help='If set nsfw-content is also downloaded.') action='store_true', default=False,
parser.add_option('--lzma', dest='lzma', help='If set nsfw-content is also downloaded.')
action='store_true', default=False, parser.add_option('--lzma', dest='lzma',
help='If set the lzma-compression module is used.') action='store_true', default=False,
return parser.parse_args() help='If set the lzma-compression module is used.')
return parser.parse_args()
def get_images(reddit_client: praw.Reddit, subreddit: str, limit: int, nsfw: bool = False):
""" def get_images(reddit_client: praw.Reddit, subreddit: str, limit: int, nsfw: bool = False):
Uses the reddit api to fetch all image posts """
:param reddit_client: instance of the reddit client Uses the reddit api to fetch all image posts
:param subreddit: reddit subreddit name :param reddit_client: instance of the reddit client
:param limit: max images to download. if set to None the maximum fetchable amout is used. :param subreddit: reddit subreddit name
:param nsfw: if set to true, nsfw-images won't be filtered :param limit: max images to download. if set to None the maximum fetchable amout is used.
:return: list of images :param nsfw: if set to true, nsfw-images won't be filtered
""" :return: list of images
print('[~] Fetching images for r/%s...' % subreddit) """
urls = [submission.url for submission in reddit_client.subreddit(subreddit).hot(limit=limit) print('[~] Fetching images for r/%s...' % subreddit)
if not submission.over_18 or nsfw] # fetches hot images and filters nsfw if set to false urls = [submission.url for submission in reddit_client.subreddit(subreddit).hot(limit=limit)
return [url for url in urls if url.split('.')[-1] in img_ext] if not submission.over_18 or nsfw] # fetches hot images and filters nsfw if set to false
return [url for url in urls if url.split('.')[-1] in img_ext]
def download_images(images: list, dl_dir: str):
""" def download_images(images: list, dl_dir: str):
Downloads a list of image urls to a folder """
:param images: list of image urls Downloads a list of image urls to a folder
:param dl_dir: destination directory :param images: list of image urls
:return: None :param dl_dir: destination directory
""" :return: None
imgcount = len(images) """
realcount = preexist = 0 imgcount = len(images)
print('[~] Downloading %s images to %s' % (imgcount, dl_dir)) realcount = preexist = 0
pb = ProgressBar(total=imgcount, prefix='[~] Downloading', suffix='Complete') print('[~] Downloading %s images to %s' % (imgcount, dl_dir))
assert_dir_exist(dl_dir) pb = ProgressBar(total=imgcount, prefix='[~] Downloading', suffix='Complete')
assert_dir_exist(dl_dir)
for img in images: # download each image if it doesn't exist
success = False for img in images: # download each image if it doesn't exist
imgname = img.split('/')[-1] success = False
name = os.path.join(dl_dir, imgname) imgname = img.split('/')[-1]
if not os.path.isfile(name): name = os.path.join(dl_dir, imgname)
success = download_file(img, name, pb) if not os.path.isfile(name):
else: success = download_file(img, name, pb)
preexist += 1 else:
if success: preexist += 1
realcount += 1 if success:
pb.tick() realcount += 1
print('[+] Successfully downloaded %s out of %s images to %s (%s already existed)' % pb.tick()
(realcount, imgcount, dl_dir, preexist)) print('[+] Successfully downloaded %s out of %s images to %s (%s already existed)' %
(realcount, imgcount, dl_dir, preexist))
def filter_zip_files(images: list, zip_fname: str):
""" def filter_zip_files(images: list, zip_fname: str):
Removes the images that already exist in the zip-file """
:param images: Removes the images that already exist in the zip-file
:param zip_fname: :param images:
:return: :param zip_fname:
""" :return:
if os.path.isfile(zip_fname): """
zfile = zipfile.ZipFile(zip_fname, 'r') if os.path.isfile(zip_fname):
zfnames = [f.filename for f in zfile.infolist()] zfile = zipfile.ZipFile(zip_fname, 'r')
print('[~] Removing entries already in zip-file') zfnames = [f.filename for f in zfile.infolist()]
return [img for img in images if img.split('/')[-1] not in zfnames] print('[~] Removing entries already in zip-file')
else: return [img for img in images if img.split('/')[-1] not in zfnames]
return images else:
return images
def compress_folder(folder: str, zip_fname: str, compression: int):
""" def compress_folder(folder: str, zip_fname: str, compression: int):
Zips the contents of a folder to the destination zipfile name. """
:param folder: the folder to zip Zips the contents of a folder to the destination zipfile name.
:param zip_fname: the name of the destination zipfile :param folder: the folder to zip
:param compression: The compression method (constant from zipfile module) :param zip_fname: the name of the destination zipfile
:return: None :param compression: The compression method (constant from zipfile module)
""" :return: None
print('[~] Compressing folder...') """
mode = 'w' print('[~] Compressing folder...')
mode = 'w'
if os.path.isfile(zip_fname): # append to the zipfile if it already exists
mode = 'a' if os.path.isfile(zip_fname): # append to the zipfile if it already exists
mode = 'a'
zfile = zipfile.ZipFile(zip_fname, mode, compression=compression)
zfile = zipfile.ZipFile(zip_fname, mode, compression=compression)
for _, _, files in os.walk(folder): # add all files of the folder to the zipfile
for file in files: for _, _, files in os.walk(folder): # add all files of the folder to the zipfile
zfile.write(os.path.join(folder, file), file) for file in files:
zfile.close() zfile.write(os.path.join(folder, file), file)
print('[+] Folder %s compressed to %s.' % (folder, zip_fname)) zfile.close()
print('[+] Folder %s compressed to %s.' % (folder, zip_fname))
def main():
""" def main():
Main entry method. Loads the settings and iterates through subreddits and downloads all images it fetched. """
If the --zip flag is set, the images will be downloaded in a .cache directory and then compressed. Main entry method. Loads the settings and iterates through subreddits and downloads all images it fetched.
""" If the --zip flag is set, the images will be downloaded in a .cache directory and then compressed.
options, subreddits = parser_init() """
with open('config.yaml', 'r') as file: # loads the config.yaml file options, subreddits = parser_init()
try: with open('config.yaml', 'r') as file: # loads the config.yaml file
settings = yaml.safe_load(file) try:
except yaml.YAMLError as err: settings = yaml.safe_load(file)
print(err) except yaml.YAMLError as err:
if settings: print(err)
if 'image-extensions' in settings: if settings:
global img_ext if 'image-extensions' in settings:
img_ext = settings['image-extensions'] global img_ext
if 'min-size' in settings: img_ext = settings['image-extensions']
global min_size if 'min-size' in settings:
min_size = int(settings['min-size']) global min_size
credentials = settings['credentials'] min_size = int(settings['min-size'])
client = praw.Reddit( credentials = settings['credentials']
client_id=credentials['client_id'], client = praw.Reddit(
client_secret=credentials['client_secret'], client_id=credentials['client_id'],
user_agent=user_agent client_secret=credentials['client_secret'],
) user_agent=user_agent
for subreddit in subreddits: )
dldest = subreddit for subreddit in subreddits:
if options.output: dldest = subreddit
dldest = options.output # uses the -o output destination if options.output:
images = get_images(client, subreddit, limit=options.count, dldest = options.output # uses the -o output destination
nsfw=options.nsfw) images = get_images(client, subreddit, limit=options.count,
if options.zip: # downloads to a cache-folder first before compressing it to zip nsfw=options.nsfw)
comp_mode = zipfile.ZIP_STORED if options.zip: # downloads to a cache-folder first before compressing it to zip
if options.lzma: comp_mode = zipfile.ZIP_STORED
comp_mode = zipfile.ZIP_LZMA if options.lzma:
cachedir = '.cache-' + dldest.split('/')[-1] comp_mode = zipfile.ZIP_LZMA
images = filter_zip_files(images, dldest+'.zip') cachedir = '.cache-' + dldest.split('/')[-1]
download_images(images, cachedir) images = filter_zip_files(images, dldest+'.zip')
compress_folder(cachedir, dldest+'.zip', compression=comp_mode) download_images(images, cachedir)
shutil.rmtree(cachedir) compress_folder(cachedir, dldest+'.zip', compression=comp_mode)
else: shutil.rmtree(cachedir)
download_images(images, dldest) else:
print('[+] All downloads finished') download_images(images, dldest)
print('[+] All downloads finished')
if __name__ == '__main__':
print('\n--- riddle.py reddit downloader by u/Trivernis ---\n') if __name__ == '__main__':
main() print('\n--- riddle.py reddit downloader by u/Trivernis ---\n')
main()

Write Preview
Loading…
Cancel
Save