diff --git a/Pipfile b/Pipfile index a41cab4..964649a 100644 --- a/Pipfile +++ b/Pipfile @@ -6,6 +6,7 @@ name = "pypi" [packages] PyYAML = "*" praw = "*" +pillow = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 3c20aa9..b76574c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e030a28963c27bc726b49ad8bc68cf9648c19fde4e1a5a76d1fc8a5955b06cd1" + "sha256": "7abb433480c00144e7e50d130155eab1a78a619f76b57ffc5464f40a6b550edd" }, "pipfile-spec": 6, "requires": { @@ -18,10 +18,10 @@ "default": { "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "chardet": { "hashes": [ @@ -37,6 +37,42 @@ ], "version": "==2.8" }, + "pillow": { + "hashes": [ + "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", + "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", + "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", + "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", + "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", + "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", + "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", + "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", + "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", + "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", + "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", + "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", + "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", + "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", + "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", + "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", + "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", + "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", + "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", + "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", + "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", + "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", + "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9", + "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1", + "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a", + "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96", + "sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132", + "sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a", + "sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5", + "sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0" + ], + "index": "pypi", + "version": "==6.2.1" + }, "praw": { "hashes": [ "sha256:2e5c98e49fe60e5308255ed147b670d350f98281f84f582df30f87de727b6de2", @@ -54,22 +90,20 @@ }, "pyyaml": { "hashes": [ - "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", - "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", - "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", - "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", - "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", - "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", - "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", - "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", - "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", - "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", - "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", - "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", - "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", + "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", + "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", + "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", + "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", + "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", + "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", + "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", + "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", + "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", + "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" ], "index": "pypi", - "version": "==5.1.2" + "version": "==5.2" }, "requests": { "hashes": [ @@ -80,10 +114,10 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "update-checker": { "hashes": [ @@ -94,17 +128,17 @@ }, "urllib3": { "hashes": [ - "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", - "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" ], - "version": "==1.25.6" + "version": "==1.25.7" }, "websocket-client": { "hashes": [ - "sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9", - "sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a" + "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549", + "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010" ], - "version": "==0.56.0" + "version": "==0.57.0" } }, "develop": {} diff --git a/README.md b/README.md index 849552e..87f1390 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ image-extensions: - jpeg min-size: 5 # minimum size in kilobytes +min-mp: 0.5 # minimum siz ein megapixels ``` ## Running diff --git a/riddle.py b/riddle.py old mode 100644 new mode 100755 index 900ae9d..7906a49 --- a/riddle.py +++ b/riddle.py @@ -9,65 +9,12 @@ import praw import optparse import zipfile import urllib.request as urlreq +from PIL import Image -user_agent = 'python:riddle:3.0 (by u/Trivernis)' # the reddit api user-agent +user_agent = 'linux:riddle:3.0 (by u/Trivernis)' # the reddit api user-agent img_ext = ['jpg', 'jpeg', 'png'] # default used extensions to filter for images min_size = 5 # minimum size in kilobytes. changeable in settings - - -def assert_dir_exist(dirpath): - """ - Creates the directory if it doesn't exist - :param dirpath: path to the directory - :return: None - """ - if not os.path.exists(dirpath): - os.mkdir(dirpath) - - -def download_file(url: str, dest: str, progressbar = None): - """ - Downloads a url to a file - :param url: download url - :param dest: download destination - :param progressbar: The progressbar instance to clear it before writing an error message - :return: Success? - """ - f = open(dest, "wb") - req = urlreq.Request(url) - success = False - try: - image = urlreq.urlopen(req) - f.write(image.read()) - success = True - except ConnectionError: - if progressbar: - progressbar.clear() - print('\r[-] Connection Error') - except urlreq.HTTPError as err: - if progressbar: - progressbar.clear() - print('\r[-] HTTPError for %s: %s' % (url, err)) - except urlreq.URLError as err: - if progressbar: - progressbar.clear() - print('\r[-] URLError for %s: %s' % (url, err)) - f.close() - try: - file_size = round(os.path.getsize(dest) / 1000) - if not success: - os.remove(dest) - elif file_size < min_size: - os.remove(dest) - success = False - if progressbar: - progressbar.clear() - print('\r[-] Removed %s: Too small (%s kb)' % (dest, file_size)) - except IOError as err: - if progressbar: - progressbar.clear() - print('\r[-] Error when removing file %s: %s' % (dest, err)) - return success +min_mp = 0.5 # minimum megapixels. changeable in the settings class ProgressBar: @@ -151,6 +98,64 @@ def parser_init(): return parser.parse_args() +def assert_dir_exist(dirpath): + """ + Creates the directory if it doesn't exist + :param dirpath: path to the directory + :return: None + """ + if not os.path.exists(dirpath): + os.mkdir(dirpath) + + +def download_file(url: str, dest: str, progressbar = None): + """ + Downloads a url to a file + :param url: download url + :param dest: download destination + :param progressbar: The progressbar instance to clear it before writing an error message + :return: Success? + """ + f = open(dest, "wb") + req = urlreq.Request(url) + success = False + try: + image = urlreq.urlopen(req) + f.write(image.read()) + success = True + except ConnectionError: + if progressbar: + progressbar.clear() + print('\r[-] Connection Error') + except urlreq.HTTPError as err: + if progressbar: + progressbar.clear() + print('\r[-] HTTPError for %s: %s' % (url, err)) + except urlreq.URLError as err: + if progressbar: + progressbar.clear() + print('\r[-] URLError for %s: %s' % (url, err)) + f.close() + try: + width, height = Image.open(dest).size + mp = (width * height)/1000000 + file_size = round(os.path.getsize(dest) / 1000) + if __name__ == '__main__': + if not success: + os.remove(dest) + elif file_size < min_size or mp < min_mp: + os.remove(dest) + success = False + if progressbar: + progressbar.clear() + print('\r[-] Removed %s: Too small (%s kb, %s MP)' % (dest, file_size, mp)) + except IOError as err: + if progressbar: + progressbar.clear() + print('\r[-] Error when removing file %s: %s' % (dest, err)) + return success + + def get_images(reddit_client: praw.Reddit, subreddit: str, limit: int, nsfw: bool = False): """ Uses the reddit api to fetch all image posts @@ -161,8 +166,14 @@ def get_images(reddit_client: praw.Reddit, subreddit: str, limit: int, nsfw: boo :return: list of images """ print('[~] Fetching images for r/%s...' % subreddit) - urls = [submission.url for submission in reddit_client.subreddit(subreddit).hot(limit=limit) - if not submission.over_18 or nsfw] # fetches hot images and filters nsfw if set to false + urls = [] + try: + for submission in reddit_client.subreddit(subreddit).hot(limit=limit): + if not submission.over_18 or nsfw: + urls.append(submission.url) + print('\r[~] %s images' % len(urls), end='\r') + except Exception as e: + print(e) return [url for url in urls if url.split('.')[-1] in img_ext] @@ -252,6 +263,9 @@ def main(): if 'min-size' in settings: global min_size min_size = int(settings['min-size']) + if 'min-mp' in settings: + global min_mp + min_mp = int(settings['min-mp']) credentials = settings['credentials'] client = praw.Reddit( client_id=credentials['client_id'],