From a47f036772388e7327192c2aa67f805a69c1006a Mon Sep 17 00:00:00 2001 From: limkokhole Date: Sun, 23 Jan 2022 06:24:46 +0800 Subject: [PATCH] --update-all option. --- README.md | 5 ++++ pinterest-downloader.py | 53 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4e27269..98811bc 100755 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Download all images/videos from Pinterest user/board/section. - [x] Unique board/log name with timestamp options. - [x] Proxies options. - [x] Speed up existing folder update without re-fetch all pages. +- [x] Update all option. ### Requirements: @@ -66,6 +67,10 @@ Download all images/videos from Pinterest user/board/section. reordered as you see in the webpage which possible newer images reorder below local highest Pin ID image and missed unless fetch all pages. + -ua, --update-all Update all in current directory. Only Options other + than -c, -j, and -rs will ignore. -c must same if + provided previously or else filename not same will re- + download. Not recommend to use -c at all. -es, --exclude-section Exclude sections if download from username or board. -ps HTTPS_PROXY, --https-proxy HTTPS_PROXY diff --git a/pinterest-downloader.py b/pinterest-downloader.py index 92d311c..60dd5b3 100755 --- a/pinterest-downloader.py +++ b/pinterest-downloader.py @@ -34,7 +34,7 @@ __status__ = 'Production' # Note: Support python 3 but not python 2 -import sys, os, traceback +import sys, os, re, traceback from pathlib import PurePath import platform plat = platform.system().lower() @@ -1248,15 +1248,57 @@ Please ensure your username/boardname/[section] or link has media item.\n') ) print() +def update_all( arg_thread_max :int, arg_cut :int, arg_rescrape :bool): + bk_cwd = os.getcwd() + imgs_f = [] + for root, dirs, files in os.walk(bk_cwd): + #print('#r: ' + repr(root) + ' #d: ' + repr(dirs) + ' #f: ' + repr(files)) + imgs_f.extend( [os.path.join(root, _) for _ in files if (_ == 'urls-pinterest-downloader.urls') ] ) + + urls_map = {} + for f in imgs_f: + r = open(f, "r") + input_url = None + folder_url = None + for line in r: + if re.search('^Input URL: ', line): + input_url = line.split('Input URL: ')[1].strip() + elif re.search('^Folder URL: ', line): + folder_url = line.split('Folder URL: ')[1].strip() + if input_url and folder_url: + cd_back_count = len(folder_url.split('https://www.pinterest.com/')[1].split('/')) + dir_origin = os.path.realpath( os.path.join(f, '../'*cd_back_count ) ) + if dir_origin in urls_map: + #print(urls_map[dir_origin][2], input_url, folder_url) + if cd_back_count < urls_map[dir_origin][1]: # cd_back_count: 4 means section, 3 means board, 2 means username + urls_map[dir_origin] = (input_url, cd_back_count) + #else: + # print('Skip child') + else: + urls_map[dir_origin] = (input_url, cd_back_count) + break + + total_str = str(len(urls_map)) + bk_cwd = os.getcwd() + for i, (dir_origin, (input_url, _)) in enumerate(urls_map.items()): + print('\nUpdating [ ' + str(i+1) + ' / ' + total_str + ' ] \nChange to directory: ' + str(dir_origin) + ' \nURL: ' + str(input_url)) + os.chdir(dir_origin) + #ime.sleep(1) + run_library_main(input_url, '.', arg_thread_max, arg_cut, False, False, False, False, arg_rescrape, False, None, None) + + # Caller script example: # import importlib # pin_dl = importlib.import_module('pinterest-downloader') # pin_dl.run_library_main('antonellomiglio/computer', '.', 0, -1, False, False, False, False, False, None, None) def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut :int , arg_board_timestamp :bool, arg_log_timestamp :bool - , arg_force :bool, arg_rescrape :bool, arg_exclude_section :bool + , arg_force :bool, arg_exclude_section :bool, arg_rescrape :bool, arg_update_all :bool , arg_https_proxy :str, arg_http_proxy :str): + if arg_update_all: + return update_all(arg_thread_max, arg_cut, arg_rescrape) + start_time = int(time.time()) if not arg_path: @@ -1470,6 +1512,9 @@ def run_direct_main(): arg_parser.add_argument('-rs', '--re-scrape', dest='rescrape', action='store_true', help='Default is only fetch new images since latest(highest) Pin ID local image to speed up update process.\n\ This option disable that behavior and re-scrape all, use it when you feel missing images somewhere or incomplete download.\n\ This issue is because Pinterest only lists reordered as you see in the webpage which possible newer images reorder below local highest Pin ID image and missed unless fetch all pages.') + arg_parser.add_argument('-ua', '--update-all', dest='update_all', action='store_true', help='Update all in current directory. Only\n\ + Options other than -c, -j, and -rs will ignore.\n\ + -c must same if provided previously or else filename not same will re-download. Not recommend to use -c at all.') arg_parser.add_argument('-es', '--exclude-section', dest='exclude_section', action='store_true', help='Exclude sections if download from username or board.') arg_parser.add_argument('-ps', '--https-proxy', help='Set proxy for https.') arg_parser.add_argument('-p', '--http-proxy', help='Set proxy for http.') @@ -1481,12 +1526,12 @@ def run_direct_main(): return quit( ['You type redundant options: ' + ' '.join(remaining) , 'Please check your command or --help to see options manual.' ]) - if not args.path: + if not args.update_all and not args.path: args.path = input('Username/Boardname/Section or Link: ').strip() return run_library_main(args.path, args.dir, args.thread_max, args.cut , args.board_timestamp, args.log_timestamp - , args.force, args.rescrape, args.exclude_section + , args.force, args.exclude_section, args.rescrape, args.update_all , args.https_proxy, args.http_proxy)