rdarius/pinterest-downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable back update all feature.

Browse Source
This commit is contained in:
limkokhole
2022-01-23 09:19:06 +08:00
parent 306f22ac11
commit bf1c9082d3
2 changed files with 29 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@@ -67,11 +67,11 @@ Download all images/videos from Pinterest user/board/section.
reordered as you see in the webpage which possible reordered as you see in the webpage which possible
newer images reorder below local highest Pin ID image newer images reorder below local highest Pin ID image
and missed unless fetch all pages. and missed unless fetch all pages.
-ua, --update-all Update all in current directory recursively. Options -ua, --update-all Update all folders in current directory recursively
other than -c, -j, and -rs will ignore. -c must same based on theirs urls-pinterest-downloader.urls.
if provided previously or else filename not same will Options other than -c, -j, and -rs will ignore. -c
re-download. Not recommend to use -c at all. must same if provided previously or else filename not
same will re-download. Not recommend to use -c at all.
-es, --exclude-section -es, --exclude-section
Exclude sections if download from username or board. Exclude sections if download from username or board.
-ps HTTPS_PROXY, --https-proxy HTTPS_PROXY -ps HTTPS_PROXY, --https-proxy HTTPS_PROXY

43
pinterest-downloader.py
View File

@@ -34,7 +34,7 @@ __status__ = 'Production'
# Note: Support python 3 but not python 2 # Note: Support python 3 but not python 2
import sys, os, re, traceback import sys, os, traceback
from pathlib import PurePath from pathlib import PurePath
import platform import platform
plat = platform.system().lower() plat = platform.system().lower()
@@ -861,6 +861,7 @@ def write_log(arg_timestamp_log, url_path, shortform, save_dir, images, pin, arg
with open(log_url_path, 'w') as f: with open(log_url_path, 'w') as f:
f.write('Pinterest Downloader: Version ' + str(__version__) + '\n\n') # Easy to recognize if future want to change something f.write('Pinterest Downloader: Version ' + str(__version__) + '\n\n') # Easy to recognize if future want to change something
# Ensure -ua same parsing format
f.write('Input URL: https://www.pinterest.com/' + url_path.rstrip('/') + '/\n') # Reuse/refer when want to update f.write('Input URL: https://www.pinterest.com/' + url_path.rstrip('/') + '/\n') # Reuse/refer when want to update
if shortform: # single pin no need if shortform: # single pin no need
f.write('Folder URL: https://www.pinterest.com/' + shortform.rstrip('/') + '/\n\n') # Reuse/refer when want to update specific folder only f.write('Folder URL: https://www.pinterest.com/' + shortform.rstrip('/') + '/\n\n') # Reuse/refer when want to update specific folder only
@@ -1158,23 +1159,22 @@ Please ensure your username/boardname/[section] or link has media item.\n') )
if img['id'].isdigit(): if img['id'].isdigit():
img_curr = img['id'] img_curr = img['id']
if img_prev and (int(img_curr) > int(img_prev)): if img_prev and (int(img_curr) > int(img_prev)):
cprint(''.join([ HIGHER_YELLOW, '%s' % ('\n[W] This images list is not sorted(Due to user reorder or alphanumeric pin ID), fallback to -rs for this list.\n\n') ]), attrs=BOLD_ONLY, end='' ) cprint(''.join([ HIGHER_YELLOW, '%s' % ('\n[W] This images list is not sorted(Due to user reorder), fallback to -rs for this list.\n\n') ]), attrs=BOLD_ONLY, end='' )
sorted_api = False sorted_api = False
reach_lastest_pin = False reach_lastest_pin = False
if on_hold_break: if on_hold_break:
imgs_round = data['resource_response']['data'] # replaced back below imgs_round = data['resource_response']['data'] # replaced back below
break break
if on_hold_break: # Need check above 2 imgs first, test with delete 1 image in unsorted list
break
if latest_pin == img_curr: if latest_pin == img_curr:
#print('\nAlready scroll to latest downloaded pin. Break.') #print('\nAlready scroll to latest downloaded pin. Break.')
#print('bookmark: ' + repr(bookmark)) #print('bookmark: ' + repr(bookmark))
imgs_round = imgs_round[:img_round_i] imgs_round = imgs_round[:img_round_i]
reach_lastest_pin = True reach_lastest_pin = True
# Next check all 25 items in current page to know owner recently has reorder habit or not
on_hold_break = True on_hold_break = True
img_prev = img_curr img_prev = img_curr
else: else:
cprint(''.join([ HIGHER_YELLOW, '%s' % ('\n[W] This images list is not sorted(Due to user reorder or alphanumeric pin ID), fallback to -rs for this list.\n\n') ]), attrs=BOLD_ONLY, end='' ) cprint(''.join([ HIGHER_YELLOW, '%s' % ('\n[W] This images list is not sorted(Due to alphanumeric pin ID), fallback to -rs for this list.\n\n') ]), attrs=BOLD_ONLY, end='' )
sorted_api = False sorted_api = False
reach_lastest_pin = False reach_lastest_pin = False
imgs_round = data['resource_response']['data'] # replaced back above imgs_round = data['resource_response']['data'] # replaced back above
@@ -1263,32 +1263,38 @@ def update_all( arg_thread_max :int, arg_cut :int, arg_rescrape :bool):
input_url = None input_url = None
folder_url = None folder_url = None
for line in r: for line in r:
if re.search('^Input URL: ', line): l_strip = line.strip()
input_url = line.split('Input URL: ')[1].strip() if l_strip.startswith('Input URL: '): #re.search('^Input URL: ', line):
elif re.search('^Folder URL: ', line): input_url = l_strip.split('Input URL: ')[1].strip()
folder_url = line.split('Folder URL: ')[1].strip() elif l_strip.startswith('Folder URL: '):
folder_url = l_strip.split('Folder URL: ')[1].strip()
if input_url and folder_url: if input_url and folder_url:
cd_back_count = len(folder_url.split('https://www.pinterest.com/')[1].split('/')) cd_back_count = len(folder_url.split('/')[3:])
if cd_back_count not in cd_back_fixed_range: if cd_back_count not in cd_back_fixed_range:
return quit( ['[E1][-ua] Input url: ' + input_url + '\nFolder url: ' + folder_url return quit( ['[E1][-ua] Input url: ' + input_url + '\nFolder url: ' + folder_url
, 'Somthing is not right. Please report this issue at https://github.com/limkokhole/pinterest-downloader/issues , thanks.']) , 'Something is not right. Please report this issue at https://github.com/limkokhole/pinterest-downloader/issues , thanks.'])
dir_origin = os.path.abspath( os.path.join(f, '../'*cd_back_count ) ) dir_origin = os.path.abspath( os.path.join(f, '../'*cd_back_count ) )
dir_split = PurePath(dir_origin).parts[:] dir_split = PurePath(dir_origin).parts[:]
#print(dir_split) #print(dir_split)
# Security checking to avoid travel to parent of current directory # Security checking to avoid travel to parent of current directory
if len(dir_split) < cwd_component_total: if len(dir_split) < cwd_component_total:
return quit( ['[E2][-ua] Input url: ' + input_url + '\nFolder url: ' + folder_url cprint(''.join([ HIGHER_YELLOW, '%s' % ('\n' + 'Update from parent directory of current directory is forbidden. Skipped.\nInput url: '
+ '\nf: ' + f + '\ncd_back_count: ' + str(cd_back_count) + '\ndir_origin: ' + dir_origin + '\ncwd: ' + bk_cwd + '\nlen(dir_split): ' + str(len(dir_split)) + '\ncwd_component_total: ' + str(cwd_component_total) + input_url + '\nFolder url: ' + folder_url
, 'Somthing is not right. Please report this issue at https://github.com/limkokhole/pinterest-downloader/issues , thanks.']) + '\nurls file: ' + f + '\ncd back count: ' + str(cd_back_count) + '\ndir origin: '
+ dir_origin + '\ncwd: ' + bk_cwd + '\ndir origin len: ' + str(len(dir_split))
+ '\ncwd component total: ' + str(cwd_component_total) + '\n\n') ])
, attrs=BOLD_ONLY, end='' )
break
if dir_origin in urls_map: if dir_origin in urls_map:
#print(urls_map[dir_origin][2], input_url, folder_url) #print(urls_map[dir_origin][2], input_url, folder_url)
if cd_back_count < urls_map[dir_origin][1]: # cd_back_count: 4 means section, 3 means board, 2 means username # cd_back_count: 4(extra trailing /) means section, 3 means board, 2 means username
if cd_back_count < urls_map[dir_origin][1]:
urls_map[dir_origin] = (input_url, cd_back_count) urls_map[dir_origin] = (input_url, cd_back_count)
#else: #else:
# print('Skip child') # print('Skip child')
else: else:
urls_map[dir_origin] = (input_url, cd_back_count) urls_map[dir_origin] = (input_url, cd_back_count)
break break # Only read headers
total_str = str(len(urls_map)) total_str = str(len(urls_map))
bk_cwd = os.getcwd() bk_cwd = os.getcwd()
@@ -1296,8 +1302,7 @@ def update_all( arg_thread_max :int, arg_cut :int, arg_rescrape :bool):
print('\nUpdating [ ' + str(i+1) + ' / ' + total_str + ' ] \nChange to directory: ' + str(dir_origin) + ' \nURL: ' + str(input_url)) print('\nUpdating [ ' + str(i+1) + ' / ' + total_str + ' ] \nChange to directory: ' + str(dir_origin) + ' \nURL: ' + str(input_url))
os.chdir(dir_origin) os.chdir(dir_origin)
#ime.sleep(1) #ime.sleep(1)
print('-ua temporary disabled.') run_library_main(input_url, '.', arg_thread_max, arg_cut, False, False, False, False, arg_rescrape, False, None, None)
#run_library_main(input_url, '.', arg_thread_max, arg_cut, False, False, False, False, arg_rescrape, False, None, None)
# Caller script example: # Caller script example:
@@ -1525,7 +1530,7 @@ def run_direct_main():
arg_parser.add_argument('-rs', '--re-scrape', dest='rescrape', action='store_true', help='Default is only fetch new images since latest(highest) Pin ID local image to speed up update process.\n\ arg_parser.add_argument('-rs', '--re-scrape', dest='rescrape', action='store_true', help='Default is only fetch new images since latest(highest) Pin ID local image to speed up update process.\n\
This option disable that behavior and re-scrape all, use it when you feel missing images somewhere or incomplete download.\n\ This option disable that behavior and re-scrape all, use it when you feel missing images somewhere or incomplete download.\n\
This issue is because Pinterest only lists reordered as you see in the webpage which possible newer images reorder below local highest Pin ID image and missed unless fetch all pages.') This issue is because Pinterest only lists reordered as you see in the webpage which possible newer images reorder below local highest Pin ID image and missed unless fetch all pages.')
arg_parser.add_argument('-ua', '--update-all', dest='update_all', action='store_true', help='Update all in current directory recursively.\n\ arg_parser.add_argument('-ua', '--update-all', dest='update_all', action='store_true', help='Update all folders in current directory recursively based on theirs urls-pinterest-downloader.urls.\n\
Options other than -c, -j, and -rs will ignore.\n\ Options other than -c, -j, and -rs will ignore.\n\
-c must same if provided previously or else filename not same will re-download. Not recommend to use -c at all.') -c must same if provided previously or else filename not same will re-download. Not recommend to use -c at all.')
arg_parser.add_argument('-es', '--exclude-section', dest='exclude_section', action='store_true', help='Exclude sections if download from username or board.') arg_parser.add_argument('-es', '--exclude-section', dest='exclude_section', action='store_true', help='Exclude sections if download from username or board.')