diff --git a/README.md b/README.md index e0ca68f..238394e 100755 --- a/README.md +++ b/README.md @@ -172,5 +172,5 @@ Download all images/videos from Pinterest user/board/section. ### You can also use another python script to run, e.g.: import importlib pin_dl = importlib.import_module('pinterest-downloader') - pin_dl.run_library_main('antonellomiglio/computer', '.', 0, -1, False, False, False, False, False, False, False, False, None, None) + pin_dl.run_library_main('antonellomiglio/computer', '.', 0, -1, False, False, False, False, False, False, False, False, None, None, None) diff --git a/pinterest-downloader.py b/pinterest-downloader.py index b539a66..9e3c6ec 100755 --- a/pinterest-downloader.py +++ b/pinterest-downloader.py @@ -79,6 +79,7 @@ import argparse import time from datetime import datetime, timedelta +from collections import OrderedDict import json import lxml.html as html @@ -223,6 +224,7 @@ def get_pin_info(pin_id, arg_timestamp_log, url_path scripts = [] is_success = False + image = None for t in (15, 30, 40, 50, 60): #print('https://www.pinterest.com/pin/{}/'.format(pin_id)) @@ -238,40 +240,51 @@ def get_pin_info(pin_id, arg_timestamp_log, url_path try: r = PIN_SESSION.get('https://www.pinterest.com/pin/{}/'.format(pin_id), timeout=(t, t), cookies=cookies) - is_success = True - break except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e: #print('[E1][pin] Failed. Retry after 5 seconds...') time.sleep(5) PIN_SESSION = get_session(0, proxies, cookies) + continue - if is_success: root = html.fromstring(r.content) #print(r.content) try: #tag = root.xpath("//script[@id='initial-state']")[0] scripts = root.xpath('//script/text()') except IndexError: #list index out of range - is_success = False + time.sleep(5) + PIN_SESSION = get_session(0, proxies, cookies) + continue + + indexErr = False + for script in scripts: + try: + data = json.loads(script) + if 'props' in data: + pins = data['props']['initialReduxState']['pins'] + try: + image = pins[list(pins.keys())[0]] + is_success = True + break + except IndexError: # Sometime `"pins":{}``, need retry + indexErr = True + except json.decoder.JSONDecodeError: + pass + + if not is_success: + if indexErr: + print('\n[Retry] Getting error pin id: ' + repr(pin_id) + '...\n\n') + continue if not is_success: if not get_data_only: # get data error show later + print('### HTML START ###') + print(r.content) + print('### HTML END ###\n\nPlease report this issue at https://github.com/limkokhole/pinterest-downloader/issues , thanks.\n\n') cprint(''.join([ HIGHER_RED, '%s %s%s' % ('\n[' + x_tag + '] Get this pin id failed :', pin_id, '\n') ]), attrs=BOLD_ONLY, end='' ) return - image = None - for script in scripts: - try: - data = json.loads(script) - if 'props' in data: - pins = data['props']['initialReduxState']['pins'] - image = pins[list(pins.keys())[0]] - #dj(image) - break - except json.decoder.JSONDecodeError: - pass - if get_data_only: return image try: @@ -653,6 +666,18 @@ def get_output_file_path(url, arg_cut, fs_f_max, image_id, human_fname, save_dir #print('final f: ' + file_path) return file_path +def isVideoExist(image): + #dj(image) + if ('videos' in image) and image['videos']: # image['videos'] may None + return 1 # Video type 1 in 'V_720P' + elif 'story_pin_data' in image and image['story_pin_data'] and ('pages' in image['story_pin_data']): # image['story_pin_data'] may null + pg = image['story_pin_data']['pages'] + if (len(pg) > 0) and ('blocks' in pg[0]): + blocks = pg[0]['blocks'] + if len(blocks) > 0 and 'video' in blocks[0] and ('video_list' in blocks[0]['video']): + return 2 # Video type 2 in 'V_EXP7' + return 0 # No video + def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, cookie_file, arg_cut, arg_el, fs_f_max): try: @@ -862,7 +887,11 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM else: pass #print('No image found in this image index. This is normal (may be 1))') - if not arg_img_only and ('videos' in image) and image['videos']: # image['videos'] may None + if not arg_img_only: + video_type = isVideoExist(image) + if video_type == 0: # No video + return + #dj(image, 'before override') # override m3u8-only data with pin details page mp4 v_pin_id = image['id'] image = get_pin_info(v_pin_id, None, None, None, False, False, None, None, None, None, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, cookie_file, True) @@ -871,7 +900,11 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM cprint(''.join([ HIGHER_RED, '%s %s%s' % ('\n[' + x_tag + '] Get this video pin id failed :', v_pin_id, '\n') ]), attrs=BOLD_ONLY, end='' ) return - v_d = image['videos']['video_list'] + if video_type == 1: + v_d = image['videos']['video_list'] + elif video_type == 2: # Already check index/key in isVideoExist() + v_d_unsort = image['story_pin_data']['pages'][0]['blocks'][0]['video']['video_list'] + v_d = OrderedDict(sorted(v_d_unsort.items(), key=lambda t: t[0])) # Sort by V_EXP{3-7} (V_HLSV3_MOBILE will ignore below since not .mp4) #dj(v_d) vDimens = [] vDimensD = {} @@ -882,6 +915,7 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM if vDimens: vDimens.sort(key=int) vurl = vDimensD[int(vDimens[-1])] + #print('\n' + vurl) #cprint('\n\n[...] Try with best quality video: {}'.format(vurl), attrs=BOLD_ONLY) file_path = get_output_file_path(vurl, arg_cut, fs_f_max, image_id, human_fname, save_dir)