From 336e1d9785107d53757e6765d385065b92c6b849 Mon Sep 17 00:00:00 2001 From: mukkino <25956452+fabiolichinchi@users.noreply.github.com> Date: Thu, 13 Oct 2022 23:49:23 +0000 Subject: [PATCH] Patch for downloading hidden galleries (cookies) Added ability of specifying a cookie file. The cookie file needs to be in the format of a token like the one generated by this extension: https://chrome.google.com/webstore/detail/get-token-cookie/naciaagbkifhpnoodlkhbejjldaiffcm Example: given that the info from the extension is saved in a file named "cookies.txt" and placed in the same folder as "pinterest-downloader.py", now we can call "python pinterest-downloader.py https://www.pinterest.co.uk/usename --cookies cookies.txt" and also hidden galleries for your user will be downloaded --- pinterest-downloader.py | 300 ++++++++++++++++++++++++++++------------ 1 file changed, 214 insertions(+), 86 deletions(-) diff --git a/pinterest-downloader.py b/pinterest-downloader.py index 93089e0..d7a9d00 100755 --- a/pinterest-downloader.py +++ b/pinterest-downloader.py @@ -71,7 +71,6 @@ HIGHER_RED = Fore.LIGHTRED_EX HIGHER_YELLOW = Fore.LIGHTYELLOW_EX BOLD_ONLY = ['bold'] - def quit(msgs, exit=True): if not isinstance(msgs, list): msgs = [msgs] @@ -83,7 +82,6 @@ def quit(msgs, exit=True): else: cprint(''.join([ HIGHER_RED, '%s' % (msg) ]), attrs=BOLD_ONLY, end='\n' ) - try: x_tag = '✖' done_tag = '✔' @@ -109,6 +107,9 @@ import requests from concurrent.futures import ThreadPoolExecutor, as_completed +from http.cookies import SimpleCookie +from requests.cookies import cookiejar_from_dict + # RIP UA, https://groups.google.com/a/chromium.org/forum/m/#!msg/blink-dev/-2JIRNMWJ7s/yHe4tQNLCgAJ UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.0.0 Safari/537.36' @@ -117,7 +118,6 @@ UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like #[DEPRECATED] no more 259 since always -el now AND Windows 259 - \\?\ == 255 normal Linux WIN_MAX_PATH = 255 # MAX_PATH 260 need exclude 1 terminating null character - # https://stackoverflow.com/a/34325723/1074998 def printProgressBar(iteration, total, prefix='', suffix='', decimals=1, length=100, fill='#'): ''' @@ -139,7 +139,6 @@ def printProgressBar(iteration, total, prefix='', suffix='', decimals=1, length= cprint(''.join([ HIGHER_GREEN, '%s' % ('\r{} |{}| {}% {}'.format(prefix, bar, percent, suffix)) ]), attrs=BOLD_ONLY, end='' ) sys.stdout.flush() - #imgs: #source_url=%2Fmistafisha%2Fanimals%2F&data=%7B%22options%22%3A%7B%22isPrefetch%22%3Afalse%2C%22board_id%22%3A #%2253761857990790784%22%2C%22board_url%22%3A%22%2Fmistafisha%2Fanimals%2F%22%2C%22field_set_key%22%3A%22react_grid_pin @@ -150,9 +149,26 @@ def printProgressBar(iteration, total, prefix='', suffix='', decimals=1, length= #,"board_url":"/mistafisha/animals/","field_set_key":"react_grid_pin","filter_section_pins":true,"sort":"default" #,"layout":"default","page_size":25,"redux_normalize_feed":true},"context":{}}&_=1592340515565 VER = (None, 'c643827', '4c8c36f') -def get_session(ver_i, proxies): +def get_session(ver_i, proxies, cookie_file): s = requests.Session() s.proxies = proxies + + try: + with open(cookie_file) as f: + rawdata = f.read() + + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + + except: + cookies = None + + try: + s.cookies = cookiejar_from_dict(cookies) + except: + pass + if ver_i == 0: s.headers = { #'Host': 'www.pinterest.com', @@ -213,31 +229,40 @@ def get_session(ver_i, proxies): return s - def dj(j, tag=None): if tag: print('### [' + tag + '] ###') print(json.dumps(j, sort_keys=True, indent=4)) - def get_pin_info(pin_id, arg_timestamp_log, url_path , arg_force_update, arg_img_only, arg_v_only , arg_dir, arg_cut, arg_el, fs_f_max , IMG_SESSION, V_SESSION, PIN_SESSION, proxies - , get_data_only): + , cookie_file, get_data_only): scripts = [] is_success = False for t in (15, 30, 40, 50, 60): #print('https://www.pinterest.com/pin/{}/'.format(pin_id)) + try: - r = PIN_SESSION.get('https://www.pinterest.com/pin/{}/'.format(pin_id), timeout=(t, t)) + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + + try: + r = PIN_SESSION.get('https://www.pinterest.com/pin/{}/'.format(pin_id), timeout=(t, t), cookies=cookies) is_success = True break except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e: #print('[E1][pin] Failed. Retry after 5 seconds...') time.sleep(5) - PIN_SESSION = get_session(0, proxies) + PIN_SESSION = get_session(0, proxies, cookies) if is_success: root = html.fromstring(r.content) @@ -276,16 +301,26 @@ def get_pin_info(pin_id, arg_timestamp_log, url_path write_log( arg_timestamp_log, url_path, None, arg_img_only, arg_v_only, arg_dir, [image], image['id'], arg_cut, False ) print('[i] Download Pin id: ' + str(image['id']) + ' into directory: ' + arg_dir.rstrip(os.sep) + os.sep) printProgressBar(0, 1, prefix='[...] Downloading:', suffix='Complete', length=50) - download_img(image, arg_dir, arg_force_update, arg_img_only, arg_v_only, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, arg_cut, arg_el, fs_f_max) + download_img(image, arg_dir, arg_force_update, arg_img_only, arg_v_only, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, cookie_file, arg_cut, arg_el, fs_f_max) printProgressBar(1, 1, prefix='[' + done_tag + '] Downloaded:', suffix='Complete ', length=50) except KeyError: return quit(traceback.format_exc()) print() - -def get_board_info(board_or_sec_path, exclude_section, section, board_path, proxies, retry=False): - s = get_session(0, proxies) - +def get_board_info(board_or_sec_path, exclude_section, section, board_path, proxies, cookie_file, retry=False): + try: + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + + s = get_session(0, proxies, cookies) + #s.cookies = cookies + #dj(data, 'board main') boards = {} sections = [] @@ -294,13 +329,22 @@ def get_board_info(board_or_sec_path, exclude_section, section, board_path, prox #print('https://www.pinterest.com/{}/'.format(board_or_sec_path)) for t in (15, 30, 40, 50, 60): try: - r = s.get('https://www.pinterest.com/{}/'.format(board_or_sec_path), timeout=(t, t)) + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + try: + r = s.get('https://www.pinterest.com/{}/'.format(board_or_sec_path), timeout=(t, t), cookies=cookies) is_success = True break except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e: #print('[E1][pin] Failed. Retry after 5 seconds...') time.sleep(5) - s = get_session(0, proxies) + s = get_session(0, proxies, cookies) if is_success: root = html.fromstring(r.content) @@ -330,7 +374,6 @@ def get_board_info(board_or_sec_path, exclude_section, section, board_path, prox else: return boards, sections - board_dk = list(board_d.keys()) if section: path_to_compare = board_path @@ -382,10 +425,21 @@ def get_board_info(board_or_sec_path, exclude_section, section, board_path, prox else: return boards, sections +def fetch_boards(uname, proxies, cookie_file): -def fetch_boards(uname, proxies): + try: + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + + s = get_session(1, proxies, cookies) + #s.cookies = cookies - s = get_session(1, proxies) bookmark = None boards = [] @@ -435,12 +489,22 @@ def fetch_boards(uname, proxies): is_success = False for t in (15, 30, 40, 50, 60): try: - r = s.get('https://www.pinterest.com/resource/BoardsResource/get/', params=post_d, timeout=(t, t)) + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + try: + r = s.get('https://www.pinterest.com/resource/BoardsResource/get/', params=post_d, timeout=(t, t), cookies=cookies) is_success = True break except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e: time.sleep(5) - s = get_session(1, proxies) + s = get_session(1, proxies, cookies) + #s.cookies = cookies if not is_success: cprint(''.join([ HIGHER_RED, '%s %s%s' % ('\n[' + x_tag + '] Get this username failed :', uname, '\n') ]), attrs=BOLD_ONLY, end='' ) @@ -460,7 +524,6 @@ def fetch_boards(uname, proxies): return boards - def sanitize(path): # trim multiple whitespaces # ".." is the responsibilities of get max path @@ -488,7 +551,6 @@ def sanitize(path): else: return '' - # The filesystem limits is 255(normal) , 242(docker) or 143((eCryptfs) bytes # So can't blindly [:] slice without encode first (which most downloaders do the wrong way) # And need decode back after slice @@ -501,6 +563,7 @@ def sanitize(path): # https://stackoverflow.com/questions/13132976 # https://stackoverflow.com/questions/50385123 # https://stackoverflow.com/questions/11820006 + def get_max_path(arg_cut, fs_f_max, fpart_excluded_immutable, immutable): #print('before f: ' + fpart_excluded_immutable) if arg_cut >= 0: @@ -532,7 +595,6 @@ def get_max_path(arg_cut, fs_f_max, fpart_excluded_immutable, immutable): .format(arg_cut, fs_f_max, repr(fpart_excluded_immutable), repr(fpart_excluded_immutable_base), immutable) ]), attrs=BOLD_ONLY, end='' ) return fpart_excluded_immutable_base - def get_output_file_path(url, arg_cut, fs_f_max, image_id, human_fname, save_dir): pin_id_str = sanitize(str(image_id)) @@ -610,8 +672,7 @@ def get_output_file_path(url, arg_cut, fs_f_max, image_id, human_fname, save_dir #print('final f: ' + file_path) return file_path - -def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, arg_cut, arg_el, fs_f_max): +def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, cookie_file, arg_cut, arg_el, fs_f_max): try: # Using threading.Lock() if necessary @@ -660,14 +721,23 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM is_ok = False for t in (15, 30, 40, 50, 60): try: - r = IMG_SESSION.get(url, stream=True, timeout=(t, t)) + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + try: + r = IMG_SESSION.get(url, stream=True, timeout=(t, t), cookies=cookies) is_ok = True #raise(requests.exceptions.ConnectionError) break except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e: # Shouldn't print bcoz quite common time.sleep(5) - IMG_SESSION = get_session(3, proxies) + IMG_SESSION = get_session(3, proxies, cookies) #cprint(''.join([ HIGHER_RED, '{}'.format('\n[' + x_tag + '] Image Timeout (Retry next).\n') ]), attrs=BOLD_ONLY, end='' ) #print(url + ' ok? ' + str(r.ok)) @@ -685,8 +755,17 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM for t in (15, 30, 40, 50, 60): time.sleep(5) try: - IMG_SESSION_RETY = get_session(3, proxies) - r = IMG_SESSION_RETY.get(url, stream=True, timeout=(t, t)) # Need higher timeout + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + try: + IMG_SESSION_RETY = get_session(3, proxies, cookies) + r = IMG_SESSION_RETY.get(url, stream=True, timeout=(t, t), cookies=cookies) # Need higher timeout with open(file_path, 'wb') as f: for chunk in r: f.write(chunk) @@ -728,15 +807,25 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM if not os.path.exists(file_path) or arg_force_update: is_ok = False for t in (15, 30, 40, 50, 60): + try: + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None try: # timeout=(connect_timeout, read_timeout) # https://github.com/psf/requests/issues/3099#issuecomment-215498005 - r = IMG_SESSION.get(url, stream=True, timeout=(t, t)) + + r = IMG_SESSION.get(url, stream=True, timeout=(t, t), cookies=cookies) is_ok = True break except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e: time.sleep(5) - IMG_SESSION = get_session(3, proxies) + IMG_SESSION = get_session(3, proxies, cookies) if is_ok and r.ok: try: @@ -749,8 +838,17 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM for t in (15, 30, 40, 50, 60): time.sleep(5) try: - IMG_SESSION_RETY = get_session(3, proxies) - r = IMG_SESSION_RETY.get(url, stream=True, timeout=(t, t)) + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + try: + IMG_SESSION_RETY = get_session(3, proxies, cookies) + r = IMG_SESSION_RETY.get(url, stream=True, timeout=(t, t), cookies=cookies) with open(file_path, 'wb') as f: for chunk in r: f.write(chunk) @@ -786,7 +884,7 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM if not arg_img_only and ('videos' in image) and image['videos']: # image['videos'] may None #dj(image, 'before override') # override m3u8-only data with pin details page mp4 v_pin_id = image['id'] - image = get_pin_info(v_pin_id, None, None, None, False, False, None, None, None, None, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, True) + image = get_pin_info(v_pin_id, None, None, None, False, False, None, None, None, None, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, cookie_file, True) #dj(image, 'after override') # [todo:0] Rich Metadata for video write to log (only pin can get) if not image: cprint(''.join([ HIGHER_RED, '%s %s%s' % ('\n[' + x_tag @@ -818,13 +916,22 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM is_ok = False for t in (15, 30, 40, 50, 60): try: - r = V_SESSION.get(vurl, stream=True, timeout=(t, t)) + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + try: + r = V_SESSION.get(vurl, stream=True, timeout=(t, t), cookies=cookies) is_ok = True break except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e: # Shouldn't print bcoz quite common time.sleep(5) #cprint(''.join([ HIGHER_RED, '{}'.format('\n[' + x_tag + '] Video Timeout (Retry next).\n') ]), attrs=BOLD_ONLY, end='' ) - V_SESSION = get_session(4, proxies) + V_SESSION = get_session(4, proxies, cookies) #print(vurl + ' ok? ' + str(r.ok)) @@ -841,8 +948,17 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM for t in (15, 30, 40, 50, 60): time.sleep(5) try: - V_SESSION_RETY = get_session(4, proxies) - r = V_SESSION_RETY.get(vurl, stream=True, timeout=(t, t)) + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None + try: + V_SESSION_RETY = get_session(4, proxies, cookies) + r = V_SESSION_RETY.get(vurl, stream=True, timeout=(t, t), cookies=cookies) with open(file_path, 'wb') as f: for chunk in r: f.write(chunk) @@ -869,7 +985,6 @@ def download_img(image, save_dir, arg_force_update, arg_img_only, arg_v_only, IM print() return quit(traceback.format_exc()) - def create_dir(save_dir): try: @@ -1019,14 +1134,12 @@ def write_log(arg_timestamp_log, url_path, shortform return got_img - def sort_func(x): prefix = x.split('.')[0].split('_')[0] if prefix.isdigit(): return int(prefix) return 0 - def get_latest_pin(save_dir): # Currently possible long non-number A8pQTwIQQLQGWEacY5vc6og pin id but should rare case and ignore/re-scrape is fine latest_pin = '0' @@ -1048,13 +1161,12 @@ def get_latest_pin(save_dir): #print('latest_pin: ' + latest_pin) return latest_pin - def fetch_imgs(board, uname, board_slug, section_slug, is_main_board , arg_timestamp, arg_timestamp_log, url_path , arg_force_update, arg_rescrape, arg_img_only, arg_v_only , arg_dir, arg_thread_max , IMGS_SESSION, IMG_SESSION, V_SESSION, PIN_SESSION, proxies - , arg_cut, arg_el, fs_f_max): + , cookie_file, arg_cut, arg_el, fs_f_max): bookmark = None images = [] @@ -1199,16 +1311,33 @@ Please ensure your username/boardname/[section] or link has media item.\n') ) for t in (15, 30, 40, 50, 60): try: if section_slug: + try: + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None r = IMGS_SESSION.get('https://www.pinterest.com/resource/BoardSectionPinsResource/get/' - , params=post_d, timeout=(t, t)) + , params=post_d, timeout=(t, t), cookies=cookies) else: + try: + with open(cookie_file) as f: + rawdata = f.read() + my_cookie = SimpleCookie() + my_cookie.load(rawdata) + cookies = {key: morsel.value for key, morsel in my_cookie.items()} + cookies = cookiejar_from_dict(cookies) + except: + cookies = None r = IMGS_SESSION.get('https://www.pinterest.com/resource/BoardFeedResource/get/' - , params=post_d, timeout=(t, t)) + , params=post_d, timeout=(t, t), cookies=cookies) break except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as e: time.sleep(5) - IMGS_SESSION = get_session(2, proxies) - + IMGS_SESSION = get_session(2, proxies, cookies) #print('Imgs url ok: ' + str(r.ok)) #print('Imgs url: ' + r.url) @@ -1306,7 +1435,7 @@ Please ensure your username/boardname/[section] or link has media item.\n') ) # Create threads futures = {executor.submit(download_img, image, save_dir, arg_force_update, arg_img_only, arg_v_only - , IMG_SESSION, V_SESSION, PIN_SESSION, proxies, arg_cut, arg_el, fs_f_max) for image in images} + , IMG_SESSION, V_SESSION, PIN_SESSION, proxies, cookie_file, arg_cut, arg_el, fs_f_max) for image in images} # as_completed() gives you the threads once finished for index, f in enumerate(as_completed(futures)): @@ -1326,7 +1455,7 @@ Please ensure your username/boardname/[section] or link has media item.\n') ) def update_all( arg_thread_max :int, arg_cut :int, arg_rescrape :bool , arg_img_only, arg_v_only - , arg_https_proxy :str, arg_http_proxy :str): + , arg_https_proxy :str, arg_http_proxy :str, arg_cookies :str): bk_cwd = os.path.abspath(os.getcwd()) cwd_component_total = len(PurePath(os.path.abspath(bk_cwd)).parts[:]) @@ -1402,7 +1531,7 @@ def update_all( arg_thread_max :int, arg_cut :int, arg_rescrape :bool #print('run URL:' + input_url) while 1: try: - run_library_main(input_url, '.', arg_thread_max, arg_cut, False, False, False, True, arg_rescrape, arg_img_only, arg_v_only, False, arg_https_proxy, arg_http_proxy) + run_library_main(input_url, '.', arg_thread_max, arg_cut, False, False, False, True, arg_rescrape, arg_img_only, arg_v_only, False, arg_https_proxy, arg_http_proxy, arg_cookies) break except requests.exceptions.ReadTimeout: cprint(''.join([ HIGHER_RED, '{}'.format('\n[' + x_tag + '] [U] Suddenly not able to connect. Please check your network.\n') ]), attrs=BOLD_ONLY, end='' ) @@ -1411,16 +1540,16 @@ def update_all( arg_thread_max :int, arg_cut :int, arg_rescrape :bool cprint(''.join([ HIGHER_RED, '{}'.format('\n[' + x_tag + '] [U] Not able to connect. Please check your network.\n') ]), attrs=BOLD_ONLY, end='' ) time.sleep(5) - # Caller script example: # import importlib # pin_dl = importlib.import_module('pinterest-downloader') # pin_dl.run_library_main('antonellomiglio/computer', '.', 0, -1, False, False, False, False, False, False, False, False, None, None) + def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut :int , arg_board_timestamp :bool, arg_log_timestamp :bool , arg_force :bool, arg_exclude_section :bool, arg_rescrape :bool , arg_img_only :bool, arg_v_only :bool, arg_update_all :bool - , arg_https_proxy :str, arg_http_proxy :str): + , arg_https_proxy :str, arg_http_proxy :str, arg_cookies :str): # Not feasible update based on latest pin if v/img only # , unless download zero size img if video only(vice-versa) which seems not desired. @@ -1428,7 +1557,7 @@ def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut : arg_rescrape = True if arg_update_all: - return update_all(arg_thread_max, arg_cut, arg_rescrape, arg_img_only, arg_v_only, arg_https_proxy, arg_http_proxy) + return update_all(arg_thread_max, arg_cut, arg_rescrape, arg_img_only, arg_v_only, arg_https_proxy, arg_http_proxy, arg_cookies) start_time = int(time.time()) @@ -1488,6 +1617,7 @@ def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut : fs_f_max = os.statvfs('.').f_namemax proxies = dict(http=arg_http_proxy, https=arg_https_proxy) + cookies = str(arg_cookies) if len(slash_path) == 2: # may copy USERNAME/boards/ links @@ -1499,10 +1629,10 @@ def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut : print('[i] Job is download video/image of single pin page.') pin_id = slash_path[-1] #bk first before reset slash_path = [] # reset for later in case exception - PIN_SESSION = get_session(0, proxies) - IMG_SESSION = get_session(3, proxies) - V_SESSION = get_session(4, proxies) - get_pin_info(pin_id.strip(), arg_log_timestamp, url_path, arg_force, arg_img_only, arg_v_only, arg_dir, arg_cut, arg_el, fs_f_max, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, False) + PIN_SESSION = get_session(0, proxies, cookies) + IMG_SESSION = get_session(3, proxies, cookies) + V_SESSION = get_session(4, proxies, cookies) + get_pin_info(pin_id.strip(), arg_log_timestamp, url_path, arg_force, arg_img_only, arg_v_only, arg_dir, arg_cut, arg_el, fs_f_max, IMG_SESSION, V_SESSION, PIN_SESSION, proxies, cookie_file, False) if len(slash_path) == 3: sec_path = '/'.join(slash_path) @@ -1511,18 +1641,18 @@ def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut : # Will err if try to create section by naming 'more_ideas' if ( slash_path[-3] in ('search', 'categories', 'topics') ) or ( slash_path[-1] in ['more_ideas'] ): return quit('{}'.format('\n[' + x_tag + '] Search, Categories, Topics, more_ideas are not supported.\n') ) - board = get_board_info(sec_path, False, slash_path[-1], board_path, proxies) # need_get_section's True/False not used + board = get_board_info(sec_path, False, slash_path[-1], board_path, proxies, cookies) # need_get_section's True/False not used try: - PIN_SESSION = get_session(0, proxies) - IMGS_SESSION = get_session(2, proxies) - IMG_SESSION = get_session(3, proxies) - V_SESSION = get_session(4, proxies) + PIN_SESSION = get_session(0, proxies, cookies) + IMGS_SESSION = get_session(2, proxies, cookies) + IMG_SESSION = get_session(3, proxies, cookies) + V_SESSION = get_session(4, proxies, cookies) fetch_imgs( board, slash_path[-3], slash_path[-2], slash_path[-1], False , arg_board_timestamp, arg_log_timestamp, url_path , arg_force, arg_rescrape, arg_img_only, arg_v_only , arg_dir, arg_thread_max , IMGS_SESSION, IMG_SESSION, V_SESSION, PIN_SESSION, proxies - , arg_cut, arg_el, fs_f_max ) + , cookies, arg_cut, arg_el, fs_f_max ) except KeyError: return quit(traceback.format_exc()) @@ -1531,30 +1661,30 @@ def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut : print('[i] Job is download single board by username/boardname: {}'.format(board_path)) if slash_path[-2] in ('search', 'categories', 'topics'): return quit('{}'.format('\n[' + x_tag + '] Search, Categories and Topics not supported.\n') ) - board, sections = get_board_info(board_path, arg_exclude_section, None, None, proxies) + board, sections = get_board_info(board_path, arg_exclude_section, None, None, proxies, cookies) try: - PIN_SESSION = get_session(0, proxies) - IMGS_SESSION = get_session(2, proxies) - IMG_SESSION = get_session(3, proxies) - V_SESSION = get_session(4, proxies) + PIN_SESSION = get_session(0, proxies, cookies) + IMGS_SESSION = get_session(2, proxies, cookies) + IMG_SESSION = get_session(3, proxies, cookies) + V_SESSION = get_session(4, proxies, cookies) fetch_imgs( board, slash_path[-2], slash_path[-1], None, False , arg_board_timestamp, arg_log_timestamp, url_path , arg_force, arg_rescrape, arg_img_only, arg_v_only , arg_dir, arg_thread_max , IMGS_SESSION, IMG_SESSION, V_SESSION, PIN_SESSION, proxies - , arg_cut, arg_el, fs_f_max ) + , cookies, arg_cut, arg_el, fs_f_max ) if (not arg_exclude_section) and sections: sec_c = len(sections) print('[i] Trying to get ' + str(sec_c) + ' section{}'.format('s' if sec_c > 1 else '')) for sec in sections: sec_path = board_path + '/' + sec['slug'] - board = get_board_info(sec_path, False, sec['slug'], board_path, proxies) # False not using bcoz sections not [] already + board = get_board_info(sec_path, False, sec['slug'], board_path, proxies, cookies) # False not using bcoz sections not [] already fetch_imgs( board, slash_path[-2], slash_path[-1], sec['slug'], False , arg_board_timestamp, arg_log_timestamp, url_path , arg_force, arg_rescrape, arg_img_only, arg_v_only , arg_dir, arg_thread_max , IMGS_SESSION, IMG_SESSION, V_SESSION, PIN_SESSION, proxies - , arg_cut, arg_el, fs_f_max ) + , cookies, arg_cut, arg_el, fs_f_max ) except KeyError: return quit(traceback.format_exc()) @@ -1564,11 +1694,11 @@ def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut : if slash_path[-1] in ('search', 'categories', 'topics'): return quit('{}'.format('\n[' + x_tag + '] Search, Categories and Topics not supported.\n') ) try: - boards = fetch_boards( slash_path[-1], proxies ) - PIN_SESSION = get_session(0, proxies) - IMGS_SESSION = get_session(2, proxies) - IMG_SESSION = get_session(3, proxies) - V_SESSION = get_session(4, proxies) + boards = fetch_boards( slash_path[-1], proxies, cookies) + PIN_SESSION = get_session(0, proxies, cookies) + IMGS_SESSION = get_session(2, proxies, cookies) + IMG_SESSION = get_session(3, proxies, cookies) + V_SESSION = get_session(4, proxies, cookies) # Multiple logs saved inside relevant board dir for index, board in enumerate(boards): if 'name' not in board: @@ -1593,22 +1723,22 @@ def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut : , arg_force, arg_rescrape, arg_img_only, arg_v_only , arg_dir, arg_thread_max , IMGS_SESSION, IMG_SESSION, V_SESSION, PIN_SESSION, proxies - , arg_cut, arg_el, fs_f_max ) + , cookies, arg_cut, arg_el, fs_f_max ) if (not arg_exclude_section) and (board['section_count'] > 0): sec_c = board['section_count'] print('[i] Trying to get ' + str(sec_c) + ' section{}'.format('s' if sec_c > 1 else '')) # ags.es placeholder below always False bcoz above already check (not arg_exclude_section) - board, sections = get_board_info(board_path, False, None, None, proxies) + board, sections = get_board_info(board_path, False, None, None, proxies, cookies) for sec in sections: sec_path = board_path + '/' + sec['slug'] - board = get_board_info(sec_path, False, sec['slug'], board_path, proxies) + board = get_board_info(sec_path, False, sec['slug'], board_path, proxies, cookies) sec_uname, sec_bname = board_path.split('/') fetch_imgs( board, sec_uname, sec_bname, sec['slug'], False , arg_board_timestamp, arg_log_timestamp, url_path , arg_force, arg_rescrape, arg_img_only, arg_v_only , arg_dir, arg_thread_max , IMGS_SESSION, IMG_SESSION, V_SESSION, PIN_SESSION, proxies - , arg_cut, arg_el, fs_f_max ) + , cookies, arg_cut, arg_el, fs_f_max ) except KeyError: return quit(traceback.format_exc()) @@ -1620,7 +1750,6 @@ def run_library_main(arg_path :str, arg_dir :str, arg_thread_max :int, arg_cut : # after 999999999 days OR ~2,739,726 years, test case: str(timedelta(seconds= 86400000000000)) print('Can you revive me please? Thanks.') - def run_direct_main(): arg_parser = argparse.ArgumentParser(description='Download ALL board/section from ' + pinterest_logo + 'interest by username, username/boardname, username/boardname/section or link. Support image and video.\n\ @@ -1643,6 +1772,7 @@ def run_direct_main(): arg_parser.add_argument('-bt', '--board-timestamp', dest='board_timestamp', action='store_true', help='Suffix board directory name with unique timestamp.') arg_parser.add_argument('-lt', '--log-timestamp', dest='log_timestamp', action='store_true', help='Suffix log-pinterest-downloader.log filename with unique timestamp. Default filename is log-pinterest-downloader.log.\n\ Note: Pin id without Title/Description/Link/Metadata/Created_at will not write to log.') + arg_parser.add_argument('-co', '--cookies', help='Set the cookies file to be used to login into Pinterest. Useful for personal Hidden boards') arg_parser.add_argument('-f', '--force', action='store_true', help='Force re-download even if image already exist. Normally used with -rs') # Need reverse images order(previously is latest to oldest) to avoid abort this need re-download in-between missing images. arg_parser.add_argument('-rs', '--re-scrape', dest='rescrape', action='store_true', help='Default is only fetch new images since latest(highest) Pin ID local image to speed up update process.\n\ @@ -1672,9 +1802,8 @@ def run_direct_main(): , args.board_timestamp, args.log_timestamp , args.force, args.exclude_section, args.rescrape , args.img_only, args.v_only, args.update_all - , args.https_proxy, args.http_proxy) + , args.https_proxy, args.http_proxy, args.cookies) - if __name__ == '__main__': try: run_direct_main() @@ -1686,4 +1815,3 @@ if __name__ == '__main__': quit('') except: quit(traceback.format_exc()) -