關於蝦皮爬蟲，對於 seller 的登入問題

python3 網頁爬蟲 crawler

zivhsiao 2022-04-19 11:57:27 ‧ 1616 瀏覽

分享至

各位大大，您好，
我有試過 selenium 或者有用 requests
selenium 會遇到要點選連結，發簡訊的問題

requests 我用 cookies 的方式
好像是不正確的，因為跑很久才會出來不正確的

回傳的信息： ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

請問是否不能透過 cookies 登入的方式？
還請各位大大是否可以告知，謝謝！

import requests
import re
import pandas as pd
import http.cookiejar
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from lxml import etree, html

import json

url = 'https://seller.shopee.tw/portal/product/list/all'
headers = requests.utils.default_headers()

headers.update({
        'Host': 'seller.shopee.tw',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.44',
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'Accept-Encoding': 'gzip, deflate, br',
        'Referer': 'https://seller.shopee.tw/account/signin?next=index.index&params=%5B%5D',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Content-Length': '175',
        'Connection': 'keep-alive',
        'cache-control': 'max-age=0',
        'Cookie':'SPC_CDS=c6d83195-36d0-44d0-8cf3-28048ab0a801; SPC_SC_SA_TK=; SPC_SC_SA_UD=; SC_DFP=8fIlJwr9WELra90iOnVKcX5q50C77BVq; SPC_F=WUoPxOooqqf7bZ4zi9YbiC51pOD8kAF4; _gcl_au=1.1.554183486.1650289319; SPC_SI=mall.yG6gp8FzRwFbCWAUMpO672W58fJwTtun; _gid=GA1.2.430041453.1650289320; SC_SSO=-; SC_SSO_U=-; _QPWSDCXHZQA=de866c47-e411-4af3-e561-2c59a7c453ec; UYOMAPJWEMDGJ=; _ga_RPSBE3TQZZ=GS1.1.1650334609.103.0.1650334609.60; _ga=GA1.1.11249714.1650289320; G_ENABLED_IDPS=google; shopee_webUnique_ccd=f70yT08jegushoSqNREyFg%3D%3D%7CFriBOLTO%2BOf6MbIucNOxDDB1AoeJ%2B4qXZCgpskp9M42zwyM1YdfDM1orDXb3iK9e7uNOj9tJ43LU3BNIcNz1%7Cd01NDUQtBfXRtBuX%7C04%7C3; SPC_IVS=; SPC_R_T_ID=PzjMC/wTquV/Q6j9BN6no6yfCGcWgIOXIWen5BjXAPpYwDtvihLaOf58sJIL9Usig5/YxC7POPfsxpfI3tAVch/zPXCa+7IOddYdlgb1Qz8=; SPC_R_T_IV=tvDRVjWvzEPbV0YRDn4z6g==; SPC_T_ID=PzjMC/wTquV/Q6j9BN6no6yfCGcWgIOXIWen5BjXAPpYwDtvihLaOf58sJIL9Usig5/YxC7POPfsxpfI3tAVch/zPXCa+7IOddYdlgb1Qz8=; SPC_T_IV=tvDRVjWvzEPbV0YRDn4z6g==; SPC_SC_TK=94ee3ddd4e5e841c5812a6f9ceed87d8; SPC_EC="ai6N7Ekh6VVCe6tJgCoVI6vY9cRXXbOyhAOvxwFGW+A8s6AdZ7x2d1jV6tW6jwQvw/0wimB6LynfafPaL0O0FQQc0tWZcnDI76rwVY244MMWRbX4NelOK/XC2E43fl3bwZ95ltlJZoZGEiyaTpA1lA=="; SPC_ST=".RzJRVnF2OVoxR0xuV2k3ThY1Hz7uXPdsCrM0pqko5udwfsPr+jrJbZR/cIHfPcWS4bBj80rmI9LNN3uPlSUr35P7cMLJ6xA59FSQRJcKZJSM9dqsCxfK4m7M9RrK/q2p6oqcWdNjnXn5Sx1Y7r5yYgHydXWqcFzd/MEveMhYU3eZFOhWkBXEqqQngVPzK0Rk"; SPC_WST="ai6N7Ekh6VVCe6tJgCoVI6vY9cRXXbOyhAOvxwFGW+A8s6AdZ7x2d1jV6tW6jwQvw/0wimB6LynfafPaL0O0FQQc0tWZcnDI76rwVY244MMWRbX4NelOK/XC2E43fl3bwZ95ltlJZoZGEiyaTpA1lA=="; SPC_SC_UD=3778846; SPC_U=3778846; SPC_STK="4SoACDMBGg2//PSvdIBpZp85qefsnIMBjxjvLEFV0NFNqYSZOV+F4PbFs0tfIRj0CSaS4lyOa1kTuGRy3rUsLHeC1VdNEk3CmHjmIwRk2NYLcdttubYa8kZu0ygQao9WHKC5wmwJrUhyfprOL25imvks2kiuNCeQi0lP/NCGFzQ="'
})

r = requests.get(url, headers=headers)
print(r.status_code)
print(r.history)
print(r.url)

zivhsiao iT邦新手 4 級 ‧ 2022-04-19 13:41:22 檢舉

裏面我寫到
headers.update({
'Host': 'seller.shopee.tw',
'referer': 'https://seller.shopee.tw/account/signin',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="100", "Microsoft Edge";v="100"',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.44',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Accept-Encoding': 'gzip, deflate, br',
'etag': 'W/"625bc6cc-18351"',
'content-type': 'text/html',
'content-encoding': 'gzip',
'Connection': 'keep-alive',
'cache-control': 'max-age=0',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'same-origin',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'cookie':'SPC_CDS=c6d83195-36d0-44d0-8cf3-28048ab0a801; SPC_SC_SA_TK=; SPC_SC_SA_UD=; SC_DFP=8fIlJwr9WELra90iOnVKcX5q50C77BVq; SPC_F=WUoPxOooqqf7bZ4zi9YbiC51pOD8kAF4; _gcl_au=1.1.554183486.1650289319; SPC_SI=mall.yG6gp8FzRwFbCWAUMpO672W58fJwTtun; _gid=GA1.2.430041453.1650289320; SC_SSO=-; SC_SSO_U=-; _QPWSDCXHZQA=de866c47-e411-4af3-e561-2c59a7c453ec; UYOMAPJWEMDGJ=; _ga_RPSBE3TQZZ=GS1.1.1650334609.103.0.1650334609.60; _ga=GA1.1.11249714.1650289320; G_ENABLED_IDPS=google; SPC_IVS=; SPC_SC_TK=94ee3ddd4e5e841c5812a6f9ceed87d8; SPC_WST="ai6N7Ekh6VVCe6tJgCoVI6vY9cRXXbOyhAOvxwFGW+A8s6AdZ7x2d1jV6tW6jwQvw/0wimB6LynfafPaL0O0FQQc0tWZcnDI76rwVY244MMWRbX4NelOK/XC2E43fl3bwZ95ltlJZoZGEiyaTpA1lA=="; SPC_SC_UD=3778846; SPC_U=3778846; SPC_STK="4SoACDMBGg2//PSvdIBpZp85qefsnIMBjxjvLEFV0NFNqYSZOV+F4PbFs0tfIRj0CSaS4lyOa1kTuGRy3rUsLHeC1VdNEk3CmHjmIwRk2NYLcdttubYa8kZu0ygQao9WHKC5wmwJrUhyfprOL25imvks2kiuNCeQi0lP/NCGFzQ="; SPC_ST=.RzJRVnF2OVoxR0xuV2k3ThY1Hz7uXPdsCrM0pqko5udwfsPr+jrJbZR/cIHfPcWS4bBj80rmI9LNN3uPlSUr35P7cMLJ6xA59FSQRJcKZJSM9dqsCxfK4m7M9RrK/q2p6oqcWdNjnXn5Sx1Y7r5yYgHydXWqcFzd/MEveMhYU3eZFOhWkBXEqqQngVPzK0Rk; SPC_CT_211f0b57="1650344275.xZ3U7QetUodk1jjefpZY3t1KfHp5thh1D7OeUXhn+nQ="; SPC_CT_854018f2="1650344294.j+FAA/ONW4rv0o8VHlMV14vv+6CsbG+9OsjCEoCyyyM="; SPC_R_T_ID=+Z5F+YcOlpTKWQ3bl62ncYNrIuhfF+gzNXhLLj3PtNF+RXm3V1nnRAeRHMWA7K9capslneAQwHmhu0gNVbb9sVwNj9PwARZjIL3cRw0wIrM=; SPC_R_T_IV=KHpzXB2WjqowpoZaP6f3TA==; SPC_T_ID=+Z5F+YcOlpTKWQ3bl62ncYNrIuhfF+gzNXhLLj3PtNF+RXm3V1nnRAeRHMWA7K9capslneAQwHmhu0gNVbb9sVwNj9PwARZjIL3cRw0wIrM=; SPC_T_IV=KHpzXB2WjqowpoZaP6f3TA==; SPC_EC=STNtRmFpckhOQ1BUOFBYMirvHkCH1ekBkGbj8e8ikPM3s4uyrDo/lfe6gT4Q/nyjamdleQ+ZNtiDMkw0O5C5rxTwb0D4JeJw6KsQAEM737X1oDU4MA5I6jNQxm51MHxJ98eGwbTEeWjmgmDyI5/iJw==; shopee_webUnique_ccd=qXlwjqlgy6vcerzZDDo8EQ%3D%3D%7CFriBOLTO%2BOf6MbIucNOxDKFhAseJ%2B4qXZJrasXpmu1F6QnhEcson8Y5hUR32GpCu4CGZ2xQO74jya2BJfdzw%7Cd01NDUQtBfXRtBuX%7C04%7C3'
})

看起來是有東西回覆，可是都是一些 js 會出來的東西
可是就是沒有 <body><div id="all"></div> 裏面的東西

裏面我寫到
headers.update({
        'Host': 'seller.shopee.tw',
        'referer': 'https://seller.shopee.tw/account/signin',
        'sec-ch-ua': '&quot; Not A;Brand&quot;;v=&quot;99&quot;, &quot;Chromium&quot;;v=&quot;100&quot;, &quot;Microsoft Edge&quot;;v=&quot;100&quot;',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.44',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'Accept-Encoding': 'gzip, deflate, br',
        'etag': 'W/&quot;625bc6cc-18351&quot;',
        'content-type': 'text/html',
        'content-encoding': 'gzip',
        'Connection': 'keep-alive',
        'cache-control': 'max-age=0',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '&quot;Windows&quot;',
        'sec-fetch-dest': 'document',
        'sec-fetch-mode': 'navigate',
        'sec-fetch-site': 'same-origin',
        'sec-fetch-user': '?1',
        'upgrade-insecure-requests': '1',
        'cookie':'SPC_CDS=c6d83195-36d0-44d0-8cf3-28048ab0a801; SPC_SC_SA_TK=; SPC_SC_SA_UD=; SC_DFP=8fIlJwr9WELra90iOnVKcX5q50C77BVq; SPC_F=WUoPxOooqqf7bZ4zi9YbiC51pOD8kAF4; _gcl_au=1.1.554183486.1650289319; SPC_SI=mall.yG6gp8FzRwFbCWAUMpO672W58fJwTtun; _gid=GA1.2.430041453.1650289320; SC_SSO=-; SC_SSO_U=-; _QPWSDCXHZQA=de866c47-e411-4af3-e561-2c59a7c453ec; UYOMAPJWEMDGJ=; _ga_RPSBE3TQZZ=GS1.1.1650334609.103.0.1650334609.60; _ga=GA1.1.11249714.1650289320; G_ENABLED_IDPS=google; SPC_IVS=; SPC_SC_TK=94ee3ddd4e5e841c5812a6f9ceed87d8; SPC_WST=&quot;ai6N7Ekh6VVCe6tJgCoVI6vY9cRXXbOyhAOvxwFGW+A8s6AdZ7x2d1jV6tW6jwQvw/0wimB6LynfafPaL0O0FQQc0tWZcnDI76rwVY244MMWRbX4NelOK/XC2E43fl3bwZ95ltlJZoZGEiyaTpA1lA==&quot;; SPC_SC_UD=3778846; SPC_U=3778846; SPC_STK=&quot;4SoACDMBGg2//PSvdIBpZp85qefsnIMBjxjvLEFV0NFNqYSZOV+F4PbFs0tfIRj0CSaS4lyOa1kTuGRy3rUsLHeC1VdNEk3CmHjmIwRk2NYLcdttubYa8kZu0ygQao9WHKC5wmwJrUhyfprOL25imvks2kiuNCeQi0lP/NCGFzQ=&quot;; SPC_ST=.RzJRVnF2OVoxR0xuV2k3ThY1Hz7uXPdsCrM0pqko5udwfsPr+jrJbZR/cIHfPcWS4bBj80rmI9LNN3uPlSUr35P7cMLJ6xA59FSQRJcKZJSM9dqsCxfK4m7M9RrK/q2p6oqcWdNjnXn5Sx1Y7r5yYgHydXWqcFzd/MEveMhYU3eZFOhWkBXEqqQngVPzK0Rk; SPC_CT_211f0b57=&quot;1650344275.xZ3U7QetUodk1jjefpZY3t1KfHp5thh1D7OeUXhn+nQ=&quot;; SPC_CT_854018f2=&quot;1650344294.j+FAA/ONW4rv0o8VHlMV14vv+6CsbG+9OsjCEoCyyyM=&quot;; SPC_R_T_ID=+Z5F+YcOlpTKWQ3bl62ncYNrIuhfF+gzNXhLLj3PtNF+RXm3V1nnRAeRHMWA7K9capslneAQwHmhu0gNVbb9sVwNj9PwARZjIL3cRw0wIrM=; SPC_R_T_IV=KHpzXB2WjqowpoZaP6f3TA==; SPC_T_ID=+Z5F+YcOlpTKWQ3bl62ncYNrIuhfF+gzNXhLLj3PtNF+RXm3V1nnRAeRHMWA7K9capslneAQwHmhu0gNVbb9sVwNj9PwARZjIL3cRw0wIrM=; SPC_T_IV=KHpzXB2WjqowpoZaP6f3TA==; SPC_EC=STNtRmFpckhOQ1BUOFBYMirvHkCH1ekBkGbj8e8ikPM3s4uyrDo/lfe6gT4Q/nyjamdleQ+ZNtiDMkw0O5C5rxTwb0D4JeJw6KsQAEM737X1oDU4MA5I6jNQxm51MHxJ98eGwbTEeWjmgmDyI5/iJw==; shopee_webUnique_ccd=qXlwjqlgy6vcerzZDDo8EQ%3D%3D%7CFriBOLTO%2BOf6MbIucNOxDKFhAseJ%2B4qXZJrasXpmu1F6QnhEcson8Y5hUR32GpCu4CGZ2xQO74jya2BJfdzw%7Cd01NDUQtBfXRtBuX%7C04%7C3'
})

看起來是有東西回覆，可是都是一些 js 會出來的東西
可是就是沒有  &lt;body&gt;&lt;div id=&quot;all&quot;&gt;&lt;/div&gt; 裏面的東西

修改

登入發表討論