Python实战系列-获取单页和多页京东评论

Python实战系列-获取单页和多页京东评论如下：

#!/usr/bin/env python
# coding: utf-8

# # 爬取单页评论

# In[2]:


from selenium import webdriver
import time

from selenium.webdriver.common.by import By

browser = webdriver.Chrome()
url = 'https://item.jd.com/100000177770.html'
browser.get(url)


# In[ ]:


time.sleep(20)  # 需要手动登录下，只有登录后才能查看到评论


# In[3]:


#browser.find_element_by_xpath('//*[@id="detail"]/div[1]/ul/li[5]').click()  # 点击“商品评价”按钮
browser.find_element(by=By.XPATH, value='//*[@id="detail"]/div[1]/ul/li[5]').click()
data = browser.page_source  # 获取此时的网页源代码
data


# In[4]:


import re
p_comment = '<p class="comment-con">(.*?)</p>'
comment = re.findall(p_comment, data)
comment
print(comment)

# In[ ]:


for i in range(len(comment)):
    comment[i] = re.sub('<.*?>', '', comment[i])  
    print(str(i+1) + '.' + comment[i])


# # 爬取多页评论

# In[13]:
time.sleep(10)

#browser.find_element_by_xpath('//*[@id="comment-0"]/div[13]/div/div/a[7]').click()  # 第一页的下一页按钮和之后的不一样
browser.find_element(by=By.XPATH, value='//*[@id="comment-0"]/div[12]/div/div/a[7]').click()

# //*[@id="askAnswer"]/div[2]/div[2]/div[12]/div/a[8]
#第一页：//*[@id="comment-0"]/div[12]/div/div/a[7]    //*[@id="comment-0"]/div[12]/div/div/a[7]
#第二页：//*[@id="comment-0"]/div[12]/div/div/a[8]
#第三页：//*[@id="comment-0"]/div[12]/div/div/a[8]
# In[17]:


for i in range(9):
    #browser.find_element_by_xpath('//*[@id="comment-0"]/div[13]/div/div/a[8]').click()
    browser.find_element(by=By.XPATH, value='//*[@id="comment-0"]/div[12]/div/div/a[8]').click()
    time.sleep(2)


# In[18]:


data_all = data
#browser.find_element_by_xpath('//*[@id="comment-0"]/div[13]/div/div/a[7]').click()  # 第一页的向下一页按钮和之后的不一样
browser.find_element(by=By.XPATH, value='//*[@id="comment-0"]/div[13]/div/div/a[7]').click()
time.sleep(3)
data = browser.page_source
data_all = data_all + data

for i in range(8):
    #browser.find_element_by_xpath('//*[@id="comment-0"]/div[13]/div/div/a[8]').click()
    browser.find_element(by=By.XPATH, value='//*[@id="comment-0"]/div[13]/div/div/a[8]').click()
    time.sleep(2)
    data = browser.page_source
    data_all = data_all + data


# In[19]:


data_all


# In[20]:


import re
p_comment = '<p class="comment-con">(.*?)</p>'
comment = re.findall(p_comment, data_all)
comment


# # 爬取负面评论

# In[21]:


#browser.find_element_by_xpath('//*[@id="comment"]/div[2]/div[2]/div[1]/ul/li[7]/a/em').click()  # 点击差评按钮
browser.find_element(by=By.XPATH, value='//*[@id="comment"]/div[2]/div[2]/div[1]/ul/li[7]/a/em').click()
# In[ ]:


# 剩下的逻辑和之前的代码一摸一样了


# In[ ]:





# In[ ]:





# # 2 京东商品评价信息 - Requests获取法

# In[46]:


import requests
import re
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'}

url = 'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100000177770&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1'
res = requests.get(url, headers=headers).text
# print(res)

p_comment = '"content":"(.*?)"'
comment = re.findall(p_comment, res)
comment

for i in range(len(comment)):
    comment[i] = comment[i].replace(r'\n', '')
    print(str(i+1) + '.' + comment[i])


# # 获取多页信息

# In[48]:


import requests
import re
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'}

res_all = ''
for i in range(10):
    url = 'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100000177770&score=0&sortType=5&pageSize=10&isShadowSku=0&fold=1&page=' + str(i)
    res = requests.get(url, headers=headers).text
    res_all = res_all + res  # 也可以简写成res_all += res
    
p_comment = '"content":"(.*?)"'
comment = re.findall(p_comment, res_all)

for i in range(len(comment)):
    comment[i] = comment[i].replace(r'\n', '') 
    print(str(i+1) + '.' + comment[i])


# # 爬取负面新闻

# 只需要把url中的score参数由0换成1即可，如果是爬取正面新闻，则是换成3

# In[50]:


#browser.find_element_by_xpath('//*[@id="noticeDetail"]/div/div[1]/div[3]/div[1]/button').click()
browser.find_element(by=By.XPATH, value='//*[@id="noticeDetail"]/div/div[1]/div[3]/div[1]/button').click()