Closed InMyDreammer closed 1 year ago
报错信息如下 MediaCrawler ERROR aweme_id: 7266050530072481076 get comments failed, error: Expecting value: line 1 column 1 (char 0), 按理说即使又抖音的反爬取机制,但是也应该有一两条数据,但是一条也没有,以下是我修改过后的保存到本地csv的代码: import json from typing import Dict, List
from tortoise import fields from tortoise.models import Model import os import config from tools import utils import pandas as pd
class DouyinBaseModel(Model): id = fields.IntField(pk=True, autoincrement=True, description="自增ID") user_id = fields.CharField(null=True, max_length=64, description="用户ID") sec_uid = fields.CharField(null=True, max_length=128, description="用户sec_uid") short_user_id = fields.CharField(null=True, max_length=64, description="用户短ID") user_unique_id = fields.CharField(null=True, max_length=64, description="用户唯一ID") nickname = fields.CharField(null=True, max_length=64, description="用户昵称") avatar = fields.CharField(null=True, max_length=255, description="用户头像地址") user_signature = fields.CharField(null=True, max_length=500, description="用户签名") ip_location = fields.CharField(null=True, max_length=255, description="评论时的IP地址") add_ts = fields.BigIntField(description="记录添加时间戳") last_modify_ts = fields.BigIntField(description="记录最后修改时间戳")
class Meta: abstract = True
class DouyinAweme(DouyinBaseModel): aweme_id = fields.CharField(max_length=64, index=True, description="视频ID") aweme_type = fields.CharField(max_length=16, description="视频类型") title = fields.CharField(null=True, max_length=500, description="视频标题") desc = fields.TextField(null=True, description="视频描述") create_time = fields.BigIntField(description="视频发布时间戳", index=True) liked_count = fields.CharField(null=True, max_length=16, description="视频点赞数") comment_count = fields.CharField(null=True, max_length=16, description="视频评论数") share_count = fields.CharField(null=True, max_length=16, description="视频分享数") collected_count = fields.CharField(null=True, max_length=16, description="视频收藏数")
class Meta: table = "douyin_aweme" table_description = "抖音视频" def __str__(self): return f"{self.aweme_id} - {self.title}"
def save_data_to_excel(data: Dict, sheet_name: str): file_path = 'D:\douyin.xlsx' if not os.path.exists(file_path): df = pd.DataFrame(columns=list(data.keys())) df.to_excel(file_path, sheet_name=sheet_name,index=False, engine='openpyxl') else: with pd.ExcelFile(file_path) as xls:
df_old = pd.read_excel(xls, sheet_name=sheet_name, engine='openpyxl') # 使用 pd.concat 替代 append 方法 df_new = pd.DataFrame([data]) df_combined = pd.concat([df_old, df_new], ignore_index=True) df_combined.to_excel(file_path, sheet_name=sheet_name, index=False, engine='openpyxl')
async def save_aweme_to_excel(aweme_data: Dict): save_data_to_excel(aweme_data, "aweme")
async def save_comment_to_excel(comment_data: Dict): save_data_to_excel(comment_data, "comments")
#
class DouyinAwemeComment(DouyinBaseModel): comment_id = fields.CharField(max_length=64, index=True, description="评论ID") aweme_id = fields.CharField(max_length=64, index=True, description="视频ID") content = fields.TextField(null=True, description="评论内容") create_time = fields.BigIntField(description="评论时间戳") sub_comment_count = fields.CharField(max_length=16, description="评论回复数")
class Meta: table = "douyin_aweme_comment" table_description = "抖音视频评论" def __str__(self): return f"{self.comment_id} - {self.content}"
async def update_douyin_aweme(aweme_item: Dict): aweme_id = aweme_item.get("aweme_id") user_info = aweme_item.get("author", {}) interact_info = aweme_item.get("statistics", {}) local_db_item = { "aweme_id": aweme_id, "aweme_type": aweme_item.get("aweme_type"), "title": aweme_item.get("desc", ""), "desc": aweme_item.get("desc", ""), "create_time": aweme_item.get("create_time"), "user_id": user_info.get("uid"), "sec_uid": user_info.get("sec_uid"), "short_user_id": user_info.get("short_id"), "user_unique_id": user_info.get("unique_id"), "user_signature": user_info.get("signature"), "nickname": user_info.get("nickname"), "avatar": user_info.get("avatar_thumb", {}).get("url_list", [""])[0], "liked_count": interact_info.get("digg_count"), "collected_count": interact_info.get("collect_count"), "comment_count": interact_info.get("comment_count"), "share_count": interact_info.get("share_count"), "ip_location": aweme_item.get("ip_label", ""), "last_modify_ts": utils.get_current_timestamp(), } print(f"douyin aweme id:{aweme_id}, title:{local_db_item.get('title')}") if config.IS_SAVED_DATABASED: if not await DouyinAweme.filter(aweme_id=aweme_id).exists(): local_db_item["add_ts"] = utils.get_current_timestamp() await DouyinAweme.create(local_db_item) else: await DouyinAweme.filter(aweme_id=aweme_id).update(local_db_item) else: await save_aweme_to_excel(local_db_item)
async def batch_update_dy_aweme_comments(aweme_id: str, comments: List[Dict]): if not comments: return for comment_item in comments: await update_dy_aweme_comment(aweme_id, comment_item)
async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict): comment_aweme_id = comment_item.get("aweme_id") if aweme_id != comment_aweme_id: print(f"comment_aweme_id: {comment_aweme_id} != aweme_id: {aweme_id}") return user_info = comment_item.get("user", {}) comment_id = comment_item.get("cid") avatar_info = user_info.get("avatar_medium", {}) or user_info.get("avatar_300x300", {}) or user_info.get( "avatar_168x168", {}) or user_info.get("avatar_thumb", {}) or {} local_db_item = { "comment_id": comment_id, "create_time": comment_item.get("create_time"), "ip_location": comment_item.get("ip_label", ""), "aweme_id": aweme_id, "content": comment_item.get("text"), "content_extra": json.dumps(comment_item.get("text_extra", [])), "user_id": user_info.get("uid"), "sec_uid": user_info.get("sec_uid"), "short_user_id": user_info.get("short_id"), "user_unique_id": user_info.get("unique_id"), "user_signature": user_info.get("signature"), "nickname": user_info.get("nickname"), "avatar": avatar_info.get("url_list", [""])[0], "sub_comment_count": comment_item.get("reply_comment_total", 0), "last_modify_ts": utils.get_current_timestamp(), } print(f"douyin aweme comment: {comment_id}, content: {local_db_item.get('content')}") if config.IS_SAVED_DATABASED: if not await DouyinAwemeComment.filter(comment_id=comment_id).exists(): local_db_item["add_ts"] = utils.get_current_timestamp() await DouyinAwemeComment.create(local_db_item) else: await DouyinAwemeComment.filter(comment_id=comment_id).update(local_db_item) else: await save_comment_to_excel(local_db_item)
抖音评论爬去失效了,等待后续更新
fix it
报错信息如下 MediaCrawler ERROR aweme_id: 7266050530072481076 get comments failed, error: Expecting value: line 1 column 1 (char 0),
按理说即使又抖音的反爬取机制,但是也应该有一两条数据,但是一条也没有,以下是我修改过后的保存到本地csv的代码: import json from typing import Dict, List
from tortoise import fields from tortoise.models import Model import os import config from tools import utils import pandas as pd
class DouyinBaseModel(Model): id = fields.IntField(pk=True, autoincrement=True, description="自增ID") user_id = fields.CharField(null=True, max_length=64, description="用户ID") sec_uid = fields.CharField(null=True, max_length=128, description="用户sec_uid") short_user_id = fields.CharField(null=True, max_length=64, description="用户短ID") user_unique_id = fields.CharField(null=True, max_length=64, description="用户唯一ID") nickname = fields.CharField(null=True, max_length=64, description="用户昵称") avatar = fields.CharField(null=True, max_length=255, description="用户头像地址") user_signature = fields.CharField(null=True, max_length=500, description="用户签名") ip_location = fields.CharField(null=True, max_length=255, description="评论时的IP地址") add_ts = fields.BigIntField(description="记录添加时间戳") last_modify_ts = fields.BigIntField(description="记录最后修改时间戳")
class DouyinAweme(DouyinBaseModel): aweme_id = fields.CharField(max_length=64, index=True, description="视频ID") aweme_type = fields.CharField(max_length=16, description="视频类型") title = fields.CharField(null=True, max_length=500, description="视频标题") desc = fields.TextField(null=True, description="视频描述") create_time = fields.BigIntField(description="视频发布时间戳", index=True) liked_count = fields.CharField(null=True, max_length=16, description="视频点赞数") comment_count = fields.CharField(null=True, max_length=16, description="视频评论数") share_count = fields.CharField(null=True, max_length=16, description="视频分享数") collected_count = fields.CharField(null=True, max_length=16, description="视频收藏数")
def save_data_to_excel(data: Dict, sheet_name: str): file_path = 'D:\douyin.xlsx' if not os.path.exists(file_path): df = pd.DataFrame(columns=list(data.keys())) df.to_excel(file_path, sheet_name=sheet_name,index=False, engine='openpyxl') else: with pd.ExcelFile(file_path) as xls:
async def save_aweme_to_excel(aweme_data: Dict): save_data_to_excel(aweme_data, "aweme")
async def save_comment_to_excel(comment_data: Dict): save_data_to_excel(comment_data, "comments")
async def save_aweme_to_excel(aweme_data: Dict):
#
file_path = 'D:\douyin.xlsx'
if not os.path.exists(file_path):
raise Exception(f"File not found: {file_path}")
if not os.path.exists(file_path):
df = pd.DataFrame(columns=list(aweme_data.keys()))
df.to_excel(file_path, sheet_name='aweme', index=False, engine='openpyxl')
else:
df = pd.read_excel(file_path, sheet_name='aweme', engine='openpyxl')
df = df.append(aweme_data, ignore_index=True)
df.to_excel(file_path, sheet_name='aweme', index=False, engine='openpyxl')
#
async def save_comment_to_excel(comment_data: Dict):
file_path = 'D:\douyin.xlsx'
if not os.path.exists(file_path):
raise Exception(f"File not found: {file_path}")
if not os.path.exists(file_path):
df = pd.DataFrame(columns=list(comment_data.keys()))
df.to_excel(file_path, sheet_name='comments', index=False, engine='openpyxl')
else:
df = pd.read_excel(file_path, sheet_name='comments', engine='openpyxl')
df = df.append(comment_data, ignore_index=True)
df.to_excel(file_path, sheet_name='comments', index=False, engine='openpyxl')
class DouyinAwemeComment(DouyinBaseModel): comment_id = fields.CharField(max_length=64, index=True, description="评论ID") aweme_id = fields.CharField(max_length=64, index=True, description="视频ID") content = fields.TextField(null=True, description="评论内容") create_time = fields.BigIntField(description="评论时间戳") sub_comment_count = fields.CharField(max_length=16, description="评论回复数")
async def update_douyin_aweme(aweme_item: Dict): aweme_id = aweme_item.get("aweme_id") user_info = aweme_item.get("author", {}) interact_info = aweme_item.get("statistics", {}) local_db_item = { "aweme_id": aweme_id, "aweme_type": aweme_item.get("aweme_type"), "title": aweme_item.get("desc", ""), "desc": aweme_item.get("desc", ""), "create_time": aweme_item.get("create_time"), "user_id": user_info.get("uid"), "sec_uid": user_info.get("sec_uid"), "short_user_id": user_info.get("short_id"), "user_unique_id": user_info.get("unique_id"), "user_signature": user_info.get("signature"), "nickname": user_info.get("nickname"), "avatar": user_info.get("avatar_thumb", {}).get("url_list", [""])[0], "liked_count": interact_info.get("digg_count"), "collected_count": interact_info.get("collect_count"), "comment_count": interact_info.get("comment_count"), "share_count": interact_info.get("share_count"), "ip_location": aweme_item.get("ip_label", ""), "last_modify_ts": utils.get_current_timestamp(), } print(f"douyin aweme id:{aweme_id}, title:{local_db_item.get('title')}") if config.IS_SAVED_DATABASED: if not await DouyinAweme.filter(aweme_id=aweme_id).exists(): local_db_item["add_ts"] = utils.get_current_timestamp() await DouyinAweme.create(local_db_item) else: await DouyinAweme.filter(aweme_id=aweme_id).update(local_db_item) else: await save_aweme_to_excel(local_db_item)
async def batch_update_dy_aweme_comments(aweme_id: str, comments: List[Dict]): if not comments: return for comment_item in comments: await update_dy_aweme_comment(aweme_id, comment_item)
async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict): comment_aweme_id = comment_item.get("aweme_id") if aweme_id != comment_aweme_id: print(f"comment_aweme_id: {comment_aweme_id} != aweme_id: {aweme_id}") return user_info = comment_item.get("user", {}) comment_id = comment_item.get("cid") avatar_info = user_info.get("avatar_medium", {}) or user_info.get("avatar_300x300", {}) or user_info.get( "avatar_168x168", {}) or user_info.get("avatar_thumb", {}) or {} local_db_item = { "comment_id": comment_id, "create_time": comment_item.get("create_time"), "ip_location": comment_item.get("ip_label", ""), "aweme_id": aweme_id, "content": comment_item.get("text"), "content_extra": json.dumps(comment_item.get("text_extra", [])), "user_id": user_info.get("uid"), "sec_uid": user_info.get("sec_uid"), "short_user_id": user_info.get("short_id"), "user_unique_id": user_info.get("unique_id"), "user_signature": user_info.get("signature"), "nickname": user_info.get("nickname"), "avatar": avatar_info.get("url_list", [""])[0], "sub_comment_count": comment_item.get("reply_comment_total", 0), "last_modify_ts": utils.get_current_timestamp(), } print(f"douyin aweme comment: {comment_id}, content: {local_db_item.get('content')}") if config.IS_SAVED_DATABASED: if not await DouyinAwemeComment.filter(comment_id=comment_id).exists(): local_db_item["add_ts"] = utils.get_current_timestamp() await DouyinAwemeComment.create(local_db_item) else: await DouyinAwemeComment.filter(comment_id=comment_id).update(local_db_item) else: await save_comment_to_excel(local_db_item)