From 16849ffed2e78fb1bf9ba28de0162f78888924ab Mon Sep 17 00:00:00 2001 From: YuanChen Date: Thu, 2 Jul 2026 00:59:09 +0800 Subject: [PATCH 1/7] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E6=96=B9?= =?UTF-8?q?=E6=B3=95=EF=BC=9A=5Fget=5Ftimestamp=5Fprefix(self,=20created?= =?UTF-8?q?=5Fat)=20=E2=80=94=20=E7=BB=9F=E4=B8=80=E7=9A=84=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=90=8D=E5=89=8D=E7=BC=80=E7=94=9F=E6=88=90=E5=99=A8?= =?UTF-8?q?=EF=BC=8C=E5=B0=86=202026-07-01T12:00:00=20=E8=BD=AC=E6=8D=A2?= =?UTF-8?q?=E4=B8=BA=202026-07-01=5F12-00-00=EF=BC=8C=E4=B8=8E=20=5Fdownlo?= =?UTF-8?q?ad=5Fweibo=5Fimages=20=E4=BD=BF=E7=94=A8=E5=AE=8C=E5=85=A8?= =?UTF-8?q?=E4=B8=80=E8=87=B4=E7=9A=84=E6=A0=BC=E5=BC=8F=E3=80=82=E5=8C=85?= =?UTF-8?q?=E5=90=AB=E5=9B=9E=E9=80=80=E9=80=BB=E8=BE=91=EF=BC=9A=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E5=A4=B1=E8=B4=A5=E6=97=B6=E9=99=8D=E7=BA=A7=E4=B8=BA?= =?UTF-8?q?=E6=97=A7=E6=A0=BC=E5=BC=8F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- weibo.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) mode change 100755 => 100644 weibo.py diff --git a/weibo.py b/weibo.py old mode 100755 new mode 100644 index b56a8705..7dbce5fb --- a/weibo.py +++ b/weibo.py @@ -1259,9 +1259,22 @@ def insert_file_sqlite(self, file_path, weibo_id, url, binary): self.sqlite_insert(con, file_data, "bins") con.close() + def _get_timestamp_prefix(self, created_at): + """根据微博发布时间生成人类可读的文件名前缀 + 格式：YYYY-MM-DD_HH-MM-SS，与 _download_weibo_images 保持一致 + """ + try: + time_obj = datetime.strptime(created_at, DTFORMAT) + date_str = time_obj.strftime("%Y-%m-%d") + time_str = time_obj.strftime("%H:%M:%S") + return f"{date_str}_{time_str.replace(':', '-')}" + except (ValueError, KeyError): + # 回退：使用旧的截断格式 + return created_at[:11].replace("-", "") + "_" + str(w["id"]) + def handle_download(self, file_type, file_dir, urls, w): """处理下载相关操作""" - file_prefix = w["created_at"][:11].replace("-", "") + "_" + str(w["id"]) + file_prefix = self._get_timestamp_prefix(w["created_at"]) if file_type == "img": if "," in urls: url_list = urls.split(",") @@ -1308,7 +1321,7 @@ def get_download_file_names(self, file_type, urls, w): if not urls: return [] - file_prefix = w["created_at"][:11].replace("-", "") + "_" + str(w["id"]) + file_prefix = self._get_timestamp_prefix(w["created_at"]) file_names = [] if file_type == "img": From c396deacf1105aa64ed67357f9e6ba52a4a4eb92 Mon Sep 17 00:00:00 2001 From: YuanChen Date: Thu, 2 Jul 2026 01:03:52 +0800 Subject: [PATCH 2/7] =?UTF-8?q?feat:=20=E5=B0=86=20get=5Flive=5Fphoto=5Fur?= =?UTF-8?q?l=20=E4=BB=8E=E8=AF=BB=E5=8F=96=E7=8B=AC=E7=AB=8B=E7=9A=84=20li?= =?UTF-8?q?ve=5Fphoto=20=E6=95=B0=E7=BB=84=E6=94=B9=E4=B8=BA=E4=BB=8E=20pi?= =?UTF-8?q?cs=20=E6=95=B0=E7=BB=84=E4=B8=AD=E6=8F=90=E5=8F=96=20type:=20li?= =?UTF-8?q?vephoto=20=E6=9D=A1=E7=9B=AE=E7=9A=84=20videoSrc=20=E5=B9=B6?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E5=85=B6=E5=9C=A8=E8=BF=87=E6=BB=A4=E5=90=8E?= =?UTF-8?q?=20pics=20=E4=B8=AD=E7=9A=84=E4=BD=8D=E7=BD=AE=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=EF=BC=8C=E4=BD=BF=20Live=20Photo=20=E8=A7=86=E9=A2=91?= =?UTF-8?q?=E7=9A=84=E6=96=87=E4=BB=B6=E5=90=8D=E5=BA=8F=E5=8F=B7=E4=B8=8E?= =?UTF-8?q?=E5=AF=B9=E5=BA=94=E5=9B=BE=E7=89=87=E6=96=87=E4=BB=B6=E5=90=8D?= =?UTF-8?q?=E5=BA=8F=E5=8F=B7=E4=BF=9D=E6=8C=81=E4=B8=80=E8=87=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- weibo.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/weibo.py b/weibo.py index 7dbce5fb..d1a9c8bd 100644 --- a/weibo.py +++ b/weibo.py @@ -1016,9 +1016,37 @@ def get_inline_image_urls(self, weibo_info): def get_live_photo_url(self, weibo_info): - """获取Live Photo视频URL""" - live_photo_list = weibo_info.get("live_photo", []) - return ";".join(live_photo_list) if live_photo_list else "" + """获取Live Photo视频URL及其在过滤后pics数组中的位置索引 + + 从 pics 数组中提取 type='livephoto' 的 videoSrc 字段， + 并使用与 get_pics() 相同的过滤逻辑，使 live_photo 的文件名索引 + 与对应的 still image 文件名索引保持一致。 + + 返回 dict: {"url": 分号分隔的URL字符串, "indices": [1-based位置列表]} + """ + pic_info = weibo_info.get("pics") or [] + urls = [] + indices = [] + + if pic_info: + filtered_pos = 0 # 在过滤后的 pics 列表中的 1-based 位置 + for pic in pic_info: + # 与 get_pics() 保持完全一致的过滤逻辑 + if not isinstance(pic, dict) or not pic.get('large'): + continue + if pic.get('type') == 'video': + continue + filtered_pos += 1 + + # 检查该 pic 是否是 livephoto 类型并有 videoSrc + if pic.get('type') == 'livephoto' and pic.get('videoSrc'): + urls.append(pic['videoSrc']) + indices.append(filtered_pos) + + return { + "url": ";".join(urls) if urls else "", + "indices": indices # 1-based 位置列表 + } def get_video_url(self, weibo_info): """获取微博普通视频URL""" @@ -1296,7 +1324,7 @@ def handle_download(self, file_type, file_dir, urls, w): file_name = file_prefix + file_suffix file_path = file_dir + os.sep + file_name self.download_one_file(urls, file_path, file_type, w["id"], w["created_at"]) - elif file_type == "video" or file_type == "live_photo": + elif file_type == "video": file_suffix = ".mp4" if ";" in urls: url_list = urls.split(";") @@ -1315,6 +1343,31 @@ def handle_download(self, file_type, file_dir, urls, w): file_name = file_prefix + file_suffix file_path = file_dir + os.sep + file_name self.download_one_file(urls, file_path, file_type, w["id"], w["created_at"]) + elif file_type == "live_photo": + # 使用 live_photo_indices 使 live_photo 文件名与对应图片文件名一致 + indices = w.get("live_photo_indices", []) + if ";" in urls: + url_list = urls.split(";") + for j, url in enumerate(url_list): + file_suffix = ".mov" if url.endswith(".mov") else ".mp4" + # 使用 indices 中的对应索引，若缺失则回退到 j+1 + idx = indices[j] if j < len(indices) else (j + 1) + file_name = file_prefix + "_" + str(idx) + file_suffix + file_path = file_dir + os.sep + file_name + self.download_one_file(url, file_path, file_type, w["id"], w["created_at"]) + if j < len(url_list) - 1: + sleep(random.uniform(1, 3)) + else: + file_suffix = ".mov" if urls.endswith(".mov") else ".mp4" + idx = indices[0] if indices else 1 + # 与图片命名保持一致：多张图片时加序号，单张图片不加序号 + pics_count = len([p for p in (w.get("pics", "").split(",")) if p]) if w.get("pics") else 0 + if pics_count > 1: + file_name = file_prefix + "_" + str(idx) + file_suffix + else: + file_name = file_prefix + file_suffix + file_path = file_dir + os.sep + file_name + self.download_one_file(urls, file_path, file_type, w["id"], w["created_at"]) def get_download_file_names(self, file_type, urls, w): """根据下载规则推导本地文件名，供 Markdown 链接使用""" @@ -1339,7 +1392,7 @@ def get_download_file_names(self, file_type, urls, w): else: file_name = file_prefix + file_suffix file_names.append(file_name) - elif file_type == "video" or file_type == "live_photo": + elif file_type == "video": url_list = urls.split(";") if ";" in urls else [urls] for i, url in enumerate(url_list): if not url: @@ -1350,6 +1403,22 @@ def get_download_file_names(self, file_type, urls, w): else: file_name = file_prefix + file_suffix file_names.append(file_name) + elif file_type == "live_photo": + indices = w.get("live_photo_indices", []) + url_list = urls.split(";") if ";" in urls else [urls] + for j, url in enumerate(url_list): + if not url: + continue + file_suffix = ".mov" if url.endswith(".mov") else ".mp4" + # 使用 indices 中的对应索引 + idx = indices[j] if j < len(indices) else (j + 1) + # 单张图片时不加序号，但仍有多个pics时需加序号以匹配对应图片 + pics_count = len([p for p in (w.get("pics", "").split(",")) if p]) if w.get("pics") else 0 + if pics_count > 1: + file_name = file_prefix + "_" + str(idx) + file_suffix + else: + file_name = file_prefix + file_suffix + file_names.append(file_name) return file_names @@ -1573,7 +1642,9 @@ def parse_weibo(self, weibo_info): weibo["article_url"] = self.get_article_url(selector) weibo["pics"] = self.get_pics(weibo_info) weibo["video_url"] = self.get_video_url(weibo_info) # 普通视频URL - weibo["live_photo_url"] = self.get_live_photo_url(weibo_info) # Live Photo视频URL + live_photo_data = self.get_live_photo_url(weibo_info) + weibo["live_photo_url"] = live_photo_data["url"] # Live Photo视频URL + weibo["live_photo_indices"] = live_photo_data["indices"] # Live Photo在过滤后pics中的1-based索引 weibo["links"] = self.get_link_urls(selector) weibo["location"] = self.get_location(selector) weibo["created_at"] = weibo_info["created_at"] @@ -2089,7 +2160,7 @@ def get_write_info(self, wrote_count): for w in self.weibo[wrote_count:]: wb = OrderedDict() for k, v in w.items(): - if k not in ["user_id", "screen_name", "retweet"]: + if k not in ["user_id", "screen_name", "retweet", "live_photo_indices"]: if k == "links": continue if "unicode" in str(type(v)): @@ -2101,7 +2172,7 @@ def get_write_info(self, wrote_count): if w.get("retweet"): wb["is_original"] = False for k2, v2 in w["retweet"].items(): - if k2 == "links": + if k2 == "links" or k2 == "live_photo_indices": continue if "unicode" in str(type(v2)): v2 = v2.encode("utf-8") From c76ee477e74acfaca0112316884df9e4546a96f4 Mon Sep 17 00:00:00 2001 From: YuanChen Date: Sat, 4 Jul 2026 16:54:55 +0800 Subject: [PATCH 3/7] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20markdown=5Fima?= =?UTF-8?q?ge=5Forder=20=E9=85=8D=E7=BD=AE=EF=BC=8C=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E4=B8=A4=E7=A7=8D=E5=9B=BE=E7=89=87=E4=B8=8E=20Live=20Photo=20?= =?UTF-8?q?=E7=9A=84=E6=8E=92=E5=88=97=E6=A8=A1=E5=BC=8F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.json | 1 + weibo.py | 85 +++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/config.json b/config.json index f3b047fa..553ec37e 100644 --- a/config.json +++ b/config.json @@ -11,6 +11,7 @@ "json" ], // 输出格式（可多选），可选项：csv, json, mysql, mongo, sqlite, post, markdown，示例：["markdown"] 或 ["markdown", "csv", "json"] "markdown_split_by": "day_by_month", // markdown文件分割方式，可选项：day（按天，平铺），day_by_month（按天，按月归档到子文件夹），month（按月），year（按年），all（所有数据在一个文件） + "markdown_image_order": "live_photo_first", // markdown中图片与Live Photo的排列顺序，可选项：live_photo_first（Live Photo排在图片前面），image_first_interleaved（图片优先，同名Live Photo插入到对应图片下方） "original_pic_download": 0, // 是否下载原创微博图片 0 = 不下载，1 = 下载 "retweet_pic_download": 0, // 是否下载转发微博图片 0 = 不下载，1 = 下载 "original_video_download": 0, // 是否下载原创微博视频 0 = 不下载，1 = 下载 diff --git a/weibo.py b/weibo.py index d1a9c8bd..bb5dc34b 100644 --- a/weibo.py +++ b/weibo.py @@ -89,6 +89,7 @@ def __init__(self, config): "write_mode" ] # 结果信息保存类型，为list形式，可包含csv、mongo和mysql三种类型 self.markdown_split_by = config.get("markdown_split_by", "day") # markdown文件分割方式，day/day_by_month/month/year/all + self.markdown_image_order = config.get("markdown_image_order", "live_photo_first") # markdown中图片与Live Photo的排列顺序，live_photo_first/image_first_interleaved self.original_pic_download = config[ "original_pic_download" ] # 取值范围为0、1, 0代表不下载原创微博图片,1代表下载 @@ -492,6 +493,12 @@ def validate_config(self, config): logger.warning("markdown_split_by值应为day、day_by_month、month、year或all,请重新输入") sys.exit() + # 验证markdown_image_order + markdown_image_order = config.get("markdown_image_order", "live_photo_first") + if markdown_image_order not in ["live_photo_first", "image_first_interleaved"]: + logger.warning("markdown_image_order值应为live_photo_first或image_first_interleaved,请重新输入") + sys.exit() + # 验证user_id_list user_id_list = config["user_id_list"] if (not isinstance(user_id_list, list)) and (not user_id_list.endswith(".txt")): @@ -3179,6 +3186,54 @@ def format_blockquote(self, text, label=None): return "\n".join(quoted_lines) + def _render_media_items(self, image_filenames, image_prefix, live_photo_files, + live_photo_prefix, live_photo_label_prefix, blockquote=False): + """根据 markdown_image_order 配置渲染图片和 Live Photo 的 Markdown 内容 + + live_photo_first: Live Photo 全部放前面，图片放后面（现有模式） + image_first_interleaved: 图片优先，同名 Live Photo 插入到对应图片下方 + """ + quote = "> " if blockquote else "" + lines = [] + + if self.markdown_image_order == "image_first_interleaved": + # 构建 Live Photo stem → (label, filename) 映射 + # stem 如 2026-07-01_22-00-00_1.mov → 2026-07-01_22-00-00_1 + lp_map = {} + for i, file_name in enumerate(live_photo_files): + stem = os.path.splitext(file_name)[0] + label = (live_photo_label_prefix if len(live_photo_files) == 1 + else f"{live_photo_label_prefix} {i + 1}") + lp_map[stem] = (label, file_name) + + # 先输出图片，每张图片后紧跟同名 Live Photo + remaining_lp_stems = set(lp_map.keys()) + for img_filename in image_filenames: + lines.append(f"{quote}![img](img/{img_filename})\n") + img_stem = os.path.splitext(img_filename)[0] + if img_stem in lp_map: + label, lp_filename = lp_map[img_stem] + # Live Photo 紧跟同名图片，不加空行 + lines.append(f"{quote}[{label}]({live_photo_prefix}{lp_filename})\n") + remaining_lp_stems.discard(img_stem) + # 每条图片（及其可能跟随的 Live Photo）之后加一个空行 + lines.append("\n") + + # 输出未能匹配到任何图片的 Live Photo（异常情况兜底） + for stem in remaining_lp_stems: + label, lp_filename = lp_map[stem] + lines.append(f"{quote}[{label}]({live_photo_prefix}{lp_filename})\n\n") + else: + # live_photo_first: Live Photo 全部在前，图片在后（现有默认模式） + for i, file_name in enumerate(live_photo_files): + label = (live_photo_label_prefix if len(live_photo_files) == 1 + else f"{live_photo_label_prefix} {i + 1}") + lines.append(f"{quote}[{label}]({live_photo_prefix}{file_name})\n\n") + for img_filename in image_filenames: + lines.append(f"{quote}![img](img/{img_filename})\n\n") + + return "".join(lines) + def generate_markdown_file(self, group_key, weibo_list): """生成单个markdown文件（增量模式）""" # 获取用户目录 @@ -3287,12 +3342,10 @@ def generate_markdown_file(self, group_key, weibo_list): live_photo_files = self.get_download_file_names( "live_photo", w.get("live_photo_url", ""), w ) - for idx, file_name in enumerate(live_photo_files, start=1): - label = "Live Photo" if len(live_photo_files) == 1 else f"Live Photo {idx}" - new_md_content += f"[{label}](原创微博Live Photo视频/{file_name})\n\n" - - for image_filename in remaining_images: - new_md_content += f"![img](img/{image_filename})\n\n" + new_md_content += self._render_media_items( + remaining_images, "img/", live_photo_files, + "原创微博Live Photo视频/", "Live Photo", blockquote=False + ) # 转发部分 retweet = w["retweet"] @@ -3324,12 +3377,10 @@ def generate_markdown_file(self, group_key, weibo_list): retweet_live_photo_files = self.get_download_file_names( "live_photo", retweet.get("live_photo_url", ""), retweet ) - for idx, file_name in enumerate(retweet_live_photo_files, start=1): - label = "转发Live Photo" if len(retweet_live_photo_files) == 1 else f"转发Live Photo {idx}" - new_md_content += f"> [{label}](转发微博Live Photo视频/{file_name})\n\n" - - for image_filename in remaining_retweet_images: - new_md_content += f"> ![img](img/{image_filename})\n\n" + new_md_content += self._render_media_items( + remaining_retweet_images, "img/", retweet_live_photo_files, + "转发微博Live Photo视频/", "转发Live Photo", blockquote=True + ) else: # 原创微博 text = w.get("text", "").strip() @@ -3355,12 +3406,10 @@ def generate_markdown_file(self, group_key, weibo_list): live_photo_files = self.get_download_file_names( "live_photo", w.get("live_photo_url", ""), w ) - for idx, file_name in enumerate(live_photo_files, start=1): - label = "Live Photo" if len(live_photo_files) == 1 else f"Live Photo {idx}" - new_md_content += f"[{label}](原创微博Live Photo视频/{file_name})\n\n" - - for image_filename in remaining_images: - new_md_content += f"![img](img/{image_filename})\n\n" + new_md_content += self._render_media_items( + remaining_images, "img/", live_photo_files, + "原创微博Live Photo视频/", "Live Photo", blockquote=False + ) # 添加分隔线 new_md_content += "---\n\n" From c482d9484547cf1a49a4f7550df62709f59213e5 Mon Sep 17 00:00:00 2001 From: YuanChen Date: Sat, 4 Jul 2026 18:44:38 +0800 Subject: [PATCH 4/7] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20markdown=5Fpre?= =?UTF-8?q?view=20=E9=85=8D=E7=BD=AE=EF=BC=8C=E6=94=AF=E6=8C=81=E5=90=84?= =?UTF-8?q?=E7=B1=BB=E5=AA=92=E4=BD=93=E6=96=87=E4=BB=B6=E5=9C=A8=20Markdo?= =?UTF-8?q?wn=20=E4=B8=AD=E7=9A=84=E9=A2=84=E8=A7=88=E5=BC=80=E5=85=B3?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.json | 10 ++++ weibo.py | 131 +++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 120 insertions(+), 21 deletions(-) diff --git a/config.json b/config.json index 553ec37e..d785b944 100644 --- a/config.json +++ b/config.json @@ -12,6 +12,16 @@ ], // 输出格式（可多选），可选项：csv, json, mysql, mongo, sqlite, post, markdown，示例：["markdown"] 或 ["markdown", "csv", "json"] "markdown_split_by": "day_by_month", // markdown文件分割方式，可选项：day（按天，平铺），day_by_month（按天，按月归档到子文件夹），month（按月），year（按年），all（所有数据在一个文件） "markdown_image_order": "live_photo_first", // markdown中图片与Live Photo的排列顺序，可选项：live_photo_first（Live Photo排在图片前面），image_first_interleaved（图片优先，同名Live Photo插入到对应图片下方） + "markdown_preview": { + "link_preview": 0, // 网页链接是否预览（!语法） 0 = 普通链接 [text](url)，1 = 预览图片 ![](url) + "original_video_preview": 0, // 原创微博视频是否预览 0 = 普通链接，1 = 预览 + "retweet_video_preview": 0, // 转发微博视频是否预览 0 = 普通链接，1 = 预览 + "original_image_preview": 1, // 原创微博图片是否预览 0 = 普通链接，1 = 预览 + "retweet_image_preview": 1, // 转发微博图片是否预览 0 = 普通链接，1 = 预览 + "original_live_photo_preview": 0, // 原创微博Live Photo是否预览 0 = 普通链接，1 = 预览 + "retweet_live_photo_preview": 0, // 转发微博Live Photo是否预览 0 = 普通链接，1 = 预览 + "inline_image_preview": 1 // 正文中"查看图片"是否预览 0 = 普通链接，1 = 预览 + }, "original_pic_download": 0, // 是否下载原创微博图片 0 = 不下载，1 = 下载 "retweet_pic_download": 0, // 是否下载转发微博图片 0 = 不下载，1 = 下载 "original_video_download": 0, // 是否下载原创微博视频 0 = 不下载，1 = 下载 diff --git a/weibo.py b/weibo.py index bb5dc34b..5f767676 100644 --- a/weibo.py +++ b/weibo.py @@ -90,6 +90,20 @@ def __init__(self, config): ] # 结果信息保存类型，为list形式，可包含csv、mongo和mysql三种类型 self.markdown_split_by = config.get("markdown_split_by", "day") # markdown文件分割方式，day/day_by_month/month/year/all self.markdown_image_order = config.get("markdown_image_order", "live_photo_first") # markdown中图片与Live Photo的排列顺序，live_photo_first/image_first_interleaved + # markdown 媒体预览配置，默认值 + default_preview = { + "link_preview": 0, + "original_video_preview": 0, + "retweet_video_preview": 0, + "original_image_preview": 1, + "retweet_image_preview": 1, + "original_live_photo_preview": 0, + "retweet_live_photo_preview": 0, + "inline_image_preview": 1, + } + md_preview = config.get("markdown_preview", {}) + # 合并用户配置，缺失的 key 使用默认值 + self.markdown_preview = {**default_preview, **md_preview} if isinstance(md_preview, dict) else default_preview self.original_pic_download = config[ "original_pic_download" ] # 取值范围为0、1, 0代表不下载原创微博图片,1代表下载 @@ -499,6 +513,23 @@ def validate_config(self, config): logger.warning("markdown_image_order值应为live_photo_first或image_first_interleaved,请重新输入") sys.exit() + # 验证markdown_preview + md_preview = config.get("markdown_preview", {}) + if isinstance(md_preview, dict): + valid_preview_keys = [ + "link_preview", "original_video_preview", "retweet_video_preview", + "original_image_preview", "retweet_image_preview", + "original_live_photo_preview", "retweet_live_photo_preview", + "inline_image_preview", + ] + for key in md_preview: + if key not in valid_preview_keys: + logger.warning("markdown_preview中存在无效键: %s, 有效键为: %s", key, ", ".join(valid_preview_keys)) + sys.exit() + if md_preview[key] not in (0, 1): + logger.warning("markdown_preview.%s 值应为0或1,请重新输入", key) + sys.exit() + # 验证user_id_list user_id_list = config["user_id_list"] if (not isinstance(user_id_list, list)) and (not user_id_list.endswith(".txt")): @@ -3156,9 +3187,15 @@ def render_markdown_text(self, text, links=None, image_filenames=None): replacement = line if stripped == "网页链接" and remaining_links: - replacement = f"[网页链接]({remaining_links.pop(0)})" + if self.markdown_preview.get("link_preview", 0): + replacement = f"![网页链接]({remaining_links.pop(0)})" + else: + replacement = f"[网页链接]({remaining_links.pop(0)})" elif stripped == "查看图片" and remaining_images: - replacement = f"![{stripped}](img/{remaining_images.pop(0)})" + if self.markdown_preview.get("inline_image_preview", 1): + replacement = f"![{stripped}](img/{remaining_images.pop(0)})" + else: + replacement = f"[{stripped}](img/{remaining_images.pop(0)})" rendered_lines.append(replacement) @@ -3186,14 +3223,30 @@ def format_blockquote(self, text, label=None): return "\n".join(quoted_lines) - def _render_media_items(self, image_filenames, image_prefix, live_photo_files, - live_photo_prefix, live_photo_label_prefix, blockquote=False): + def _render_media_items(self, image_filenames, live_photo_files, + live_photo_prefix, live_photo_label_prefix, + blockquote=False, image_preview=True, live_photo_preview=False): """根据 markdown_image_order 配置渲染图片和 Live Photo 的 Markdown 内容 live_photo_first: Live Photo 全部放前面，图片放后面（现有模式） image_first_interleaved: 图片优先，同名 Live Photo 插入到对应图片下方 + + image_preview / live_photo_preview: 控制是否使用 ! 语法实现直接预览 """ quote = "> " if blockquote else "" + + def _img_link(filename): + if image_preview: + return f"{quote}![img](img/{filename})\n" + else: + return f"{quote}[img](img/{filename})\n" + + def _lp_link(label, filename): + if live_photo_preview: + return f"{quote}![{label}]({live_photo_prefix}{filename})\n" + else: + return f"{quote}[{label}]({live_photo_prefix}{filename})\n" + lines = [] if self.markdown_image_order == "image_first_interleaved": @@ -3209,12 +3262,12 @@ def _render_media_items(self, image_filenames, image_prefix, live_photo_files, # 先输出图片，每张图片后紧跟同名 Live Photo remaining_lp_stems = set(lp_map.keys()) for img_filename in image_filenames: - lines.append(f"{quote}![img](img/{img_filename})\n") + lines.append(_img_link(img_filename)) img_stem = os.path.splitext(img_filename)[0] if img_stem in lp_map: label, lp_filename = lp_map[img_stem] # Live Photo 紧跟同名图片，不加空行 - lines.append(f"{quote}[{label}]({live_photo_prefix}{lp_filename})\n") + lines.append(_lp_link(label, lp_filename)) remaining_lp_stems.discard(img_stem) # 每条图片（及其可能跟随的 Live Photo）之后加一个空行 lines.append("\n") @@ -3222,15 +3275,15 @@ def _render_media_items(self, image_filenames, image_prefix, live_photo_files, # 输出未能匹配到任何图片的 Live Photo（异常情况兜底） for stem in remaining_lp_stems: label, lp_filename = lp_map[stem] - lines.append(f"{quote}[{label}]({live_photo_prefix}{lp_filename})\n\n") + lines.append(_lp_link(label, lp_filename) + "\n") else: # live_photo_first: Live Photo 全部在前，图片在后（现有默认模式） for i, file_name in enumerate(live_photo_files): label = (live_photo_label_prefix if len(live_photo_files) == 1 else f"{live_photo_label_prefix} {i + 1}") - lines.append(f"{quote}[{label}]({live_photo_prefix}{file_name})\n\n") + lines.append(_lp_link(label, file_name) + "\n") for img_filename in image_filenames: - lines.append(f"{quote}![img](img/{img_filename})\n\n") + lines.append(_img_link(img_filename) + "\n") return "".join(lines) @@ -3330,21 +3383,33 @@ def generate_markdown_file(self, group_key, weibo_list): new_md_content += f"{rendered_text}\n\n" for link in remaining_links: - new_md_content += f"[网页链接]({link})\n\n" + if self.markdown_preview.get("link_preview", 0): + new_md_content += f"![网页链接]({link})\n\n" + else: + new_md_content += f"[网页链接]({link})\n\n" + video_preview = self.markdown_preview.get("original_video_preview", 0) video_files = self.get_download_file_names( "video", w.get("video_url", ""), w ) for idx, file_name in enumerate(video_files, start=1): label = "视频" if len(video_files) == 1 else f"视频 {idx}" - new_md_content += f"[{label}](原创微博视频/{file_name})\n\n" + if video_preview: + new_md_content += f"![{label}](原创微博视频/{file_name})\n\n" + else: + new_md_content += f"[{label}](原创微博视频/{file_name})\n\n" + live_photo_preview = self.markdown_preview.get("original_live_photo_preview", 0) live_photo_files = self.get_download_file_names( "live_photo", w.get("live_photo_url", ""), w ) + image_preview = self.markdown_preview.get("original_image_preview", 1) new_md_content += self._render_media_items( - remaining_images, "img/", live_photo_files, - "原创微博Live Photo视频/", "Live Photo", blockquote=False + remaining_images, live_photo_files, + "原创微博Live Photo视频/", "Live Photo", + blockquote=False, + image_preview=image_preview, + live_photo_preview=live_photo_preview, ) # 转发部分 @@ -3365,21 +3430,33 @@ def generate_markdown_file(self, group_key, weibo_list): new_md_content += f"{quoted_retweet}\n\n" for link in remaining_retweet_links: - new_md_content += f"> [网页链接]({link})\n\n" + if self.markdown_preview.get("link_preview", 0): + new_md_content += f"> ![网页链接]({link})\n\n" + else: + new_md_content += f"> [网页链接]({link})\n\n" + retweet_video_preview = self.markdown_preview.get("retweet_video_preview", 0) retweet_video_files = self.get_download_file_names( "video", retweet.get("video_url", ""), retweet ) for idx, file_name in enumerate(retweet_video_files, start=1): label = "转发视频" if len(retweet_video_files) == 1 else f"转发视频 {idx}" - new_md_content += f"> [{label}](转发微博视频/{file_name})\n\n" + if retweet_video_preview: + new_md_content += f"> ![{label}](转发微博视频/{file_name})\n\n" + else: + new_md_content += f"> [{label}](转发微博视频/{file_name})\n\n" + retweet_live_photo_preview = self.markdown_preview.get("retweet_live_photo_preview", 0) retweet_live_photo_files = self.get_download_file_names( "live_photo", retweet.get("live_photo_url", ""), retweet ) + retweet_image_preview = self.markdown_preview.get("retweet_image_preview", 1) new_md_content += self._render_media_items( - remaining_retweet_images, "img/", retweet_live_photo_files, - "转发微博Live Photo视频/", "转发Live Photo", blockquote=True + remaining_retweet_images, retweet_live_photo_files, + "转发微博Live Photo视频/", "转发Live Photo", + blockquote=True, + image_preview=retweet_image_preview, + live_photo_preview=retweet_live_photo_preview, ) else: # 原创微博 @@ -3394,21 +3471,33 @@ def generate_markdown_file(self, group_key, weibo_list): new_md_content += f"{rendered_text}\n\n" for link in remaining_links: - new_md_content += f"[网页链接]({link})\n\n" + if self.markdown_preview.get("link_preview", 0): + new_md_content += f"![网页链接]({link})\n\n" + else: + new_md_content += f"[网页链接]({link})\n\n" + video_preview = self.markdown_preview.get("original_video_preview", 0) video_files = self.get_download_file_names( "video", w.get("video_url", ""), w ) for idx, file_name in enumerate(video_files, start=1): label = "视频" if len(video_files) == 1 else f"视频 {idx}" - new_md_content += f"[{label}](原创微博视频/{file_name})\n\n" + if video_preview: + new_md_content += f"![{label}](原创微博视频/{file_name})\n\n" + else: + new_md_content += f"[{label}](原创微博视频/{file_name})\n\n" + live_photo_preview = self.markdown_preview.get("original_live_photo_preview", 0) live_photo_files = self.get_download_file_names( "live_photo", w.get("live_photo_url", ""), w ) + image_preview = self.markdown_preview.get("original_image_preview", 1) new_md_content += self._render_media_items( - remaining_images, "img/", live_photo_files, - "原创微博Live Photo视频/", "Live Photo", blockquote=False + remaining_images, live_photo_files, + "原创微博Live Photo视频/", "Live Photo", + blockquote=False, + image_preview=image_preview, + live_photo_preview=live_photo_preview, ) # 添加分隔线 From f363b978774a5de895ade6a539e254133f157f63 Mon Sep 17 00:00:00 2001 From: YuanChen Date: Sat, 4 Jul 2026 20:01:23 +0800 Subject: [PATCH 5/7] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20markdown=5Ffil?= =?UTF-8?q?e=5Fdir=20=E4=B8=8E=20download=5Ffolder=5Fname=20=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E6=94=AF=E6=8C=81=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E5=AA=92=E4=BD=93=E6=96=87=E4=BB=B6=E7=9A=84=E5=AD=98=E5=82=A8?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E5=90=8D=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.json | 47 ++++++++++++++++++++ weibo.py | 120 +++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 146 insertions(+), 21 deletions(-) diff --git a/config.json b/config.json index d785b944..57066720 100644 --- a/config.json +++ b/config.json @@ -22,6 +22,53 @@ "retweet_live_photo_preview": 0, // 转发微博Live Photo是否预览 0 = 普通链接，1 = 预览 "inline_image_preview": 1 // 正文中"查看图片"是否预览 0 = 普通链接，1 = 预览 }, + // ========================================================================== + // 为什么会有两个配置块（markdown_file_dir 和 download_folder_name）？ + // + // 这取决于是否在 write_mode 中启用了 "markdown"。 + // + // —————————————————————————————————————————————————————————————————————— + // 场景一：启用了 "markdown"（write_mode 含 "markdown"） + // + // 程序会生成 .md 文件，并下载媒体文件到本地。 + // + // 您应该修改以下两个配置块的全部项，并让相同媒体的值保持一致： + // ● markdown_file_dir → 决定 .md 文件里链接的路径 + // 同时决定图片的下载目录 + // ● download_folder_name → 决定视频和 Live Photo 的下载目录 + // （其中图片项在 md 模式下不生效，可忽略） + // + // —————————————————————————————————————————————————————————————————————— + // 场景二：没有启用 "markdown"（write_mode 不含 "markdown"） + // + // 程序不会生成 .md 文件，只下载媒体文件到本地。 + // + // 你只需修改 download_folder_name 的全部六项： + // ● download_folder_name → 决定所有媒体文件的下载目录 + // ● markdown_file_dir → 不参与任何逻辑，完全忽略 + // + // —————————————————————————————————————————————————————————————————————— + // + // 图片的下载目录在 md 模式下改由 markdown_file_dir 接管； + // 视频和 Live Photo 的下载目录在所有模式下统一由 download_folder_name 接管。 + // + // ========================================================================== + "markdown_file_dir": { + "original_image_dir": "img", // md 中图片链接路径 & 图片下载目录 + "retweet_image_dir": "img", + "original_video_dir": "原创微博视频", // md 中视频链接路径 + "retweet_video_dir": "转发微博视频", + "original_live_photo_dir": "原创微博Live Photo视频", // md 中 Live Photo 链接路径 + "retweet_live_photo_dir": "转发微博Live Photo视频" + }, + "download_folder_name": { + "original_image_folder": "原创微博图片", // 图片下载目录（仅非 md 模式生效） + "retweet_image_folder": "转发微博图片", + "original_video_folder": "原创微博视频", // 视频下载目录（所有模式生效） + "retweet_video_folder": "转发微博视频", + "original_live_photo_folder": "原创微博Live Photo视频", // Live Photo 下载目录（所有模式生效） + "retweet_live_photo_folder": "转发微博Live Photo视频" + }, "original_pic_download": 0, // 是否下载原创微博图片 0 = 不下载，1 = 下载 "retweet_pic_download": 0, // 是否下载转发微博图片 0 = 不下载，1 = 下载 "original_video_download": 0, // 是否下载原创微博视频 0 = 不下载，1 = 下载 diff --git a/weibo.py b/weibo.py index 5f767676..84437444 100644 --- a/weibo.py +++ b/weibo.py @@ -104,6 +104,30 @@ def __init__(self, config): md_preview = config.get("markdown_preview", {}) # 合并用户配置，缺失的 key 使用默认值 self.markdown_preview = {**default_preview, **md_preview} if isinstance(md_preview, dict) else default_preview + + # markdown 中各类媒体文件的相对路径目录名 + default_md_dir = { + "original_image_dir": "img", + "retweet_image_dir": "img", + "original_video_dir": "原创微博视频", + "retweet_video_dir": "转发微博视频", + "original_live_photo_dir": "原创微博Live Photo视频", + "retweet_live_photo_dir": "转发微博Live Photo视频", + } + md_file_dir = config.get("markdown_file_dir", {}) + self.markdown_file_dir = {**default_md_dir, **md_file_dir} if isinstance(md_file_dir, dict) else default_md_dir + + # 下载后本地存储的文件夹名称 + default_dl_folder = { + "original_image_folder": "原创微博图片", + "retweet_image_folder": "转发微博图片", + "original_video_folder": "原创微博视频", + "retweet_video_folder": "转发微博视频", + "original_live_photo_folder": "原创微博Live Photo视频", + "retweet_live_photo_folder": "转发微博Live Photo视频", + } + dl_folder = config.get("download_folder_name", {}) + self.download_folder_name = {**default_dl_folder, **dl_folder} if isinstance(dl_folder, dict) else default_dl_folder self.original_pic_download = config[ "original_pic_download" ] # 取值范围为0、1, 0代表不下载原创微博图片,1代表下载 @@ -530,6 +554,32 @@ def validate_config(self, config): logger.warning("markdown_preview.%s 值应为0或1,请重新输入", key) sys.exit() + # 验证markdown_file_dir + md_file_dir = config.get("markdown_file_dir", {}) + if isinstance(md_file_dir, dict): + valid_md_dir_keys = [ + "original_image_dir", "retweet_image_dir", + "original_video_dir", "retweet_video_dir", + "original_live_photo_dir", "retweet_live_photo_dir", + ] + for key in md_file_dir: + if key not in valid_md_dir_keys: + logger.warning("markdown_file_dir中存在无效键: %s, 有效键为: %s", key, ", ".join(valid_md_dir_keys)) + sys.exit() + + # 验证download_folder_name + dl_folder = config.get("download_folder_name", {}) + if isinstance(dl_folder, dict): + valid_dl_keys = [ + "original_image_folder", "retweet_image_folder", + "original_video_folder", "retweet_video_folder", + "original_live_photo_folder", "retweet_live_photo_folder", + ] + for key in dl_folder: + if key not in valid_dl_keys: + logger.warning("download_folder_name中存在无效键: %s, 有效键为: %s", key, ", ".join(valid_dl_keys)) + sys.exit() + # 验证user_id_list user_id_list = config["user_id_list"] if (not isinstance(user_id_list, list)) and (not user_id_list.endswith(".txt")): @@ -1466,15 +1516,20 @@ def download_files(self, file_type, weibo_type, wrote_count): if file_type == "img": describe = "图片" key = "pics" + folder_key = "original_image_folder" if weibo_type == "original" else "retweet_image_folder" elif file_type == "video": describe = "视频" key = "video_url" + folder_key = "original_video_folder" if weibo_type == "original" else "retweet_video_folder" elif file_type == "live_photo": describe = "Live Photo视频" key = "live_photo_url" + folder_key = "original_live_photo_folder" if weibo_type == "original" else "retweet_live_photo_folder" else: return - + + # 使用自定义的下载文件夹名称 + folder_name = self.download_folder_name.get(folder_key, describe) if weibo_type == "original": describe = "原创微博" + describe else: @@ -1526,7 +1581,7 @@ def download_files(self, file_type, weibo_type, wrote_count): # 创建月份子目录下的文件目录（使用父微博的月份） month_dir = os.path.join(base_dir, month_folder) - file_dir = os.path.join(month_dir, describe) + file_dir = os.path.join(month_dir, folder_name) if not os.path.isdir(file_dir): os.makedirs(file_dir) @@ -1536,7 +1591,7 @@ def download_files(self, file_type, weibo_type, wrote_count): else: # 原有逻辑：所有文件放在同一目录 file_dir = self.get_filepath(file_type) - file_dir = file_dir + os.sep + describe + file_dir = file_dir + os.sep + folder_name if not os.path.isdir(file_dir): os.makedirs(file_dir) @@ -2238,7 +2293,7 @@ def get_filepath(self, type): if type in ["img", "video", "live_photo"]: file_dir = file_dir + os.sep + type elif type == "markdown": - # Markdown文件保存在用户目录下，图片在用户目录的img子目录中 + # Markdown文件保存在用户目录下，图片和视频在用户目录的子目录中 file_dir = file_dir if not os.path.isdir(file_dir): os.makedirs(file_dir) @@ -3076,6 +3131,7 @@ def download_markdown_images(self, wrote_count): # 对于 day_by_month 模式，按月分组图片 if self.markdown_split_by == "day_by_month": # 按月分组微博，然后为每个月创建img目录 + img_folder_name = self.markdown_file_dir.get("original_image_dir", "img") for w in self.weibo[wrote_count:]: created_at = w.get("created_at", "") if not created_at: @@ -3085,9 +3141,9 @@ def download_markdown_images(self, wrote_count): month_folder = time_obj.strftime("%Y-%m") except ValueError: continue - + month_dir = os.path.join(file_dir, month_folder) - img_dir = os.path.join(month_dir, "img") + img_dir = os.path.join(month_dir, img_folder_name) if not os.path.isdir(img_dir): os.makedirs(img_dir) @@ -3103,7 +3159,8 @@ def download_markdown_images(self, wrote_count): self._download_weibo_images(retweet, img_dir, is_retweet=True) else: # 其他模式：所有图片放在同一个 img 目录 - img_dir = os.path.join(file_dir, "img") + img_folder_name = self.markdown_file_dir.get("original_image_dir", "img") + img_dir = os.path.join(file_dir, img_folder_name) if not os.path.isdir(img_dir): os.makedirs(img_dir) @@ -3192,10 +3249,11 @@ def render_markdown_text(self, text, links=None, image_filenames=None): else: replacement = f"[网页链接]({remaining_links.pop(0)})" elif stripped == "查看图片" and remaining_images: + original_img_dir = self.markdown_file_dir.get("original_image_dir", "img") if self.markdown_preview.get("inline_image_preview", 1): - replacement = f"![{stripped}](img/{remaining_images.pop(0)})" + replacement = f"![{stripped}]({original_img_dir}/{remaining_images.pop(0)})" else: - replacement = f"[{stripped}](img/{remaining_images.pop(0)})" + replacement = f"[{stripped}]({original_img_dir}/{remaining_images.pop(0)})" rendered_lines.append(replacement) @@ -3225,21 +3283,23 @@ def format_blockquote(self, text, label=None): def _render_media_items(self, image_filenames, live_photo_files, live_photo_prefix, live_photo_label_prefix, - blockquote=False, image_preview=True, live_photo_preview=False): + image_dir_name, blockquote=False, + image_preview=True, live_photo_preview=False): """根据 markdown_image_order 配置渲染图片和 Live Photo 的 Markdown 内容 live_photo_first: Live Photo 全部放前面，图片放后面（现有模式） image_first_interleaved: 图片优先，同名 Live Photo 插入到对应图片下方 image_preview / live_photo_preview: 控制是否使用 ! 语法实现直接预览 + image_dir_name: 图片文件所在的目录名（如 "img"） """ quote = "> " if blockquote else "" def _img_link(filename): if image_preview: - return f"{quote}![img](img/{filename})\n" + return f"{quote}![img]({image_dir_name}/{filename})\n" else: - return f"{quote}[img](img/{filename})\n" + return f"{quote}[img]({image_dir_name}/{filename})\n" def _lp_link(label, filename): if live_photo_preview: @@ -3371,6 +3431,12 @@ def generate_markdown_file(self, group_key, weibo_list): # 处理转发微博 if not self.only_crawl_original and w.get("retweet"): + original_img_dir = self.markdown_file_dir.get("original_image_dir", "img") + original_video_dir = self.markdown_file_dir.get("original_video_dir", "原创微博视频") + original_lp_dir = self.markdown_file_dir.get("original_live_photo_dir", "原创微博Live Photo视频") + retweet_img_dir = self.markdown_file_dir.get("retweet_image_dir", "img") + retweet_video_dir = self.markdown_file_dir.get("retweet_video_dir", "转发微博视频") + retweet_lp_dir = self.markdown_file_dir.get("retweet_live_photo_dir", "转发微博Live Photo视频") # 原创部分 text = w.get("text", "").strip() links = w.get("links") or [] @@ -3395,10 +3461,12 @@ def generate_markdown_file(self, group_key, weibo_list): for idx, file_name in enumerate(video_files, start=1): label = "视频" if len(video_files) == 1 else f"视频 {idx}" if video_preview: - new_md_content += f"![{label}](原创微博视频/{file_name})\n\n" + new_md_content += f"![{label}]({original_video_dir}/{file_name})\n\n" else: - new_md_content += f"[{label}](原创微博视频/{file_name})\n\n" + new_md_content += f"[{label}]({original_video_dir}/{file_name})\n\n" + original_lp_dir_ref = self.markdown_file_dir.get("original_live_photo_dir", "原创微博Live Photo视频") + live_photo_prefix = original_lp_dir_ref + "/" live_photo_preview = self.markdown_preview.get("original_live_photo_preview", 0) live_photo_files = self.get_download_file_names( "live_photo", w.get("live_photo_url", ""), w @@ -3406,7 +3474,8 @@ def generate_markdown_file(self, group_key, weibo_list): image_preview = self.markdown_preview.get("original_image_preview", 1) new_md_content += self._render_media_items( remaining_images, live_photo_files, - "原创微博Live Photo视频/", "Live Photo", + live_photo_prefix, "Live Photo", + original_img_dir, blockquote=False, image_preview=image_preview, live_photo_preview=live_photo_preview, @@ -3442,10 +3511,12 @@ def generate_markdown_file(self, group_key, weibo_list): for idx, file_name in enumerate(retweet_video_files, start=1): label = "转发视频" if len(retweet_video_files) == 1 else f"转发视频 {idx}" if retweet_video_preview: - new_md_content += f"> ![{label}](转发微博视频/{file_name})\n\n" + new_md_content += f"> ![{label}]({retweet_video_dir}/{file_name})\n\n" else: - new_md_content += f"> [{label}](转发微博视频/{file_name})\n\n" + new_md_content += f"> [{label}]({retweet_video_dir}/{file_name})\n\n" + retweet_lp_dir_ref = self.markdown_file_dir.get("retweet_live_photo_dir", "转发微博Live Photo视频") + retweet_live_photo_prefix = retweet_lp_dir_ref + "/" retweet_live_photo_preview = self.markdown_preview.get("retweet_live_photo_preview", 0) retweet_live_photo_files = self.get_download_file_names( "live_photo", retweet.get("live_photo_url", ""), retweet @@ -3453,13 +3524,17 @@ def generate_markdown_file(self, group_key, weibo_list): retweet_image_preview = self.markdown_preview.get("retweet_image_preview", 1) new_md_content += self._render_media_items( remaining_retweet_images, retweet_live_photo_files, - "转发微博Live Photo视频/", "转发Live Photo", + retweet_live_photo_prefix, "转发Live Photo", + retweet_img_dir, blockquote=True, image_preview=retweet_image_preview, live_photo_preview=retweet_live_photo_preview, ) else: # 原创微博 + original_img_dir = self.markdown_file_dir.get("original_image_dir", "img") + original_video_dir = self.markdown_file_dir.get("original_video_dir", "原创微博视频") + original_lp_dir = self.markdown_file_dir.get("original_live_photo_dir", "原创微博Live Photo视频") text = w.get("text", "").strip() links = w.get("links") or [] image_filenames = self.get_markdown_image_filenames(w, created_at) @@ -3483,10 +3558,12 @@ def generate_markdown_file(self, group_key, weibo_list): for idx, file_name in enumerate(video_files, start=1): label = "视频" if len(video_files) == 1 else f"视频 {idx}" if video_preview: - new_md_content += f"![{label}](原创微博视频/{file_name})\n\n" + new_md_content += f"![{label}]({original_video_dir}/{file_name})\n\n" else: - new_md_content += f"[{label}](原创微博视频/{file_name})\n\n" + new_md_content += f"[{label}]({original_video_dir}/{file_name})\n\n" + original_lp_dir_ref = self.markdown_file_dir.get("original_live_photo_dir", "原创微博Live Photo视频") + live_photo_prefix = original_lp_dir_ref + "/" live_photo_preview = self.markdown_preview.get("original_live_photo_preview", 0) live_photo_files = self.get_download_file_names( "live_photo", w.get("live_photo_url", ""), w @@ -3494,7 +3571,8 @@ def generate_markdown_file(self, group_key, weibo_list): image_preview = self.markdown_preview.get("original_image_preview", 1) new_md_content += self._render_media_items( remaining_images, live_photo_files, - "原创微博Live Photo视频/", "Live Photo", + live_photo_prefix, "Live Photo", + original_img_dir, blockquote=False, image_preview=image_preview, live_photo_preview=live_photo_preview, From 96fbf025a97a16cd5be7fbe6972311b4a024310f Mon Sep 17 00:00:00 2001 From: YuanChen Date: Sat, 4 Jul 2026 20:25:01 +0800 Subject: [PATCH 6/7] =?UTF-8?q?refactor:=20=E6=96=B0=E5=A2=9E=20config.jso?= =?UTF-8?q?n.example=20=E6=A8=A1=E6=9D=BF=EF=BC=8C=E5=B0=86=20config.json?= =?UTF-8?q?=20=E4=BB=8E=E7=89=88=E6=9C=AC=E6=8E=A7=E5=88=B6=E4=B8=AD?= =?UTF-8?q?=E7=A7=BB=E9=99=A4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + config.json => config.json.example | 0 weibo.py | 31 +++++++++++++++++++++++------- 3 files changed, 25 insertions(+), 7 deletions(-) rename config.json => config.json.example (100%) diff --git a/.gitignore b/.gitignore index 8b5ac535..3372478b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ ### Project ### log/ weibo/ +config.json # Created by https://www.toptal.com/developers/gitignore/api/python,jetbrains,visualstudiocode,windows,macos,linux # Edit at https://www.toptal.com/developers/gitignore?templates=python,jetbrains,visualstudiocode,windows,macos,linux diff --git a/config.json b/config.json.example similarity index 100% rename from config.json rename to config.json.example diff --git a/weibo.py b/weibo.py index 84437444..f6db868a 100644 --- a/weibo.py +++ b/weibo.py @@ -3822,14 +3822,31 @@ def handle_config_renaming(config, oldName, newName): del config[oldName] def get_config(): - """获取配置文件信息（支持JSON5格式）""" - config_path = os.path.split(os.path.realpath(__file__))[0] + os.sep + "config.json" + """获取配置文件信息（支持JSON5格式） + + 配置文件优先级： + 1. config.json（用户已从 config.json.example 复制并重命名） + 2. config.json.example（版本控制中的默认配置模板） + """ + base_dir = os.path.split(os.path.realpath(__file__))[0] + os.sep + config_path = base_dir + "config.json" + example_path = base_dir + "config.json.example" + + # 优先使用 config.json，若不存在则使用 config.json.example if not os.path.isfile(config_path): - logger.warning( - "当前路径：%s 不存在配置文件config.json", - (os.path.split(os.path.realpath(__file__))[0] + os.sep), - ) - sys.exit() + if os.path.isfile(example_path): + logger.warning( + "未找到 config.json。正在使用 config.json.example（模板配置）。\n" + "建议将 config.json.example 复制为 config.json 并填写自己的 Cookie 等配置：\n" + " cp config.json.example config.json" + ) + config_path = example_path + else: + logger.warning( + "当前路径：%s 不存在配置文件 config.json 和 config.json.example", + base_dir, + ) + sys.exit() try: with open(config_path, encoding="utf-8") as f: config_content = f.read() From 5bb45e8352cdb3003ab4aaca2b4e1e1893ebdcc4 Mon Sep 17 00:00:00 2001 From: YuanChen Date: Sat, 4 Jul 2026 22:20:31 +0800 Subject: [PATCH 7/7] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20markdown=5Fsor?= =?UTF-8?q?t=5Forder=20=E4=B8=8E=20missing=5Fweibo=5Freport=20=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E4=BF=AE=E5=A4=8D=E8=BD=AC=E5=8F=91=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E4=B8=8B=E8=BD=BD=E4=B8=8D=E5=B0=8A=E9=87=8D=20retwee?= =?UTF-8?q?t=5Fpic=5Fdownload=20=E7=9A=84=20bug=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.json.example | 2 + weibo.py | 114 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index 57066720..70acad4b 100644 --- a/config.json.example +++ b/config.json.example @@ -11,7 +11,9 @@ "json" ], // 输出格式（可多选），可选项：csv, json, mysql, mongo, sqlite, post, markdown，示例：["markdown"] 或 ["markdown", "csv", "json"] "markdown_split_by": "day_by_month", // markdown文件分割方式，可选项：day（按天，平铺），day_by_month（按天，按月归档到子文件夹），month（按月），year（按年），all（所有数据在一个文件） + "markdown_sort_order": "new_to_old", // markdown中同一分组内多条微博的排序方式，可选项：new_to_old（从新到旧，最新在前），old_to_new（从旧到新，最早在前） "markdown_image_order": "live_photo_first", // markdown中图片与Live Photo的排列顺序，可选项：live_photo_first（Live Photo排在图片前面），image_first_interleaved（图片优先，同名Live Photo插入到对应图片下方） + "missing_weibo_report": 0, // 是否在每次爬取结束后，在 output_directory 下生成本次爬取较上次缺失（已删除/未爬到）的微博ID报告 0 = 不生成，1 = 生成 "markdown_preview": { "link_preview": 0, // 网页链接是否预览（!语法） 0 = 普通链接 [text](url)，1 = 预览图片 ![](url) "original_video_preview": 0, // 原创微博视频是否预览 0 = 普通链接，1 = 预览 diff --git a/weibo.py b/weibo.py index f6db868a..44135d53 100644 --- a/weibo.py +++ b/weibo.py @@ -89,6 +89,7 @@ def __init__(self, config): "write_mode" ] # 结果信息保存类型，为list形式，可包含csv、mongo和mysql三种类型 self.markdown_split_by = config.get("markdown_split_by", "day") # markdown文件分割方式，day/day_by_month/month/year/all + self.markdown_sort_order = config.get("markdown_sort_order", "new_to_old") # markdown中微博排序方式，new_to_old/old_to_new self.markdown_image_order = config.get("markdown_image_order", "live_photo_first") # markdown中图片与Live Photo的排列顺序，live_photo_first/image_first_interleaved # markdown 媒体预览配置，默认值 default_preview = { @@ -166,6 +167,8 @@ def __init__(self, config): self.output_directory = config.get( "output_directory", "weibo" ) # 输出目录配置，默认为"weibo" + + self.missing_weibo_report = config.get("missing_weibo_report", 0) # 是否生成缺失微博报告 # Cookie支持：优先使用环境变量WEIBO_COOKIE，其次使用config.json中的配置 cookie_config = config.get("cookie") @@ -496,6 +499,7 @@ def validate_config(self, config): "retweet_live_photo_download", "download_comment", "download_repost", + "missing_weibo_report", ] for argument in argument_list: # 使用 get() 获取值，新增字段默认为0 @@ -537,6 +541,12 @@ def validate_config(self, config): logger.warning("markdown_image_order值应为live_photo_first或image_first_interleaved,请重新输入") sys.exit() + # 验证markdown_sort_order + markdown_sort_order = config.get("markdown_sort_order", "new_to_old") + if markdown_sort_order not in ["new_to_old", "old_to_new"]: + logger.warning("markdown_sort_order值应为new_to_old或old_to_new,请重新输入") + sys.exit() + # 验证markdown_preview md_preview = config.get("markdown_preview", {}) if isinstance(md_preview, dict): @@ -3121,8 +3131,101 @@ def group_weibo_by_config(self, wrote_count): logger.warning(f"无法解析微博日期: {created_at}") continue + # 每个分组内按 markdown_sort_order 排序 + descending = self.markdown_sort_order != "old_to_new" + for key in weibo_by_group: + weibo_by_group[key].sort(key=lambda w: int(w.get("id", 0)), reverse=descending) + return weibo_by_group + def generate_missing_weibo_report(self): + """生成缺失微博报告 + + 比较本次爬取到的微博ID与markdown文件中已有的微博ID， + 列出在md中存在但本次未爬取到的微博（可能已被删除或漏爬）。 + 报告按user_id区分，以结束爬取时间为suffix命名。 + """ + if not self.missing_weibo_report: + return + + # 确保self.user已初始化 + if not self.user or not self.user_config: + return + + # 收集本次爬取到的微博ID（直接从self.weibo中获取） + crawled_ids = set() + for w in self.weibo: + weibo_id = w.get("id") + if weibo_id: + crawled_ids.add(int(weibo_id)) + + if not crawled_ids: + return + + # 收集所有md文件中已有的微博ID + file_dir = self.get_filepath("markdown") + existing_ids = set() + try: + md_files = self._find_md_files(file_dir) + for md_path in md_files: + with open(md_path, "r", encoding="utf-8") as f: + content = f.read() + ids = re.findall(r"", content) + existing_ids.update(int(i) for i in ids) + except Exception as e: + logger.warning(f"读取md文件收集已有微博ID失败: {e}") + return + + # 找出缺失的微博（md中有但本次没爬到） + missing_ids = existing_ids - crawled_ids + if not missing_ids: + logger.info("本次爬取无缺失微博，所有已有微博均已再次爬取到") + return + + # 构建输出目录和文件路径 + base_dir = os.path.split(os.path.realpath(__file__))[0] + os.sep + self.output_directory + user_screen_name = self.user.get("screen_name", self.user_config.get("user_id", "unknown")) + user_id = self.user_config.get("user_id", "unknown") + end_time = datetime.now().strftime("%Y%m%d_%H%M%S") + report_filename = f"missing_weibo_{user_id}_{user_screen_name}_{end_time}.txt" + report_path = os.path.join(base_dir, report_filename) + + try: + if not os.path.isdir(base_dir): + os.makedirs(base_dir) + with open(report_path, "w", encoding="utf-8") as f: + f.write(f"缺失微博报告\n") + f.write(f"用户: {user_screen_name} (ID: {user_id})\n") + f.write(f"生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + f.write(f"md中已有微博数: {len(existing_ids)}\n") + f.write(f"本次爬取到微博数: {len(crawled_ids)}\n") + f.write(f"缺失（已删除/未爬到）微博数: {len(missing_ids)}\n") + f.write("=" * 60 + "\n") + f.write("以下微博ID在md文件中存在，但本次未爬取到（可能已被删除）：\n\n") + for mid in sorted(missing_ids, reverse=True): + f.write(f" https://m.weibo.cn/detail/{mid}\n") + f.write("\n") + f.write("=" * 60 + "\n") + f.write(f"报告完毕。\n") + logger.info( + f"缺失微博报告已生成: {report_path} " + f"({user_screen_name}: {len(missing_ids)} 条缺失)" + ) + except Exception as e: + logger.error(f"生成缺失微博报告失败: {e}") + + @staticmethod + def _find_md_files(dir_path): + """递归查找目录下所有 .md 文件""" + md_files = [] + if not os.path.isdir(dir_path): + return md_files + for root, dirs, files in os.walk(dir_path): + for f in files: + if f.endswith('.md'): + md_files.append(os.path.join(root, f)) + return md_files + def download_markdown_images(self, wrote_count): """为Markdown格式下载图片，使用指定的命名规则""" # 获取用户目录 @@ -3152,7 +3255,7 @@ def download_markdown_images(self, wrote_count): self._download_weibo_images(w, img_dir, is_retweet=False) # 处理转发微博图片（使用父微博的月份文件夹） - if not self.only_crawl_original and w.get("retweet"): + if not self.only_crawl_original and self.retweet_pic_download and w.get("retweet"): retweet = w["retweet"] if retweet.get("pics"): # 转发微博的图片保存到父微博的月份文件夹中 @@ -3171,7 +3274,7 @@ def download_markdown_images(self, wrote_count): self._download_weibo_images(w, img_dir, is_retweet=False) # 处理转发微博图片 - if not self.only_crawl_original and w.get("retweet"): + if not self.only_crawl_original and self.retweet_pic_download and w.get("retweet"): retweet = w["retweet"] if retweet.get("pics"): self._download_weibo_images(retweet, img_dir, is_retweet=True) @@ -3401,6 +3504,10 @@ def generate_markdown_file(self, group_key, weibo_list): logger.info(f"分组 {group_key} 没有新微博需要写入") return + # 按 markdown_sort_order 排序 + descending = self.markdown_sort_order != "old_to_new" + new_weibo_list.sort(key=lambda w: int(w.get("id", 0)), reverse=descending) + # 构建新微博的markdown内容 new_md_content = "" for w in new_weibo_list: @@ -3806,6 +3913,9 @@ def start(self): # 当前用户所有微博和评论抓取完毕后，再导出该用户的评论 CSV self.export_comments_to_csv_for_current_user() + # 生成缺失微博报告 + self.generate_missing_weibo_report() + logger.info("信息抓取完毕") logger.info("*" * 100) if self.user_config_file_path and self.user: