From 646ce6305760e4acfcfa41a7119054da83daf172 Mon Sep 17 00:00:00 2001 From: JonathanJu Date: Sun, 17 May 2026 17:29:34 +0800 Subject: [PATCH 1/2] =?UTF-8?q?refactor(database):=20=E5=B0=86=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E5=88=9D=E5=A7=8B=E5=8C=96=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E7=BB=9F=E4=B8=80=E6=95=B4=E5=90=88=E5=88=B0=20database=20?= =?UTF-8?q?=E5=8C=85=20-=20=E6=96=B0=E5=A2=9E=20app/database/=5F=5Finit=5F?= =?UTF-8?q?=5F.py=EF=BC=8C=E6=95=B4=E5=90=88=E5=BB=BA=E8=A1=A8=E3=80=81?= =?UTF-8?q?=E7=A7=8D=E5=AD=90=E6=95=B0=E6=8D=AE=E5=92=8C=E5=9F=BA=E7=BA=BF?= =?UTF-8?q?=E8=AE=A1=E7=AE=97=E4=B8=BA=20init=5Fdatabase()=20-=20=E5=B0=86?= =?UTF-8?q?=20scripts/seed=5Fdata.py=20=E5=92=8C=20scripts/compute=5Fbasel?= =?UTF-8?q?ine.py=20=E8=BF=81=E7=A7=BB=E5=88=B0=20app/database/=20-=20?= =?UTF-8?q?=E5=88=A0=E9=99=A4=20scripts/init=5Fdb.py=EF=BC=8C=E5=BB=BA?= =?UTF-8?q?=E8=A1=A8=E9=80=BB=E8=BE=91=E5=B7=B2=E5=90=88=E5=B9=B6=E5=88=B0?= =?UTF-8?q?=20database=20=E5=8C=85=20-=20=E7=A7=8D=E5=AD=90=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=92=8C=E5=9F=BA=E7=BA=BF=E8=AE=A1=E7=AE=97=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E5=B9=82=E7=AD=89=E6=80=A7=E6=A3=80=E6=9F=A5=EF=BC=8C?= =?UTF-8?q?=E4=BB=85=E5=9C=A8=E7=A9=BA=E8=A1=A8=E6=97=B6=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=20-=20main.py=20=E5=90=AF=E5=8A=A8=E6=97=B6=E8=B0=83=E7=94=A8?= =?UTF-8?q?=20init=5Fdatabase()=20=E6=9B=BF=E4=BB=A3=E5=86=85=E8=81=94?= =?UTF-8?q?=E5=BB=BA=E8=A1=A8=E4=BB=A3=E7=A0=81=20-=20=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=20pathlib.Path=20=E6=9B=BF=E4=BB=A3=20os.path=20=E5=A4=84?= =?UTF-8?q?=E7=90=86=E8=B7=AF=E5=BE=84=20-=20=E7=AE=80=E5=8C=96=20CI=20?= =?UTF-8?q?=E6=B5=81=E7=A8=8B=E5=92=8C=E9=83=A8=E7=BD=B2=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E6=95=B0=E6=8D=AE=E5=BA=93=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E6=AD=A5=E9=AA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 5 +- Makefile | 4 +- backend/app/database/__init__.py | 106 ++++++++++++++++++ .../app/database/baseline.py | 23 ++-- .../app/database}/seed_data.py | 27 ++--- backend/app/local_memory.py | 3 +- backend/app/main.py | 66 +---------- deploy_backend.py | 5 +- scripts/init_db.py | 82 -------------- 9 files changed, 133 insertions(+), 188 deletions(-) create mode 100644 backend/app/database/__init__.py rename scripts/compute_baseline.py => backend/app/database/baseline.py (94%) rename {scripts => backend/app/database}/seed_data.py (95%) delete mode 100644 scripts/init_db.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0245856..a6869a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,7 @@ jobs: - run: python -c "from app.main import app; print('import ok')" - name: Init test DB run: | - cd .. - python scripts/init_db.py - python scripts/seed_data.py - python scripts/compute_baseline.py + python -c "from app.database import init_database; from app.main import DB_PATH; init_database(DB_PATH)" - run: python -m pytest tests/ -v --tb=short frontend: diff --git a/Makefile b/Makefile index 7af39b4..9effbfa 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,7 @@ # 一键初始化数据库并生成 baseline data: - python3 scripts/init_db.py - python3 scripts/seed_data.py - python3 scripts/compute_baseline.py + @echo "数据库表 / 种子数据 / 基线指标 已由 init_database() 自动处理" # 安装所有依赖 install: diff --git a/backend/app/database/__init__.py b/backend/app/database/__init__.py new file mode 100644 index 0000000..14c2273 --- /dev/null +++ b/backend/app/database/__init__.py @@ -0,0 +1,106 @@ +import sqlite3 +from pathlib import Path + + +def init_database(db_path: Path): + """创建数据库表结构（如不存在）""" + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS notes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT NOT NULL, + title TEXT NOT NULL, + title_length INTEGER, + content TEXT, + tags TEXT, + publish_hour INTEGER, + likes INTEGER DEFAULT 0, + collects INTEGER DEFAULT 0, + comments INTEGER DEFAULT 0, + followers INTEGER DEFAULT 0, + is_viral INTEGER DEFAULT 0, + cover_has_face INTEGER DEFAULT 0, + cover_text_ratio REAL DEFAULT 0, + cover_saturation REAL DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_notes_category ON notes(category)") + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_notes_viral ON notes(category, is_viral)" + ) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS baseline_stats ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT NOT NULL, + metric_name TEXT NOT NULL, + metric_value REAL, + metric_json TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(category, metric_name) + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS diagnosis_history ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + category TEXT NOT NULL, + overall_score REAL, + grade TEXT, + report_json TEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_history_created + ON diagnosis_history(created_at DESC) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS usage_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL, + action TEXT NOT NULL DEFAULT 'diagnose', + title TEXT DEFAULT '', + category TEXT DEFAULT '', + total_tokens INTEGER DEFAULT 0, + duration_sec REAL DEFAULT 0, + status TEXT DEFAULT 'ok', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_usage_created ON usage_log(created_at DESC)" + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_ip ON usage_log(ip)") + + conn.execute(""" + CREATE TABLE IF NOT EXISTS visit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + visitor_hash TEXT NOT NULL, + user_agent_hash TEXT DEFAULT '', + path TEXT NOT NULL, + referrer TEXT DEFAULT '', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_visit_created ON visit_log(created_at DESC)" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_visit_visitor ON visit_log(visitor_hash)" + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_path ON visit_log(path)") + + conn.commit() + conn.close() + + from app.database.seed_data import seed_data + from app.database.baseline import compute_baseline + + seed_data(db_path) + compute_baseline(db_path) diff --git a/scripts/compute_baseline.py b/backend/app/database/baseline.py similarity index 94% rename from scripts/compute_baseline.py rename to backend/app/database/baseline.py index 8b7fc6d..325a80e 100644 --- a/scripts/compute_baseline.py +++ b/backend/app/database/baseline.py @@ -1,15 +1,7 @@ -""" -基于 notes 表数据，预计算各垂类的 baseline 统计指标并写入 baseline_stats 表。 - -Usage: - python scripts/compute_baseline.py -""" import sqlite3 import json -import os from collections import Counter - -DB_PATH = os.path.join(os.path.dirname(__file__), "..", "backend", "data", "baseline.db") +from pathlib import Path def upsert_stat(cursor, category, metric_name, metric_value=None, metric_json=None): @@ -174,12 +166,15 @@ def compute_for_category(cursor, category): print(f" [{category}] 已计算 baseline 指标（含粉丝分层与标签分桶）") -def main(): +def compute_baseline(db_path: Path): """计算所有垂类的 baseline 统计指标""" - conn = sqlite3.connect(DB_PATH) + conn = sqlite3.connect(db_path) cursor = conn.cursor() - cursor.execute("DELETE FROM baseline_stats") + cursor.execute("SELECT COUNT(*) FROM baseline_stats") + if cursor.fetchone()[0] > 0: + conn.close() + return for cat in ["food", "fashion", "tech", "travel", "beauty", "fitness", "lifestyle", "home"]: compute_for_category(cursor, cat) @@ -187,7 +182,3 @@ def main(): conn.commit() conn.close() print("所有 baseline 统计指标已计算完毕") - - -if __name__ == "__main__": - main() diff --git a/scripts/seed_data.py b/backend/app/database/seed_data.py similarity index 95% rename from scripts/seed_data.py rename to backend/app/database/seed_data.py index 3354721..5d7f001 100644 --- a/scripts/seed_data.py +++ b/backend/app/database/seed_data.py @@ -1,16 +1,8 @@ -""" -生成模拟 baseline 种子数据用于开发和演示。 -实际比赛前应替换为真实采集的小红书笔记数据。 - -Usage: - python scripts/seed_data.py -""" import sqlite3 import json import random -import os +from pathlib import Path -DB_PATH = os.path.join(os.path.dirname(__file__), "..", "backend", "data", "baseline.db") FOOD_TITLES = [ "手把手教你做日式溏心蛋！零失败！", "一周减脂餐分享｜好吃不胖", @@ -41,7 +33,7 @@ "这个APP改变了我的学习方式", "数码产品年度盘点｜好用到哭", "iPad学习法｜从学渣到学霸", "耳机横评｜千元内最值得买的5款", "NAS入门指南｜打造私人云存储", "手机摄影技巧｜拍出电影质感", - "机械键盘入坑指南｜新手必看", "二手数码避坑指南‼️", + "机械键盘入坑指南｜新手必看", "二手数码避坑指南‼️", "AI工具合集｜效率提升10倍", "极简桌面布置｜打造高效工作台", ] @@ -153,12 +145,15 @@ def generate_notes(category, titles, tags_pool, count=500): return notes -def seed(): - """写入种子数据""" - conn = sqlite3.connect(DB_PATH) +def seed_data(db_path: Path): + """若 notes 表为空，则填充种子数据""" + conn = sqlite3.connect(db_path) cursor = conn.cursor() - cursor.execute("DELETE FROM notes") + cursor.execute("SELECT COUNT(*) FROM notes") + if cursor.fetchone()[0] > 0: + conn.close() + return all_notes = [] all_notes.extend(generate_notes("food", FOOD_TITLES, FOOD_TAGS, 500)) @@ -181,7 +176,3 @@ def seed(): conn.commit() print(f"已插入 {len(all_notes)} 条种子数据") conn.close() - - -if __name__ == "__main__": - seed() diff --git a/backend/app/local_memory.py b/backend/app/local_memory.py index 4131cab..b477a31 100644 --- a/backend/app/local_memory.py +++ b/backend/app/local_memory.py @@ -12,10 +12,11 @@ import logging import os from datetime import datetime +from pathlib import Path logger = logging.getLogger("noterx.local_memory") -_DATA_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data")) +_DATA_ROOT = Path(__file__).parent.parent.parent / "data" WORKSPACE_ROOT = os.path.join(_DATA_ROOT, "noterx_workspace") MEMORY_MD = os.path.join(WORKSPACE_ROOT, "MEMORY.md") MEMORY_DIR = os.path.join(WORKSPACE_ROOT, "memory") diff --git a/backend/app/main.py b/backend/app/main.py index 113bf9d..d389304 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,7 @@ """ import logging import os -import sqlite3 +from pathlib import Path from contextlib import asynccontextmanager from fastapi import FastAPI @@ -13,69 +13,17 @@ from app.api.routes import router as api_router from app import local_memory +from app.database import init_database +DB_PATH = Path(__file__).parent.parent.parent / "data" / "baseline.db" FRONTEND_DIST = os.path.join(os.path.dirname(__file__), "..", "..", "frontend", "dist") -DB_PATH = os.path.join(os.path.dirname(__file__), "..", "data", "baseline.db") - - -def _ensure_history_table(): - """启动时自动创建 diagnosis_history 表（如不存在）""" - os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) - conn = sqlite3.connect(DB_PATH) - conn.execute(""" - CREATE TABLE IF NOT EXISTS diagnosis_history ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL, - category TEXT NOT NULL, - overall_score REAL, - grade TEXT, - report_json TEXT NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_history_created - ON diagnosis_history(created_at DESC) - """) - # Usage tracking table - conn.execute(""" - CREATE TABLE IF NOT EXISTS usage_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOT NULL, - action TEXT NOT NULL DEFAULT 'diagnose', - title TEXT DEFAULT '', - category TEXT DEFAULT '', - total_tokens INTEGER DEFAULT 0, - duration_sec REAL DEFAULT 0, - status TEXT DEFAULT 'ok', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_created ON usage_log(created_at DESC)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_ip ON usage_log(ip)") - conn.execute(""" - CREATE TABLE IF NOT EXISTS visit_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - visitor_hash TEXT NOT NULL, - user_agent_hash TEXT DEFAULT '', - path TEXT NOT NULL, - referrer TEXT DEFAULT '', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_created ON visit_log(created_at DESC)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_visitor ON visit_log(visitor_hash)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_path ON visit_log(path)") - conn.commit() - conn.close() - local_memory.ensure_memory_md() - @asynccontextmanager async def lifespan(_app: FastAPI): """应用生命周期：启动时自动建表""" - _ensure_history_table() + init_database(DB_PATH) + local_memory.ensure_memory_md() yield logging.basicConfig( @@ -178,12 +126,10 @@ async def serve_app(): async def health(): """详细健康检查，含数据库探测""" import sqlite3 - import os - db_path = os.path.join(os.path.dirname(__file__), "..", "data", "baseline.db") db_ok = False note_count = 0 try: - conn = sqlite3.connect(db_path) + conn = sqlite3.connect(DB_PATH) cur = conn.cursor() cur.execute("SELECT COUNT(*) FROM notes") note_count = cur.fetchone()[0] diff --git a/deploy_backend.py b/deploy_backend.py index b12907a..ae3a379 100644 --- a/deploy_backend.py +++ b/deploy_backend.py @@ -89,10 +89,7 @@ def run(ssh, cmd, check=True): run(ssh, f"{REMOTE_DIR}/backend/venv/bin/pip install -r {REMOTE_DIR}/backend/requirements.txt") # Init DB -print(" Initializing database...") -run(ssh, f"cd {REMOTE_DIR} && {REMOTE_DIR}/backend/venv/bin/python scripts/init_db.py", check=False) -run(ssh, f"cd {REMOTE_DIR} && {REMOTE_DIR}/backend/venv/bin/python scripts/seed_data.py", check=False) -run(ssh, f"cd {REMOTE_DIR} && {REMOTE_DIR}/backend/venv/bin/python scripts/compute_baseline.py", check=False) +print(" Database tables / seed data / baseline will be handled by app startup") # Upload .env print(" Uploading .env...") diff --git a/scripts/init_db.py b/scripts/init_db.py deleted file mode 100644 index ecf746c..0000000 --- a/scripts/init_db.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -初始化 SQLite 数据库，创建 baseline 数据表结构。 - -Usage: - python scripts/init_db.py -""" -import sqlite3 -import os - -DB_PATH = os.path.join(os.path.dirname(__file__), "..", "backend", "data", "baseline.db") - - -def init_database(): - """创建数据库表结构""" - os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) - conn = sqlite3.connect(DB_PATH) - cursor = conn.cursor() - - cursor.execute(""" - CREATE TABLE IF NOT EXISTS notes ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - category TEXT NOT NULL, -- food / fashion / tech - title TEXT NOT NULL, - title_length INTEGER, - content TEXT, - tags TEXT, -- JSON array - publish_hour INTEGER, -- 0-23 - likes INTEGER DEFAULT 0, - collects INTEGER DEFAULT 0, - comments INTEGER DEFAULT 0, - followers INTEGER DEFAULT 0, - is_viral INTEGER DEFAULT 0, -- 1=爆款, 0=普通 - cover_has_face INTEGER DEFAULT 0, - cover_text_ratio REAL DEFAULT 0, - cover_saturation REAL DEFAULT 0, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - - cursor.execute(""" - CREATE TABLE IF NOT EXISTS baseline_stats ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - category TEXT NOT NULL, - metric_name TEXT NOT NULL, -- e.g. avg_title_length - metric_value REAL, - metric_json TEXT, -- JSON for complex metrics - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(category, metric_name) - ) - """) - - cursor.execute(""" - CREATE TABLE IF NOT EXISTS diagnosis_history ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL, - category TEXT NOT NULL, - overall_score REAL, - grade TEXT, - report_json TEXT NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - - cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_history_created - ON diagnosis_history(created_at DESC) - """) - - cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_notes_category ON notes(category) - """) - cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_notes_viral ON notes(category, is_viral) - """) - - conn.commit() - conn.close() - print(f"数据库已初始化: {os.path.abspath(DB_PATH)}") - - -if __name__ == "__main__": - init_database() From 42bdbdba1e3ec000f4a29ef985804b1badb5c084 Mon Sep 17 00:00:00 2001 From: JonathanJu Date: Wed, 27 May 2026 22:50:00 +0800 Subject: [PATCH 2/2] =?UTF-8?q?refactor(database):=20=E6=8F=90=E5=8F=96?= =?UTF-8?q?=E5=BB=BA=E8=A1=A8SQL=E5=88=B0=20schema.py=EF=BC=8C=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=20force=20=E5=8F=82=E6=95=B0=E5=92=8C=20data-refresh?= =?UTF-8?q?=20=E5=85=A5=E5=8F=A3=20-=20=E5=B0=86=E5=BB=BA=E8=A1=A8/?= =?UTF-8?q?=E5=BB=BA=E7=B4=A2=E5=BC=95=E9=80=BB=E8=BE=91=E4=BB=8E=20=5F=5F?= =?UTF-8?q?init=5F=5F.py=20=E6=8A=BD=E5=8F=96=E5=88=B0=E7=8B=AC=E7=AB=8B?= =?UTF-8?q?=E7=9A=84=20schema.py=20-=20seed=5Fdata=20/=20compute=5Fbaselin?= =?UTF-8?q?e=20/=20init=5Fdatabase=20=E5=A2=9E=E5=8A=A0=20force=20?= =?UTF-8?q?=E5=8F=82=E6=95=B0=EF=BC=8C=E6=94=AF=E6=8C=81=E5=BC=BA=E5=88=B6?= =?UTF-8?q?=E6=B8=85=E7=A9=BA=E5=90=8E=E9=87=8D=E5=86=99=20-=20Makefile=20?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=20data-refresh=20=E7=9B=AE=E6=A0=87=EF=BC=8C?= =?UTF-8?q?=E8=B0=83=E7=94=A8=20init=5Fdatabase(force=3DTrue)=20-=20?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20DB=5FPATH=20/=20=5FDATA=5FROOT=20=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E5=B1=82=E7=BA=A7=E9=94=99=E8=AF=AF=EF=BC=88=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E6=A0=B9=E7=9B=AE=E5=BD=95=20=E2=86=92=20backend/data?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 6 +- backend/app/database/__init__.py | 101 +++--------------------------- backend/app/database/baseline.py | 15 +++-- backend/app/database/schema.py | 84 +++++++++++++++++++++++++ backend/app/database/seed_data.py | 15 +++-- backend/app/local_memory.py | 2 +- backend/app/main.py | 2 +- 7 files changed, 116 insertions(+), 109 deletions(-) create mode 100644 backend/app/database/schema.py diff --git a/Makefile b/Makefile index 9effbfa..70a7e7d 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,11 @@ # 一键初始化数据库并生成 baseline data: - @echo "数据库表 / 种子数据 / 基线指标 已由 init_database() 自动处理" + cd backend && python3 -c "from app.database import init_database; from pathlib import Path; init_database(Path('data') / 'baseline.db')" + +# 清空种子和基线后重新初始化 +data-refresh: + cd backend && python3 -c "from app.database import init_database; from pathlib import Path; init_database(Path('data') / 'baseline.db', force=True)" # 安装所有依赖 install: diff --git a/backend/app/database/__init__.py b/backend/app/database/__init__.py index 14c2273..788c60e 100644 --- a/backend/app/database/__init__.py +++ b/backend/app/database/__init__.py @@ -1,106 +1,19 @@ import sqlite3 from pathlib import Path +from app.database.schema import create_tables -def init_database(db_path: Path): - """创建数据库表结构（如不存在）""" + +def init_database(db_path: Path, force: bool = False): + """创建数据库表结构（如不存在）；force=True 时清空种子和基线后重写""" db_path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(db_path) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS notes ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - category TEXT NOT NULL, - title TEXT NOT NULL, - title_length INTEGER, - content TEXT, - tags TEXT, - publish_hour INTEGER, - likes INTEGER DEFAULT 0, - collects INTEGER DEFAULT 0, - comments INTEGER DEFAULT 0, - followers INTEGER DEFAULT 0, - is_viral INTEGER DEFAULT 0, - cover_has_face INTEGER DEFAULT 0, - cover_text_ratio REAL DEFAULT 0, - cover_saturation REAL DEFAULT 0, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute("CREATE INDEX IF NOT EXISTS idx_notes_category ON notes(category)") - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_notes_viral ON notes(category, is_viral)" - ) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS baseline_stats ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - category TEXT NOT NULL, - metric_name TEXT NOT NULL, - metric_value REAL, - metric_json TEXT, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(category, metric_name) - ) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS diagnosis_history ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL, - category TEXT NOT NULL, - overall_score REAL, - grade TEXT, - report_json TEXT NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_history_created - ON diagnosis_history(created_at DESC) - """) - - conn.execute(""" - CREATE TABLE IF NOT EXISTS usage_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOT NULL, - action TEXT NOT NULL DEFAULT 'diagnose', - title TEXT DEFAULT '', - category TEXT DEFAULT '', - total_tokens INTEGER DEFAULT 0, - duration_sec REAL DEFAULT 0, - status TEXT DEFAULT 'ok', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_usage_created ON usage_log(created_at DESC)" - ) - conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_ip ON usage_log(ip)") - - conn.execute(""" - CREATE TABLE IF NOT EXISTS visit_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - visitor_hash TEXT NOT NULL, - user_agent_hash TEXT DEFAULT '', - path TEXT NOT NULL, - referrer TEXT DEFAULT '', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_visit_created ON visit_log(created_at DESC)" - ) - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_visit_visitor ON visit_log(visitor_hash)" - ) - conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_path ON visit_log(path)") - + create_tables(conn) conn.commit() conn.close() from app.database.seed_data import seed_data from app.database.baseline import compute_baseline - seed_data(db_path) - compute_baseline(db_path) + seed_data(db_path, force=force) + compute_baseline(db_path, force=force) diff --git a/backend/app/database/baseline.py b/backend/app/database/baseline.py index 325a80e..5ce98fa 100644 --- a/backend/app/database/baseline.py +++ b/backend/app/database/baseline.py @@ -166,15 +166,18 @@ def compute_for_category(cursor, category): print(f" [{category}] 已计算 baseline 指标（含粉丝分层与标签分桶）") -def compute_baseline(db_path: Path): - """计算所有垂类的 baseline 统计指标""" +def compute_baseline(db_path: Path, force: bool = False): + """计算所有垂类的 baseline 统计指标；force=True 时清空后重算""" conn = sqlite3.connect(db_path) cursor = conn.cursor() - cursor.execute("SELECT COUNT(*) FROM baseline_stats") - if cursor.fetchone()[0] > 0: - conn.close() - return + if force: + cursor.execute("DELETE FROM baseline_stats") + else: + cursor.execute("SELECT COUNT(*) FROM baseline_stats") + if cursor.fetchone()[0] > 0: + conn.close() + return for cat in ["food", "fashion", "tech", "travel", "beauty", "fitness", "lifestyle", "home"]: compute_for_category(cursor, cat) diff --git a/backend/app/database/schema.py b/backend/app/database/schema.py new file mode 100644 index 0000000..ad3a734 --- /dev/null +++ b/backend/app/database/schema.py @@ -0,0 +1,84 @@ +import sqlite3 + + +def create_tables(conn: sqlite3.Connection): + conn.execute(""" + CREATE TABLE IF NOT EXISTS notes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT NOT NULL, + title TEXT NOT NULL, + title_length INTEGER, + content TEXT, + tags TEXT, + publish_hour INTEGER, + likes INTEGER DEFAULT 0, + collects INTEGER DEFAULT 0, + comments INTEGER DEFAULT 0, + followers INTEGER DEFAULT 0, + is_viral INTEGER DEFAULT 0, + cover_has_face INTEGER DEFAULT 0, + cover_text_ratio REAL DEFAULT 0, + cover_saturation REAL DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_notes_category ON notes(category)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_notes_viral ON notes(category, is_viral)") + + conn.execute(""" + CREATE TABLE IF NOT EXISTS baseline_stats ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT NOT NULL, + metric_name TEXT NOT NULL, + metric_value REAL, + metric_json TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(category, metric_name) + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS diagnosis_history ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + category TEXT NOT NULL, + overall_score REAL, + grade TEXT, + report_json TEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_history_created + ON diagnosis_history(created_at DESC) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS usage_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL, + action TEXT NOT NULL DEFAULT 'diagnose', + title TEXT DEFAULT '', + category TEXT DEFAULT '', + total_tokens INTEGER DEFAULT 0, + duration_sec REAL DEFAULT 0, + status TEXT DEFAULT 'ok', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_created ON usage_log(created_at DESC)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_ip ON usage_log(ip)") + + conn.execute(""" + CREATE TABLE IF NOT EXISTS visit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + visitor_hash TEXT NOT NULL, + user_agent_hash TEXT DEFAULT '', + path TEXT NOT NULL, + referrer TEXT DEFAULT '', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_created ON visit_log(created_at DESC)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_visitor ON visit_log(visitor_hash)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_path ON visit_log(path)") diff --git a/backend/app/database/seed_data.py b/backend/app/database/seed_data.py index 5d7f001..23946d4 100644 --- a/backend/app/database/seed_data.py +++ b/backend/app/database/seed_data.py @@ -145,15 +145,18 @@ def generate_notes(category, titles, tags_pool, count=500): return notes -def seed_data(db_path: Path): - """若 notes 表为空，则填充种子数据""" +def seed_data(db_path: Path, force: bool = False): + """若 notes 表为空，则填充种子数据；force=True 时清空后重写""" conn = sqlite3.connect(db_path) cursor = conn.cursor() - cursor.execute("SELECT COUNT(*) FROM notes") - if cursor.fetchone()[0] > 0: - conn.close() - return + if force: + cursor.execute("DELETE FROM notes") + else: + cursor.execute("SELECT COUNT(*) FROM notes") + if cursor.fetchone()[0] > 0: + conn.close() + return all_notes = [] all_notes.extend(generate_notes("food", FOOD_TITLES, FOOD_TAGS, 500)) diff --git a/backend/app/local_memory.py b/backend/app/local_memory.py index b477a31..774cdd2 100644 --- a/backend/app/local_memory.py +++ b/backend/app/local_memory.py @@ -16,7 +16,7 @@ logger = logging.getLogger("noterx.local_memory") -_DATA_ROOT = Path(__file__).parent.parent.parent / "data" +_DATA_ROOT = Path(__file__).parent.parent / "data" WORKSPACE_ROOT = os.path.join(_DATA_ROOT, "noterx_workspace") MEMORY_MD = os.path.join(WORKSPACE_ROOT, "MEMORY.md") MEMORY_DIR = os.path.join(WORKSPACE_ROOT, "memory") diff --git a/backend/app/main.py b/backend/app/main.py index d389304..3bbeb04 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -15,7 +15,7 @@ from app import local_memory from app.database import init_database -DB_PATH = Path(__file__).parent.parent.parent / "data" / "baseline.db" +DB_PATH = Path(__file__).parent.parent / "data" / "baseline.db" FRONTEND_DIST = os.path.join(os.path.dirname(__file__), "..", "..", "frontend", "dist")