Repository navigation
Expand file tree
/
Copy pathtesting.py
More file actions
106 lines (92 loc) · 3.75 KB
/
Copy pathtesting.py
File metadata and controls
106 lines (92 loc) · 3.75 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
import json
from markdown import markdown
import requests
import os
from Document_processor import html, md, txt
from shared import normalize_url, save_meta_raw, add_meta, META_RAW, import_meta_raw, document_queue, filter_url
import data_processing as processor
import requests
FILE_NAME = "example.html"
INPUT = "D:\\Thanh\\WLU\\Spring 2026\\CP423\\Project\\Full-Stack-RAG\\Data\\raw\\doc_0000198.md"
PROCESSED = "D:\\Thanh\\WLU\\Spring 2026\\CP423\\Project\\Full-Stack-RAG\\Data\\processed.jsonl"
CHUNKED = "D:\\Thanh\\WLU\\Spring 2026\\CP423\\Project\\Full-Stack-RAG\\Data\\chunked.jsonl"
MAX = 5000
not_matched = [53]
meta = import_meta_raw() # Load existing metadata
docs = {}
with open(PROCESSED, "r", encoding="utf-8") as f:
for line in f:
temp = json.loads(line)
docs[int(temp["docno"])] = {
"title": temp["title"],
"content": temp["content"],
}
# for i in range(MAX):
# # print(f"Processing document {i}...")
# url = meta[i]['url']
# ext = meta[i]['ext']
# # print(url, ext)
# response = requests.get(
# url,
# headers={"User-Agent": "MyCRAWLER"},
# timeout=5
# )
# # print(f"Response status code for document {i}: {response.status_code}")
# response.raise_for_status()
# content = links = title = None
# raw_content = response.content.decode("utf-8", errors="ignore") # Decode the content to a string
# # Choose how to decode and what extension to save based on content type
# if ext == ".md":
# content, links, title = md.process_markdown(raw_content, url)
# content = md.extract_text_from_md(content)
# elif ext == ".html":
# content, links, title = html.process_html(raw_content, url)
# content = html.extract_text_from_html(content)
# elif ext == ".txt":
# content, links, title = txt.process_txt(raw_content, url)
# content = txt.extract_text_from_txt(content)
# # print(f"Document {i} content: {content[:100]}")
# if (content == docs[i]["content"] and title == docs[i]["title"]):
# print(f"Document {i} content and title match.")
# pass
# elif (title != docs[i]["title"]):
# print(f"Document {i} title does not match.")
# not_matched.append(i)
# elif (content != docs[i]["content"]):
# print(f"Document {i} content does not match.")
# not_matched.append(i)
print(f"Documents that did not match: {not_matched}")
# for i in not_matched:
# print(f"Processing document {i}...")
# url = meta[i]['url']
# ext = meta[i]['ext']
# # print(url, ext)
# response = requests.get(
# url,
# headers={"User-Agent": "MyCRAWLER"},
# timeout=5
# )
# # print(f"Response status code for document {i}: {response.status_code}")
# response.raise_for_status()
# content = links = title = None
# raw_content = response.content.decode("utf-8", errors="ignore") # Decode the content to a string
# # Choose how to decode and what extension to save based on content type
# if ext == ".md":
# content, links, title = md.process_markdown(raw_content, url)
# content = md.extract_text_from_md(content)
# elif ext == ".html":
# content, links, title = html.process_html(raw_content, url)
# content = html.extract_text_from_html(content)
# elif ext == ".txt":
# content, links, title = txt.process_txt(raw_content, url)
# content = txt.extract_text_from_txt(content)
# docs[i]["content"] = content
# docs[i]["title"] = title
# for i in range(MAX):
# document = {
# "docno": i,
# "title": docs[i]["title"],
# "content": docs[i]["content"]
# }
# processor.save_processed_data(document)
# processor.chunk_text(document)