-
Notifications
You must be signed in to change notification settings - Fork 9.9k
feat(route): add Fars News showcase #21999
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from 3 commits
dc8dea6
057e044
8fa7840
297f8fb
61aa8d3
63c631c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| import type { Namespace } from '@/types'; | ||
|
|
||
| export const namespace: Namespace = { | ||
| name: 'Fars News', | ||
| url: 'farsnews.ir', | ||
| lang: 'fa', | ||
| }; |
| Original file line number | Diff line number | Diff line change | ||
|---|---|---|---|---|
| @@ -0,0 +1,80 @@ | ||||
| import { load } from 'cheerio'; | ||||
Check failureCode scanning / oxlint simple-import-sort(imports) Error
Run autofix to sort these imports!
|
||||
|
github-advanced-security[bot] marked this conversation as resolved.
Fixed
|
||||
| import type { Route } from '@/types'; | ||||
|
|
||||
| import cache from '@/utils/cache'; | ||||
| import got from '@/utils/got'; | ||||
| import { parseDate } from '@/utils/parse-date'; | ||||
|
|
||||
| export const route: Route = { | ||||
| path: '/showcase/:category?', | ||||
| categories: ['traditional-media'], | ||||
| example: '/farsnews/showcase', | ||||
| parameters: { category: 'Category slug from farsnews.ir/showcase URL' }, | ||||
| features: { | ||||
| requireConfig: false, | ||||
| requirePuppeteer: false, | ||||
| antiCrawler: true, | ||||
| supportBT: false, | ||||
| supportPodcast: false, | ||||
| supportScihub: false, | ||||
| }, | ||||
| radar: [{ | ||||
| source: ['farsnews.ir/showcase'], | ||||
| target: '/showcase', | ||||
| }], | ||||
| name: 'Showcase', | ||||
| maintainers: ['github-oysl'], | ||||
| handler, | ||||
| description: 'Fars News showcase articles. Persian news agency.', | ||||
| }; | ||||
|
|
||||
| async function handler(ctx) { | ||||
| const category = ctx.req.param('category') ?? ''; | ||||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Unncessary RSSHub/lib/routes/farsnews/showcase.ts Line 53 in 63c631c
|
||||
| const baseUrl = 'https://farsnews.ir'; | ||||
| const currentUrl = category ? `${baseUrl}/showcase/${category}` : `${baseUrl}/showcase`; | ||||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The site places |
||||
|
|
||||
| const response = await got({ method: 'get', url: currentUrl }); | ||||
| const $ = load(response.data); | ||||
|
|
||||
| const items = $('a[href^="/"]') | ||||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a textbook example of the first anti-pattern listed. |
||||
| .toArray() | ||||
| .map((item) => { | ||||
| item = $(item); | ||||
| const href = item.attr('href'); | ||||
| const title = item.find('h2, h3').first().text().trim() || item.text().trim(); | ||||
|
|
||||
| if (!href || !title || !/^\/[^/]+\/\d+\//.test(href)) { | ||||
| return null; | ||||
| } | ||||
|
|
||||
| return { | ||||
| title, | ||||
| link: `${baseUrl}${href}`, | ||||
| }; | ||||
| }) | ||||
| .filter((item) => item !== null) | ||||
| .filter((item, index, self) => self.findIndex((i) => i.link === item.link) === index); | ||||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use the embeded data from |
||||
|
|
||||
| const processedItems = await Promise.all( | ||||
| items.map((item) => | ||||
| cache.tryGet(item.link, async () => { | ||||
| const detailResponse = await got({ method: 'get', url: item.link }); | ||||
| const detail$ = load(detailResponse.data); | ||||
|
|
||||
| const desc = detail$('meta[name="description"]').attr('content') || ''; | ||||
| item.description = desc; | ||||
|
|
||||
| const timeText = detail$('time').attr('datetime') || detail$('.text-gray-400').first().text(); | ||||
| if (timeText) { | ||||
| item.pubDate = parseDate(timeText); | ||||
| } | ||||
| return item; | ||||
| }) | ||||
| ) | ||||
| ); | ||||
|
Comment on lines
+55
to
+143
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Turns out the site has an API, so use the official API endpoint instead. import { randomBytes, webcrypto } from 'node:crypto';
import { Packr } from 'msgpackr';
const subtle = webcrypto.subtle;
const packr = new Packr({ encodeUndefinedAsNil: true, useRecords: false });function generateDuid() {
const id = Date.now().toString(32) + Math.floor(10000000 * Math.random() + 1000000).toString(32) + '-web';
return id.padStart(16, '0');
}
function deriveKey(duid) {
const raw = new TextEncoder().encode(duid).slice(0, 16);
return subtle.importKey('raw', raw, { name: 'AES-GCM' }, true, ['encrypt', 'decrypt']);
}
async function encrypt(key, plaintext) {
const iv = randomBytes(12);
const ct = await subtle.encrypt({ name: 'AES-GCM', tagLength: 128, iv }, key, plaintext);
const ctArray = new Uint8Array(ct);
const out = new Uint8Array(iv.length + ctArray.length);
out.set(iv, 0);
out.set(ctArray, iv.length);
return out;
}
async function decrypt(key, payload) {
const iv = payload.slice(0, 12);
const ct = payload.slice(12);
const pt = await subtle.decrypt({ name: 'AES-GCM', tagLength: 128, iv }, key, ct);
return new Uint8Array(pt);
}async function handler(ctx) {
// ...
const duid = generateDuid();
const key = await deriveKey(duid);
const headers = {
'accept-language': 'fa',
duid,
platform: 'web',
os: 'macOS',
'app-version': '1',
'api-version': '1',
};
const body: {
location: string;
showcaseType: 'global' | 'user';
userID?: string;
} = {
location: 'showcase',
showcaseType: category ? 'user' : 'global',
};
if (category) {
const packed = packr.pack({
usernames: [category],
});
const encryptedBody = await encrypt(key, packed);
const response = await ofetch('https://api.farsnews.ir/user/info', {
headers,
method: 'POST',
body: encryptedBody,
responseType: 'arrayBuffer',
});
const decrypted = await decrypt(key, response);
const { data } = packr.unpack(decrypted);
body.userID = data.users[0].userID;
}
const packed = packr.pack(body);
// encrypt
const response = await ofetch('same base api url as above/showcase/block/listV2', {
// same options as the category request
});
// decrypt and unpack
const list = data.centerBlocks
.find((b) => b.type === 'card-list')
.tabs[0].contents.map((c) => {
const post = data.showcaseContents.find((p) => p.postID === c.postID);
const author = data.usersSummary.find((u) => u.userID === post.userID);
return {
title: // ...,
description: // ...,
link: `${baseUrl}/${author.username}/${c.postID}`,
pubDate: // ...,
author: // ...,
postId: c.postID,
};
});and if you want full article const packed = packr.pack({
postID: item.postId,
checkPublicVisibility: false,
});
// encrypt
const response = await ofetch('same base api url as above/post/get', {
// same options as the category request
});
// decrypt and unpack |
||||
|
|
||||
| return { | ||||
| title: 'Fars News - Showcase', | ||||
| link: currentUrl, | ||||
| item: processedItems, | ||||
| }; | ||||
| } | ||||
Uh oh!
There was an error while loading. Please reload this page.