From ff6e32ca85238e094b36cec45bac805007a6a8ba Mon Sep 17 00:00:00 2001 From: il3ven Date: Thu, 1 Jun 2023 00:32:15 +0530 Subject: [PATCH 01/15] use timestamp instead of block number to maintain `last_updated_at` View the discussion here for more context: https://github.com/orgs/neume-network/discussions/29#discussioncomment-5938752 --- database/tracks.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/database/tracks.ts b/database/tracks.ts index ead3dcc..ef32003 100644 --- a/database/tracks.ts +++ b/database/tracks.ts @@ -33,7 +33,7 @@ export class Tracks { return Boolean(rows.length); }; - upsertTrack = async (track: Track, blockNumber: number) => { + upsertTrack = async (track: Track, timestamp: number = Date.now()) => { this.db.transaction(async (trx) => { await trx("tracks") .insert({ @@ -51,7 +51,7 @@ export class Tracks { erc721_metadata: track.erc721.metadata, erc721_uri: track.erc721.uri, uid: track.uid, - lastUpdatedAt: blockNumber, + lastUpdatedAt: timestamp, }) .onConflict(["uid"]) .merge(); @@ -102,10 +102,10 @@ export class Tracks { }), ); - const inputs = [track, blockNumber]; + const inputs = [track, timestamp]; await this.log.put( - `${track.platform.name}/${this.encodeNumber(blockNumber)}/${hashCode( + `${track.platform.name}/${this.encodeNumber(timestamp)}/${hashCode( JSON.stringify(inputs), )}`, { @@ -121,10 +121,10 @@ export class Tracks { tokenId: string, owner: Owner, platform: string, - blockNumber: number, + timestamp: number = Date.now(), ) => { - const inputs = [uid, tokenId, owner, blockNumber]; - await this.db("tracks").update({ lastUpdatedAt: blockNumber }).where("uid", "=", uid); + const inputs = [uid, tokenId, owner, timestamp]; + await this.db("tracks").update({ lastUpdatedAt: timestamp }).where("uid", "=", uid); await this.db("owners") .insert({ @@ -140,7 +140,7 @@ export class Tracks { .merge(); await this.log.put( - `${platform}/${this.encodeNumber(blockNumber)}/${hashCode(JSON.stringify(inputs))}`, + `${platform}/${this.encodeNumber(timestamp)}/${hashCode(JSON.stringify(inputs))}`, { operation: "upsertOwner", inputs, @@ -253,7 +253,7 @@ export class Tracks { // will break for numbers greater than maximum digits. In the // above example, the solution will break for numbers greater than 100. encodeNumber(num: Number) { - const MAX_LENGTH = 10; // TODO: increase this number for polygon + const MAX_LENGTH = 20; if (num.toString().length > MAX_LENGTH) throw new Error(`Database cannot encode number greater than 10 digits`); return num.toString().padStart(MAX_LENGTH, "0"); From 06c1e6f5b66f294788ef188dbd9c11aaedde7475 Mon Sep 17 00:00:00 2001 From: il3ven Date: Thu, 1 Jun 2023 00:39:51 +0530 Subject: [PATCH 02/15] add localstorage LocalStorage is similar to the local-storage present in browsers. It is a simple key-value database powered by LevelDB. Each strategy can have its own localstorage and use it to store any data other than tracks. Strategies are free to use it however they wish. The purpose of localstorage is to store intermediate data. Here tracks are the final data. For example, a strategy like SoundProtocol needs to keep track of contracts which can mint NFTs and listen for updates on them. LocalStorage can be used here. Earlier, we created a JSON file to store the data required by SoundProtocol. LocalStorage replaces that JSON file as it is more generic and scalable. --- database/localstorage.ts | 52 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 database/localstorage.ts diff --git a/database/localstorage.ts b/database/localstorage.ts new file mode 100644 index 0000000..273ca16 --- /dev/null +++ b/database/localstorage.ts @@ -0,0 +1,52 @@ +import path from "path"; +import { Level } from "level"; + +export class LocalStorage { + level: Level; + + constructor(dbPath: string) { + this.level = new Level(path.resolve(dbPath, "./localstorage"), { + valueEncoding: "json", + }); + } + + async insert(key: string, value: any, prefix?: string) { + prefix = prefix ?? ""; + return this.level.put(`${prefix}-${key}`, value); + } + + async del(key: string, prefix?: string) { + return this.level.del(`${prefix}-${key}`); + } + + async get(key: string, prefix?: string) { + prefix = prefix ?? ""; + return this.level.get(`${prefix}-${key}`); + } +} + +export const localStorage = new Level(path.resolve("./data", "./localstorage"), { + valueEncoding: "json", +}); + +export async function getLocalStorage(sublevelName: string) { + const sublevel = localStorage.sublevel(sublevelName, { valueEncoding: "json" }); + const all = await sublevel.iterator({}).all(); + return all; +} + +export async function saveLocalStorage(sublevelName: string, entries: Array<[string, any]>) { + const sublevel = localStorage.sublevel(sublevelName, { valueEncoding: "json" }); + const operations = entries.map((e) => { + return { + type: "put" as "put", + key: e[0], + value: e[1], + }; + }); + await sublevel.batch(operations); +} + +process.on("exit", async () => { + await localStorage.close(); +}); From 93a5af2477ea627c41f5a632ce289dcdc5bc20b4 Mon Sep 17 00:00:00 2001 From: il3ven Date: Thu, 1 Jun 2023 20:29:09 +0530 Subject: [PATCH 03/15] add lens strategy and refactor core structure of strategies Major changes: - The Strategy class has been changed. All strategies implement this class. - Introduce a concept of *components*. - Add Lens strategy. - Refactor Sound Protocol strategy according to the new strucutre. - Introduce multi-chain strategies. - Changes in daemon.ts, neume.ts to run the new refactored strategies. What are components? Each strategy is a class and it can define functions. What if we want to share some functions among strategies. The answer to do that is components. For example, `ethGetLogs` is a component that can be used to fetch logs. Difference between components and utils: Utils are simple functions which can be called from anywhere. Components can only be called from inside strategies. We use the `this` keyword to enforce it. TypeScript helps here by defining the type of `this`. Why can't we have utils instead of components? Components are aware that they will be called from a strategy. Therefore, they can use the `this` keyword to fetch information about the strategy and act upon it. For example, the `ethGetLogs` component can use `this.chain` to get the chain of the strategy and fetch logs from the appropriate chain. If `ethGetLogs` was a simple function we would have to pass the chain as an argument. It is okay if it is just chain but functions often need more information than just chain. Passing 10 parameters is not ideal and problematic from a refactor point of view. About `handleTransfer` component: It is a component that can be used by any strategy to listen for ERC721 transfer events. The component is heavily opnionated. --- commands/daemon.ts | 68 +- neume.ts | 16 +- package-lock.json | 2191 +++++++++++++++++++++-------- package.json | 2 + src/components/call-owner.ts | 16 +- src/components/call-tokenuri.ts | 11 +- src/components/eth-get-logs.ts | 46 + src/components/handle-transfer.ts | 167 +++ src/state.ts | 17 +- src/strategies/lens/components.ts | 199 +++ src/strategies/lens/lens.ts | 400 ++++++ src/strategies/sound_protocol.ts | 171 +-- src/strategies/strategy.types.ts | 42 +- src/types.ts | 53 +- src/utils.ts | 40 +- tsconfig.json | 27 +- 16 files changed, 2641 insertions(+), 825 deletions(-) create mode 100644 src/components/eth-get-logs.ts create mode 100644 src/components/handle-transfer.ts create mode 100644 src/strategies/lens/components.ts create mode 100644 src/strategies/lens/lens.ts diff --git a/commands/daemon.ts b/commands/daemon.ts index 5835d28..f613bd9 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -1,62 +1,76 @@ import { fastify as Fastify } from "fastify"; import { JSONRPCServer, JSONRPCErrorException } from "json-rpc-2.0"; -import { Config } from "../src/types.js"; -import { getLatestBlockNumber, getStrategies, getUserContracts } from "../src/utils.js"; -import crawl from "./crawl.js"; -import filter_contracts from "./filter_contracts.js"; -import { db } from "../database/index.js"; +import { CHAINS, Config } from "../src/types.js"; +import { getLatestBlockNumber, getStrategies } from "../src/utils.js"; import { DaemonJsonrpcType } from "./daemon/daemon-jsonrpc-type.js"; import { daemonJsonrpcSchema } from "./daemon/daemon-jsonrpc-schema.js"; import { getLastCrawledBlock, saveLastCrawledBlock } from "../src/state.js"; +import { tracksDB } from "../database/tracks.js"; +import { getLocalStorage } from "../database/localstorage.js"; +import ExtractionWorker from "@neume-network/extraction-worker"; const fastify = Fastify(); export default async function daemon( - _from: number | undefined, crawlFlag: boolean, recrawl: boolean, port: number, config: Config, strategyNames: string[], ) { - const RANGE_FOR_CRAWL = 5000; - let from = _from ?? (await getLastCrawledBlock()); - let to = Math.min(from + RANGE_FOR_CRAWL, await getLatestBlockNumber(config.rpc[0])); - let strategies = getStrategies(strategyNames, from, to); + const worker = ExtractionWorker(config.worker); + const allStrategies = getStrategies(strategyNames).map((s) => new s(worker, config)); - const task = async () => { - const latestBlockNumber = await getLatestBlockNumber(config.rpc[0]); - to = Math.min(from + RANGE_FOR_CRAWL, latestBlockNumber); + const task = async (chain: CHAINS) => { + const { crawlStep } = config.chain[chain]; + const latestBlockNumber = await getLatestBlockNumber(config.chain[chain].rpc[0]); + const from = await getLastCrawledBlock(chain); + const to = Math.min(from + crawlStep, latestBlockNumber); - console.log(`\n\n***** Starting a crawl cycle from ${from} to ${to} *****\n`); + const strategies = allStrategies.filter( + (s) => + s.createdAtBlock <= from && + to <= (s.deprecatedAtBlock ?? Number.MAX_VALUE) && + s.chain === chain, + ); - strategies = getStrategies(strategyNames, from, to); - await filter_contracts(from, to, recrawl, config, strategies); - await crawl(from, to, recrawl, config, strategies); + await Promise.all( + strategies.map(async (strategy) => { + console.log("Calling strategy", strategy.constructor.name, "from", from, "to", to); + await strategy.crawl(from, to, recrawl); + }), + ); - await saveLastCrawledBlock(to); - from = to; + await saveLastCrawledBlock(chain, to); const nextTaskWaitTime = to === latestBlockNumber ? config.breatheTimeMS : 0; - setTimeout(task, nextTaskWaitTime); + + setTimeout(task.bind({}, chain), nextTaskWaitTime); }; - if (crawlFlag) task(); + if (crawlFlag) { + Object.values(CHAINS).forEach((c) => { + // Do not call the task for this chain if no strategies are present + if (allStrategies.filter((s) => s.chain === c).length) task(c); + }); + } + await startServer(port); } async function startServer(port: number) { const server = new JSONRPCServer(); - server.addMethod("getIdsChanged_fill", async ([from, to]) => { - if (to - from > 5000) - return new JSONRPCErrorException("Block range should be less than 5000", -32600); - const res = await db.getIdsChanged_fill(from, to); + server.addMethod("getTracks", async ({ from, to, platform }) => { + // TODO: Make it per platform + // if (to - from > 5000) + // return new JSONRPCErrorException("Block range should be less than 5000", -32600); + const res = await tracksDB.getTracksChanged(from, to, platform); return res; }); - server.addMethod("getUserContracts", async () => { - return getUserContracts(); + server.addMethod("getLocalStorage", async ({ platform }) => { + return getLocalStorage(platform); }); fastify.route<{ Body: DaemonJsonrpcType }>({ diff --git a/neume.ts b/neume.ts index dc73ff8..9fe3841 100644 --- a/neume.ts +++ b/neume.ts @@ -7,12 +7,12 @@ import yargs from "yargs"; import { hideBin } from "yargs/helpers"; import path from "path"; -import crawl from "./commands/crawl.js"; -import dump from "./commands/dump.js"; -import filterContracts from "./commands/filter_contracts.js"; +// import crawl from "./commands/crawl.js"; +// import dump from "./commands/dump.js"; +// import filterContracts from "./commands/filter_contracts.js"; import { getLatestBlockNumber, getStrategies } from "./src/utils.js"; import daemon from "./commands/daemon.js"; -import sync from "./commands/sync.js"; +// import sync from "./commands/sync.js"; import init from "./commands/init.js"; import { db } from "./database/index.js"; import runMigration from "./database/runMigration.js"; @@ -37,7 +37,7 @@ const argv = yargs(hideBin(process.argv)) ) .command( "crawl", - "Find new NFTs from the list of already known contracts", + "Find new NFTs from the list of already known contracts [Out of date]", { from: { type: "number", @@ -64,7 +64,7 @@ const argv = yargs(hideBin(process.argv)) ) .command( "filter-contracts", - "Find new contracts", + "Find new contracts [Out of date]", { from: { type: "number", @@ -91,7 +91,7 @@ const argv = yargs(hideBin(process.argv)) ) .command( "dump", - "Export database as JSON", + "Export database as JSON [Out of date]", { at: { type: "number", @@ -135,7 +135,7 @@ const argv = yargs(hideBin(process.argv)) }, async (argv) => { const { config, strategies: strategyNames } = await import(path.resolve("./config.js")); - await daemon(argv.from, argv.crawl, argv.recrawl, argv.port, config, strategyNames); + await daemon(argv.crawl, argv.recrawl, argv.port, config, strategyNames); }, ) .command( diff --git a/package-lock.json b/package-lock.json index 911cc53..8192083 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@neume-network/extraction-worker": "github:neume-network/extraction-worker", "@neume-network/schema": "github:neume-network/schema", "ava": "^5.1.0", + "better-sqlite3": "^8.2.0", "dotenv": "^16.0.3", "eth-fun": "^0.9.2", "fastify": "^4.10.2", @@ -19,6 +20,7 @@ "json-canonicalize": "^1.0.4", "json-rpc-2.0": "^1.4.2", "json-schema-to-typescript": "^11.0.2", + "knex": "^2.4.2", "level": "^8.0.0", "p-map": "^5.5.0", "yargs": "^17.6.2" @@ -31,33 +33,42 @@ "@types/yargs": "^17.0.14", "ts-node": "^10.9.1", "typescript": "^4.8.4" + }, + "engines": { + "node": "16" + } + }, + "../music-os-schema": { + "name": "@neume-network/schema", + "version": "0.8.1", + "extraneous": true, + "license": "LGPL-3.0-only", + "devDependencies": { + "ajv": "8.11.0", + "ajv-formats": "2.1.1", + "ava": "4.2.0", + "husky": "7.0.4", + "json-schema-to-typescript": "11.0.2", + "lint-staged": "12.4.0", + "mime-db": "1.52.0", + "prettier": "2.6.2" } }, "node_modules/@bcherny/json-schema-ref-parser": { - "version": "9.0.9", - "resolved": "https://registry.npmjs.org/@bcherny/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz", - "integrity": "sha512-vmEmnJCfpkLdas++9OYg6riIezTYqTHpqUTODJzHLzs5UnXujbOJW9VwcVCnyo1mVRt32FRr23iXBx/sX8YbeQ==", + "version": "10.0.5-fork", + "resolved": "https://registry.npmjs.org/@bcherny/json-schema-ref-parser/-/json-schema-ref-parser-10.0.5-fork.tgz", + "integrity": "sha512-E/jKbPoca1tfUPj3iSbitDZTGnq6FUFjkH6L8U2oDwSuwK1WhnnVtCG7oFOTg/DDnyoXbQYUiUiGOibHqaGVnw==", "dependencies": { "@jsdevtools/ono": "^7.1.3", "@types/json-schema": "^7.0.6", "call-me-maybe": "^1.0.1", "js-yaml": "^4.1.0" - } - }, - "node_modules/@bcherny/json-schema-ref-parser/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, - "node_modules/@bcherny/json-schema-ref-parser/node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dependencies": { - "argparse": "^2.0.1" }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/philsturgeon" } }, "node_modules/@cspotcode/source-map-support": { @@ -513,14 +524,6 @@ "multiformats": "9.9.0" } }, - "node_modules/@neume-network/extraction-worker/node_modules/cross-fetch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", - "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", - "dependencies": { - "node-fetch": "2.6.7" - } - }, "node_modules/@neume-network/extraction-worker/node_modules/dotenv": { "version": "16.0.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.0.tgz", @@ -531,7 +534,7 @@ }, "node_modules/@neume-network/schema": { "version": "0.8.1", - "resolved": "git+ssh://git@github.com/neume-network/schema.git#546037ed9db83969686a2338e0ca287dfd387276", + "resolved": "git+ssh://git@github.com/neume-network/schema.git#f2f22cb4f37a27ba8a02ff7b1852ff386e833949", "license": "LGPL-3.0-only" }, "node_modules/@nodelib/fs.scandir": { @@ -605,9 +608,9 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==" }, "node_modules/@types/lodash": { - "version": "4.14.191", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz", - "integrity": "sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==" + "version": "4.14.192", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.192.tgz", + "integrity": "sha512-km+Vyn3BYm5ytMO13k9KTp27O75rbQ0NFw+U//g+PX7VZyjCioXaRFisqSIJRECljcTv73G3i6BpglNGHgUQ5A==" }, "node_modules/@types/minimatch": { "version": "5.1.2", @@ -615,9 +618,9 @@ "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==" }, "node_modules/@types/node": { - "version": "18.11.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", - "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==" + "version": "18.15.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz", + "integrity": "sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q==" }, "node_modules/@types/prettier": { "version": "2.7.2", @@ -625,9 +628,9 @@ "integrity": "sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg==" }, "node_modules/@types/yargs": { - "version": "17.0.14", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.14.tgz", - "integrity": "sha512-9Pj7abXoW1RSTcZaL2Hk6G2XyLMlp5ECdVC/Zf2p/KBjC3srijLGgRAXOBjtFrJoIrvxdTKyKDA14bEcbxBaWw==", + "version": "17.0.24", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", + "integrity": "sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==", "dev": true, "dependencies": { "@types/yargs-parser": "*" @@ -673,9 +676,9 @@ "integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==" }, "node_modules/acorn": { - "version": "8.8.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", - "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", + "version": "8.8.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", + "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", "bin": { "acorn": "bin/acorn" }, @@ -738,22 +741,22 @@ } }, "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", "engines": { - "node": ">=8" + "node": ">=12" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" @@ -788,12 +791,9 @@ "dev": true }, "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dependencies": { - "sprintf-js": "~1.0.2" - } + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "node_modules/array-find-index": { "version": "1.0.2", @@ -839,9 +839,9 @@ } }, "node_modules/ava": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/ava/-/ava-5.1.0.tgz", - "integrity": "sha512-e5VFrSQ0WBPyZJWRXVrO7RFOizFeNM0t2PORwrPvWtApgkORI6cvGnY3GX1G+lzpd0HjqNx5Jus22AhxVnUMNA==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ava/-/ava-5.2.0.tgz", + "integrity": "sha512-W8yxFXJr/P68JP55eMpQIa6AiXhCX3VeuajM8nolyWNExcMDD6rnIWKTjw0B/+GkFHBIaN6Jd0LtcMThcoqVfg==", "dependencies": { "acorn": "^8.8.1", "acorn-walk": "^8.2.0", @@ -850,10 +850,10 @@ "arrify": "^3.0.0", "callsites": "^4.0.0", "cbor": "^8.1.0", - "chalk": "^5.1.2", + "chalk": "^5.2.0", "chokidar": "^3.5.3", "chunkd": "^2.0.1", - "ci-info": "^3.6.1", + "ci-info": "^3.7.1", "ci-parallel-vars": "^1.0.1", "clean-yaml-object": "^0.1.0", "cli-truncate": "^3.1.0", @@ -865,7 +865,7 @@ "del": "^7.0.0", "emittery": "^1.0.1", "figures": "^5.0.0", - "globby": "^13.1.2", + "globby": "^13.1.3", "ignore-by-default": "^2.1.0", "indent-string": "^5.0.0", "is-error": "^2.2.2", @@ -904,51 +904,10 @@ } } }, - "node_modules/ava/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ava/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/ava/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "node_modules/ava/node_modules/strip-ansi": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", - "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, "node_modules/avvio": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/avvio/-/avvio-8.2.0.tgz", - "integrity": "sha512-bbCQdg7bpEv6kGH41RO/3B2/GMMmJSo2iBK+X8AWN9mujtfUipMDfIjsgHCfpnKqoGEQrrmCDKSa5OQ19+fDmg==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/avvio/-/avvio-8.2.1.tgz", + "integrity": "sha512-TAlMYvOuwGyLK3PfBb5WKBXZmXz2fVCgv23d6zZFdle/q3gPjmxBaeuC0pY0Dzs5PWMSgfqqEZkrye19GlDTgw==", "dependencies": { "archy": "^1.0.0", "debug": "^4.0.0", @@ -979,6 +938,16 @@ } ] }, + "node_modules/better-sqlite3": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-8.2.0.tgz", + "integrity": "sha512-8eTzxGk9535SB3oSNu0tQ6I4ZffjVCBUjKHN9QeeIFtphBX0sEd0NxAuglBNR9TO5ThnxBB7GqzfcYo9kjadJQ==", + "hasInstallScript": true, + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.0" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -987,6 +956,60 @@ "node": ">=8" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/blueimp-md5": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.19.0.tgz", @@ -1133,15 +1156,26 @@ "fsevents": "~2.3.2" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, "node_modules/chunkd": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/chunkd/-/chunkd-2.0.1.tgz", "integrity": "sha512-7d58XsFmOq0j6el67Ug9mHf9ELUXsQXYJBkyxhH/k+6Ke0qXRnv0kbemx+Twc6fRJ07C49lcbdgm9FL1Ei/6SQ==" }, "node_modules/ci-info": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.7.0.tgz", - "integrity": "sha512-2CpRNYmImPx+RXKLq6jko/L07phmS9I02TyqkcNU20GCF/GgaWvc58hPtjxDX8lPpkdwc9sNh72V9k00S7ezog==", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", + "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], "engines": { "node": ">=8" } @@ -1219,63 +1253,62 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/cli-truncate/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, "engines": { "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, - "node_modules/cli-truncate/node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } }, - "node_modules/cli-truncate/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/cliui/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=8" } }, - "node_modules/cli-truncate/node_modules/strip-ansi": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", - "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dependencies": { - "ansi-regex": "^6.0.1" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" }, "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" + "node": ">=8" } }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" + "ansi-regex": "^5.0.1" }, "engines": { - "node": ">=12" + "node": ">=8" } }, "node_modules/code-excerpt": { @@ -1305,6 +1338,19 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/colorette": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", + "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==" + }, + "node_modules/commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "engines": { + "node": "^12.20.0 || >=14" + } + }, "node_modules/common-path-prefix": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz", @@ -1333,14 +1379,6 @@ "node": ">=10.18.0 <11 || >=12.14.0 <13 || >=14" } }, - "node_modules/content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/convert-to-spaces": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz", @@ -1364,19 +1402,11 @@ "dev": true }, "node_modules/cross-fetch": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.4.tgz", - "integrity": "sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", "dependencies": { - "node-fetch": "2.6.1" - } - }, - "node_modules/cross-fetch/node_modules/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", - "engines": { - "node": "4.x || >=6.0.0" + "node-fetch": "2.6.7" } }, "node_modules/currently-unhandled": { @@ -1426,23 +1456,39 @@ } } }, + "node_modules/debug/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", "engines": { "node": ">=0.10" } }, "node_modules/decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "dependencies": { - "mimic-response": "^1.0.0" + "mimic-response": "^3.1.0" }, "engines": { - "node": ">=4" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" } }, "node_modules/del": { @@ -1477,6 +1523,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", + "engines": { + "node": ">=8" + } + }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -1546,9 +1600,17 @@ } }, "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } }, "node_modules/es5-ext": { "version": "0.10.62", @@ -1613,6 +1675,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", + "engines": { + "node": ">=6" + } + }, "node_modules/esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", @@ -1646,6 +1716,22 @@ "node": ">=14 <=16" } }, + "node_modules/eth-fun/node_modules/cross-fetch": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.4.tgz", + "integrity": "sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==", + "dependencies": { + "node-fetch": "2.6.1" + } + }, + "node_modules/eth-fun/node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "engines": { + "node": "4.x || >=6.0.0" + } + }, "node_modules/eth-lib": { "version": "0.2.8", "resolved": "https://registry.npmjs.org/eth-lib/-/eth-lib-0.2.8.tgz", @@ -1712,6 +1798,14 @@ "node": ">=0.8.x" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "engines": { + "node": ">=6" + } + }, "node_modules/ext": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", @@ -1725,6 +1819,11 @@ "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" }, + "node_modules/fast-content-type-parse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-1.0.0.tgz", + "integrity": "sha512-Xbc4XcysUXcsP5aHUU7Nq3OwvHq97C+WnbkeIefpeYLX+ryzFJlU6OStFJhs6Ol0LkUGpcK+wL0JwfM+FCU5IA==" + }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -1756,9 +1855,9 @@ } }, "node_modules/fast-json-stringify": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.5.0.tgz", - "integrity": "sha512-rmw2Z8/mLkND8zI+3KTYIkNPEoF5v6GqDP/o+g7H3vjdWjBwuKpgAYFHIzL6ORRB+iqDjjtJnLIW9Mzxn5szOA==", + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.7.0.tgz", + "integrity": "sha512-sBVPTgnAZseLu1Qgj6lUbQ0HfjFhZWXAmpZ5AaSGkyLh5gAXBga/uPJjQPHpDFjC9adWIpdOcCLSDTgrZ7snoQ==", "dependencies": { "@fastify/deepmerge": "^1.0.0", "ajv": "^8.10.0", @@ -1769,9 +1868,9 @@ } }, "node_modules/fast-querystring": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-querystring/-/fast-querystring-1.0.0.tgz", - "integrity": "sha512-3LQi62IhQoDlmt4ULCYmh17vRO2EtS7hTSsG4WwoKWgV7GLMKBOecEh+aiavASnLx8I2y89OD33AGLo0ccRhzA==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/fast-querystring/-/fast-querystring-1.1.1.tgz", + "integrity": "sha512-qR2r+e3HvhEFmpdHMv//U8FnFlnYjaC6QKDuaXALDkw2kvHO8WDjxH+f/rHGR4Me4pnk8p9JAkRNTjYHAKRn2Q==", "dependencies": { "fast-decode-uri-component": "^1.0.1" } @@ -1790,17 +1889,17 @@ "integrity": "sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg==" }, "node_modules/fastify": { - "version": "4.10.2", - "resolved": "https://registry.npmjs.org/fastify/-/fastify-4.10.2.tgz", - "integrity": "sha512-0T+4zI6N3S8ex0LCZi3H4FasJR4AzWw834fUkPWvV8r6GBJkLmAOfFxH8f5V29Plef24IK0QSQD/tz1Nx+1UOA==", + "version": "4.15.0", + "resolved": "https://registry.npmjs.org/fastify/-/fastify-4.15.0.tgz", + "integrity": "sha512-m/CaRN8nf5uyYdrDe2qqq+0z3oGyE+A++qlKQoLJTI4WI0nWK9D6R3FxXQ3MVwt/md977GMR4F43pE9oqrS2zw==", "dependencies": { - "@fastify/ajv-compiler": "^3.3.1", + "@fastify/ajv-compiler": "^3.5.0", "@fastify/error": "^3.0.0", - "@fastify/fast-json-stringify-compiler": "^4.1.0", + "@fastify/fast-json-stringify-compiler": "^4.2.0", "abstract-logging": "^2.0.1", "avvio": "^8.2.0", - "content-type": "^1.0.4", - "find-my-way": "^7.3.0", + "fast-content-type-parse": "^1.0.0", + "find-my-way": "^7.6.0", "light-my-request": "^5.6.1", "pino": "^8.5.0", "process-warning": "^2.0.0", @@ -1834,6 +1933,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -1846,9 +1950,9 @@ } }, "node_modules/find-my-way": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-7.3.1.tgz", - "integrity": "sha512-kGvM08SOkqvheLcuQ8GW9t/H901Qb9rZEbcNWbXopzy4jDRoaJpJoObPSKf4MnQLZ20ZTp7rL5MpF6rf+pqmyg==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-7.6.0.tgz", + "integrity": "sha512-H7berWdHJ+5CNVr4ilLWPai4ml7Y2qAsxjw3pfeBxPigZmaDTzF0wjJLj90xRCmGcWYcyt050yN+34OZDJm1eQ==", "dependencies": { "fast-deep-equal": "^3.1.3", "fast-querystring": "^1.0.0", @@ -1881,6 +1985,11 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1899,6 +2008,11 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1907,6 +2021,14 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/get-stdin": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz", @@ -1918,6 +2040,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/getopts": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", + "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==" + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -1976,9 +2108,9 @@ } }, "node_modules/globby": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", - "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.3.tgz", + "integrity": "sha512-8krCNHXvlCgHDpegPzleMq07yMYTO2sXKASmZmquEYWEmCx6J5UTRbp5RwMJkTJGtcQ44YpiUYUiN0b9mzy8Bw==", "dependencies": { "dir-glob": "^3.0.1", "fast-glob": "^3.2.11", @@ -2005,9 +2137,20 @@ } }, "node_modules/graceful-fs": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } }, "node_modules/hash.js": { "version": "1.1.7", @@ -2048,9 +2191,9 @@ ] }, "node_modules/ignore": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.1.tgz", - "integrity": "sha512-d2qQLzTJ9WxQftPAuEQpSPmKqzxePjzVbpAVv62AQ64NTL+wR4JkrVqR/LqFsFEUsHDAiId52mJteHDFuDkElA==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", "engines": { "node": ">= 4" } @@ -2096,6 +2239,19 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "node_modules/interpret": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -2110,9 +2266,9 @@ "integrity": "sha512-oHrL0x2MbBlEdr1g+wBORq50q119sTV1ib5jVeq5ktYYZa/qy1eJfKgT70YGSn+jOuy/GQ2KqY+g44ynsyPoag==" }, "node_modules/irregular-plurals": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.3.0.tgz", - "integrity": "sha512-MVBLKUTangM3EfRPFROhmWQQKRDsrgI83J8GS3jXy+OwYqiR2/aoWndYQ5416jLE3uaGgLH7ncme3X9y09gZ3g==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.5.0.tgz", + "integrity": "sha512-1ANGLZ+Nkv1ptFb2pa8oG8Lem4krflKuX/gINiHJHjJUKaJHk/SXk5x6K3J+39/p0h1RQ2saROclJJ+QLvETCQ==", "engines": { "node": ">=8" } @@ -2150,6 +2306,17 @@ "node": ">=4" } }, + "node_modules/is-core-module": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-error": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/is-error/-/is-error-2.2.2.tgz", @@ -2164,11 +2331,14 @@ } }, "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz", + "integrity": "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==", "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/is-function": { @@ -2264,12 +2434,11 @@ } }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -2281,16 +2450,16 @@ "integrity": "sha512-YNr/ePzgReHwlnAm3EVV1pcimwesI+1DZr5v7WBKOc1zE1t7pjxWAPRxJFT3ll6flLIdRe0DPia/8cl2FLAZNA==" }, "node_modules/json-rpc-2.0": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/json-rpc-2.0/-/json-rpc-2.0-1.4.2.tgz", - "integrity": "sha512-oqiMRhWN4Q+2ySitZR5GA9vtiNOK19GTDDad0yn8/T/ND+Ap81QP6+3UMl738H4bmRbCEwhQkkBruNqx7EXfUQ==" + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/json-rpc-2.0/-/json-rpc-2.0-1.5.1.tgz", + "integrity": "sha512-ZY/vYl/uUgKN3tNrZMq7w+CGLcoUT+8AzDO/HJZVa+K4XcwgfgES1QDa5y7ieAeh4NgRo3hLexMxgdaiEiK9aA==" }, "node_modules/json-schema-to-typescript": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/json-schema-to-typescript/-/json-schema-to-typescript-11.0.2.tgz", - "integrity": "sha512-XRyeXBJeo/IH4eTP5D1ptX78vCvH86nMDt2k3AxO28C3uYWEDmy4mgPyMpb8bLJ/pJMElOGuQbnKR5Y6NSh3QQ==", + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/json-schema-to-typescript/-/json-schema-to-typescript-11.0.5.tgz", + "integrity": "sha512-ZNlvngzlPzjYYECbR+uJ9aUWo25Gw/VuwUytvcuKiwc6NaiZhMyf7qBsxZE2eixmj8AoQEQJhSRG7btln0sUDw==", "dependencies": { - "@bcherny/json-schema-ref-parser": "9.0.9", + "@bcherny/json-schema-ref-parser": "10.0.5-fork", "@types/json-schema": "^7.0.11", "@types/lodash": "^4.14.182", "@types/prettier": "^2.6.1", @@ -2322,6 +2491,56 @@ "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.2.0.tgz", "integrity": "sha512-4TikKSf+Gb+Et5SnA4ppyrxLSf9qWFq+SqfdDdrgHE1KLwSch/Zi1AQB0TrE4ppYjZdUrHnwdx+6dyx0cx/HyA==" }, + "node_modules/knex": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/knex/-/knex-2.4.2.tgz", + "integrity": "sha512-tMI1M7a+xwHhPxjbl/H9K1kHX+VncEYcvCx5K00M16bWvpYPKAZd6QrCu68PtHAdIZNQPWZn0GVhqVBEthGWCg==", + "dependencies": { + "colorette": "2.0.19", + "commander": "^9.1.0", + "debug": "4.3.4", + "escalade": "^3.1.1", + "esm": "^3.2.25", + "get-package-type": "^0.1.0", + "getopts": "2.3.0", + "interpret": "^2.2.0", + "lodash": "^4.17.21", + "pg-connection-string": "2.5.0", + "rechoir": "^0.8.0", + "resolve-from": "^5.0.0", + "tarn": "^3.0.2", + "tildify": "2.0.0" + }, + "bin": { + "knex": "bin/cli.js" + }, + "engines": { + "node": ">=12" + }, + "peerDependenciesMeta": { + "better-sqlite3": { + "optional": true + }, + "mysql": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "pg-native": { + "optional": true + }, + "sqlite3": { + "optional": true + }, + "tedious": { + "optional": true + } + } + }, "node_modules/level": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/level/-/level-8.0.0.tgz", @@ -2359,9 +2578,9 @@ } }, "node_modules/light-my-request": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.8.0.tgz", - "integrity": "sha512-4BtD5C+VmyTpzlDPCZbsatZMJVgUIciSOwYhJDCbLffPZ35KoDkDj4zubLeHDEb35b4kkPeEv5imbh+RJxK/Pg==", + "version": "5.9.1", + "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.9.1.tgz", + "integrity": "sha512-UT7pUk8jNCR1wR7w3iWfIjx32DiB2f3hFdQSOwy3/EPQ3n3VocyipUxcyRZR0ahoev+fky69uA+GejPa9KuHKg==", "dependencies": { "cookie": "^0.5.0", "process-warning": "^2.0.0", @@ -2388,9 +2607,9 @@ } }, "node_modules/locate-path": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.1.1.tgz", - "integrity": "sha512-vJXaRMJgRVD3+cUZs3Mncj2mxpt5mP0EmNOsxRSZRMlbqjvxzDEOIUWXGmavo0ZC9+tNZCBLQ66reA11nbpHZg==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", "dependencies": { "p-locate": "^6.0.0" }, @@ -2534,11 +2753,14 @@ } }, "node_modules/mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", "engines": { - "node": ">=4" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/min-document": { @@ -2571,9 +2793,9 @@ } }, "node_modules/minimist": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz", - "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==", + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -2589,6 +2811,11 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "node_modules/module-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz", @@ -2598,9 +2825,9 @@ } }, "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/multiformats": { "version": "9.9.0", @@ -2617,6 +2844,11 @@ "thenify-all": "^1.0.0" } }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "node_modules/napi-macros": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", @@ -2627,6 +2859,17 @@ "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" }, + "node_modules/node-abi": { + "version": "3.34.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.34.0.tgz", + "integrity": "sha512-O5sNsdgxptez/bSXk2CfpTcVu4yTiFW1YcMHIVn2uAY8MksXWQeReMx63krFrj/QSyjRJ5/jIBkWvJ3/ZimdcA==", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -2647,9 +2890,9 @@ } }, "node_modules/node-gyp-build": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.5.0.tgz", - "integrity": "sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg==", + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", + "integrity": "sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==", "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", @@ -2818,6 +3061,11 @@ "node": ">=0.10.0" } }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -2826,6 +3074,11 @@ "node": ">=8" } }, + "node_modules/pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -2838,9 +3091,9 @@ } }, "node_modules/pino": { - "version": "8.8.0", - "resolved": "https://registry.npmjs.org/pino/-/pino-8.8.0.tgz", - "integrity": "sha512-cF8iGYeu2ODg2gIwgAHcPrtR63ILJz3f7gkogaHC/TXVVXxZgInmNYiIpDYEwgEkxZti2Se6P2W2DxlBIZe6eQ==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.11.0.tgz", + "integrity": "sha512-Z2eKSvlrl2rH8p5eveNUnTdd4AjJk8tAsLkHYZQKGHP4WTh2Gi1cOSOs3eWPqaj+niS3gj4UkoreoaWgF3ZWYg==", "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", @@ -2868,9 +3121,9 @@ } }, "node_modules/pino-std-serializers": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.0.0.tgz", - "integrity": "sha512-mMMOwSKrmyl+Y12Ri2xhH1lbzQxwwpuru9VjyJpgFIH4asSj88F2csdMwN6+M5g1Ll4rmsYghHLQJw81tgZ7LQ==" + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.1.0.tgz", + "integrity": "sha512-KO0m2f1HkrPe9S0ldjx7za9BJjeHqBku5Ch8JyxETxT8dEFGz1PwgrHaOQupVYitpzbFSYm7nnljxD8dik2c+g==" }, "node_modules/pkg-conf": { "version": "4.0.0", @@ -2901,10 +3154,35 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prettier": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.1.tgz", - "integrity": "sha512-lqGoSJBQNJidqCHE80vqZJHWHRFoNYsSpP9AjFhlhi9ODCJA541svILes/+/1GM3VaL/abZi7cpFzOpdR9UPKg==", + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.7.tgz", + "integrity": "sha512-yPngTo3aXUUmyuTjeTUT75txrf+aMh9FiD7q9ZE/i6r0bPb22g4FsE6Y338PQX1bmfy08i9QQCB7/rcUAVntfw==", "bin": { "prettier": "bin-prettier.js" }, @@ -2954,10 +3232,19 @@ "node": ">= 0.10" } }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", "engines": { "node": ">=6" } @@ -3007,10 +3294,24 @@ "safe-buffer": "^5.1.0" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, "node_modules/readable-stream": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.2.0.tgz", - "integrity": "sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.3.0.tgz", + "integrity": "sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ==", "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -3040,6 +3341,17 @@ "node": ">= 12.13.0" } }, + "node_modules/rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "dependencies": { + "resolve": "^1.20.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3056,6 +3368,22 @@ "node": ">=0.10.0" } }, + "node_modules/resolve": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "dependencies": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", @@ -3191,17 +3519,17 @@ } }, "node_modules/safe-stable-stringify": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.1.tgz", - "integrity": "sha512-dVHE6bMtS/bnL2mwualjc6IxEv1F+OCUpA46pKUj6F8uDbUM0jCCulPqRNPSnWwGNKx5etqMjZYdXtrm5KJZGA==", + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==", "engines": { "node": ">=10" } }, "node_modules/secure-json-parse": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.6.0.tgz", - "integrity": "sha512-B9osKohb6L+EZ6Kve3wHKfsAClzOC/iISA2vSuCe5Jx5NAKiwitfxx8ZKYapHXr0sYRj7UZInT7pLb3rp2Yx6A==" + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" }, "node_modules/semver": { "version": "7.3.8", @@ -3232,9 +3560,9 @@ } }, "node_modules/set-cookie-parser": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.5.1.tgz", - "integrity": "sha512-1jeBGaKNGdEq4FgIrORu/N570dwoPYio8lSoYLWmX7sQ//0JY08Xh9o5pBcgmHQ/MbsYp/aZnOe1s1lIsbLprQ==" + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz", + "integrity": "sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==" }, "node_modules/signal-exit": { "version": "3.0.7", @@ -3261,11 +3589,25 @@ ] }, "node_modules/simple-get": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.2.tgz", - "integrity": "sha512-Ijd/rV5o+mSBBs4F/x9oDPtTx9Zb6X9brmnXvMW4J7IR15ngi9q5xxqWBKU744jTZiaXtxaPL7uHG6vtN8kUkw==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], "dependencies": { - "decompress-response": "^3.3.0", + "decompress-response": "^6.0.0", "once": "^1.3.1", "simple-concat": "^1.0.0" } @@ -3293,40 +3635,18 @@ "url": "https://github.com/chalk/slice-ansi?sponsor=1" } }, - "node_modules/slice-ansi/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz", - "integrity": "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/sonic-boom": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.2.1.tgz", - "integrity": "sha512-iITeTHxy3B9FGu8aVdiDXUVAcHMF9Ss0cCsAOo2HfCrmVGT3/DT5oYaeu0M/YKZDlKTvChEyPq0zI9Hf33EX6A==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.3.0.tgz", + "integrity": "sha512-LYxp34KlZ1a2Jb8ZQgFCK3niIHzibdwtwNUWKg0qQRzsDoJ3Gfgkf8KdBTFU3SkejDEIlWwnSnpVdOZIhFMl/g==", "dependencies": { "atomic-sleep": "^1.0.0" } }, "node_modules/split2": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", - "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", "engines": { "node": ">= 10.x" } @@ -3363,28 +3683,42 @@ "node": ">=0.10.0" } }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" + "safe-buffer": "~5.2.0" } }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", "dependencies": { - "ansi-regex": "^5.0.1" + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" }, "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", + "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, "node_modules/strip-hex-prefix": { @@ -3399,6 +3733,14 @@ "npm": ">=3" } }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/supertap": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/supertap/-/supertap-3.0.1.tgz", @@ -3413,29 +3755,82 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, - "node_modules/supertap/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "node_modules/supertap/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/supertap/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", "engines": { - "node": ">=12" + "node": ">= 0.4" }, "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" + "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/supertap/node_modules/strip-ansi": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", - "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", "dependencies": { - "ansi-regex": "^6.0.1" + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" }, "engines": { - "node": ">=12" + "node": ">=6" + } + }, + "node_modules/tar-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" + "engines": { + "node": ">= 6" + } + }, + "node_modules/tarn": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", + "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", + "engines": { + "node": ">=8.0.0" } }, "node_modules/temp-dir": { @@ -3466,13 +3861,21 @@ } }, "node_modules/thread-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.2.0.tgz", - "integrity": "sha512-rUkv4/fnb4rqy/gGy7VuqK6wE1+1DOCOWy4RMeaV69ZHMP11tQKZvZSip1yTgrKCMZzEMcCL/bKfHvSfDHx+iQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.3.0.tgz", + "integrity": "sha512-kaDqm1DET9pp3NXwR8382WHbnpXnRkN9xGN9dQt3B2+dmXiW8X1SOwmFOxAErEQ47ObhZ96J6yhZNXuyCOL7KA==", "dependencies": { "real-require": "^0.2.0" } }, + "node_modules/tildify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", + "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==", + "engines": { + "node": ">=8" + } + }, "node_modules/time-zone": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/time-zone/-/time-zone-1.0.0.tgz", @@ -3499,11 +3902,11 @@ } }, "node_modules/tiny-lru": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-10.0.1.tgz", - "integrity": "sha512-Vst+6kEsWvb17Zpz14sRJV/f8bUWKhqm6Dc+v08iShmIJ/WxqWytHzCTd6m88pS33rE2zpX34TRmOpAJPloNCA==", + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-10.4.1.tgz", + "integrity": "sha512-buLIzw7ppqymuO3pt10jHk/6QMeZLbidihMQU+N6sogF6EnBzG0qtDWIHuhw1x3dyNgVL/KTGIZsTK81+yCzLg==", "engines": { - "node": ">=6" + "node": ">=12" } }, "node_modules/to-regex-range": { @@ -3565,6 +3968,17 @@ } } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", @@ -3582,9 +3996,9 @@ } }, "node_modules/typescript": { - "version": "4.8.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", - "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -3617,6 +4031,11 @@ "resolved": "https://registry.npmjs.org/utf8/-/utf8-3.0.0.tgz", "integrity": "sha512-E8VjFIQ/TyQgp+TZfS6l8yp/xWppSAHzidGiRrqe4bK4XP9pTRyKFgGJpO3SN7zdX4DeomTrwaseCHovfpFcqQ==" }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -3697,6 +4116,65 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -3747,6 +4225,35 @@ "xhr-request": "^1.1.0" } }, + "node_modules/xhr-request/node_modules/decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", + "dependencies": { + "mimic-response": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/xhr-request/node_modules/mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/xhr-request/node_modules/simple-get": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.2.tgz", + "integrity": "sha512-Ijd/rV5o+mSBBs4F/x9oDPtTx9Zb6X9brmnXvMW4J7IR15ngi9q5xxqWBKU744jTZiaXtxaPL7uHG6vtN8kUkw==", + "dependencies": { + "decompress-response": "^3.3.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -3769,9 +4276,9 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yargs": { - "version": "17.6.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", - "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", + "version": "17.7.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.1.tgz", + "integrity": "sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw==", "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -3793,6 +4300,51 @@ "node": ">=12" } }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/yargs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -3816,29 +4368,14 @@ }, "dependencies": { "@bcherny/json-schema-ref-parser": { - "version": "9.0.9", - "resolved": "https://registry.npmjs.org/@bcherny/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz", - "integrity": "sha512-vmEmnJCfpkLdas++9OYg6riIezTYqTHpqUTODJzHLzs5UnXujbOJW9VwcVCnyo1mVRt32FRr23iXBx/sX8YbeQ==", + "version": "10.0.5-fork", + "resolved": "https://registry.npmjs.org/@bcherny/json-schema-ref-parser/-/json-schema-ref-parser-10.0.5-fork.tgz", + "integrity": "sha512-E/jKbPoca1tfUPj3iSbitDZTGnq6FUFjkH6L8U2oDwSuwK1WhnnVtCG7oFOTg/DDnyoXbQYUiUiGOibHqaGVnw==", "requires": { "@jsdevtools/ono": "^7.1.3", "@types/json-schema": "^7.0.6", "call-me-maybe": "^1.0.1", "js-yaml": "^4.1.0" - }, - "dependencies": { - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, - "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "requires": { - "argparse": "^2.0.1" - } - } } }, "@cspotcode/source-map-support": { @@ -4117,14 +4654,6 @@ "multiformats": "9.9.0" }, "dependencies": { - "cross-fetch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", - "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", - "requires": { - "node-fetch": "2.6.7" - } - }, "dotenv": { "version": "16.0.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.0.tgz", @@ -4133,7 +4662,7 @@ } }, "@neume-network/schema": { - "version": "git+ssh://git@github.com/neume-network/schema.git#546037ed9db83969686a2338e0ca287dfd387276", + "version": "git+ssh://git@github.com/neume-network/schema.git#f2f22cb4f37a27ba8a02ff7b1852ff386e833949", "from": "@neume-network/schema@github:neume-network/schema" }, "@nodelib/fs.scandir": { @@ -4198,9 +4727,9 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==" }, "@types/lodash": { - "version": "4.14.191", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz", - "integrity": "sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==" + "version": "4.14.192", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.192.tgz", + "integrity": "sha512-km+Vyn3BYm5ytMO13k9KTp27O75rbQ0NFw+U//g+PX7VZyjCioXaRFisqSIJRECljcTv73G3i6BpglNGHgUQ5A==" }, "@types/minimatch": { "version": "5.1.2", @@ -4208,9 +4737,9 @@ "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==" }, "@types/node": { - "version": "18.11.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", - "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==" + "version": "18.15.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz", + "integrity": "sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q==" }, "@types/prettier": { "version": "2.7.2", @@ -4218,9 +4747,9 @@ "integrity": "sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg==" }, "@types/yargs": { - "version": "17.0.14", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.14.tgz", - "integrity": "sha512-9Pj7abXoW1RSTcZaL2Hk6G2XyLMlp5ECdVC/Zf2p/KBjC3srijLGgRAXOBjtFrJoIrvxdTKyKDA14bEcbxBaWw==", + "version": "17.0.24", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", + "integrity": "sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==", "dev": true, "requires": { "@types/yargs-parser": "*" @@ -4260,9 +4789,9 @@ "integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==" }, "acorn": { - "version": "8.8.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", - "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==" + "version": "8.8.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", + "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==" }, "acorn-walk": { "version": "8.2.0", @@ -4298,17 +4827,14 @@ } }, "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" }, "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" }, "any-promise": { "version": "1.3.0", @@ -4336,12 +4862,9 @@ "dev": true }, "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "array-find-index": { "version": "1.0.2", @@ -4372,9 +4895,9 @@ "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" }, "ava": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/ava/-/ava-5.1.0.tgz", - "integrity": "sha512-e5VFrSQ0WBPyZJWRXVrO7RFOizFeNM0t2PORwrPvWtApgkORI6cvGnY3GX1G+lzpd0HjqNx5Jus22AhxVnUMNA==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ava/-/ava-5.2.0.tgz", + "integrity": "sha512-W8yxFXJr/P68JP55eMpQIa6AiXhCX3VeuajM8nolyWNExcMDD6rnIWKTjw0B/+GkFHBIaN6Jd0LtcMThcoqVfg==", "requires": { "acorn": "^8.8.1", "acorn-walk": "^8.2.0", @@ -4383,10 +4906,10 @@ "arrify": "^3.0.0", "callsites": "^4.0.0", "cbor": "^8.1.0", - "chalk": "^5.1.2", + "chalk": "^5.2.0", "chokidar": "^3.5.3", "chunkd": "^2.0.1", - "ci-info": "^3.6.1", + "ci-info": "^3.7.1", "ci-parallel-vars": "^1.0.1", "clean-yaml-object": "^0.1.0", "cli-truncate": "^3.1.0", @@ -4398,7 +4921,7 @@ "del": "^7.0.0", "emittery": "^1.0.1", "figures": "^5.0.0", - "globby": "^13.1.2", + "globby": "^13.1.3", "ignore-by-default": "^2.1.0", "indent-string": "^5.0.0", "is-error": "^2.2.2", @@ -4421,37 +4944,12 @@ "temp-dir": "^3.0.0", "write-file-atomic": "^5.0.0", "yargs": "^17.6.2" - }, - "dependencies": { - "ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" - }, - "ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "strip-ansi": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", - "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", - "requires": { - "ansi-regex": "^6.0.1" - } - } } }, "avvio": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/avvio/-/avvio-8.2.0.tgz", - "integrity": "sha512-bbCQdg7bpEv6kGH41RO/3B2/GMMmJSo2iBK+X8AWN9mujtfUipMDfIjsgHCfpnKqoGEQrrmCDKSa5OQ19+fDmg==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/avvio/-/avvio-8.2.1.tgz", + "integrity": "sha512-TAlMYvOuwGyLK3PfBb5WKBXZmXz2fVCgv23d6zZFdle/q3gPjmxBaeuC0pY0Dzs5PWMSgfqqEZkrye19GlDTgw==", "requires": { "archy": "^1.0.0", "debug": "^4.0.0", @@ -4468,11 +4966,59 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, + "better-sqlite3": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-8.2.0.tgz", + "integrity": "sha512-8eTzxGk9535SB3oSNu0tQ6I4ZffjVCBUjKHN9QeeIFtphBX0sEd0NxAuglBNR9TO5ThnxBB7GqzfcYo9kjadJQ==", + "requires": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.0" + } + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==" }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "requires": { + "file-uri-to-path": "1.0.0" + } + }, + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "blueimp-md5": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.19.0.tgz", @@ -4573,15 +5119,20 @@ "readdirp": "~3.6.0" } }, + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, "chunkd": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/chunkd/-/chunkd-2.0.1.tgz", "integrity": "sha512-7d58XsFmOq0j6el67Ug9mHf9ELUXsQXYJBkyxhH/k+6Ke0qXRnv0kbemx+Twc6fRJ07C49lcbdgm9FL1Ei/6SQ==" }, "ci-info": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.7.0.tgz", - "integrity": "sha512-2CpRNYmImPx+RXKLq6jko/L07phmS9I02TyqkcNU20GCF/GgaWvc58hPtjxDX8lPpkdwc9sNh72V9k00S7ezog==" + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", + "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==" }, "ci-parallel-vars": { "version": "1.0.1", @@ -4632,48 +5183,53 @@ "requires": { "slice-ansi": "^5.0.0", "string-width": "^5.0.0" + } + }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" }, "dependencies": { "ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" }, "emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, "string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "requires": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" } }, "strip-ansi": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", - "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "requires": { - "ansi-regex": "^6.0.1" + "ansi-regex": "^5.0.1" } } } }, - "cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - } - }, "code-excerpt": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-4.0.0.tgz", @@ -4695,6 +5251,16 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "colorette": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", + "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==" + }, + "commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==" + }, "common-path-prefix": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz", @@ -4720,11 +5286,6 @@ "well-known-symbols": "^2.0.0" } }, - "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" - }, "convert-to-spaces": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz", @@ -4742,18 +5303,11 @@ "dev": true }, "cross-fetch": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.4.tgz", - "integrity": "sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==", - "requires": { - "node-fetch": "2.6.1" - }, - "dependencies": { - "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" - } + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "requires": { + "node-fetch": "2.6.7" } }, "currently-unhandled": { @@ -4787,21 +5341,33 @@ "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "requires": { "ms": "2.1.2" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } } }, "decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==" + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==" }, "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "requires": { - "mimic-response": "^1.0.0" + "mimic-response": "^3.1.0" } }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, "del": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/del/-/del-7.0.0.tgz", @@ -4824,6 +5390,11 @@ } } }, + "detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==" + }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -4880,9 +5451,17 @@ "integrity": "sha512-2ID6FdrMD9KDLldGesP6317G78K7km/kMcwItRtVFva7I/cSEOIaLpewaUb+YLXVwdAp3Ctfxh/V5zIl1sj7dQ==" }, "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } }, "es5-ext": { "version": "0.10.62", @@ -4934,6 +5513,11 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==" }, + "esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" + }, "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", @@ -4951,6 +5535,21 @@ "async-retry": "1.3.3", "cross-fetch": "3.1.4", "web3-eth-abi": "1.4.0" + }, + "dependencies": { + "cross-fetch": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.4.tgz", + "integrity": "sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==", + "requires": { + "node-fetch": "2.6.1" + } + }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + } } }, "eth-lib": { @@ -5013,6 +5612,11 @@ "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==" + }, "ext": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", @@ -5028,6 +5632,11 @@ } } }, + "fast-content-type-parse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-1.0.0.tgz", + "integrity": "sha512-Xbc4XcysUXcsP5aHUU7Nq3OwvHq97C+WnbkeIefpeYLX+ryzFJlU6OStFJhs6Ol0LkUGpcK+wL0JwfM+FCU5IA==" + }, "fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -5056,9 +5665,9 @@ } }, "fast-json-stringify": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.5.0.tgz", - "integrity": "sha512-rmw2Z8/mLkND8zI+3KTYIkNPEoF5v6GqDP/o+g7H3vjdWjBwuKpgAYFHIzL6ORRB+iqDjjtJnLIW9Mzxn5szOA==", + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.7.0.tgz", + "integrity": "sha512-sBVPTgnAZseLu1Qgj6lUbQ0HfjFhZWXAmpZ5AaSGkyLh5gAXBga/uPJjQPHpDFjC9adWIpdOcCLSDTgrZ7snoQ==", "requires": { "@fastify/deepmerge": "^1.0.0", "ajv": "^8.10.0", @@ -5069,9 +5678,9 @@ } }, "fast-querystring": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-querystring/-/fast-querystring-1.0.0.tgz", - "integrity": "sha512-3LQi62IhQoDlmt4ULCYmh17vRO2EtS7hTSsG4WwoKWgV7GLMKBOecEh+aiavASnLx8I2y89OD33AGLo0ccRhzA==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/fast-querystring/-/fast-querystring-1.1.1.tgz", + "integrity": "sha512-qR2r+e3HvhEFmpdHMv//U8FnFlnYjaC6QKDuaXALDkw2kvHO8WDjxH+f/rHGR4Me4pnk8p9JAkRNTjYHAKRn2Q==", "requires": { "fast-decode-uri-component": "^1.0.1" } @@ -5087,17 +5696,17 @@ "integrity": "sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg==" }, "fastify": { - "version": "4.10.2", - "resolved": "https://registry.npmjs.org/fastify/-/fastify-4.10.2.tgz", - "integrity": "sha512-0T+4zI6N3S8ex0LCZi3H4FasJR4AzWw834fUkPWvV8r6GBJkLmAOfFxH8f5V29Plef24IK0QSQD/tz1Nx+1UOA==", + "version": "4.15.0", + "resolved": "https://registry.npmjs.org/fastify/-/fastify-4.15.0.tgz", + "integrity": "sha512-m/CaRN8nf5uyYdrDe2qqq+0z3oGyE+A++qlKQoLJTI4WI0nWK9D6R3FxXQ3MVwt/md977GMR4F43pE9oqrS2zw==", "requires": { - "@fastify/ajv-compiler": "^3.3.1", + "@fastify/ajv-compiler": "^3.5.0", "@fastify/error": "^3.0.0", - "@fastify/fast-json-stringify-compiler": "^4.1.0", + "@fastify/fast-json-stringify-compiler": "^4.2.0", "abstract-logging": "^2.0.1", "avvio": "^8.2.0", - "content-type": "^1.0.4", - "find-my-way": "^7.3.0", + "fast-content-type-parse": "^1.0.0", + "find-my-way": "^7.6.0", "light-my-request": "^5.6.1", "pino": "^8.5.0", "process-warning": "^2.0.0", @@ -5125,6 +5734,11 @@ "is-unicode-supported": "^1.2.0" } }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -5134,9 +5748,9 @@ } }, "find-my-way": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-7.3.1.tgz", - "integrity": "sha512-kGvM08SOkqvheLcuQ8GW9t/H901Qb9rZEbcNWbXopzy4jDRoaJpJoObPSKf4MnQLZ20ZTp7rL5MpF6rf+pqmyg==", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-7.6.0.tgz", + "integrity": "sha512-H7berWdHJ+5CNVr4ilLWPai4ml7Y2qAsxjw3pfeBxPigZmaDTzF0wjJLj90xRCmGcWYcyt050yN+34OZDJm1eQ==", "requires": { "fast-deep-equal": "^3.1.3", "fast-querystring": "^1.0.0", @@ -5157,6 +5771,11 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -5168,16 +5787,36 @@ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "optional": true }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, + "get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==" + }, "get-stdin": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz", "integrity": "sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg==" }, + "getopts": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", + "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==" + }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -5217,9 +5856,9 @@ } }, "globby": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", - "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.3.tgz", + "integrity": "sha512-8krCNHXvlCgHDpegPzleMq07yMYTO2sXKASmZmquEYWEmCx6J5UTRbp5RwMJkTJGtcQ44YpiUYUiN0b9mzy8Bw==", "requires": { "dir-glob": "^3.0.1", "fast-glob": "^3.2.11", @@ -5236,9 +5875,17 @@ } }, "graceful-fs": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } }, "hash.js": { "version": "1.1.7", @@ -5265,9 +5912,9 @@ "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, "ignore": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.1.tgz", - "integrity": "sha512-d2qQLzTJ9WxQftPAuEQpSPmKqzxePjzVbpAVv62AQ64NTL+wR4JkrVqR/LqFsFEUsHDAiId52mJteHDFuDkElA==" + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==" }, "ignore-by-default": { "version": "2.1.0", @@ -5298,6 +5945,16 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "interpret": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==" + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -5309,9 +5966,9 @@ "integrity": "sha512-oHrL0x2MbBlEdr1g+wBORq50q119sTV1ib5jVeq5ktYYZa/qy1eJfKgT70YGSn+jOuy/GQ2KqY+g44ynsyPoag==" }, "irregular-plurals": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.3.0.tgz", - "integrity": "sha512-MVBLKUTangM3EfRPFROhmWQQKRDsrgI83J8GS3jXy+OwYqiR2/aoWndYQ5416jLE3uaGgLH7ncme3X9y09gZ3g==" + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.5.0.tgz", + "integrity": "sha512-1ANGLZ+Nkv1ptFb2pa8oG8Lem4krflKuX/gINiHJHjJUKaJHk/SXk5x6K3J+39/p0h1RQ2saROclJJ+QLvETCQ==" }, "is-binary-path": { "version": "2.1.0", @@ -5326,6 +5983,14 @@ "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" }, + "is-core-module": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", + "requires": { + "has": "^1.0.3" + } + }, "is-error": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/is-error/-/is-error-2.2.2.tgz", @@ -5337,9 +6002,9 @@ "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==" }, "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz", + "integrity": "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==" }, "is-function": { "version": "1.0.2", @@ -5400,12 +6065,11 @@ "integrity": "sha512-Smw4xcfIQ5LVjAOuJCvN/zIodzA/BBSsluuoSykP+lUvScIi4U6RJLfwHet5cxFnCswUjISV8oAXaqaJDY3chg==" }, "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" } }, "json-canonicalize": { @@ -5414,16 +6078,16 @@ "integrity": "sha512-YNr/ePzgReHwlnAm3EVV1pcimwesI+1DZr5v7WBKOc1zE1t7pjxWAPRxJFT3ll6flLIdRe0DPia/8cl2FLAZNA==" }, "json-rpc-2.0": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/json-rpc-2.0/-/json-rpc-2.0-1.4.2.tgz", - "integrity": "sha512-oqiMRhWN4Q+2ySitZR5GA9vtiNOK19GTDDad0yn8/T/ND+Ap81QP6+3UMl738H4bmRbCEwhQkkBruNqx7EXfUQ==" + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/json-rpc-2.0/-/json-rpc-2.0-1.5.1.tgz", + "integrity": "sha512-ZY/vYl/uUgKN3tNrZMq7w+CGLcoUT+8AzDO/HJZVa+K4XcwgfgES1QDa5y7ieAeh4NgRo3hLexMxgdaiEiK9aA==" }, "json-schema-to-typescript": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/json-schema-to-typescript/-/json-schema-to-typescript-11.0.2.tgz", - "integrity": "sha512-XRyeXBJeo/IH4eTP5D1ptX78vCvH86nMDt2k3AxO28C3uYWEDmy4mgPyMpb8bLJ/pJMElOGuQbnKR5Y6NSh3QQ==", + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/json-schema-to-typescript/-/json-schema-to-typescript-11.0.5.tgz", + "integrity": "sha512-ZNlvngzlPzjYYECbR+uJ9aUWo25Gw/VuwUytvcuKiwc6NaiZhMyf7qBsxZE2eixmj8AoQEQJhSRG7btln0sUDw==", "requires": { - "@bcherny/json-schema-ref-parser": "9.0.9", + "@bcherny/json-schema-ref-parser": "10.0.5-fork", "@types/json-schema": "^7.0.11", "@types/lodash": "^4.14.182", "@types/prettier": "^2.6.1", @@ -5449,6 +6113,27 @@ "resolved": "https://registry.npmjs.org/just-performance/-/just-performance-4.2.0.tgz", "integrity": "sha512-4TikKSf+Gb+Et5SnA4ppyrxLSf9qWFq+SqfdDdrgHE1KLwSch/Zi1AQB0TrE4ppYjZdUrHnwdx+6dyx0cx/HyA==" }, + "knex": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/knex/-/knex-2.4.2.tgz", + "integrity": "sha512-tMI1M7a+xwHhPxjbl/H9K1kHX+VncEYcvCx5K00M16bWvpYPKAZd6QrCu68PtHAdIZNQPWZn0GVhqVBEthGWCg==", + "requires": { + "colorette": "2.0.19", + "commander": "^9.1.0", + "debug": "4.3.4", + "escalade": "^3.1.1", + "esm": "^3.2.25", + "get-package-type": "^0.1.0", + "getopts": "2.3.0", + "interpret": "^2.2.0", + "lodash": "^4.17.21", + "pg-connection-string": "2.5.0", + "rechoir": "^0.8.0", + "resolve-from": "^5.0.0", + "tarn": "^3.0.2", + "tildify": "2.0.0" + } + }, "level": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/level/-/level-8.0.0.tgz", @@ -5473,9 +6158,9 @@ } }, "light-my-request": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.8.0.tgz", - "integrity": "sha512-4BtD5C+VmyTpzlDPCZbsatZMJVgUIciSOwYhJDCbLffPZ35KoDkDj4zubLeHDEb35b4kkPeEv5imbh+RJxK/Pg==", + "version": "5.9.1", + "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.9.1.tgz", + "integrity": "sha512-UT7pUk8jNCR1wR7w3iWfIjx32DiB2f3hFdQSOwy3/EPQ3n3VocyipUxcyRZR0ahoev+fky69uA+GejPa9KuHKg==", "requires": { "cookie": "^0.5.0", "process-warning": "^2.0.0", @@ -5496,9 +6181,9 @@ "integrity": "sha512-Gnxj3ev3mB5TkVBGad0JM6dmLiQL+o0t23JPBZ9sd+yvSLk05mFoqKBw5N8gbbkU4TNXyqCgIrl/VM17OgUIgQ==" }, "locate-path": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.1.1.tgz", - "integrity": "sha512-vJXaRMJgRVD3+cUZs3Mncj2mxpt5mP0EmNOsxRSZRMlbqjvxzDEOIUWXGmavo0ZC9+tNZCBLQ66reA11nbpHZg==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", "requires": { "p-locate": "^6.0.0" } @@ -5605,9 +6290,9 @@ "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==" }, "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" }, "min-document": { "version": "2.19.0", @@ -5636,24 +6321,29 @@ } }, "minimist": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz", - "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==" + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" }, "mkdirp": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "module-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/module-error/-/module-error-1.0.2.tgz", "integrity": "sha512-0yuvsqSCv8LbaOKhnsQ/T5JhyFlCYLPXK3U2sgV10zoKQwzs/MyfuQUOZQ1V/6OCOJsK/TRgNVrPuPDqtdMFtA==" }, "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "multiformats": { "version": "9.9.0", @@ -5670,6 +6360,11 @@ "thenify-all": "^1.0.0" } }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "napi-macros": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", @@ -5680,6 +6375,14 @@ "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" }, + "node-abi": { + "version": "3.34.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.34.0.tgz", + "integrity": "sha512-O5sNsdgxptez/bSXk2CfpTcVu4yTiFW1YcMHIVn2uAY8MksXWQeReMx63krFrj/QSyjRJ5/jIBkWvJ3/ZimdcA==", + "requires": { + "semver": "^7.3.5" + } + }, "node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -5689,9 +6392,9 @@ } }, "node-gyp-build": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.5.0.tgz", - "integrity": "sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg==" + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", + "integrity": "sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==" }, "nofilter": { "version": "3.1.0", @@ -5799,20 +6502,30 @@ "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, "path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==" }, + "pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" + }, "picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" }, "pino": { - "version": "8.8.0", - "resolved": "https://registry.npmjs.org/pino/-/pino-8.8.0.tgz", - "integrity": "sha512-cF8iGYeu2ODg2gIwgAHcPrtR63ILJz3f7gkogaHC/TXVVXxZgInmNYiIpDYEwgEkxZti2Se6P2W2DxlBIZe6eQ==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.11.0.tgz", + "integrity": "sha512-Z2eKSvlrl2rH8p5eveNUnTdd4AjJk8tAsLkHYZQKGHP4WTh2Gi1cOSOs3eWPqaj+niS3gj4UkoreoaWgF3ZWYg==", "requires": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", @@ -5837,9 +6550,9 @@ } }, "pino-std-serializers": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.0.0.tgz", - "integrity": "sha512-mMMOwSKrmyl+Y12Ri2xhH1lbzQxwwpuru9VjyJpgFIH4asSj88F2csdMwN6+M5g1Ll4rmsYghHLQJw81tgZ7LQ==" + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.1.0.tgz", + "integrity": "sha512-KO0m2f1HkrPe9S0ldjx7za9BJjeHqBku5Ch8JyxETxT8dEFGz1PwgrHaOQupVYitpzbFSYm7nnljxD8dik2c+g==" }, "pkg-conf": { "version": "4.0.0", @@ -5858,10 +6571,29 @@ "irregular-plurals": "^3.3.0" } }, + "prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "requires": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + } + }, "prettier": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.1.tgz", - "integrity": "sha512-lqGoSJBQNJidqCHE80vqZJHWHRFoNYsSpP9AjFhlhi9ODCJA541svILes/+/1GM3VaL/abZi7cpFzOpdR9UPKg==" + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.7.tgz", + "integrity": "sha512-yPngTo3aXUUmyuTjeTUT75txrf+aMh9FiD7q9ZE/i6r0bPb22g4FsE6Y338PQX1bmfy08i9QQCB7/rcUAVntfw==" }, "pretty-ms": { "version": "8.0.0", @@ -5890,10 +6622,19 @@ "ipaddr.js": "1.9.1" } }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==" }, "query-string": { "version": "5.1.1", @@ -5923,10 +6664,21 @@ "safe-buffer": "^5.1.0" } }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + } + }, "readable-stream": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.2.0.tgz", - "integrity": "sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.3.0.tgz", + "integrity": "sha512-MuEnA0lbSi7JS8XM+WNJlWZkHAAdm7gETHdFK//Q/mChGyj2akEFtdLZh32jSdkWGbRwCW9pn6g3LWDdDeZnBQ==", "requires": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -5947,6 +6699,14 @@ "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==" }, + "rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "requires": { + "resolve": "^1.20.0" + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -5957,6 +6717,16 @@ "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" }, + "resolve": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "requires": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, "resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", @@ -6028,14 +6798,14 @@ } }, "safe-stable-stringify": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.1.tgz", - "integrity": "sha512-dVHE6bMtS/bnL2mwualjc6IxEv1F+OCUpA46pKUj6F8uDbUM0jCCulPqRNPSnWwGNKx5etqMjZYdXtrm5KJZGA==" + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==" }, "secure-json-parse": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.6.0.tgz", - "integrity": "sha512-B9osKohb6L+EZ6Kve3wHKfsAClzOC/iISA2vSuCe5Jx5NAKiwitfxx8ZKYapHXr0sYRj7UZInT7pLb3rp2Yx6A==" + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" }, "semver": { "version": "7.3.8", @@ -6054,9 +6824,9 @@ } }, "set-cookie-parser": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.5.1.tgz", - "integrity": "sha512-1jeBGaKNGdEq4FgIrORu/N570dwoPYio8lSoYLWmX7sQ//0JY08Xh9o5pBcgmHQ/MbsYp/aZnOe1s1lIsbLprQ==" + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz", + "integrity": "sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==" }, "signal-exit": { "version": "3.0.7", @@ -6069,11 +6839,11 @@ "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" }, "simple-get": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.2.tgz", - "integrity": "sha512-Ijd/rV5o+mSBBs4F/x9oDPtTx9Zb6X9brmnXvMW4J7IR15ngi9q5xxqWBKU744jTZiaXtxaPL7uHG6vtN8kUkw==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", "requires": { - "decompress-response": "^3.3.0", + "decompress-response": "^6.0.0", "once": "^1.3.1", "simple-concat": "^1.0.0" } @@ -6090,32 +6860,20 @@ "requires": { "ansi-styles": "^6.0.0", "is-fullwidth-code-point": "^4.0.0" - }, - "dependencies": { - "ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" - }, - "is-fullwidth-code-point": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz", - "integrity": "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==" - } } }, "sonic-boom": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.2.1.tgz", - "integrity": "sha512-iITeTHxy3B9FGu8aVdiDXUVAcHMF9Ss0cCsAOo2HfCrmVGT3/DT5oYaeu0M/YKZDlKTvChEyPq0zI9Hf33EX6A==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.3.0.tgz", + "integrity": "sha512-LYxp34KlZ1a2Jb8ZQgFCK3niIHzibdwtwNUWKg0qQRzsDoJ3Gfgkf8KdBTFU3SkejDEIlWwnSnpVdOZIhFMl/g==", "requires": { "atomic-sleep": "^1.0.0" } }, "split2": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", - "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==" + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" }, "sprintf-js": { "version": "1.0.3", @@ -6142,22 +6900,30 @@ "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", "integrity": "sha512-R3f198pcvnB+5IpnBlRkphuE9n46WyVl8I39W/ZUTZLz4nqSP/oLYUrcnJrw462Ds8he4YKMov2efsTIw1BDGQ==" }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" } }, "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", + "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", "requires": { - "ansi-regex": "^5.0.1" + "ansi-regex": "^6.0.1" } }, "strip-hex-prefix": { @@ -6168,6 +6934,11 @@ "is-hex-prefixed": "1.0.0" } }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==" + }, "supertap": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/supertap/-/supertap-3.0.1.tgz", @@ -6179,21 +6950,70 @@ "strip-ansi": "^7.0.1" }, "dependencies": { - "ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } }, - "strip-ansi": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", - "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + } + } + }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" + }, + "tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "requires": { - "ansi-regex": "^6.0.1" + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" } } } }, + "tarn": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", + "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==" + }, "temp-dir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-3.0.0.tgz", @@ -6216,13 +7036,18 @@ } }, "thread-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.2.0.tgz", - "integrity": "sha512-rUkv4/fnb4rqy/gGy7VuqK6wE1+1DOCOWy4RMeaV69ZHMP11tQKZvZSip1yTgrKCMZzEMcCL/bKfHvSfDHx+iQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.3.0.tgz", + "integrity": "sha512-kaDqm1DET9pp3NXwR8382WHbnpXnRkN9xGN9dQt3B2+dmXiW8X1SOwmFOxAErEQ47ObhZ96J6yhZNXuyCOL7KA==", "requires": { "real-require": "^0.2.0" } }, + "tildify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", + "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==" + }, "time-zone": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/time-zone/-/time-zone-1.0.0.tgz", @@ -6243,9 +7068,9 @@ } }, "tiny-lru": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-10.0.1.tgz", - "integrity": "sha512-Vst+6kEsWvb17Zpz14sRJV/f8bUWKhqm6Dc+v08iShmIJ/WxqWytHzCTd6m88pS33rE2zpX34TRmOpAJPloNCA==" + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-10.4.1.tgz", + "integrity": "sha512-buLIzw7ppqymuO3pt10jHk/6QMeZLbidihMQU+N6sogF6EnBzG0qtDWIHuhw1x3dyNgVL/KTGIZsTK81+yCzLg==" }, "to-regex-range": { "version": "5.0.1", @@ -6281,6 +7106,14 @@ "yn": "3.1.1" } }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "type": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", @@ -6292,9 +7125,9 @@ "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==" }, "typescript": { - "version": "4.8.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", - "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "dev": true }, "underscore": { @@ -6320,6 +7153,11 @@ "resolved": "https://registry.npmjs.org/utf8/-/utf8-3.0.0.tgz", "integrity": "sha512-E8VjFIQ/TyQgp+TZfS6l8yp/xWppSAHzidGiRrqe4bK4XP9pTRyKFgGJpO3SN7zdX4DeomTrwaseCHovfpFcqQ==" }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -6385,6 +7223,49 @@ "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + } } }, "wrappy": { @@ -6424,6 +7305,31 @@ "timed-out": "^4.0.1", "url-set-query": "^1.0.0", "xhr": "^2.0.4" + }, + "dependencies": { + "decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", + "requires": { + "mimic-response": "^1.0.0" + } + }, + "mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" + }, + "simple-get": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.2.tgz", + "integrity": "sha512-Ijd/rV5o+mSBBs4F/x9oDPtTx9Zb6X9brmnXvMW4J7IR15ngi9q5xxqWBKU744jTZiaXtxaPL7uHG6vtN8kUkw==", + "requires": { + "decompress-response": "^3.3.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + } } }, "xhr-request-promise": { @@ -6450,9 +7356,9 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "yargs": { - "version": "17.6.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", - "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", + "version": "17.7.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.1.tgz", + "integrity": "sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw==", "requires": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -6461,6 +7367,41 @@ "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + } } }, "yargs-parser": { diff --git a/package.json b/package.json index 39f672a..e77ed21 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "@neume-network/extraction-worker": "github:neume-network/extraction-worker", "@neume-network/schema": "github:neume-network/schema", "ava": "^5.1.0", + "better-sqlite3": "^8.2.0", "dotenv": "^16.0.3", "eth-fun": "^0.9.2", "fastify": "^4.10.2", @@ -28,6 +29,7 @@ "json-canonicalize": "^1.0.4", "json-rpc-2.0": "^1.4.2", "json-schema-to-typescript": "^11.0.2", + "knex": "^2.4.2", "level": "^8.0.0", "p-map": "^5.5.0", "yargs": "^17.6.2" diff --git a/src/components/call-owner.ts b/src/components/call-owner.ts index 9a83e00..a77c160 100644 --- a/src/components/call-owner.ts +++ b/src/components/call-owner.ts @@ -1,19 +1,11 @@ -import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; import { toHex, encodeFunctionSignature, decodeParameters } from "eth-fun"; -import { Config } from "../types.js"; +import { Strategy } from "../strategies/strategy.types.js"; import { randomItem } from "../utils.js"; -export async function callOwner( - worker: ExtractionWorkerHandler, - config: Config, - to: string, - blockNumber: number, -): Promise { - if (!config.rpc.length) throw new Error("Atleast one RPC host is required"); - - const rpc = randomItem(config.rpc); +export async function callOwner(this: Strategy, to: string, blockNumber: number): Promise { + const rpc = randomItem(this.config.chain[this.chain].rpc); const data = encodeFunctionSignature("owner()"); - const msg = await worker({ + const msg = await this.worker({ type: "json-rpc", commissioner: "", version: "0.0.1", diff --git a/src/components/call-tokenuri.ts b/src/components/call-tokenuri.ts index 5aabd96..355da53 100644 --- a/src/components/call-tokenuri.ts +++ b/src/components/call-tokenuri.ts @@ -1,13 +1,12 @@ -import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; import { Jsonrpc } from "@neume-network/schema"; import { encodeFunctionCall, decodeParameters, toHex } from "eth-fun"; +import { Strategy } from "../strategies/strategy.types.js"; -import { Config, NFT } from "../types.js"; +import { NFT } from "../types.js"; import { randomItem } from "../utils.js"; export async function callTokenUri( - worker: ExtractionWorkerHandler, - config: Config, + this: Strategy, blockNumber: number, nft: NFT, overrideSignature?: Record, @@ -22,7 +21,7 @@ export async function callTokenUri( }, ], }; - const rpc = randomItem(config.rpc); + const rpc = randomItem(this.config.chain[this.chain].rpc); const options = { url: rpc.url, ...(rpc.key && { @@ -52,7 +51,7 @@ export async function callTokenUri( toHex(blockNumber), ], }; - const ret = await worker(msg); + const ret = await this.worker(msg); if (ret.error) throw new Error(`Error while calling tokenURI on contract: ${JSON.stringify(ret, null, 2)}`); diff --git a/src/components/eth-get-logs.ts b/src/components/eth-get-logs.ts new file mode 100644 index 0000000..cb5fea5 --- /dev/null +++ b/src/components/eth-get-logs.ts @@ -0,0 +1,46 @@ +import { toHex } from "eth-fun"; +import { Strategy } from "../strategies/strategy.types.js"; +import { JsonRpcLog } from "../types.js"; +import { randomItem } from "../utils.js"; + +export async function ethGetLogs( + this: Strategy, + from: number, + to: number, + topics: Array<(string | number) | (string | number)[]>, + address?: Array, +): Promise { + const rpcHost = randomItem(this.config.chain[this.chain].rpc); + + const msg = await this.worker({ + type: "json-rpc", + commissioner: "", + method: "eth_getLogs", + options: { + url: rpcHost.url, + headers: { + ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), + }, + retry: { + retries: 3, + }, + }, + params: [ + { + fromBlock: toHex(from), + toBlock: toHex(to), + ...(address && { address: address }), + topics: topics, + }, + ], + version: "0.0.1", + }); + + if (msg.error) { + throw new Error( + `Error occured while fetching Transfer events: ${JSON.stringify(msg, null, 2)}`, + ); + } + + return msg.results as JsonRpcLog[]; +} diff --git a/src/components/handle-transfer.ts b/src/components/handle-transfer.ts new file mode 100644 index 0000000..0a1fbaa --- /dev/null +++ b/src/components/handle-transfer.ts @@ -0,0 +1,167 @@ +import { decodeLog } from "eth-fun"; +import SoundProtocol from "../strategies/sound_protocol.js"; +import { JsonRpcLog, NFT } from "../types.js"; +import { ethGetLogs } from "./eth-get-logs.js"; +import { tracksDB } from "../../database/tracks.js"; +import Lens from "../strategies/lens/lens.js"; + +export async function handleTransfer( + this: Lens | SoundProtocol, + from: number, + to: number, + recrawl: boolean, +) { + // `from - to` should be smaller than crawlStep but just in case + // it is not, call handleTransfer multiple times + const { crawlStep } = this.config.chain[this.chain]; + for (let i = from; i <= to; i += crawlStep + 1) { + const fromBlock = i; + const toBlock = Math.min(to, i + crawlStep); + await _handleTransfer.call(this, fromBlock, toBlock, recrawl); + } +} + +async function _handleTransfer( + this: Lens | SoundProtocol, + from: number, + to: number, + recrawl: boolean, +) { + const contractsStorage = this.localStorage.sublevel("contracts", {}); + const { getLogsBlockSpanSize, getLogsAddressSize } = this.config.chain[this.chain]; + const iterator = contractsStorage.iterator(); + const entries = await iterator.all(); + const addresses = entries.map((e) => e[0]); + + const mintNFTsPromise: Promise[] = []; + const allTransferNFTs: NFT[] = []; + + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) { + const fromBlock = i; + const toBlock = Math.min(to, i + getLogsBlockSpanSize); + + for (let j = 0; j < addresses.length; j += getLogsAddressSize) { + const addressSlice = addresses.slice(j, j + getLogsAddressSize); + + const logs = await ethGetLogs.call( + this, + fromBlock, + toBlock, + [SoundProtocol.TRANSFER_EVENT_SELECTOR], + addressSlice, + ); + + let nfts = logs.map((log) => prepareNFT(log)); + + // Partition NFTs into mints and transfers + const { mintNfts, transferNfts } = nfts.reduce( + (nfts, nft) => { + if (nft.erc721.transaction.from === "0x0000000000000000000000000000000000000000") { + nfts.mintNfts.push(nft); + } else { + nfts.transferNfts.push(nft); + } + return nfts; + }, + { mintNfts: [] as NFT[], transferNfts: [] as NFT[] }, + ); + + const promises = mintNfts.map(async (nft) => { + if (!recrawl) { + const uid = await this.nftToUid(nft); + if (await tracksDB.isTrackPresent(uid)) return; + } + + const track = await this.fetchMetadata(nft); + + if (track) { + console.log( + "Found track:", + track?.title, + track?.platform.version, + track?.platform.name, + "at", + nft.erc721.blockNumber, + ); + + await tracksDB.upsertTrack(track); + } + }); + + mintNFTsPromise.push(...promises); + allTransferNFTs.push(...transferNfts); + } + } + + await Promise.all(mintNFTsPromise); + + await Promise.all( + allTransferNFTs.map(async (nft) => { + let alias; + let uid = await this.nftToUid(nft); + + if (!recrawl && (await tracksDB.isTrackPresent(uid))) return; + + await tracksDB.upsertOwner( + uid, + nft.erc721.token.id, + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: alias ?? undefined, + }, + this.constructor.name, + ); + + console.log( + "Update ownership of", + nft.erc721.address, + "at", + nft.erc721.blockNumber, + "from", + nft.erc721.transaction.from, + "to", + nft.erc721.transaction.to, + ); + }), + ); +} + +function prepareNFT(log: JsonRpcLog): NFT { + if (!log.topics[3] || !log.transactionHash || !log.blockNumber) { + throw new Error(`log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`); + } + + const decodedTopics = decodeLog( + [ + { indexed: true, name: "from", type: "address" }, + { indexed: true, name: "to", type: "address" }, + { indexed: true, name: "tokenId", type: "uint256" }, + ], + log.data, + log.topics.slice(1), + ); + + return { + platform: { + name: "", + version: "1.0", + }, + erc721: { + blockNumber: parseInt(log.blockNumber, 16), + address: log.address, + transaction: { + from: decodedTopics[0], + to: decodedTopics[1], + transactionHash: log.transactionHash, + blockNumber: parseInt(log.blockNumber, 16), + }, + token: { + id: BigInt(log.topics[3]).toString(10), + }, + }, + metadata: {}, + }; +} diff --git a/src/state.ts b/src/state.ts index bd7f944..9e1c3fb 100644 --- a/src/state.ts +++ b/src/state.ts @@ -4,19 +4,24 @@ import fs from "fs/promises"; import path from "path"; -import { CONSTANTS } from "./types.js"; +import { CHAINS, CONSTANTS } from "./types.js"; -export async function getLastCrawledBlock() { - const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL); +export async function getLastCrawledBlock(chain: CHAINS) { + const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL, chain); const fileExists = await fs .access(location, fs.constants.F_OK) .then(() => true) .catch(() => false); - if (!fileExists) await saveLastCrawledBlock(CONSTANTS.FIRST_BLOCK); + if (!fileExists) await saveLastCrawledBlock(chain, CONSTANTS.FIRST_BLOCK[chain]); return fs.readFile(location, "utf-8").then(parseInt); } -export async function saveLastCrawledBlock(blockNumber: number) { - const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL); +export async function saveLastCrawledBlock(chain: CHAINS, blockNumber: number) { + const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL, chain); + const fileExists = await fs + .access(path.dirname(location), fs.constants.F_OK) + .then(() => true) + .catch(() => false); + if (!fileExists) await fs.mkdir(path.dirname(location), { recursive: true }); return fs.writeFile(location, blockNumber.toString(), "utf-8"); } diff --git a/src/strategies/lens/components.ts b/src/strategies/lens/components.ts new file mode 100644 index 0000000..e2e4c6f --- /dev/null +++ b/src/strategies/lens/components.ts @@ -0,0 +1,199 @@ +import { toHex, decodeLog, encodeFunctionCall, decodeParameters } from "eth-fun"; + +import { randomItem } from "../../utils.js"; +import Lens from "./lens.js"; +import { NFT } from "../../types.js"; + +export async function getHandle(this: Lens, profileId: number, blockNumber: number) { + const rpcHost = randomItem(this.config.chain[this.chain].rpc); + + const data = encodeFunctionCall( + { + name: "getHandle", + type: "function", + inputs: [ + { + type: "uint256", + name: "", + }, + ], + }, + [profileId], + ); + + const msg = await this.worker({ + type: "json-rpc", + commissioner: "", + method: "eth_call", + options: { + url: rpcHost.url, + headers: { + ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), + }, + retry: { + retries: 3, + }, + }, + params: [ + { + to: Lens.LENS_HUB_ADDRESS, + data, + }, + toHex(blockNumber), + ], + version: "0.0.1", + }); + + if (msg.error) + throw new Error(`Error while calling getHandle on contract: ${JSON.stringify(msg, null, 2)}`); + + const handle = decodeParameters(["string"], msg.results)[0]; + + if (typeof handle !== "string") + throw new Error(`Invalid result of getHandle for contract: ${JSON.stringify(msg, null, 2)}`); + + return handle; +} + +export async function getCollectNFT( + this: Lens, + profileId: number, + pubId: number, + blockNumber: number, +) { + const rpcHost = randomItem(this.config.chain[this.chain].rpc); + + const data = encodeFunctionCall( + { + name: "getCollectNFT", + type: "function", + inputs: [ + { + type: "uint256", + name: "", + }, + { + type: "uint256", + name: "", + }, + ], + }, + [profileId, pubId], + ); + + const msg = await this.worker({ + type: "json-rpc", + commissioner: "", + method: "eth_call", + options: { + url: rpcHost.url, + headers: { + ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), + }, + retry: { + retries: 3, + }, + }, + params: [ + { + to: Lens.LENS_HUB_ADDRESS, + data, + }, + toHex(blockNumber), + ], + version: "0.0.1", + }); + + if (msg.error) + throw new Error( + `Error while calling getContractNFT on contract: ${JSON.stringify(msg, null, 2)}`, + ); + + const address = decodeParameters(["address"], msg.results)[0]; + + if (typeof address !== "string") + throw new Error( + `invalid result of getContractNFT for contract: ${JSON.stringify(msg, null, 2)}`, + ); + + return address; +} + +// The address may own multiple profiles but we are currently +// only interested in one of them. Hence, the zero index. +export async function getHandleByAddress(this: Lens, address: string, blockNumber: number) { + const rpcHost = randomItem(this.config.chain[this.chain].rpc); + + const data = encodeFunctionCall( + { + name: "tokenOfOwnerByIndex", + type: "function", + inputs: [ + { + type: "address", + name: "", + }, + { + type: "uint256", + name: "", + }, + ], + }, + [address, 0], + ); + + const msg = await this.worker({ + type: "json-rpc", + commissioner: "", + method: "eth_call", + options: { + url: rpcHost.url, + headers: { + ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), + }, + retry: { + retries: 3, + }, + }, + params: [ + { + to: Lens.LENS_HUB_ADDRESS, + data, + }, + toHex(blockNumber), + ], + version: "0.0.1", + }); + + if (msg.error) + throw new Error( + `Error while calling tokenOwnerByIndex on contract: ${JSON.stringify( + msg, + null, + 2, + )} \n ${address}`, + ); + + const tokenId = parseInt(decodeParameters(["uint256"], msg.results)[0]); + + if (typeof tokenId !== "number" || Number.isNaN(tokenId)) + throw new Error( + `Invalid result of tokenOwnerByIndex for contract: ${JSON.stringify(msg, null, 2)}`, + ); + + const handle = await getHandle.call(this, tokenId, blockNumber); + + return handle; +} + +export async function getAlias(this: Lens, nft: NFT): Promise { + let handle = null; + + // Not every address will have an alias. Therefore, ignoring + // failures + try { + handle = await getHandleByAddress.call(this, nft.erc721.transaction.to, nft.erc721.blockNumber); + } catch (err) {} + + return handle; +} diff --git a/src/strategies/lens/lens.ts b/src/strategies/lens/lens.ts new file mode 100644 index 0000000..db0b689 --- /dev/null +++ b/src/strategies/lens/lens.ts @@ -0,0 +1,400 @@ +import ExtractionWorker, { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; +import { Track } from "@neume-network/schema"; +import { toHex, decodeLog, encodeParameters, decodeParameters } from "eth-fun"; + +import { CHAINS, Config, NFT, PROTOCOLS } from "../../types.js"; +import { Strategy } from "../strategy.types.js"; +import { getProtocol } from "../../utils.js"; +import { localStorage } from "../../../database/localstorage.js"; +import { getArweaveTokenUri } from "../../components/get-arweave-tokenuri.js"; +import { getIpfsTokenUri } from "../../components/get-ipfs-tokenuri.js"; +import { fetchTokenUri } from "../../components/fetch-tokenuri.js"; +import { ethGetLogs } from "../../components/eth-get-logs.js"; +import { AbstractSublevel } from "abstract-level"; +import { Level } from "level"; +import { tracksDB } from "../../../database/tracks.js"; +import { handleTransfer } from "../../components/handle-transfer.js"; +import { getAlias, getCollectNFT, getHandle } from "./components.js"; + +// Post from Lens +type Post = { + profileId: number; + pubId: number; + contentURI: string; + collectModule: string; + collectModuleReturnData: any; + referenceModule: string; + referenceModuleReturnData: any; + timestamp: number; + blockNumber: number; +}; + +export default class Lens implements Strategy { + public static version = "1.0.0"; + public deprecatedAtBlock = null; + public createdAtBlock = 0; + public worker: ExtractionWorkerHandler; + public config: Config; + public chain = CHAINS.polygon; + public localStorage: AbstractSublevel< + Level, + string | Buffer | Uint8Array, + string, + any + >; + + public static LENS_HUB_ADDRESS = "0xDb46d1Dc155634FbC732f92E853b10B288AD5a1d"; + public static POST_CREATED_EVENT_SELECTOR = + "0xc672c38b4d26c3c978228e99164105280410b144af24dd3ed8e4f9d211d96a50"; + public static COLLECT_NFT_DEPLOYED = + "0x0b227b550ffed48af813b32e246f787e99581ee13206ba8f9d90d63615269b3f"; + public static TRANSFER_EVENT_SELECTOR = + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; + + // `${post.profileId}-${post.pubId}` + public static ignoredPosts = ["39133-682", "88863-16"]; + + /** Contracts where NFTs are published */ + public contracts: AbstractSublevel; + /** Lens protocol IDs to listen for contracts */ + public trackedIds: AbstractSublevel; + /** A mapping between NFT contract address and Lens ID */ + public addressToId: AbstractSublevel; + + constructor(worker: ExtractionWorkerHandler, config: Config) { + this.worker = worker; + this.config = config; + this.localStorage = localStorage.sublevel(Lens.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + this.trackedIds = this.localStorage.sublevel("trackedIds", { + valueEncoding: "json", + }); + this.addressToId = this.localStorage.sublevel("addressToId", { + valueEncoding: "json", + }); + } + + async crawl(from: number, to: number, recrawl: boolean) { + console.time(`handlePostCreated: ${from}-${to}`); + await this.handlePostCreated(from, to, recrawl); + console.timeEnd(`handlePostCreated: ${from}-${to}`); + + console.time(`handleCollectNftDeployed: ${from}-${to}`); + await this.handleCollectNftDeployed(from, to, recrawl); + console.timeEnd(`handleCollectNftDeployed: ${from}-${to}`); + + console.time(`handleTransfer: ${from}-${to}`); + await handleTransfer.call(this, from, to, recrawl); + console.timeEnd(`handleTransfer: ${from}-${to}`); + } + + async handleCollectNftDeployed(from: number, to: number, recrawl: boolean) { + const promises = []; + + const { getLogsBlockSpanSize, getLogsAddressSize } = this.config.chain[this.chain]; + const MAX_TOPICS = getLogsAddressSize; + + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) { + const fromBlock = i; + const toBlock = Math.min(to, i + getLogsBlockSpanSize); + + const iter = this.trackedIds.iterator(); + const pendingEnteries = []; + + while (true) { + const entries: [string, any][] = [...pendingEnteries, ...(await iter.nextv(MAX_TOPICS))]; + + if (entries.length === 0) { + break; + } + + // prepare topics filter. we have to take care of max topics. + const profileIds = new Set(); + const pubIds = new Set(); + + for (let j = 0; j < entries.length; j++) { + const [profileId, pubId] = entries[j][0].split("-"); + + if (profileIds.size >= MAX_TOPICS - 1 || pubIds.size >= MAX_TOPICS - 1) { + pendingEnteries.push(entries[j]); + } else { + profileIds.add(profileId); + pubIds.add(pubId); + } + } + + const promise = ethGetLogs + .call( + this, + fromBlock, + toBlock, + [ + Lens.COLLECT_NFT_DEPLOYED, + Array.from(profileIds).map((i) => encodeParameters(["uint256"], [i])), + Array.from(pubIds).map((i) => encodeParameters(["uint256"], [i])), + ], + [Lens.LENS_HUB_ADDRESS], + ) + .then(async (logs) => { + await Promise.all( + logs.map(async (log) => { + if (!log.transactionHash || !log.blockNumber) { + throw new Error( + `log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`, + ); + } + + const decodedTopics = decodeLog( + [ + { indexed: true, name: "profileId", type: "uint256" }, + { indexed: true, name: "pubId", type: "uint256" }, + { indexed: true, name: "collectNFT", type: "address" }, + { indexed: false, name: "timestamp", type: "uint256" }, + ], + log.data, + log.topics.slice(1), + ); + + let { profileId, pubId, collectNFT } = decodedTopics; + collectNFT = collectNFT.toLowerCase(); + + console.log("found collect nft", collectNFT, profileId, pubId); + + await this.contracts.put(collectNFT, { + name: Lens.name, + version: Lens.version, + }); + await this.addressToId.put(collectNFT, `${this.chain}/${profileId}/${pubId}`); + await this.trackedIds.del(`${profileId}-${pubId}`); + }), + ); + }); + + promises.push(promise); + } + } + + await Promise.all(promises); + } + + async handlePostCreated(from: number, to: number, recrawl: boolean) { + // We are searching for PostCreatedEvents and not directly for + // CollectedNFTDeployed event because a song maybe posted that + // does not have any collectors + + const _handlePostCreated = async (from: number, to: number, recrawl: boolean) => { + const logs = await ethGetLogs.call( + this, + from, + to, + [Lens.POST_CREATED_EVENT_SELECTOR], + [Lens.LENS_HUB_ADDRESS], + ); + + const posts = (await Promise.all( + logs.map(async (log) => { + if (!log.transactionHash || !log.blockNumber) { + throw new Error( + `log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`, + ); + } + + const decodedTopics = decodeLog( + [ + { indexed: true, name: "profileId", type: "uint256" }, + { indexed: true, name: "pubId", type: "uint256" }, + { indexed: false, name: "contentURI", type: "string" }, + { indexed: false, name: "collectModule", type: "address" }, + { indexed: false, name: "collectModuleReturnData", type: "bytes" }, + { indexed: false, name: "referenceModule", type: "address" }, + { indexed: false, name: "referenceModuleReturnData", type: "bytes" }, + { indexed: false, name: "timestamp", type: "uint256" }, + ], + log.data, + log.topics.slice(1), + ); + + return { + profileId: parseInt(decodedTopics[0]), + pubId: parseInt(decodedTopics[1]), + contentURI: decodedTopics[2], + collectModule: decodedTopics[3], + collectModuleReturnData: decodedTopics[4], + referenceModule: decodedTopics[5], + referenceModuleReturnData: decodedTopics[6], + timestamp: parseInt(decodedTopics[7]), + blockNumber: parseInt(log.blockNumber), + }; + }), + )) as Post[]; + + await Promise.all( + posts.map(async (post) => { + let track; + try { + if ( + !recrawl && + (await tracksDB.isTrackPresent(`${this.chain}/${post.profileId}/${post.pubId}`)) + ) + return; + track = await this.processPost(post); + } catch (err) { + console.log(post); + throw err; + } + + if (track) { + await tracksDB.upsertTrack(track); + await this.trackedIds.put(`${post.profileId}-${post.pubId}`, 1); + + console.dir(track, { depth: null }); + } + }), + ); + }; + + const { getLogsBlockSpanSize } = this.config.chain[this.chain]; + const promises = []; + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) { + const fromBlock = i; + const toBlock = Math.min(to, i + getLogsBlockSpanSize); + + promises.push(_handlePostCreated(fromBlock, toBlock, recrawl)); + } + + await Promise.all(promises); + } + + // This is called when a new NFT is minted in Lens + fetchMetadata = async (nft: NFT): Promise => { + let uid; + try { + uid = await this.addressToId.get(nft.erc721.address); + } catch (err) { + console.log("Error for", nft); + throw err; + } + const track = await tracksDB.getTrack(uid); + const alias = await getAlias.call(this, nft); + + track.erc721.tokens.push({ + id: nft.erc721.token.id, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: alias ?? undefined, + }, + ], + }); + + return track; + }; + + async processPost(post: Post): Promise { + // console.log("Processing new post with contentURI", post.contentURI); + if (Lens.ignoredPosts.includes(`${post.profileId}-${post.pubId}`)) { + console.log("This post is ignored; skipping"); + return null; + } + + const protocol = getProtocol(post.contentURI); + + let datum: Record; + try { + if (protocol === PROTOCOLS.arweave) { + if (!/ar:\/\/[a-zA-Z0-9-_]{43}.*/.test(post.contentURI)) { + console.log( + `Ignoring post id: ${post.profileId}-${post.pubId} because the content URI is invalid:`, + post.contentURI, + ); + return null; + } + datum = await getArweaveTokenUri(post.contentURI, this.worker, this.config); + } else if (protocol === PROTOCOLS.ipfs) { + datum = await getIpfsTokenUri(post.contentURI, this.worker, this.config); + } else if (protocol === PROTOCOLS.https) { + datum = await fetchTokenUri(post.contentURI, this.worker); + } else { + throw new Error(`Invalid Protocl for ${post.contentURI}`); + } + } catch (err: any) { + if (err.message.includes("status: 4") || err.message.includes("Invalid CID")) { + return null; + } + throw err; + } + + if (!datum || !datum.media) { + // console.log("No media; skipping"); + return null; + } + + const media = datum.media.find((m: any) => m.type.includes("audio")); + + if (!media) { + // console.log("No audio in media; skipping"); + return null; + } + + const collectNftAdsress = ( + await getCollectNFT.call(this, post.profileId, post.pubId, post.blockNumber) + ).toLowerCase(); + + const artistHandle = await getHandle.call(this, post.profileId, post.blockNumber); + + const track = { + version: Lens.version, + title: datum.name, + uid: `${this.chain}/${post.profileId}/${post.pubId}`, + artist: { + version: Lens.version, + name: artistHandle, // TODO: We can add profile's alias here + address: post.profileId.toString(), + }, + platform: { + version: Lens.version, + name: Lens.name, + uri: "https://lens.xyz", + }, + erc721: { + version: Lens.version, + address: collectNftAdsress, + tokens: [], + metadata: { + ...datum, + name: datum.name, + description: datum.content, + image: datum.image, + }, + }, + manifestations: [ + { + version: Lens.version, + uri: media.item, + mimetype: "audio", + }, + ], + }; + + // datum.image can be undefined + if (datum?.image) + track.manifestations.push({ + version: Lens.version, + uri: datum.image, + mimetype: "image", + }); + + return track; + } + + nftToUid = async (nft: NFT) => { + return this.addressToId.get(nft.erc721.address) as Promise; + }; +} diff --git a/src/strategies/sound_protocol.ts b/src/strategies/sound_protocol.ts index 6944fb4..f67d1cc 100644 --- a/src/strategies/sound_protocol.ts +++ b/src/strategies/sound_protocol.ts @@ -1,72 +1,57 @@ import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; -import { decodeLog, toHex } from "eth-fun"; +import { Track } from "@neume-network/schema"; +import { Level } from "level"; +import { decodeLog } from "eth-fun"; +import { AbstractSublevel } from "abstract-level"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getArweaveTokenUri } from "../components/get-arweave-tokenuri.js"; import { callOwner } from "../components/call-owner.js"; -import { Config, JsonRpcLog, NFT } from "../types.js"; -import { Strategy } from "./strategy.types.js"; -import { randomItem } from "../utils.js"; - -export default class SoundProtocol implements Strategy { - public static version = "2.0.0"; - public static createdAtBlock = 15570834; - public static deprecatedAtBlock = null; - public static invalidIDs = []; - private worker: ExtractionWorkerHandler; - private config: Config; +import { CHAINS, Config, NFT } from "../types.js"; +import { ERC721Strategy } from "./strategy.types.js"; +import { ethGetLogs } from "../components/eth-get-logs.js"; +import { localStorage } from "../../database/localstorage.js"; +import { handleTransfer } from "../components/handle-transfer.js"; + +export default class SoundProtocol implements ERC721Strategy { + static version = "2.0.0"; + createdAtBlock = 15570834; + deprecatedAtBlock = null; + static invalidIDs = []; + static TRANSFER_EVENT_SELECTOR = + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; + static EDITION_CREATED_SELECTOR = + "0x405098db99342b699216d8150e930dbbf2f686f5a43485aed1e69219dafd4935"; + + static chain = CHAINS.eth; + chain = SoundProtocol.chain; + worker: ExtractionWorkerHandler; + config: Config; + localStorage: AbstractSublevel, string | Buffer | Uint8Array, string, any>; constructor(worker: ExtractionWorkerHandler, config: Config) { this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(SoundProtocol.name, { + valueEncoding: "json", + }); } - filterContracts = async (from: number, to: number) => { - const editionCreatedSelector = - "0x405098db99342b699216d8150e930dbbf2f686f5a43485aed1e69219dafd4935"; - - const rpcHost = randomItem(this.config.rpc); - const options = { - url: rpcHost.url, - headers: { - ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), - }, - retry: { - retries: 3, - }, - }; - - const fromBlock = toHex(from); - const toBlock = toHex(to); - - const message = await this.worker({ - type: "json-rpc", - method: "eth_getLogs", - commissioner: SoundProtocol.name, - params: [ - { - fromBlock, - toBlock, - topics: [[editionCreatedSelector]], - }, - ], - version: "0.0.1", - options, - }); + crawl = async (from: number, to: number, recrawl: boolean) => { + const { getLogsBlockSpanSize } = this.config.chain[this.chain]; - if (message.error) { - throw new Error( - `Error occured while filtering ${SoundProtocol.name} contracts: \n${JSON.stringify( - message, - null, - 2, - )}`, - ); - } + const handleEditionCreatedPromises = []; + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) + handleEditionCreatedPromises.push(this.handleEditionCreated(i, i + getLogsBlockSpanSize)); + await Promise.all(handleEditionCreatedPromises); - const logs = message.results as any as Array; + await this.handleTransfer(from, to, recrawl); + }; + + handleEditionCreated = async (from: number, to: number) => { + const logs = await ethGetLogs.call(this, from, to, [[SoundProtocol.EDITION_CREATED_SELECTOR]]); - return logs.map((log) => { + const contracts = logs.map((log) => { const topics = log.topics; topics.shift(); const result = decodeLog( @@ -87,9 +72,26 @@ export default class SoundProtocol implements Strategy { version: SoundProtocol.version, }; }); + + const contractsStorage = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + + await Promise.all( + contracts.map(async (c) => { + console.log("Found a SoundProtocol contract", c.address); + // Save contract address that is to be checked for NFTs in future + await contractsStorage.put(c.address, { name: c.name, version: c.version }); + }), + ); }; - crawl = async (nft: NFT) => { + handleTransfer = handleTransfer.bind(this); + + nftToUid = async (nft: NFT) => + `${this.chain}/${SoundProtocol.name}/${nft.erc721.address.toLowerCase()}`; + + fetchMetadata = async (nft: NFT): Promise => { if ( SoundProtocol.invalidIDs.filter((id) => `${nft.erc721.address}/${nft.erc721.token.id}`.match(id), @@ -101,12 +103,7 @@ export default class SoundProtocol implements Strategy { return null; } - nft.erc721.token.uri = await callTokenUri( - this.worker, - this.config, - nft.erc721.blockNumber, - nft, - ); + nft.erc721.token.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft); if (!nft.erc721.token.uri.includes("ar://")) { console.log( @@ -129,12 +126,7 @@ export default class SoundProtocol implements Strategy { throw err; } - nft.creator = await callOwner( - this.worker, - this.config, - nft.erc721.address, - nft.erc721.blockNumber, - ); + nft.creator = await callOwner.call(this, nft.erc721.address, nft.erc721.blockNumber); try { const datum = nft.erc721.token.uriContent as any; @@ -142,6 +134,7 @@ export default class SoundProtocol implements Strategy { return { version: SoundProtocol.version, title: datum.name, + uid: await this.nftToUid(nft), artist: { version: SoundProtocol.version, name: datum.artist, @@ -155,21 +148,28 @@ export default class SoundProtocol implements Strategy { erc721: { version: SoundProtocol.version, createdAt: nft.erc721.blockNumber, - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, address: nft.erc721.address, - tokenId: nft.erc721.token.id, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.name, - description: datum.description, - image: datum.image, - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.name, + description: datum.description, + image: datum.image, + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -193,6 +193,9 @@ export default class SoundProtocol implements Strategy { return null; } }; - - updateOwner(nft: NFT) {} } + +// const { config }: { config: Config } = await import(path.resolve("./config.js")); +// const soundProtocol = new SoundProtocol(ExtractionWorker(config.worker), config); +// await soundProtocol.crawl(16_01_0000, 16_05_0000, false); +// process.exit(0); diff --git a/src/strategies/strategy.types.ts b/src/strategies/strategy.types.ts index cbca466..0a689b1 100644 --- a/src/strategies/strategy.types.ts +++ b/src/strategies/strategy.types.ts @@ -1,39 +1,51 @@ import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; import { Track } from "@neume-network/schema"; -import { Config, Contract, NFT } from "../types.js"; +import { AbstractSublevel } from "abstract-level"; +import { Level } from "level"; +import { CHAINS, Config, Contract, NFT } from "../types.js"; export declare class Strategy { public static version: string; + + // We have both static and non-static variable. Both should be equal. + // static is used to get chain without initialising the class (eg. ClassName.chain) + // non-static is used for `this.chain` + // public static chain: keyof typeof CHAINS; + public chain: keyof typeof CHAINS; /** * Neume will not include the strategy in the crawl * if the range of the crawl is not included in between * createdAtBlock and deprecatedAtBlock, both inclusive. */ - public static createdAtBlock: number; + public createdAtBlock: number; /** * Neume will not include the strategy in the crawl * if the range of the crawl is not included in between * createdAtBlock and deprecatedAtBlock, both inclusive. */ - public static deprecatedAtBlock: number | null; + public deprecatedAtBlock: number | null; + public worker: ExtractionWorkerHandler; + public config: Config; + public localStorage: AbstractSublevel< + Level, + string | Buffer | Uint8Array, + string, + any + >; constructor(worker: ExtractionWorkerHandler, config: Config); /** - * Find new contracts to crawl for the given block range. - * Particularly useful for factory patterns. + * This is the entrypoint for the strategy. It will be called periodically + * with newer block numbers. * - * @returns Array of new contracts found + * @argument recrawl: If true, process all data even if it has been processed before */ - filterContracts?: (from: number, to: number) => Promise; + crawl: (from: number, to: number, recrawl: boolean) => Promise; - /** - * Given an incomplete NFT, query the blockchain to complete it. - * - * @returns A neume schema compatible track. null is returned if the - * NFT needs to be skipped. - */ - crawl: (nft: NFT) => Promise; + nftToUid: (nft: NFT) => Promise; +} - updateOwner: (nft: NFT) => void; +export declare class ERC721Strategy extends Strategy { + fetchMetadata: (nft: NFT) => Promise; } diff --git a/src/types.ts b/src/types.ts index 5bf44f7..689b1da 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,16 @@ // All common types are declared here -import { Config as ExtractionWorkerConfig, Transaction } from "@neume-network/schema"; +import { Config as ExtractionWorkerConfig } from "@neume-network/schema"; + +export enum CHAINS { + "eth" = "eth", + "polygon" = "polygon", +} + +export enum PROTOCOLS { + "arweave" = "arweave", + "https" = "https", + "ipfs" = "ipfs", +} export const CONSTANTS = { DATA_DIR: "data", @@ -7,7 +18,10 @@ export const CONSTANTS = { LAST_SYNC: "last_synced_block", LAST_CRAWL: "last_crawled_block", }, - FIRST_BLOCK: 11000000, + FIRST_BLOCK: { + [CHAINS.eth]: 11000000, + [CHAINS.polygon]: 11000000, + }, }; export type RpcConfig = { @@ -20,16 +34,7 @@ export type IpfsConfig = { httpsGatewayKey: string; }; -export type Config = { - rpc: RpcConfig[]; - ipfs?: IpfsConfig; - arweave?: { - httpsGateway: string; - }; - /** - * In order not to overwhelm the crawler we crawl in steps of block number. - */ - crawlStep: number; +type ChainConfig = { /** * RPC endpoints will not fetch events for a large block span. getLogsBlockSpanSize * is the maximum size limit enforced by the RPC endpoint. @@ -40,6 +45,23 @@ export type Config = { * expresses the maximum size limit enforced by the RPC endpoint. */ getLogsAddressSize: number; + rpc: RpcConfig[]; + crawlStep: number; +}; + +export type Config = { + ipfs?: IpfsConfig; + arweave?: { + httpsGateway: string; + }; + chain: { + [CHAINS.eth]: ChainConfig; + [CHAINS.polygon]: ChainConfig; + }; + /** + * In order not to overwhelm the crawler we crawl in steps of block number. + */ + crawlStep: number; /** * The time to wait (in milliseconds) before starting the next * crawl cycle. If the crawler has not crawled up to the latest block @@ -78,7 +100,12 @@ export type NFT = { erc721: { blockNumber: number; address: string; - transaction: Transaction; + transaction: { + from: string; + to: string; + transactionHash: string; + blockNumber: number; + }; token: { id: string; uri?: string; diff --git a/src/utils.ts b/src/utils.ts index c80e46c..a8e63a0 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -3,14 +3,15 @@ import path from "path"; import https from "https"; import { Strategy } from "./strategies/strategy.types.js"; -import { Contracts, RpcConfig } from "./types.js"; +import { CHAINS, Contracts, PROTOCOLS, RpcConfig } from "./types.js"; -import Sound from "./strategies/sound.js"; +// import Sound from "./strategies/sound.js"; import SoundProtocol from "./strategies/sound_protocol.js"; -import Zora from "./strategies/zora.js"; -import CatalogV2 from "./strategies/catalog_v2.js"; -import MintSongsV2 from "./strategies/mintsongs_v2.js"; -import Noizd from "./strategies/noizd.js"; +// import Zora from "./strategies/zora.js"; +// import CatalogV2 from "./strategies/catalog_v2.js"; +// import MintSongsV2 from "./strategies/mintsongs_v2.js"; +// import Noizd from "./strategies/noizd.js"; +import Lens from "./strategies/lens/lens.js"; export function randomItem(arr: Array): T { return arr[Math.floor(Math.random() * arr.length)]; @@ -76,20 +77,23 @@ export async function getAllContracts(): Promise { /** * New strategies should be added here. */ -export function getStrategies(strategyNames: string[], from: number, to: number) { +export function getStrategies(strategyNames: string[]) { const strategies: Array = [ - Sound, + // Sound, + Lens, SoundProtocol, - Zora, - CatalogV2, - MintSongsV2, - Noizd, + // Zora, + // CatalogV2, + // MintSongsV2, + // Noizd, ]; - return strategies.filter( - (s) => - s.createdAtBlock <= from && - to <= (s.deprecatedAtBlock ?? Number.MAX_VALUE) && - strategyNames.includes(s.name), - ); + return strategies.filter((s) => strategyNames.includes(s.name)); +} + +export function getProtocol(uri: string): PROTOCOLS { + if (uri.includes("ar://")) return PROTOCOLS.arweave; + else if (uri.includes("ipfs://")) return PROTOCOLS.ipfs; + else if (uri.includes("http://") || uri.includes("https://")) return PROTOCOLS.https; + throw new Error(`Invalid Protocl for ${uri}`); } diff --git a/tsconfig.json b/tsconfig.json index 3c1c7df..15aed16 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,7 +11,7 @@ // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ /* Language and Environment */ - "target": "ES2020", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "target": "ES2020" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ @@ -25,13 +25,16 @@ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ /* Modules */ - "module": "Node16", /* Specify what module code is generated. */ + "module": "Node16" /* Specify what module code is generated. */, // "rootDir": "./", /* Specify the root folder within your source files. */ - "moduleResolution": "Node16", /* Specify how TypeScript looks up a file from a given module specifier. */ + "moduleResolution": "Node16" /* Specify how TypeScript looks up a file from a given module specifier. */, // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ - "typeRoots": ["./node_modules/@types", "./@types"], /* Specify multiple folders that act like './node_modules/@types'. */ + "typeRoots": [ + "./node_modules/@types", + "./@types" + ] /* Specify multiple folders that act like './node_modules/@types'. */, // "types": [], /* Specify type package names to be included without being referenced in a source file. */ // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ @@ -39,7 +42,7 @@ // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ /* JavaScript Support */ - "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + "allowJs": true /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */, // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ @@ -49,7 +52,7 @@ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./dist", /* Specify an output folder for all emitted files. */ + "outDir": "./dist" /* Specify an output folder for all emitted files. */, // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ @@ -71,12 +74,12 @@ /* Interop Constraints */ // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ - "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */, // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ - "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, /* Type Checking */ - "strict": true, /* Enable all strict type-checking options. */ + "strict": true /* Enable all strict type-checking options. */, // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ @@ -98,6 +101,8 @@ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ - "skipLibCheck": true /* Skip type checking all .d.ts files. */ - } + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + // We want to only compile the needed files + "include": ["./neume.ts"] } From 515e89361ccfb3f464df8fa5a7c1e8fbf0d66265 Mon Sep 17 00:00:00 2001 From: il3ven Date: Mon, 31 Jul 2023 16:58:05 +0530 Subject: [PATCH 04/15] update database module - use WAL mode for sqlite Under heavy load, I was facing knex timeout errors. WAL mode fixes it. - update test cases - more utility functions for the database and bug fixes --- database/knexfile.js | 12 +++++++ database/tracks.test.ts | 23 ++++++------ database/tracks.ts | 79 +++++++++++++++++++++++++++++++---------- 3 files changed, 84 insertions(+), 30 deletions(-) diff --git a/database/knexfile.js b/database/knexfile.js index 8b4cad1..a56ef59 100644 --- a/database/knexfile.js +++ b/database/knexfile.js @@ -1,12 +1,24 @@ +import Knex from "knex"; import path from "path"; +/** + * @type {Knex.Knex.Config} + */ const config = { client: "better-sqlite3", connection: { // path.resolve means that a .sqlite3 will be created at the current working directory filename: path.resolve("./data/neume.sqlite3"), }, + // Under heavy load acquiring connection may require more time + acquireConnectionTimeout: 120_000, useNullAsDefault: true, + pool: { + afterCreate: function (conn, done) { + conn.pragma("journal_mode = WAL"); + done(); + }, + }, }; export default config; diff --git a/database/tracks.test.ts b/database/tracks.test.ts index 12730ce..95d8964 100644 --- a/database/tracks.test.ts +++ b/database/tracks.test.ts @@ -198,17 +198,16 @@ test.serial("should be able to get changed tracks", async (t) => { await tracksDB.upsertTrack(sample[0], 0); await tracksDB.upsertTrack(sample[1], 5); - let ret = await tracksDB.getTracksChanged(0, 0, sample[0].platform.name); - t.is(ret.length, 1); - t.deepEqual(ret[0], sample[0]); + let { tracks } = await tracksDB.getTracksChanged(0, sample[0].platform.name); + t.is(tracks.length, 2); + t.deepEqual(tracks[0], sample[0]); - ret = await tracksDB.getTracksChanged(0, 5, sample[0].platform.name); - t.is(ret.length, 2); - t.deepEqual(ret, sample); + ({ tracks } = await tracksDB.getTracksChanged(5, sample[0].platform.name)); + t.is(tracks.length, 1); + t.deepEqual(tracks[0], sample[1]); - ret = await tracksDB.getTracksChanged(5, 5, sample[1].platform.name); - t.is(ret.length, 1); - t.deepEqual(ret[0], sample[1]); + ({ tracks } = await tracksDB.getTracksChanged(6, sample[1].platform.name)); + t.is(tracks.length, 0); const newOwner = { from: "0x77a395A6f7c6E91192697Abb207ea3c171F4B338", @@ -222,10 +221,10 @@ test.serial("should be able to get changed tracks", async (t) => { sample[0].erc721.tokens[0].id, newOwner, sample[0].platform.name, - 5, + 6, ); - ret = await tracksDB.getTracksChanged(5, 5, sample[0].platform.name); - t.is(ret.length, 2); + ({ tracks } = await tracksDB.getTracksChanged(6, sample[0].platform.name)); + t.is(tracks.length, 1); }); test("should be able to update track", async (t) => { diff --git a/database/tracks.ts b/database/tracks.ts index ef32003..ad5017a 100644 --- a/database/tracks.ts +++ b/database/tracks.ts @@ -11,21 +11,28 @@ type LogValue = { inputs: Array; }; +// Checkout this discussion for alternative Database options https://github.com/orgs/neume-network/discussions/29 + /** * SQL database to store and retrieve tracks. */ export class Tracks { - public db: Knex; + private db: Knex; // Used to log all operations. It can be used to regenerate the DB at a particular block number. - public log: Level; + private log?: Level; constructor() { this.db = knex.default(config); - this.log = new Level(resolve("./data/log"), { - valueEncoding: "json", - }); + /** + * The idea behind log is to record all database operations. + * In theory, these operations could be used to replicate the DB's state + * at any given time. + */ + // this.log = new Level(resolve("./data/log"), { + // valueEncoding: "json", + // }); } isTrackPresent = async (uid: string): Promise => { @@ -33,8 +40,14 @@ export class Tracks { return Boolean(rows.length); }; + /** Given a track and it's tokenID return true if the token ID is present in the DB. */ + isTokenPresent = async (uid: string, tokenId: string) => { + const rows = await this.db("tokens").select().where({ uid, id: tokenId }); + return Boolean(rows.length); + }; + upsertTrack = async (track: Track, timestamp: number = Date.now()) => { - this.db.transaction(async (trx) => { + return this.db.transaction(async (trx) => { await trx("tracks") .insert({ version: track.version, @@ -104,7 +117,7 @@ export class Tracks { const inputs = [track, timestamp]; - await this.log.put( + await this.log?.put( `${track.platform.name}/${this.encodeNumber(timestamp)}/${hashCode( JSON.stringify(inputs), )}`, @@ -139,7 +152,7 @@ export class Tracks { .onConflict(["uid", "id", "transactionHash", "to"]) .merge(); - await this.log.put( + await this.log?.put( `${platform}/${this.encodeNumber(timestamp)}/${hashCode(JSON.stringify(inputs))}`, { operation: "upsertOwner", @@ -148,6 +161,14 @@ export class Tracks { ); }; + isOwnerPresent = async (uid: string, tokenId: string, owner: Owner) => { + const rows = await this.db("owners") + .select("*") + .where({ uid, id: tokenId, transactionHash: owner.transactionHash, to: owner.to }); + + return Boolean(rows.length); + }; + getTrack = async (uid: string): Promise => { const tokensRaw = await this.db("tokens") .select("*") @@ -191,8 +212,8 @@ export class Tracks { .select("*") .where("manifestations.uid", "=", uid); - const tracksRaw = await this.db("tracks").select("*").where("tracks.uid", "=", uid).limit(1); - const r = tracksRaw[0]; + const trackRaw = await this.db("tracks").select("*").where("tracks.uid", "=", uid).limit(1); + const r = trackRaw[0]; return { version: r.version, @@ -224,14 +245,18 @@ export class Tracks { }; }; - getTracksChanged = async (from: number, to: number, platform: string): Promise => { + getTracksChanged = async ( + since: number, + platform: string, + ): Promise<{ tracks: Track[]; nextTimestamp: number | undefined }> => { + const MAX_TRACKS = 500; + const uids = await this.db("tracks") .select("uid") - .where("lastUpdatedAt", ">=", from) - .andWhere("lastUpdatedAt", "<=", to) - .andWhere("platform_name", "=", platform); - - console.log(uids); + .where("lastUpdatedAt", ">=", since) + .andWhere("platform_name", "=", platform) + .orderBy("lastUpdatedAt", "asc") + .limit(MAX_TRACKS); const tracks = await Promise.all( uids.map(async ({ uid }) => { @@ -239,7 +264,17 @@ export class Tracks { }), ); - return tracks; + const nextTimestampRaw = await this.db("tracks") + .select("lastUpdatedAt") + .where("lastUpdatedAt", ">=", since) + .andWhere("platform_name", "=", platform) + .orderBy("lastUpdatedAt", "asc") + .offset(MAX_TRACKS) + .limit(1); + + const nextTimestamp = nextTimestampRaw[0]?.lastUpdatedAt; + + return { tracks, nextTimestamp }; }; // LevelDB stores keys in lexicographical order. Therefore, @@ -262,6 +297,10 @@ export class Tracks { decodeNumber(num: string) { return Number(num).toString(); } + + async close() { + return this.db.destroy(); + } } function hashCode(str: string) { @@ -275,4 +314,8 @@ function hashCode(str: string) { } export const tracksDB = new Tracks(); -// console.dir(await tracksDB.getTrack("polygon/106643/194"), { depth: null }); +// console.dir(await tracksDB.isTokenPresent("polygon/13/492", "1823"), { depth: null }); + +process.on("exit", async () => { + await tracksDB.close(); +}); From bd7a1d00c44253a97cffc911fb7ac05df4686e07 Mon Sep 17 00:00:00 2001 From: il3ven Date: Tue, 1 Aug 2023 00:50:47 +0530 Subject: [PATCH 05/15] update strategies - update remaining strategies according to the new Strategy interface - catch a plethora of bugs; introduce many checks to catch incorrect tracks - block numbers are now saved per strategy --- src/components/get-ipfs-tokenuri.ts | 16 +-- src/components/handle-transfer.ts | 95 +++++++++----- src/state.ts | 14 ++- src/strategies/catalog_v2.ts | 96 +++++++++----- src/strategies/lens/lens.ts | 186 +++++++++++++++++++--------- src/strategies/mintsongs_v2.ts | 96 +++++++++----- src/strategies/noizd.ts | 94 +++++++++----- src/strategies/sound.ts | 111 +++++++++++------ src/strategies/sound_protocol.ts | 81 ++++++++---- src/strategies/strategy.types.ts | 22 +++- src/strategies/zora.ts | 93 +++++++++----- src/types.ts | 1 - src/utils.ts | 58 +++------ 13 files changed, 615 insertions(+), 348 deletions(-) diff --git a/src/components/get-ipfs-tokenuri.ts b/src/components/get-ipfs-tokenuri.ts index 8af66b4..f117fe9 100644 --- a/src/components/get-ipfs-tokenuri.ts +++ b/src/components/get-ipfs-tokenuri.ts @@ -1,21 +1,15 @@ -import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; +import { Strategy } from "../strategies/strategy.types.js"; -import { Config } from "../types.js"; +export async function getIpfsTokenUri(this: Strategy, uri: string): Promise> { + if (!this.config.ipfs) throw new Error(`IPFS configuration is required for getIpfsTokenUri`); -export async function getIpfsTokenUri( - uri: string, - worker: ExtractionWorkerHandler, - config: Config, -): Promise> { - if (!config.ipfs) throw new Error(`IPFS configuration is required for getIpfsTokenUri`); - - const msg = await worker({ + const msg = await this.worker({ type: "ipfs", version: "0.0.1", commissioner: "", options: { uri: uri, - gateway: config.ipfs.httpsGateway, + gateway: this.config.ipfs.httpsGateway, retry: { retries: 3, }, diff --git a/src/components/handle-transfer.ts b/src/components/handle-transfer.ts index 0a1fbaa..1b60b8a 100644 --- a/src/components/handle-transfer.ts +++ b/src/components/handle-transfer.ts @@ -1,12 +1,36 @@ import { decodeLog } from "eth-fun"; -import SoundProtocol from "../strategies/sound_protocol.js"; import { JsonRpcLog, NFT } from "../types.js"; import { ethGetLogs } from "./eth-get-logs.js"; import { tracksDB } from "../../database/tracks.js"; import Lens from "../strategies/lens/lens.js"; +import { ERC721Strategy } from "../strategies/strategy.types.js"; + +const TRANSFER_EVENT_SELECTOR = + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; + +function debug(fn: (...args: any[]) => Promise, name: string) { + let labelAdded = false; + + const timeout = setTimeout(() => { + console.time(name); + labelAdded = true; + }, 30_000); + + const interval = setInterval(() => { + console.timeLog(name); + }, 60_000); + + return async function (this: any, ...args: any[]): Promise { + const resp = await fn.call(this, ...args); + clearTimeout(timeout); + clearInterval(interval); + if (labelAdded) console.timeEnd(name); + return resp; + }; +} export async function handleTransfer( - this: Lens | SoundProtocol, + this: Lens | ERC721Strategy, from: number, to: number, recrawl: boolean, @@ -17,17 +41,20 @@ export async function handleTransfer( for (let i = from; i <= to; i += crawlStep + 1) { const fromBlock = i; const toBlock = Math.min(to, i + crawlStep); - await _handleTransfer.call(this, fromBlock, toBlock, recrawl); + await debug( + _handleTransfer, + `_handleTransfer is hung up for ${this.constructor.name} ${fromBlock}-${toBlock}`, + ).call(this, fromBlock, toBlock, recrawl); } } async function _handleTransfer( - this: Lens | SoundProtocol, + this: Lens | ERC721Strategy, from: number, to: number, recrawl: boolean, ) { - const contractsStorage = this.localStorage.sublevel("contracts", {}); + const contractsStorage = this.contracts; const { getLogsBlockSpanSize, getLogsAddressSize } = this.config.chain[this.chain]; const iterator = contractsStorage.iterator(); const entries = await iterator.all(); @@ -41,15 +68,15 @@ async function _handleTransfer( const toBlock = Math.min(to, i + getLogsBlockSpanSize); for (let j = 0; j < addresses.length; j += getLogsAddressSize) { + // console.log( + // `handle-transfer for ${this.constructor.name} from ${fromBlock} to ${toBlock} [j=${j}]`, + // ); const addressSlice = addresses.slice(j, j + getLogsAddressSize); - const logs = await ethGetLogs.call( - this, - fromBlock, - toBlock, - [SoundProtocol.TRANSFER_EVENT_SELECTOR], - addressSlice, - ); + const logs = await debug( + ethGetLogs, + `eth-getLogs is hung up for ${this.constructor.name} ${fromBlock}-${toBlock}-${j}`, + ).call(this, fromBlock, toBlock, [TRANSFER_EVENT_SELECTOR], addressSlice); let nfts = logs.map((log) => prepareNFT(log)); @@ -67,17 +94,20 @@ async function _handleTransfer( ); const promises = mintNfts.map(async (nft) => { + let uid; if (!recrawl) { - const uid = await this.nftToUid(nft); - if (await tracksDB.isTrackPresent(uid)) return; + uid = await this.nftToUid(nft); + if (await tracksDB.isTokenPresent(uid, nft.erc721.token.id)) return; } + // console.log(`fetching metadata for ${this.constructor.name}`, uid); const track = await this.fetchMetadata(nft); if (track) { console.log( - "Found track:", + "Found new NFT (could be a new track):", track?.title, + nft.erc721.token.id, track?.platform.version, track?.platform.name, "at", @@ -99,21 +129,26 @@ async function _handleTransfer( allTransferNFTs.map(async (nft) => { let alias; let uid = await this.nftToUid(nft); - - if (!recrawl && (await tracksDB.isTrackPresent(uid))) return; - - await tracksDB.upsertOwner( - uid, - nft.erc721.token.id, - { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - alias: alias ?? undefined, - }, - this.constructor.name, - ); + let isTrackPresent = await tracksDB.isTrackPresent(uid); + + // Track has not been crawled. Most probably we ignored it. + // Makes no sense to record ownership transfer. + if (!isTrackPresent) return; + + const owner = { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: alias ?? undefined, + }; + + if (!recrawl && isTrackPresent) { + const isOwnerPresent = await tracksDB.isOwnerPresent(uid, nft.erc721.token.id, owner); + if (isOwnerPresent) return; + } + + await tracksDB.upsertOwner(uid, nft.erc721.token.id, owner, this.constructor.name); console.log( "Update ownership of", diff --git a/src/state.ts b/src/state.ts index 9e1c3fb..1dcfad3 100644 --- a/src/state.ts +++ b/src/state.ts @@ -4,20 +4,22 @@ import fs from "fs/promises"; import path from "path"; -import { CHAINS, CONSTANTS } from "./types.js"; +import { CONSTANTS } from "./types.js"; +import { getStrategies } from "./utils.js"; -export async function getLastCrawledBlock(chain: CHAINS) { - const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL, chain); +export async function getLastCrawledBlock(strategy: string) { + const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL, strategy); + const { createdAtBlock } = getStrategies([strategy])[0]; const fileExists = await fs .access(location, fs.constants.F_OK) .then(() => true) .catch(() => false); - if (!fileExists) await saveLastCrawledBlock(chain, CONSTANTS.FIRST_BLOCK[chain]); + if (!fileExists) await saveLastCrawledBlock(strategy, createdAtBlock); return fs.readFile(location, "utf-8").then(parseInt); } -export async function saveLastCrawledBlock(chain: CHAINS, blockNumber: number) { - const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL, chain); +export async function saveLastCrawledBlock(strategy: string, blockNumber: number) { + const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL, strategy); const fileExists = await fs .access(path.dirname(location), fs.constants.F_OK) .then(() => true) diff --git a/src/strategies/catalog_v2.ts b/src/strategies/catalog_v2.ts index e5e0d4f..baaa73b 100644 --- a/src/strategies/catalog_v2.ts +++ b/src/strategies/catalog_v2.ts @@ -7,36 +7,55 @@ import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; import { toHex, encodeFunctionCall, decodeParameters } from "eth-fun"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getIpfsTokenUri } from "../components/get-ipfs-tokenuri.js"; -import { Config, NFT } from "../types.js"; -import { Strategy } from "./strategy.types.js"; +import { CHAINS, Config, Contract, NFT } from "../types.js"; +import { ERC721Strategy } from "./strategy.types.js"; import { randomItem } from "../utils.js"; +import { AbstractSublevel } from "abstract-level"; +import { Level } from "level"; +import { localStorage } from "../../database/localstorage.js"; +import { handleTransfer } from "../components/handle-transfer.js"; -export default class CatalogV2 implements Strategy { +export default class CatalogV2 implements ERC721Strategy { public static version = "2.0.0"; - public static createdAtBlock = 0; - public static deprecatedAtBlock = null; - private worker: ExtractionWorkerHandler; - private config: Config; + // The Catalog contract was deployed at https://etherscan.io/tx/0x65a0c575267dae42937363299c58cb0d30e35b0a6741ff0dc079ffd927c8e1b2 + static createdAtBlock = 14566826; + createdAtBlock = CatalogV2.createdAtBlock; + deprecatedAtBlock = null; + chain = CHAINS.eth; + worker: ExtractionWorkerHandler; + config: Config; + localStorage: AbstractSublevel, string | Buffer | Uint8Array, string, any>; + contracts: AbstractSublevel; constructor(worker: ExtractionWorkerHandler, config: Config) { this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(CatalogV2.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + + const CATALOG_NFT_CONTRACT = "0x0bC2A24ce568DAd89691116d5B34DEB6C203F342"; + this.contracts.put(CATALOG_NFT_CONTRACT, { + name: CatalogV2.name, + version: CatalogV2.version, + }); } - crawl = async (nft: NFT) => { - nft.erc721.token.uri = await callTokenUri( - this.worker, - this.config, - nft.erc721.blockNumber, - nft, - ); + crawl = async (from: number, to: number, recrawl: boolean) => { + await handleTransfer.call(this, from, to, recrawl); + }; + + fetchMetadata = async (nft: NFT) => { + nft.erc721.token.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft); try { - nft.erc721.token.uriContent = await getIpfsTokenUri( + nft.erc721.token.uriContent = (await getIpfsTokenUri.call( + this, nft.erc721.token.uri, - this.worker, - this.config, - ); + )) as Record; } catch (err: any) { if (err.message.includes("Invalid CID")) { console.warn("Invalid CID: Ignoring the given track.", JSON.stringify(nft, null, 2)); @@ -69,6 +88,7 @@ export default class CatalogV2 implements Strategy { version: CatalogV2.version, title: datum.title, duration, + uid: await this.nftToUid(nft), artist: { version: CatalogV2.version, name: datum.artist, @@ -80,23 +100,30 @@ export default class CatalogV2 implements Strategy { uri: "https://catalog.works", }, erc721: { - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, version: CatalogV2.version, createdAt: nft.erc721.blockNumber, - tokenId: nft.erc721.token.id, address: nft.erc721.address, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.name, - description: datum.description, - image: datum.image, - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.name, + description: datum.description, + image: datum.image, + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -113,14 +140,15 @@ export default class CatalogV2 implements Strategy { }; }; - updateOwner(nft: NFT) {} + nftToUid = async (nft: NFT) => + `${this.chain}/${CatalogV2.name}/${nft.erc721.address.toLowerCase()}/${nft.erc721.token.id}`; private callCreator = async ( to: string, blockNumber: number, tokenId: string, ): Promise => { - const rpc = randomItem(this.config.rpc); + const rpc = randomItem(this.config.chain[this.chain].rpc); const data = encodeFunctionCall( { name: "creator", diff --git a/src/strategies/lens/lens.ts b/src/strategies/lens/lens.ts index db0b689..c7d4c07 100644 --- a/src/strategies/lens/lens.ts +++ b/src/strategies/lens/lens.ts @@ -1,8 +1,12 @@ +/** + * Current lens first song - 33474641 + */ + import ExtractionWorker, { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; import { Track } from "@neume-network/schema"; import { toHex, decodeLog, encodeParameters, decodeParameters } from "eth-fun"; -import { CHAINS, Config, NFT, PROTOCOLS } from "../../types.js"; +import { CHAINS, Config, Contract, NFT, PROTOCOLS } from "../../types.js"; import { Strategy } from "../strategy.types.js"; import { getProtocol } from "../../utils.js"; import { localStorage } from "../../../database/localstorage.js"; @@ -15,6 +19,7 @@ import { Level } from "level"; import { tracksDB } from "../../../database/tracks.js"; import { handleTransfer } from "../../components/handle-transfer.js"; import { getAlias, getCollectNFT, getHandle } from "./components.js"; +import { z } from "zod"; // Post from Lens type Post = { @@ -31,8 +36,10 @@ type Post = { export default class Lens implements Strategy { public static version = "1.0.0"; + // The lens hub was created at this block: https://polygonscan.com/tx/0xca69b18b7e2daf4695c6d614e263d6aa9bdee44bee91bee7e0e6e5e5e4262fca + public static createdAtBlock = 28384641; + public createdAtBlock = Lens.createdAtBlock; public deprecatedAtBlock = null; - public createdAtBlock = 0; public worker: ExtractionWorkerHandler; public config: Config; public chain = CHAINS.polygon; @@ -52,14 +59,28 @@ export default class Lens implements Strategy { "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; // `${post.profileId}-${post.pubId}` - public static ignoredPosts = ["39133-682", "88863-16"]; + public static ignoredPosts = [ + "39133-682", + "88863-16", + "18497-28", + "3834-24", + "40863-4", + "40635-1", + "49754-2", + ]; + // The following transactions can't be decoded + public static ignoredTransactions = [ + "0x52c63367c36eb24a08654c89dc647267a9f1171af3962dafe5cefab31e21293d", + ]; /** Contracts where NFTs are published */ - public contracts: AbstractSublevel; + public contracts: AbstractSublevel; /** Lens protocol IDs to listen for contracts */ public trackedIds: AbstractSublevel; /** A mapping between NFT contract address and Lens ID */ public addressToId: AbstractSublevel; + /** Posts that we have already seen. Either crawled or ignored. Useful to not recrawl. */ + public seenPosts: AbstractSublevel; constructor(worker: ExtractionWorkerHandler, config: Config) { this.worker = worker; @@ -76,20 +97,23 @@ export default class Lens implements Strategy { this.addressToId = this.localStorage.sublevel("addressToId", { valueEncoding: "json", }); + this.seenPosts = this.localStorage.sublevel("crawledPosts", { + valueEncoding: "json", + }); } async crawl(from: number, to: number, recrawl: boolean) { - console.time(`handlePostCreated: ${from}-${to}`); + console.time(`${Lens.name} handlePostCreated: ${from}-${to}`); await this.handlePostCreated(from, to, recrawl); - console.timeEnd(`handlePostCreated: ${from}-${to}`); + console.timeEnd(`${Lens.name} handlePostCreated: ${from}-${to}`); - console.time(`handleCollectNftDeployed: ${from}-${to}`); + console.time(`${Lens.name} handleCollectNftDeployed: ${from}-${to}`); await this.handleCollectNftDeployed(from, to, recrawl); - console.timeEnd(`handleCollectNftDeployed: ${from}-${to}`); + console.timeEnd(`${Lens.name} handleCollectNftDeployed: ${from}-${to}`); - console.time(`handleTransfer: ${from}-${to}`); + console.time(`${Lens.name} handleTransfer: ${from}-${to}`); await handleTransfer.call(this, from, to, recrawl); - console.timeEnd(`handleTransfer: ${from}-${to}`); + console.timeEnd(`${Lens.name} handleTransfer: ${from}-${to}`); } async handleCollectNftDeployed(from: number, to: number, recrawl: boolean) { @@ -162,7 +186,16 @@ export default class Lens implements Strategy { let { profileId, pubId, collectNFT } = decodedTopics; collectNFT = collectNFT.toLowerCase(); - console.log("found collect nft", collectNFT, profileId, pubId); + try { + await this.trackedIds.get(`${profileId}-${pubId}`); + } catch (err: any) { + if (err.code === "LEVEL_NOT_FOUND") { + // We have ignored this particular track but a combination of this + // profileId and pubId is present in topics that is why we are here + return; + } + throw err; + } await this.contracts.put(collectNFT, { name: Lens.name, @@ -170,6 +203,10 @@ export default class Lens implements Strategy { }); await this.addressToId.put(collectNFT, `${this.chain}/${profileId}/${pubId}`); await this.trackedIds.del(`${profileId}-${pubId}`); + + const track = await tracksDB.getTrack(`${this.chain}/${profileId}/${pubId}`); + track.erc721.address = collectNFT; // We didn't have erc721.address at the time of crawl. Updating it now. + await tracksDB.upsertTrack(track); }), ); }); @@ -195,53 +232,61 @@ export default class Lens implements Strategy { [Lens.LENS_HUB_ADDRESS], ); - const posts = (await Promise.all( - logs.map(async (log) => { - if (!log.transactionHash || !log.blockNumber) { - throw new Error( - `log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`, + const posts = ( + await Promise.all( + logs.map(async (log) => { + if (!log.transactionHash || !log.blockNumber) { + throw new Error( + `log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`, + ); + } + + if (Lens.ignoredTransactions.includes(log.transactionHash)) { + return null; + } + + const decodedTopics = decodeLog( + [ + { indexed: true, name: "profileId", type: "uint256" }, + { indexed: true, name: "pubId", type: "uint256" }, + { indexed: false, name: "contentURI", type: "string" }, + { indexed: false, name: "collectModule", type: "address" }, + { indexed: false, name: "collectModuleReturnData", type: "bytes" }, + { indexed: false, name: "referenceModule", type: "address" }, + { indexed: false, name: "referenceModuleReturnData", type: "bytes" }, + { indexed: false, name: "timestamp", type: "uint256" }, + ], + log.data, + log.topics.slice(1), ); - } - const decodedTopics = decodeLog( - [ - { indexed: true, name: "profileId", type: "uint256" }, - { indexed: true, name: "pubId", type: "uint256" }, - { indexed: false, name: "contentURI", type: "string" }, - { indexed: false, name: "collectModule", type: "address" }, - { indexed: false, name: "collectModuleReturnData", type: "bytes" }, - { indexed: false, name: "referenceModule", type: "address" }, - { indexed: false, name: "referenceModuleReturnData", type: "bytes" }, - { indexed: false, name: "timestamp", type: "uint256" }, - ], - log.data, - log.topics.slice(1), - ); - - return { - profileId: parseInt(decodedTopics[0]), - pubId: parseInt(decodedTopics[1]), - contentURI: decodedTopics[2], - collectModule: decodedTopics[3], - collectModuleReturnData: decodedTopics[4], - referenceModule: decodedTopics[5], - referenceModuleReturnData: decodedTopics[6], - timestamp: parseInt(decodedTopics[7]), - blockNumber: parseInt(log.blockNumber), - }; - }), - )) as Post[]; + return { + profileId: parseInt(decodedTopics[0]), + pubId: parseInt(decodedTopics[1]), + contentURI: decodedTopics[2], + collectModule: decodedTopics[3], + collectModuleReturnData: decodedTopics[4], + referenceModule: decodedTopics[5], + referenceModuleReturnData: decodedTopics[6], + timestamp: parseInt(decodedTopics[7]), + blockNumber: parseInt(log.blockNumber), + }; + }), + ) + ).filter((post) => post !== null) as Post[]; await Promise.all( posts.map(async (post) => { - let track; + let trackAlreadyPresent = await this.seenPosts + .get(`${post.profileId}-${post.pubId}`) + .then(() => true) + .catch(() => false); + + let track: Track | null; try { - if ( - !recrawl && - (await tracksDB.isTrackPresent(`${this.chain}/${post.profileId}/${post.pubId}`)) - ) - return; + if (!recrawl && trackAlreadyPresent) return; track = await this.processPost(post); + await this.seenPosts.put(`${post.profileId}-${post.pubId}`, {}); } catch (err) { console.log(post); throw err; @@ -304,8 +349,18 @@ export default class Lens implements Strategy { return null; } + // Regex for valid URIs; from: https://github.com/ajv-validator/ajv-formats/blob/4dd65447575b35d0187c6b125383366969e6267e/src/formats.ts#L229C12 + const URI = + /^(?:[a-z][a-z0-9+\-.]*:)(?:\/?\/(?:(?:[a-z0-9\-._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:(?:[0-9a-f]{1,4}:){6}|::(?:[0-9a-f]{1,4}:){5}|(?:[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){4}|(?:(?:[0-9a-f]{1,4}:){0,1}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){3}|(?:(?:[0-9a-f]{1,4}:){0,2}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){2}|(?:(?:[0-9a-f]{1,4}:){0,3}[0-9a-f]{1,4})?::[0-9a-f]{1,4}:|(?:(?:[0-9a-f]{1,4}:){0,4}[0-9a-f]{1,4})?::)(?:[0-9a-f]{1,4}:[0-9a-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))|(?:(?:[0-9a-f]{1,4}:){0,5}[0-9a-f]{1,4})?::[0-9a-f]{1,4}|(?:(?:[0-9a-f]{1,4}:){0,6}[0-9a-f]{1,4})?::)|[Vv][0-9a-f]+\.[a-z0-9\-._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)|(?:[a-z0-9\-._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)(?:\?(?:[a-z0-9\-._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[a-z0-9\-._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; + if (!post.contentURI || !URI.test(post.contentURI)) return null; + const protocol = getProtocol(post.contentURI); + if (!protocol) { + // console.log("Invalid protocol; skipping", post.contentURI); + return null; + } + let datum: Record; try { if (protocol === PROTOCOLS.arweave) { @@ -318,25 +373,29 @@ export default class Lens implements Strategy { } datum = await getArweaveTokenUri(post.contentURI, this.worker, this.config); } else if (protocol === PROTOCOLS.ipfs) { - datum = await getIpfsTokenUri(post.contentURI, this.worker, this.config); + datum = await getIpfsTokenUri.call(this, post.contentURI); } else if (protocol === PROTOCOLS.https) { datum = await fetchTokenUri(post.contentURI, this.worker); } else { - throw new Error(`Invalid Protocl for ${post.contentURI}`); + throw new Error(`Invalid Protocol for ${post.contentURI}`); } } catch (err: any) { - if (err.message.includes("status: 4") || err.message.includes("Invalid CID")) { + if ( + err.message.includes("status: 4") || + err.message.includes("Invalid CID") || + err.message.includes("ECONNREFUSED") + ) { return null; } throw err; } - if (!datum || !datum.media) { - // console.log("No media; skipping"); + if (!datum || !datum.media || datum.version !== "2.0.0") { + // console.log("No media; skipping", datum.media, datum.version); return null; } - const media = datum.media.find((m: any) => m.type.includes("audio")); + const media = datum.media.find?.((m: any) => m?.type?.includes("audio")); if (!media) { // console.log("No audio in media; skipping"); @@ -349,6 +408,19 @@ export default class Lens implements Strategy { const artistHandle = await getHandle.call(this, post.profileId, post.blockNumber); + try { + const schema = z.object({ + name: z.string(), + content: z.string(), + image: z.string().optional(), + }); + + schema.passthrough().parse(datum); + } catch { + // The required fields are not present in the metadata. Hence, ignoring it. + return null; + } + const track = { version: Lens.version, title: datum.name, diff --git a/src/strategies/mintsongs_v2.ts b/src/strategies/mintsongs_v2.ts index 094c95a..3542045 100644 --- a/src/strategies/mintsongs_v2.ts +++ b/src/strategies/mintsongs_v2.ts @@ -12,29 +12,52 @@ import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; import { toHex, encodeFunctionCall, decodeParameters } from "eth-fun"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getIpfsTokenUri } from "../components/get-ipfs-tokenuri.js"; -import { Config, NFT } from "../types.js"; +import { CHAINS, Config, Contract, NFT } from "../types.js"; import { randomItem } from "../utils.js"; -import { Strategy } from "./strategy.types.js"; +import { ERC721Strategy, Strategy } from "./strategy.types.js"; +import { AbstractSublevel } from "abstract-level"; +import { Level } from "level"; +import { handleTransfer } from "../components/handle-transfer.js"; +import { localStorage } from "../../database/localstorage.js"; -export default class MintSongsV2 implements Strategy { +export default class MintSongsV2 implements ERC721Strategy { public static version = "2.0.0"; // Oldest NFT mint found using OpenSea: https://etherscan.io/tx/0x4dd17de92c1d1ae0a7d17c127c57d99fd509f1b22dd176a483e5587fddf7e0a0 - public static createdAtBlock = 14799837; - public static deprecatedAtBlock = null; - public static invalidIDs = [ + static createdAtBlock = 14799837; + createdAtBlock = MintSongsV2.createdAtBlock; + deprecatedAtBlock = null; + static invalidIDs = [ /^0x2b5426a5b98a3e366230eba9f95a24f09ae4a584\/13$/, // Ignore track because URI contains a space at the end /^0x2b5426a5b98a3e366230eba9f95a24f09ae4a584\/113$/, // NFT has been burned ]; - - private worker: ExtractionWorkerHandler; - private config: Config; + chain = CHAINS.eth; + worker: ExtractionWorkerHandler; + config: Config; + localStorage: AbstractSublevel, string | Buffer | Uint8Array, string, any>; + contracts: AbstractSublevel; constructor(worker: ExtractionWorkerHandler, config: Config) { this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(MintSongsV2.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + + const MINGSONGS_NFT_CONTRACT = "0x2b5426a5b98a3e366230eba9f95a24f09ae4a584"; + this.contracts.put(MINGSONGS_NFT_CONTRACT, { + name: MintSongsV2.name, + version: MintSongsV2.version, + }); } - crawl = async (nft: NFT) => { + crawl = async (from: number, to: number, recrawl: boolean) => { + await handleTransfer.call(this, from, to, recrawl); + }; + + fetchMetadata = async (nft: NFT) => { // Crawling MintSongs at this block number or higher // because the contract is broken at the block the NFTs // were minted. Contract was upgraded later many times. @@ -51,18 +74,16 @@ export default class MintSongsV2 implements Strategy { return null; } - nft.erc721.token.uri = await callTokenUri( - this.worker, - this.config, + nft.erc721.token.uri = await callTokenUri.call( + this, Math.max(nft.erc721.blockNumber, BLOCK_NUMBER), nft, ); try { - nft.erc721.token.uriContent = await getIpfsTokenUri( + nft.erc721.token.uriContent = (await getIpfsTokenUri.call( + this, nft.erc721.token.uri, - this.worker, - this.config, - ); + )) as Record; } catch (err: any) { if (err.message.includes("Invalid CID")) { console.warn("Invalid CID: Ignoring the given track.", JSON.stringify(nft, null, 2)); @@ -94,6 +115,7 @@ export default class MintSongsV2 implements Strategy { version: MintSongsV2.version, title: datum.title, duration, + uid: await this.nftToUid(nft), artist: { version: MintSongsV2.version, name: datum.artist, @@ -105,23 +127,30 @@ export default class MintSongsV2 implements Strategy { uri: "https://www.mintsongs.com/", }, erc721: { - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, version: MintSongsV2.version, createdAt: nft.erc721.blockNumber, - tokenId: nft.erc721.token.id, address: nft.erc721.address, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.title, - description: datum.description, - image: datum.image, - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.title, + description: datum.description, + image: datum.image, + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -138,14 +167,15 @@ export default class MintSongsV2 implements Strategy { }; }; - updateOwner(nft: NFT) {} + nftToUid = async (nft: NFT) => + `${this.chain}/${MintSongsV2.name}/${nft.erc721.address.toLowerCase()}/${nft.erc721.token.id}`; private callTokenCreator = async ( to: string, blockNumber: number, tokenId: string, ): Promise => { - const rpc = randomItem(this.config.rpc); + const rpc = randomItem(this.config.chain[this.chain].rpc); const data = encodeFunctionCall( { name: "tokenCreator", diff --git a/src/strategies/noizd.ts b/src/strategies/noizd.ts index 5083c77..939c548 100644 --- a/src/strategies/noizd.ts +++ b/src/strategies/noizd.ts @@ -7,38 +7,55 @@ */ import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; +import { AbstractSublevel } from "abstract-level"; +import { Level } from "level"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getIpfsTokenUri } from "../components/get-ipfs-tokenuri.js"; -import { Config, NFT } from "../types.js"; +import { handleTransfer } from "../components/handle-transfer.js"; +import { CHAINS, Config, Contract, NFT } from "../types.js"; +import { localStorage } from "../../database/localstorage.js"; +import { ERC721Strategy } from "./strategy.types.js"; -import { Strategy } from "./strategy.types.js"; - -export default class Noizd implements Strategy { +export default class Noizd implements ERC721Strategy { public static version = "1.0.0"; // Oldest NFT mint found using OpenSea: https://etherscan.io/tx/0x9cd2b56dadc49a3c6ddb5f130de9c932a78a0ccb21c930ab978ce51cc5819901 - public static createdAtBlock = 13493464; - public static deprecatedAtBlock = null; - private worker: ExtractionWorkerHandler; - private config: Config; + static createdAtBlock = 13493464; + createdAtBlock = Noizd.createdAtBlock; + deprecatedAtBlock = null; + chain = CHAINS.eth; + worker: ExtractionWorkerHandler; + config: Config; + localStorage: AbstractSublevel, string | Buffer | Uint8Array, string, any>; + contracts: AbstractSublevel; constructor(worker: ExtractionWorkerHandler, config: Config) { this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(Noizd.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + + const NOIZD_NFT_CONTRACT = "0xf5819e27b9bad9f97c177bf007c1f96f26d91ca6"; + this.contracts.put(NOIZD_NFT_CONTRACT, { + name: Noizd.name, + version: Noizd.version, + }); } - crawl = async (nft: NFT) => { - nft.erc721.token.uri = await callTokenUri( - this.worker, - this.config, - nft.erc721.blockNumber, - nft, - ); + crawl = async (from: number, to: number, recrawl: boolean) => { + await handleTransfer.call(this, from, to, recrawl); + }; + + fetchMetadata = async (nft: NFT) => { + nft.erc721.token.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft); try { - nft.erc721.token.uriContent = await getIpfsTokenUri( + nft.erc721.token.uriContent = (await getIpfsTokenUri.call( + this, nft.erc721.token.uri, - this.worker, - this.config, - ); + )) as Record; } catch (err: any) { if (err.message.includes("Invalid CID")) { console.warn("Invalid CID: Ignoring the given track.", JSON.stringify(nft, null, 2)); @@ -65,6 +82,7 @@ export default class Noizd implements Strategy { version: Noizd.version, title: datum.name, duration, + uid: await this.nftToUid(nft), artist: { version: Noizd.version, name: datum.artist_name, @@ -76,23 +94,30 @@ export default class Noizd implements Strategy { uri: "https://noizd.com", }, erc721: { - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, version: Noizd.version, createdAt: nft.erc721.blockNumber, - tokenId: nft.erc721.token.id, address: nft.erc721.address, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.name, - description: datum.description, - image: datum.image, - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.name, + description: datum.description, + image: datum.image, + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -109,5 +134,6 @@ export default class Noizd implements Strategy { }; }; - updateOwner(nft: NFT) {} + nftToUid = async (nft: NFT) => + `${this.chain}/${Noizd.name}/${nft.erc721.address.toLowerCase()}/${nft.erc721.token.id}`; } diff --git a/src/strategies/sound.ts b/src/strategies/sound.ts index 32fb4e6..15d45c6 100644 --- a/src/strategies/sound.ts +++ b/src/strategies/sound.ts @@ -7,31 +7,55 @@ import { decodeLog, toHex } from "eth-fun"; import { callTokenUri } from "../components/call-tokenuri.js"; import { fetchTokenUri } from "../components/fetch-tokenuri.js"; import { callOwner } from "../components/call-owner.js"; - -import { Config, JsonRpcLog, NFT } from "../types.js"; -import { Strategy } from "./strategy.types.js"; +import { AbstractSublevel } from "abstract-level"; +import { Level } from "level"; +import { CHAINS, Config, Contract, JsonRpcLog, NFT } from "../types.js"; +import { ERC721Strategy } from "./strategy.types.js"; import { randomItem } from "../utils.js"; import { ifIpfsConvertToNativeIpfs } from "ipfs-uri-utils"; +import { localStorage } from "../../database/localstorage.js"; +import { handleTransfer } from "../components/handle-transfer.js"; -export default class Sound implements Strategy { +export default class Sound implements ERC721Strategy { public static version = "1.0.0"; - public static createdAtBlock = 13725566; - public static deprecatedAtBlock = null; - public static invalidIDs = []; - - private worker: ExtractionWorkerHandler; - private config: Config; + static createdAtBlock = 13725566; + createdAtBlock = Sound.createdAtBlock; + deprecatedAtBlock = null; + static invalidIDs = []; + chain = CHAINS.eth; + worker: ExtractionWorkerHandler; + config: Config; + localStorage: AbstractSublevel, string | Buffer | Uint8Array, string, any>; + contracts: AbstractSublevel; constructor(worker: ExtractionWorkerHandler, config: Config) { this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(Sound.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); } - filterContracts = async (from: number, to: number) => { + crawl = async (from: number, to: number, recrawl: boolean) => { + const { getLogsBlockSpanSize } = this.config.chain[this.chain]; + + const handleArtistCreatedPromises: Array> = []; + + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) + handleArtistCreatedPromises.push(this.handleArtistCreated(i, i + getLogsBlockSpanSize)); + await Promise.all(handleArtistCreatedPromises); + + await handleTransfer.call(this, from, to, recrawl); + }; + + handleArtistCreated = async (from: number, to: number) => { const artistCreatedSelector = "0x23748b43b77f98380e738976c6324996908ffc1989994dd3c68631c87a65a7c0"; - const rpcHost = randomItem(this.config.rpc); + const rpcHost = randomItem(this.config.chain[this.chain].rpc); const options = { url: rpcHost.url, headers: { @@ -105,10 +129,17 @@ export default class Sound implements Strategy { }; }); - return contracts; + await Promise.all( + contracts.map(async (c) => { + // Save contract address that is to be checked for NFTs in future + await this.contracts.put(c.address, { name: c.name, version: c.version }); + }), + ); }; - crawl = async (nft: NFT) => { + nftToUid = async (nft: NFT) => `${this.chain}/${Sound.name}/${nft.erc721.address.toLowerCase()}`; + + fetchMetadata = async (nft: NFT) => { // Instead of querying at the block number soundxyz NFT // was minted, we query at a higher block number because // soundxyz changed their tokenURI and the previous one @@ -128,27 +159,22 @@ export default class Sound implements Strategy { return null; } - nft.erc721.token.uri = await callTokenUri( - this.worker, - this.config, + nft.erc721.token.uri = (await callTokenUri.call( + this, Math.max(nft.erc721.blockNumber, WORKING_AFTER_BLOCK), nft, - ); + )) as string; nft.erc721.token.uriContent = await fetchTokenUri(nft.erc721.token.uri, this.worker); - nft.creator = await callOwner( - this.worker, - this.config, - nft.erc721.address, - nft.erc721.blockNumber, - ); + nft.creator = await callOwner.call(this, nft.erc721.address, nft.erc721.blockNumber); const datum = nft.erc721.token.uriContent; return { version: Sound.version, title: datum.name, + uid: await this.nftToUid(nft), artist: { version: Sound.version, name: datum.artist_name, @@ -162,21 +188,28 @@ export default class Sound implements Strategy { erc721: { version: Sound.version, createdAt: nft.erc721.blockNumber, - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, address: nft.erc721.address, - tokenId: nft.erc721.token.id, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.name, - description: datum.description, - image: datum.image, - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.name, + description: datum.description, + image: datum.image, + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -197,6 +230,4 @@ export default class Sound implements Strategy { ], }; }; - - updateOwner(nft: NFT) {} } diff --git a/src/strategies/sound_protocol.ts b/src/strategies/sound_protocol.ts index f67d1cc..d783019 100644 --- a/src/strategies/sound_protocol.ts +++ b/src/strategies/sound_protocol.ts @@ -7,19 +7,20 @@ import { AbstractSublevel } from "abstract-level"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getArweaveTokenUri } from "../components/get-arweave-tokenuri.js"; import { callOwner } from "../components/call-owner.js"; -import { CHAINS, Config, NFT } from "../types.js"; +import { CHAINS, Config, Contract, NFT } from "../types.js"; import { ERC721Strategy } from "./strategy.types.js"; import { ethGetLogs } from "../components/eth-get-logs.js"; import { localStorage } from "../../database/localstorage.js"; import { handleTransfer } from "../components/handle-transfer.js"; +import { z } from "zod"; +import { tracksDB } from "../../database/tracks.js"; export default class SoundProtocol implements ERC721Strategy { static version = "2.0.0"; - createdAtBlock = 15570834; + static createdAtBlock = 15570834; + createdAtBlock = SoundProtocol.createdAtBlock; deprecatedAtBlock = null; static invalidIDs = []; - static TRANSFER_EVENT_SELECTOR = - "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; static EDITION_CREATED_SELECTOR = "0x405098db99342b699216d8150e930dbbf2f686f5a43485aed1e69219dafd4935"; @@ -28,6 +29,7 @@ export default class SoundProtocol implements ERC721Strategy { worker: ExtractionWorkerHandler; config: Config; localStorage: AbstractSublevel, string | Buffer | Uint8Array, string, any>; + contracts: AbstractSublevel; constructor(worker: ExtractionWorkerHandler, config: Config) { this.worker = worker; @@ -35,17 +37,24 @@ export default class SoundProtocol implements ERC721Strategy { this.localStorage = localStorage.sublevel(SoundProtocol.name, { valueEncoding: "json", }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); } crawl = async (from: number, to: number, recrawl: boolean) => { const { getLogsBlockSpanSize } = this.config.chain[this.chain]; const handleEditionCreatedPromises = []; + console.time(`${SoundProtocol.name} handleEditionCreated: ${from}-${to}`); for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) handleEditionCreatedPromises.push(this.handleEditionCreated(i, i + getLogsBlockSpanSize)); await Promise.all(handleEditionCreatedPromises); + console.timeEnd(`${SoundProtocol.name} handleEditionCreated: ${from}-${to}`); + console.time(`${SoundProtocol.name} handleTransfer: ${from}-${to}`); await this.handleTransfer(from, to, recrawl); + console.timeEnd(`${SoundProtocol.name} handleTransfer: ${from}-${to}`); }; handleEditionCreated = async (from: number, to: number) => { @@ -73,15 +82,11 @@ export default class SoundProtocol implements ERC721Strategy { }; }); - const contractsStorage = this.localStorage.sublevel("contracts", { - valueEncoding: "json", - }); - await Promise.all( contracts.map(async (c) => { console.log("Found a SoundProtocol contract", c.address); // Save contract address that is to be checked for NFTs in future - await contractsStorage.put(c.address, { name: c.name, version: c.version }); + await this.contracts.put(c.address, { name: c.name, version: c.version }); }), ); }; @@ -97,12 +102,34 @@ export default class SoundProtocol implements ERC721Strategy { `${nft.erc721.address}/${nft.erc721.token.id}`.match(id), ).length != 0 ) { - console.log( - `Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because it is blacklisted`, - ); + // console.log( + // `Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because it is blacklisted`, + // ); return null; } + const uid = await this.nftToUid(nft); + + if (await tracksDB.isTrackPresent(uid)) { + // Metadata already present, don't fetch from arweave again. + const track = await tracksDB.getTrack(uid); + + track.erc721.tokens.push({ + id: nft.erc721.token.id, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }); + + return track; + } + nft.erc721.token.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft); if (!nft.erc721.token.uri.includes("ar://")) { @@ -131,6 +158,16 @@ export default class SoundProtocol implements ERC721Strategy { try { const datum = nft.erc721.token.uriContent as any; + const schema = z.object({ + name: z.string(), + artist: z.string(), + description: z.string(), + image: z.string(), + losslessAudio: z.string(), + }); + + schema.passthrough().parse(datum); + return { version: SoundProtocol.version, title: datum.name, @@ -142,7 +179,7 @@ export default class SoundProtocol implements ERC721Strategy { }, platform: { version: SoundProtocol.version, - name: "Sound Protocol", + name: SoundProtocol.name, uri: "https://sound.xyz", }, erc721: { @@ -152,13 +189,6 @@ export default class SoundProtocol implements ERC721Strategy { tokens: [ { id: nft.erc721.token.id, - uri: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.name, - description: datum.description, - image: datum.image, - }, owners: [ { from: nft.erc721.transaction.from, @@ -170,6 +200,13 @@ export default class SoundProtocol implements ERC721Strategy { ], }, ], + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.name, + description: datum.description, + image: datum.image, + }, }, manifestations: [ { @@ -184,11 +221,11 @@ export default class SoundProtocol implements ERC721Strategy { }, ], }; - } catch { + } catch (err: any) { // Failed to transform the track. Most probably the metadata is // incorrectly formatted. Ignoring the track. console.log( - `Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because of incorrect metadata`, + `Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because of incorrect metadata - ${err.code}`, ); return null; } diff --git a/src/strategies/strategy.types.ts b/src/strategies/strategy.types.ts index 0a689b1..75f75ea 100644 --- a/src/strategies/strategy.types.ts +++ b/src/strategies/strategy.types.ts @@ -6,7 +6,7 @@ import { CHAINS, Config, Contract, NFT } from "../types.js"; export declare class Strategy { public static version: string; - + // We have both static and non-static variable. Both should be equal. // static is used to get chain without initialising the class (eg. ClassName.chain) // non-static is used for `this.chain` @@ -16,8 +16,9 @@ export declare class Strategy { * Neume will not include the strategy in the crawl * if the range of the crawl is not included in between * createdAtBlock and deprecatedAtBlock, both inclusive. - */ - public createdAtBlock: number; + */ + public createdAtBlock: number; + public static createdAtBlock: number; /** * Neume will not include the strategy in the crawl * if the range of the crawl is not included in between @@ -26,7 +27,7 @@ export declare class Strategy { public deprecatedAtBlock: number | null; public worker: ExtractionWorkerHandler; public config: Config; - public localStorage: AbstractSublevel< + public localStorage?: AbstractSublevel< Level, string | Buffer | Uint8Array, string, @@ -47,5 +48,18 @@ export declare class Strategy { } export declare class ERC721Strategy extends Strategy { + /** + * ERC721 Contracts where NFTs are published. + * Addresses added to this storage will be crawled by `handleTransfer`. + * + * **Implementation**: + * ``` + * this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + ``` + **/ + contracts: AbstractSublevel; + fetchMetadata: (nft: NFT) => Promise; } diff --git a/src/strategies/zora.ts b/src/strategies/zora.ts index 626a227..117ee65 100644 --- a/src/strategies/zora.ts +++ b/src/strategies/zora.ts @@ -10,32 +10,48 @@ import { toHex, encodeFunctionCall, decodeParameters } from "eth-fun"; import { anyIpfsToNativeIpfs } from "ipfs-uri-utils"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getIpfsTokenUri } from "../components/get-ipfs-tokenuri.js"; -import { NFT, Config } from "../types.js"; +import { NFT, Config, CHAINS, Contract } from "../types.js"; import { randomItem } from "../utils.js"; -import { Strategy } from "./strategy.types.js"; +import { ERC721Strategy } from "./strategy.types.js"; +import { handleTransfer } from "../components/handle-transfer.js"; +import { AbstractSublevel } from "abstract-level"; +import { Level } from "level"; +import { localStorage } from "../../database/localstorage.js"; -export default class Zora implements Strategy { - public static version = "1.0.0"; - public static createdAtBlock = 11996516; // First catalog song: https://etherscan.io/nft/0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7/1678 +export default class Zora implements ERC721Strategy { + static version = "1.0.0"; + static createdAtBlock = 11996516; + createdAtBlock = Zora.createdAtBlock; // First catalog song: https://etherscan.io/nft/0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7/1678 // Last song on Zora contract: https://beta.catalog.works/lucalush/velvet-girls // https://cloud.hasura.io/public/graphiql?endpoint=https%3A%2F%2Fcatalog-prod.hasura.app%2Fv1%2Fgraphql&query=query+MyQuery+%7B%0A++tracks%28%0A++++where%3A+%7Bcontract_address%3A+%7B_iregex%3A+%220xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7%22%7D%7D%0A++++order_by%3A+%7Bcreated_at%3A+desc%7D%0A++%29+%7B%0A++++created_at%0A++++contract_address%0A++++short_url%0A++++title%0A++++nft_id%0A++%7D%0A%7D%0A - public static deprecatedAtBlock = null; - private worker: ExtractionWorkerHandler; - private config: Config; + deprecatedAtBlock = null; + worker: ExtractionWorkerHandler; + config: Config; + chain = CHAINS.eth; + localStorage: AbstractSublevel, string | Buffer | Uint8Array, string, any>; + contracts: AbstractSublevel; constructor(worker: ExtractionWorkerHandler, config: Config) { this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(Zora.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + + const ZORA_NFT_CONTRACT = "0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7"; + this.contracts.put(ZORA_NFT_CONTRACT, { name: Zora.name, version: Zora.version }); } - async crawl(nft: NFT) { - nft.erc721.token.uri = await callTokenUri( - this.worker, - this.config, - nft.erc721.blockNumber, - nft, - ); + crawl = async (from: number, to: number, recrawl: boolean) => { + await handleTransfer.call(this, from, to, recrawl); + }; + + fetchMetadata = async (nft: NFT) => { + nft.erc721.token.uri = (await callTokenUri.call(this, nft.erc721.blockNumber, nft)) as string; try { nft.erc721.token.uri = anyIpfsToNativeIpfs(nft.erc721.token.uri); @@ -48,7 +64,7 @@ export default class Zora implements Strategy { return null; } - nft.metadata.uri = await callTokenUri(this.worker, this.config, nft.erc721.blockNumber, nft, { + nft.metadata.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft, { name: "tokenMetadataURI", type: "function", inputs: [ @@ -71,7 +87,7 @@ export default class Zora implements Strategy { } try { - nft.metadata.uriContent = await getIpfsTokenUri(nft.metadata.uri, this.worker, this.config); + nft.metadata.uriContent = await getIpfsTokenUri.call(this, nft.metadata.uri); } catch (err: any) { if (err.message.includes("Invalid CID")) { console.warn("Invalid CID: Ignoring the given track.", JSON.stringify(nft, null, 2)); @@ -115,6 +131,7 @@ export default class Zora implements Strategy { version: Zora.version, title, duration, + uid: await this.nftToUid(nft), artist: { version: Zora.version, name: artist, @@ -128,21 +145,28 @@ export default class Zora implements Strategy { erc721: { version: Zora.version, createdAt: nft.erc721.blockNumber, - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, address: nft.erc721.address, - tokenId: nft.erc721.token.id, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: title, - description, - // TODO: add image here - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: title, + description, + // TODO: add image here + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -157,16 +181,17 @@ export default class Zora implements Strategy { }, ], } as Track; - } + }; - updateOwner(nft: NFT) {} + nftToUid = async (nft: NFT) => + `${this.chain}/${Zora.name}/${nft.erc721.address.toLowerCase()}/${nft.erc721.token.id}`; private callTokenCreator = async ( to: string, blockNumber: number, tokenId: string, ): Promise => { - const rpc = randomItem(this.config.rpc); + const rpc = randomItem(this.config.chain[this.chain].rpc); const data = encodeFunctionCall( { name: "tokenCreators", diff --git a/src/types.ts b/src/types.ts index 689b1da..5fdea3c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -87,7 +87,6 @@ export type JsonRpcLog = { }; export type Contract = { - address: string; name: string; version: string; }; diff --git a/src/utils.ts b/src/utils.ts index a8e63a0..ae12d86 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,16 +1,14 @@ -import { readFile } from "fs/promises"; -import path from "path"; import https from "https"; import { Strategy } from "./strategies/strategy.types.js"; -import { CHAINS, Contracts, PROTOCOLS, RpcConfig } from "./types.js"; +import { PROTOCOLS, RpcConfig } from "./types.js"; -// import Sound from "./strategies/sound.js"; +import Sound from "./strategies/sound.js"; import SoundProtocol from "./strategies/sound_protocol.js"; -// import Zora from "./strategies/zora.js"; -// import CatalogV2 from "./strategies/catalog_v2.js"; -// import MintSongsV2 from "./strategies/mintsongs_v2.js"; -// import Noizd from "./strategies/noizd.js"; +import Zora from "./strategies/zora.js"; +import CatalogV2 from "./strategies/catalog_v2.js"; +import MintSongsV2 from "./strategies/mintsongs_v2.js"; +import Noizd from "./strategies/noizd.js"; import Lens from "./strategies/lens/lens.js"; export function randomItem(arr: Array): T { @@ -50,50 +48,26 @@ export function getLatestBlockNumber(rpcHost: RpcConfig): Promise { }); } -export async function getDefaultContracts(): Promise { - const defaultContractsPath = new URL("../assets/contracts.hardcode.json", import.meta.url); - - return JSON.parse(await readFile(defaultContractsPath, "utf-8")); -} - -export async function getUserContracts(): Promise { - const userContractsPath = path.resolve("./data/contracts.json"); - - return JSON.parse(await readFile(userContractsPath, "utf-8")); -} - -/** - * User's contracts.json contains the new found addresses - * Neume's contracts.hardcode.json contains hardcoded addresses - * This function reads and merge them both. - */ -export async function getAllContracts(): Promise { - return { - ...(await getDefaultContracts()), - ...(await getUserContracts()), - }; -} - /** * New strategies should be added here. */ export function getStrategies(strategyNames: string[]) { const strategies: Array = [ - // Sound, + Sound, Lens, SoundProtocol, - // Zora, - // CatalogV2, - // MintSongsV2, - // Noizd, + Zora, + CatalogV2, + MintSongsV2, + Noizd, ]; return strategies.filter((s) => strategyNames.includes(s.name)); } -export function getProtocol(uri: string): PROTOCOLS { - if (uri.includes("ar://")) return PROTOCOLS.arweave; - else if (uri.includes("ipfs://")) return PROTOCOLS.ipfs; - else if (uri.includes("http://") || uri.includes("https://")) return PROTOCOLS.https; - throw new Error(`Invalid Protocl for ${uri}`); +export function getProtocol(uri: string): PROTOCOLS | null { + if (uri.startsWith("ar://")) return PROTOCOLS.arweave; + else if (uri.startsWith("ipfs://")) return PROTOCOLS.ipfs; + else if (uri.startsWith("http://") || uri.startsWith("https://")) return PROTOCOLS.https; + return null; } From 400791aeb5445d82c29b8e5054cd4bf375a90ab8 Mon Sep 17 00:00:00 2001 From: il3ven Date: Tue, 1 Aug 2023 01:01:50 +0530 Subject: [PATCH 06/15] update commands - Remove crawl and filter-contracts command. The daemon command now handles both functionalities. - Update init command. - Daemon now runs each strategy individually. Earlier, if a strategy A was at block number X then strategy B was at X too. Now, all strategies run independently. Very obvious but significant change. - Update sync command according to the new database. - Update JSON RPC endpoints for syncing. --- assets/config.sample.js | 46 +++- assets/contracts.hardcode.json | 18 -- commands/crawl.ts | 315 ----------------------- commands/daemon.ts | 49 ++-- commands/daemon/daemon-jsonrpc-schema.js | 120 +++------ commands/daemon/daemon-jsonrpc-type.d.ts | 26 +- commands/dump.ts | 4 + commands/filter_contracts.ts | 44 ---- commands/init.ts | 10 +- commands/sync.ts | 92 ++++--- neume.ts | 81 +----- 11 files changed, 185 insertions(+), 620 deletions(-) delete mode 100644 assets/contracts.hardcode.json delete mode 100644 commands/crawl.ts delete mode 100644 commands/filter_contracts.ts diff --git a/assets/config.sample.js b/assets/config.sample.js index 2eae60a..fc2d04d 100644 --- a/assets/config.sample.js +++ b/assets/config.sample.js @@ -3,44 +3,54 @@ * - Configuration common for all commands * - Configuration too verbose for CLI * - Sensitive information - * + * * neume will read values from .env. Therefore, you can do * env.RPC_API_KEYS - * + * */ import { env } from "process"; -const rpcHosts = [ - { url: "https://rpc.ankr.com/eth" }, - { url: "https://cloudflare-eth.com/" }, -]; +const rpcHosts = [{ url: "https://rpc.ankr.com/eth" }, { url: "https://cloudflare-eth.com/" }]; +const polygonRpcHosts = [{ url: "https://rpc.ankr.com/polygon" }]; /** * A list of strategies to enable. Remove entries from the list to run * selected strategies. - * + * * Note: * - The name should match the class name of the strategy. * - The name is case-sensitive. * */ -export const strategies = ["Sound", "SoundProtocol", "MintSongsV2", "CatalogV2", "Zora"]; - +export const strategies = ["Sound", "SoundProtocol", "MintSongsV2", "CatalogV2", "Zora", "Lens"]; /** * Configuration for neume - * + * * TypeScipt type for configuration can be found at: * https://github.com/neume-network/crawler/blob/main/types.ts#L25 */ export const config = { - rpc: rpcHosts, arweave: { httpsGateway: "https://arweave.net", }, - crawlStep: 5000, - getLogsBlockSpanSize: 799, - getLogsAddressSize: 100, + ipfs: { + httpsGateway: "https://ipfs.io/ipfs/", + }, + chain: { + eth: { + crawlStep: 5000, + rpc: rpcHosts, + getLogsBlockSpanSize: 799, + getLogsAddressSize: 100, + }, + polygon: { + crawlStep: 10_000, + rpc: polygonRpcHosts, + getLogsBlockSpanSize: 2000, + getLogsAddressSize: 799, + }, + }, breatheTimeMS: 900_000, // 15 mins worker: { queue: { @@ -57,6 +67,14 @@ export const config = { }; return prevValue; }, {}), + ...polygonRpcHosts.reduce((prevValue, host) => { + prevValue[host.url] = { + timeout: 120_000, + requestsPerUnit: 300, + unit: "second", + }; + return prevValue; + }, {}), "https://arweave.net": { timeout: 120_000, requestsPerUnit: 1000, diff --git a/assets/contracts.hardcode.json b/assets/contracts.hardcode.json deleted file mode 100644 index 6b1e8d4..0000000 --- a/assets/contracts.hardcode.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7": { - "name": "Zora", - "version": "1.0.0" - }, - "0x0bc2a24ce568dad89691116d5b34deb6c203f342": { - "name": "CatalogV2", - "version": "2.0.0" - }, - "0x2b5426a5b98a3e366230eba9f95a24f09ae4a584": { - "name": "MintSongsV2", - "version": "2.0.0" - }, - "0xf5819e27b9bad9f97c177bf007c1f96f26d91ca6": { - "name": "Noizd", - "version": "1.0.0" - } -} \ No newline at end of file diff --git a/commands/crawl.ts b/commands/crawl.ts deleted file mode 100644 index 90910a9..0000000 --- a/commands/crawl.ts +++ /dev/null @@ -1,315 +0,0 @@ -import ExtractionWorker, { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; -import { toHex, decodeLog } from "eth-fun"; - -import { db } from "../database/index.js"; -import { JsonRpcLog, NFT, Config, Contracts } from "../src/types.js"; -import { getAllContracts, randomItem } from "../src/utils.js"; -import { Strategy } from "../src/strategies/strategy.types.js"; -import { Track } from "@neume-network/schema"; - -const TRANSFER_EVENT_SELECTOR = - "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; -const CHAIN_ID = "1"; - -// from and to are supposed to be included -export default async function ( - from: number, - to: number, - recrawl: boolean, - config: Config, - _strategies: typeof Strategy[], -) { - const allContracts = await getAllContracts(); - const contracts = Object.entries(allContracts).reduce((prevValue, [addr, info]) => { - if (_strategies.filter((s) => s.name === info.name).length) - prevValue = { ...prevValue, ...{ [addr]: info } }; - return prevValue; - }, {} as Contracts); - const worker = ExtractionWorker(config.worker); - const strategies = _strategies.map((s) => new s(worker, config)); - - for (let i = from; i <= to; i += config.crawlStep + 1) { - const fromBlock = i; - const toBlock = Math.min(to, i + config.crawlStep); - await crawl(fromBlock, toBlock, recrawl, config, strategies, contracts, worker); - } - - console.log("Exiting from crawl command"); -} - -// (from - to) must not be too big -// from and to are supposed to be included -async function crawl( - from: number, - to: number, - recrawl: boolean, - config: Config, - strategies: Strategy[], - contracts: Contracts, - worker: ExtractionWorkerHandler, -) { - console.log("Crawling from", from, "to", to); - - /** - * We start crawling the NFT if the log is for a mint, else if - * the log is for a transfer we wait for all the mints to be - * processed. - * - * This is because to process a transfer we update the owner - * field and not re-fetch the metadata. - */ - - /** - * We have processed all the mints if all the promises in this - * array have been resolved. - */ - const mintNFTsPromise: Promise[] = []; - - /** - * We store all logs/NFTs beloging to a transfer event in this - * array to process the later that is after all the mints. - */ - const allTransferNFTs: NFT[] = []; - - for (let i = from; i <= to; i += config.getLogsBlockSpanSize + 1) { - const fromBlock = i; - const toBlock = Math.min(to, i + config.getLogsBlockSpanSize); - - for (let j = 0; j < Object.keys(contracts).length; j += config.getLogsAddressSize) { - const contractsSlice = Object.keys(contracts).slice(j, j + config.getLogsAddressSize); - const rpcHost = randomItem(config.rpc); - - const msg = await worker({ - type: "json-rpc", - commissioner: "", - method: "eth_getLogs", - options: { - url: rpcHost.url, - headers: { - ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), - }, - retry: { - retries: 3, - }, - }, - params: [ - { - fromBlock: toHex(fromBlock), - toBlock: toHex(toBlock), - address: contractsSlice, - topics: [TRANSFER_EVENT_SELECTOR], - }, - ], - version: "0.0.1", - }); - - if (msg.error) { - console.error(msg); - throw new Error(`Error occured while fetching Transfer events`); - } - - const logs = msg.results as any as JsonRpcLog[]; - let nfts = logs.map((log) => prepareNFT(contracts, log)); - - // Partition NFTs into mints and transfers - const { mintNfts, transferNfts } = nfts.reduce( - (nfts, nft) => { - if (nft.erc721.transaction.from === "0x0000000000000000000000000000000000000000") { - nfts.mintNfts.push(nft); - } else { - nfts.transferNfts.push(nft); - } - return nfts; - }, - { mintNfts: [] as NFT[], transferNfts: [] as NFT[] }, - ); - - const promises = mintNfts.map(async (nft) => { - let nftExists = false; - - try { - nftExists = !!(await db.level.get( - db.datumToKey({ - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber, - }), - )); - } catch (err: any) { - if (err.code !== "LEVEL_NOT_FOUND") throw err; - } - - if (!recrawl && nftExists) return; - - const strategy = strategies.find((s) => s.constructor.name === nft.platform.name); - - if (!strategy) { - throw new Error( - `Couldn't find any strategy with the name of ${nft.platform.name} for address ${nft.erc721.address}`, - ); - } - - await fetchMetadata(nft, strategy); - }); - - mintNFTsPromise.push(...promises); - allTransferNFTs.push(...transferNfts); - } - } - - await Promise.all(mintNFTsPromise); - - await Promise.all( - allTransferNFTs.map(async (nft) => { - let nftExists = false; - - try { - nftExists = !!(await db.level.get( - db.datumToKey({ - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber, - }), - )); - } catch (err: any) { - if (err.code !== "LEVEL_NOT_FOUND") throw err; - } - - if (!recrawl && nftExists) return; - - const strategy = strategies.find((s) => s.constructor.name === nft.platform.name); - - if (!strategy) { - throw new Error( - `Couldn't find any strategy with the name of ${nft.platform.name} for address ${nft.erc721.address}`, - ); - } - - await updateOwnership(nft, strategy); - }), - ); -} - -function prepareNFT(contracts: Contracts, log: JsonRpcLog): NFT { - if (!log.topics[3] || !log.transactionHash || !log.blockNumber) { - throw new Error(`log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`); - } - - const decodedTopics = decodeLog( - [ - { indexed: true, name: "from", type: "address" }, - { indexed: true, name: "to", type: "address" }, - { indexed: true, name: "tokenId", type: "uint256" }, - ], - log.data, - log.topics.slice(1), - ); - - return { - platform: { - ...contracts[log.address], - }, - erc721: { - blockNumber: parseInt(log.blockNumber, 16), - address: log.address, - transaction: { - from: decodedTopics[0], - to: decodedTopics[1], - transactionHash: log.transactionHash, - blockNumber: parseInt(log.blockNumber, 16), - }, - token: { - id: BigInt(log.topics[3]).toString(10), - }, - }, - metadata: {}, - }; -} - -async function fetchMetadata(nft: NFT, strategy: Strategy) { - let track = null; - try { - track = await strategy?.crawl(nft); - } catch (err) { - console.log(track); - console.error(`Error occurured while crawling\n`, err, JSON.stringify(nft, null, 2)); - throw err; // Re-throwing to stop the application - } - - if (track !== null) { - await db.insert( - { - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber, - }, - track, - ); - - console.log( - "Found track:", - track?.title, - track?.platform.version, - track?.platform.name, - "at", - track?.erc721.createdAt, - ); - } -} - -async function updateOwnership(nft: NFT, strategy: Strategy) { - let track: Track | undefined = undefined; - - try { - // Get the last track - track = ( - await db.getOne({ - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber - 1, - }) - ).value; - } catch (err: any) { - if (err.code !== "LEVEL_NOT_FOUND") throw err; - } - - // If the NFT hasn't been crawled before we can't update its owner - if (track) { - track.erc721.transaction = { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }; - - // If in future we have to update strategy specific - // ownership data we can call updateOwner function - - await db.insert( - { - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber, - }, - track, - ); - - console.log( - "Update ownership of", - track.title, - track?.platform.name, - track?.platform.version, - "at", - nft.erc721.blockNumber, - "from", - nft.erc721.transaction.from, - "to", - nft.erc721.transaction.to, - ); - } -} diff --git a/commands/daemon.ts b/commands/daemon.ts index f613bd9..01428ee 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -1,7 +1,7 @@ import { fastify as Fastify } from "fastify"; -import { JSONRPCServer, JSONRPCErrorException } from "json-rpc-2.0"; +import { JSONRPCServer } from "json-rpc-2.0"; -import { CHAINS, Config } from "../src/types.js"; +import { Config } from "../src/types.js"; import { getLatestBlockNumber, getStrategies } from "../src/utils.js"; import { DaemonJsonrpcType } from "./daemon/daemon-jsonrpc-type.js"; import { daemonJsonrpcSchema } from "./daemon/daemon-jsonrpc-schema.js"; @@ -9,6 +9,7 @@ import { getLastCrawledBlock, saveLastCrawledBlock } from "../src/state.js"; import { tracksDB } from "../database/tracks.js"; import { getLocalStorage } from "../database/localstorage.js"; import ExtractionWorker from "@neume-network/extraction-worker"; +import { Strategy } from "../src/strategies/strategy.types.js"; const fastify = Fastify(); @@ -22,36 +23,31 @@ export default async function daemon( const worker = ExtractionWorker(config.worker); const allStrategies = getStrategies(strategyNames).map((s) => new s(worker, config)); - const task = async (chain: CHAINS) => { - const { crawlStep } = config.chain[chain]; - const latestBlockNumber = await getLatestBlockNumber(config.chain[chain].rpc[0]); - const from = await getLastCrawledBlock(chain); + const task = async (strategy: Strategy) => { + const { crawlStep } = config.chain[strategy.chain]; + const latestBlockNumber = await getLatestBlockNumber(config.chain[strategy.chain].rpc[0]); + const from = await getLastCrawledBlock(strategy.constructor.name); const to = Math.min(from + crawlStep, latestBlockNumber); - const strategies = allStrategies.filter( - (s) => - s.createdAtBlock <= from && - to <= (s.deprecatedAtBlock ?? Number.MAX_VALUE) && - s.chain === chain, - ); + if (from >= (strategy.deprecatedAtBlock ?? Number.MAX_VALUE)) { + console.log( + `Removing ${strategy.constructor.name} from queue because we have reached deprecatedAtBlock (${strategy.deprecatedAtBlock})`, + ); + return; + } - await Promise.all( - strategies.map(async (strategy) => { - console.log("Calling strategy", strategy.constructor.name, "from", from, "to", to); - await strategy.crawl(from, to, recrawl); - }), - ); + console.log("Calling strategy", strategy.constructor.name, "from", from, "to", to); + await strategy.crawl(from, to, recrawl); - await saveLastCrawledBlock(chain, to); - const nextTaskWaitTime = to === latestBlockNumber ? config.breatheTimeMS : 0; + await saveLastCrawledBlock(strategy.constructor.name, to); + const nextTaskWaitTime = to === latestBlockNumber ? config.breatheTimeMS ?? 1 : 1; - setTimeout(task.bind({}, chain), nextTaskWaitTime); + setTimeout(task.bind({}, strategy), nextTaskWaitTime); }; if (crawlFlag) { - Object.values(CHAINS).forEach((c) => { - // Do not call the task for this chain if no strategies are present - if (allStrategies.filter((s) => s.chain === c).length) task(c); + allStrategies.forEach((s) => { + task(s); }); } @@ -61,11 +57,10 @@ export default async function daemon( async function startServer(port: number) { const server = new JSONRPCServer(); - server.addMethod("getTracks", async ({ from, to, platform }) => { - // TODO: Make it per platform + server.addMethod("getTracks", async ({ since, platform }) => { // if (to - from > 5000) // return new JSONRPCErrorException("Block range should be less than 5000", -32600); - const res = await tracksDB.getTracksChanged(from, to, platform); + const res = await tracksDB.getTracksChanged(since, platform); return res; }); diff --git a/commands/daemon/daemon-jsonrpc-schema.js b/commands/daemon/daemon-jsonrpc-schema.js index a3fd22a..97e0bf8 100644 --- a/commands/daemon/daemon-jsonrpc-schema.js +++ b/commands/daemon/daemon-jsonrpc-schema.js @@ -1,97 +1,59 @@ import { compile } from "json-schema-to-typescript"; import { fileURLToPath } from "url"; -const _daemonJsonrpcSchema = { - "type": "object", - "title": 'DaemonJsonrpcType', - "oneOf": [ - { - "$ref": "#/$defs/getIdsChanged_fill", - }, - { - "$ref": "#/$defs/getAllContracts", - }, - ], - "required": ["jsonrpc", "id", "method"], - "$defs": { - "getIdsChanged_fill": { - "type": "object", - "properties": { - "jsonrpc": { "type": "string", "const": "2.0" }, - "id": { "type": "string" }, - "method": { "type": "string", "const": "getIdsChanged_fill" }, - "params": { - "type": "array", - "minItems": 2, - "maxItems": 2, - "items": { - "type": "string" - } - } - }, - "required": ["jsonrpc", "id", "method", "params"] - }, - "getAllContracts": { - "type": "object", - "properties": { - "jsonrpc": { "type": "string", "const": "2.0" }, - "id": { "type": "string" }, - "method": { "type": "string", "const": "getAllContracts" }, - }, - "required": ["jsonrpc", "id", "method"] - } - }, -} - export const daemonJsonrpcSchema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": 'DaemonJsonrpcType', - "oneOf": [ - { "$ref": "#/$defs/request" }, - { "type": "array", "items": { "$ref": "#/$defs/request" } } - ], - "$defs": { - "request": { - "type": "object", - "oneOf": [ + $schema: "http://json-schema.org/draft-07/schema#", + title: "DaemonJsonrpcType", + oneOf: [{ $ref: "#/$defs/request" }, { type: "array", items: { $ref: "#/$defs/request" } }], + $defs: { + request: { + type: "object", + oneOf: [ { - "$ref": "#/$defs/getIdsChanged_fill", + $ref: "#/$defs/getTracks", }, { - "$ref": "#/$defs/getUserContracts", + $ref: "#/$defs/getLocalStorage", }, ], }, - "getIdsChanged_fill": { - "type": "object", - "properties": { - "jsonrpc": { "type": "string", "const": "2.0" }, - "id": { "type": "string" }, - "method": { "type": "string", "const": "getIdsChanged_fill" }, - "params": { - "type": "array", - "minItems": 2, - "maxItems": 2, - "items": { - "type": "number" - } - } + getTracks: { + type: "object", + properties: { + jsonrpc: { type: "string", const: "2.0" }, + id: { type: "string" }, + method: { type: "string", const: "getTracks" }, + params: { + type: "object", + properties: { + since: { type: "number", $comment: "Since is a unix timestamp" }, + platform: { type: "string" }, + }, + required: ["since", "platform"], + }, }, - "required": ["jsonrpc", "id", "method", "params"] + required: ["jsonrpc", "id", "method", "params"], }, - "getUserContracts": { - "type": "object", - "properties": { - "jsonrpc": { "type": "string", "const": "2.0" }, - "id": { "type": "string" }, - "method": { "type": "string", "const": "getUserContracts" }, + getLocalStorage: { + type: "object", + properties: { + jsonrpc: { type: "string", const: "2.0" }, + id: { type: "string" }, + method: { type: "string", const: "getLocalStorage" }, + params: { + type: "object", + properties: { + platform: { type: "string" }, + }, + required: ["platform"], + }, }, - "required": ["jsonrpc", "id", "method"] - } + required: ["jsonrpc", "id", "method", "params"], + }, }, -} +}; // This is called from package.json to compile the above type if (process.argv[1] === fileURLToPath(import.meta.url)) { - compile(daemonJsonrpcSchema).then(console.log) + compile(daemonJsonrpcSchema).then(console.log); } diff --git a/commands/daemon/daemon-jsonrpc-type.d.ts b/commands/daemon/daemon-jsonrpc-type.d.ts index bb4f2a6..72252c4 100644 --- a/commands/daemon/daemon-jsonrpc-type.d.ts +++ b/commands/daemon/daemon-jsonrpc-type.d.ts @@ -1,4 +1,4 @@ -/* tslint:disable */ +/* eslint-disable */ /** * This file was automatically generated by json-schema-to-typescript. * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, @@ -6,23 +6,27 @@ */ export type DaemonJsonrpcType = Request | Request[]; -export type Request = GetIdsChangedFill | GetUserContracts; +export type Request = GetTracks | GetLocalStorage; -export interface GetIdsChangedFill { +export interface GetTracks { jsonrpc: "2.0"; id: string; - method: "getIdsChanged_fill"; - /** - * @minItems 2 - * @maxItems 2 - */ - params: [number, number]; + method: "getTracks"; + params: { + since: number; + platform: string; + [k: string]: unknown; + }; [k: string]: unknown; } -export interface GetUserContracts { +export interface GetLocalStorage { jsonrpc: "2.0"; id: string; - method: "getUserContracts"; + method: "getLocalStorage"; + params: { + platform: string; + [k: string]: unknown; + }; [k: string]: unknown; } diff --git a/commands/dump.ts b/commands/dump.ts index f69f458..0fc34fe 100644 --- a/commands/dump.ts +++ b/commands/dump.ts @@ -1,3 +1,7 @@ +/** + * This file is outdated. + */ + import { Track } from "@neume-network/schema"; import fs from "fs"; import { writeFile, mkdir } from "fs/promises"; diff --git a/commands/filter_contracts.ts b/commands/filter_contracts.ts deleted file mode 100644 index eb6389c..0000000 --- a/commands/filter_contracts.ts +++ /dev/null @@ -1,44 +0,0 @@ -import ExtractionWorker from "@neume-network/extraction-worker"; -import path from "path"; -import { readFile, writeFile } from "fs/promises"; - -import { Config } from "../src/types.js"; -import { Strategy } from "../src/strategies/strategy.types.js"; -import { getUserContracts } from "../src/utils.js"; - -export default async function ( - from: number, - to: number, - recrawl: boolean, - config: Config, - _strategies: typeof Strategy[], -) { - if (!config.rpc.length) throw new Error("Atleast one RPC host is required"); - - const userContracts = await getUserContracts(); - const worker = ExtractionWorker(config.worker); - const strategies = _strategies.map((s) => new s(worker, config)); - - for (let i = from; i <= to; i += config.getLogsBlockSpanSize) { - const fromBlock = i; - const toBlock = Math.min(to, i + config.getLogsBlockSpanSize); - console.log("Finding contracts from", fromBlock, toBlock); - - await Promise.all( - strategies.map(async (strategy) => { - if (!strategy.filterContracts) return; - - const newContracts = await strategy.filterContracts(fromBlock, toBlock); - newContracts.forEach((contract) => { - userContracts[contract.address] = { - name: contract.name, - version: contract.version, - }; - }); - }), - ); - } - - await writeFile(path.resolve("./data/contracts.json"), JSON.stringify(userContracts, null, 2)); - console.log("Exiting from filter-contracts command"); -} diff --git a/commands/init.ts b/commands/init.ts index 16e32d7..156ed3e 100644 --- a/commands/init.ts +++ b/commands/init.ts @@ -1,7 +1,8 @@ import path from "path"; import fs from "fs/promises"; import { saveLastCrawledBlock } from "../src/state.js"; -import { CONSTANTS } from "../src/types.js"; +import { CHAINS, CONSTANTS } from "../src/types.js"; +import runMigration from "../database/runMigration.js"; export default async function init() { await fs.copyFile(new URL("../assets/.env-copy", import.meta.url), path.resolve(".env")); @@ -9,10 +10,7 @@ export default async function init() { new URL("../assets/config.sample.js", import.meta.url), path.resolve("./config.js"), ); - // Will create file if it does not exist - await fs.writeFile(path.resolve("./data/contracts.json"), "{}", { - flag: "w", - }); // Create the last_crawled_block file in ./data - await saveLastCrawledBlock(CONSTANTS.FIRST_BLOCK); + await saveLastCrawledBlock(CHAINS.eth, CONSTANTS.FIRST_BLOCK[CHAINS.eth]); + await runMigration("up"); } diff --git a/commands/sync.ts b/commands/sync.ts index 975fa04..3f014f8 100644 --- a/commands/sync.ts +++ b/commands/sync.ts @@ -1,19 +1,21 @@ import { JSONRPCClient } from "json-rpc-2.0"; import ExtractionWorker from "@neume-network/extraction-worker"; -import fs from "fs/promises"; -import { db, ReturnValue } from "../database/index.js"; -import { Config, CONSTANTS } from "../src/types.js"; -import path from "path"; -import { getUserContracts } from "../src/utils.js"; +import { Config } from "../src/types.js"; +import { Strategy } from "../src/strategies/strategy.types.js"; +import { localStorage, saveLocalStorage } from "../database/localstorage.js"; +import { tracksDB } from "../database/tracks.js"; +import { Track } from "@neume-network/schema"; -async function getLastSyncedBlock() { - const lastId = await db.changeIndex.iterator({ reverse: true, limit: 1 }).next(); - return lastId ? parseInt(lastId[0].split("/")[0]) : 15000000; -} - -export default async function (from: number | undefined, to: number, url: string, config: Config) { +const sync = async function ( + _since: number | undefined, + url: string, + config: Config, + strategies: (typeof Strategy)[], +) { const worker = ExtractionWorker(config.worker); + const storage = localStorage.sublevel("sync", {}); + const normalizedUrl = new URL(url).host; let client: JSONRPCClient; let id = 0; client = new JSONRPCClient( @@ -41,31 +43,53 @@ export default async function (from: number | undefined, to: number, url: string () => (++id).toString(), // HACK because of a bug in JSON-RPC-Client ); - let syncFrom = from ?? (await getLastSyncedBlock()); - - console.log("Will sync from", syncFrom, "to", to); + // Assuming localstorage won't be too big in size. We can + // ask for everything and update our localstorage. + const syncLocalStorage = async (platform: string) => { + const localStorage = await client.request("getLocalStorage", { + platform, + }); - // Sync contracts - const userContractsNew = await client.request("getUserContracts", null); - const userContractsOld = await getUserContracts(); - const userContracts = { ...userContractsNew, ...userContractsOld }; - const userContractsPath = path.resolve("./data/contracts.json"); - await fs.writeFile(userContractsPath, JSON.stringify(userContracts, null, 2)); - console.log("Updated local list of contracts"); + await saveLocalStorage(platform, localStorage); + }; - for (let syncedTill = syncFrom; syncedTill <= to; syncedTill += 5000) { - console.log(`Syncing from ${syncedTill} to ${syncedTill + 5000}`); - const returnValues = (await client.request("getIdsChanged_fill", [ - syncedTill, - syncedTill + 5000, - ])) as ReturnValue[]; + const syncTracks = async (platform: string, since: number) => { + const { tracks, nextTimestamp } = (await client.request("getTracks", { + since, + platform, + })) as { + tracks: Track[]; + nextTimestamp: number; + }; - await Promise.all( - returnValues.map(async (r) => { - await db.insert(r.id, r.value); - }), + // This will either create a new track or merge with the existing track + await Promise.all(tracks.map(async (track) => tracksDB.upsertTrack(track))); + console.log( + `Upserted ${tracks.length} tracks for ${platform}. startTimestamp=${since} nextTimestamp=${nextTimestamp}`, ); + if (nextTimestamp) { + await storage.put(`${normalizedUrl}-${platform}`, nextTimestamp); + return nextTimestamp; + } + }; + + await Promise.all( + strategies.map(async (strategy) => { + let lastSync: number | undefined; + + try { + lastSync = await storage.get(`${normalizedUrl}-${strategy.name}`); + } catch (err: any) { + if (err.code !== "LEVEL_NOT_FOUND") throw err; + } + + let since: number | undefined = _since ?? lastSync ?? 0; + syncLocalStorage(strategy.name); + do { + since = await syncTracks(strategy.name, since); + } while (since); + }), + ); +}; - console.log(`Wrote ${returnValues.length} entries to database`); - } -} +export default sync; diff --git a/neume.ts b/neume.ts index 9fe3841..e0a4c5d 100644 --- a/neume.ts +++ b/neume.ts @@ -1,21 +1,17 @@ #!/usr/bin/env node -//@ts-nocheck - import "dotenv/config"; import yargs from "yargs"; import { hideBin } from "yargs/helpers"; import path from "path"; -// import crawl from "./commands/crawl.js"; -// import dump from "./commands/dump.js"; -// import filterContracts from "./commands/filter_contracts.js"; import { getLatestBlockNumber, getStrategies } from "./src/utils.js"; import daemon from "./commands/daemon.js"; -// import sync from "./commands/sync.js"; +import sync from "./commands/sync.js"; import init from "./commands/init.js"; import { db } from "./database/index.js"; import runMigration from "./database/runMigration.js"; +import { tracksDB } from "./database/tracks.js"; const argv = yargs(hideBin(process.argv)) .usage("Usage: $0 ") @@ -35,60 +31,6 @@ const argv = yargs(hideBin(process.argv)) await runMigration(argv.type); }, ) - .command( - "crawl", - "Find new NFTs from the list of already known contracts [Out of date]", - { - from: { - type: "number", - describe: "From block number", - demandOption: true, - }, - to: { - type: "number", - describe: "To block number", - }, - recrawl: { - type: "boolean", - describe: "Re-crawl an NFT if they already exist", - default: false, - }, - }, - async (argv) => { - const { config, strategies: strategyNames } = await import(path.resolve("./config.js")); - const from = argv.from; - const to = argv.to ?? (await getLatestBlockNumber(config.rpc[0])); - await crawl(from, to, argv.recrawl, config, getStrategies(strategyNames, from, to)); - process.exit(0); - }, - ) - .command( - "filter-contracts", - "Find new contracts [Out of date]", - { - from: { - type: "number", - describe: "From block number", - demandOption: true, - }, - to: { - type: "number", - describe: "From block number", - }, - recrawl: { - type: "boolean", - describe: "Re-crawl an NFT if they already exist", - default: false, - }, - }, - async (argv) => { - const { config, strategies: strategyNames } = await import(path.resolve("./config.js")); - const from = argv.from; - const to = argv.to ?? (await getLatestBlockNumber(config.rpc[0])); - await filterContracts(from, to, argv.recrawl, config, getStrategies(strategyNames, from, to)); - process.exit(0); - }, - ) .command( "dump", "Export database as JSON [Out of date]", @@ -100,8 +42,10 @@ const argv = yargs(hideBin(process.argv)) }, }, async (argv) => { + throw new Error("Not Implemented"); const { config } = await import(path.resolve("./config.js")); const at = argv.at ?? (await getLatestBlockNumber(config.rpc[0])); + // @ts-ignore return dump(at); }, ) @@ -147,23 +91,16 @@ const argv = yargs(hideBin(process.argv)) describe: "An endpoint that is running the neume-network daemon", demandOption: true, }, - from: { + since: { type: "number", - describe: "From block number", - defaultDescription: "Uses the database to calculate the last synced block", - }, - to: { - type: "number", - describe: "To block number", - defaultDescription: "Syncs to the latest block number", + describe: "`since` is a timestamp. Neume will fetch changes since the provided timestamp.", + defaultDescription: "Uses the database to find the last synced timestamp", }, }, async (argv) => { const { config, strategies: strategyNames } = await import(path.resolve("./config.js")); - const to = argv.to ?? (await getLatestBlockNumber(config.rpc[0])); - // @ts-ignore - await sync(argv.from, to, argv.url, config); - process.exit(0); + await sync(argv.since, argv.url, config, getStrategies(strategyNames)); + await tracksDB.close(); }, ) .command("create-change-index", "Create change index from primary database", async (argv) => { From d42504a12578bbcc3e3545b2bba90063945d80b3 Mon Sep 17 00:00:00 2001 From: il3ven Date: Tue, 1 Aug 2023 02:24:45 +0530 Subject: [PATCH 07/15] build and update packages --- dist/assets/config.sample.js | 46 ++- dist/commands/crawl.js | 223 ----------- dist/commands/daemon.js | 62 ++-- dist/commands/daemon/daemon-jsonrpc-schema.js | 115 ++---- dist/commands/dump.js | 38 -- dist/commands/filter_contracts.js | 29 -- dist/commands/init.js | 10 +- dist/commands/sync.js | 71 ++-- dist/database/index.test.js | 248 ------------- dist/database/knexfile.js | 22 ++ dist/database/localstorage.js | 42 +++ .../migrations/20230318200321_schema.js | 73 ++++ dist/database/runMigration.js | 18 + dist/database/tracks.js | 241 ++++++++++++ dist/neume.js | 73 +--- dist/ownership-history.mjs | 56 --- dist/src/components/call-owner.js | 8 +- dist/src/components/call-tokenuri.js | 6 +- dist/src/components/eth-get-logs.js | 32 ++ dist/src/components/get-ipfs-tokenuri.js | 8 +- dist/src/components/handle-transfer.js | 133 +++++++ dist/src/state.js | 18 +- dist/src/strategies/catalog.js | 55 --- dist/src/strategies/catalog_v2.js | 70 ++-- dist/src/strategies/lens/components.js | 147 ++++++++ dist/src/strategies/lens/lens.js | 347 ++++++++++++++++++ dist/src/strategies/mintsongs_v2.js | 67 +++- dist/src/strategies/noizd.js | 65 +++- dist/src/strategies/sound.js | 74 ++-- dist/src/strategies/sound_protocol.js | 147 +++++--- dist/src/strategies/zora.js | 236 ++++++------ dist/src/types.js | 16 +- dist/src/utils.js | 39 +- package-lock.json | 98 ++--- package.json | 9 +- 35 files changed, 1755 insertions(+), 1187 deletions(-) delete mode 100644 dist/commands/crawl.js delete mode 100644 dist/commands/dump.js delete mode 100644 dist/commands/filter_contracts.js delete mode 100644 dist/database/index.test.js create mode 100644 dist/database/knexfile.js create mode 100644 dist/database/localstorage.js create mode 100644 dist/database/migrations/20230318200321_schema.js create mode 100644 dist/database/runMigration.js create mode 100644 dist/database/tracks.js delete mode 100644 dist/ownership-history.mjs create mode 100644 dist/src/components/eth-get-logs.js create mode 100644 dist/src/components/handle-transfer.js delete mode 100644 dist/src/strategies/catalog.js create mode 100644 dist/src/strategies/lens/components.js create mode 100644 dist/src/strategies/lens/lens.js diff --git a/dist/assets/config.sample.js b/dist/assets/config.sample.js index 2eae60a..fc2d04d 100644 --- a/dist/assets/config.sample.js +++ b/dist/assets/config.sample.js @@ -3,44 +3,54 @@ * - Configuration common for all commands * - Configuration too verbose for CLI * - Sensitive information - * + * * neume will read values from .env. Therefore, you can do * env.RPC_API_KEYS - * + * */ import { env } from "process"; -const rpcHosts = [ - { url: "https://rpc.ankr.com/eth" }, - { url: "https://cloudflare-eth.com/" }, -]; +const rpcHosts = [{ url: "https://rpc.ankr.com/eth" }, { url: "https://cloudflare-eth.com/" }]; +const polygonRpcHosts = [{ url: "https://rpc.ankr.com/polygon" }]; /** * A list of strategies to enable. Remove entries from the list to run * selected strategies. - * + * * Note: * - The name should match the class name of the strategy. * - The name is case-sensitive. * */ -export const strategies = ["Sound", "SoundProtocol", "MintSongsV2", "CatalogV2", "Zora"]; - +export const strategies = ["Sound", "SoundProtocol", "MintSongsV2", "CatalogV2", "Zora", "Lens"]; /** * Configuration for neume - * + * * TypeScipt type for configuration can be found at: * https://github.com/neume-network/crawler/blob/main/types.ts#L25 */ export const config = { - rpc: rpcHosts, arweave: { httpsGateway: "https://arweave.net", }, - crawlStep: 5000, - getLogsBlockSpanSize: 799, - getLogsAddressSize: 100, + ipfs: { + httpsGateway: "https://ipfs.io/ipfs/", + }, + chain: { + eth: { + crawlStep: 5000, + rpc: rpcHosts, + getLogsBlockSpanSize: 799, + getLogsAddressSize: 100, + }, + polygon: { + crawlStep: 10_000, + rpc: polygonRpcHosts, + getLogsBlockSpanSize: 2000, + getLogsAddressSize: 799, + }, + }, breatheTimeMS: 900_000, // 15 mins worker: { queue: { @@ -57,6 +67,14 @@ export const config = { }; return prevValue; }, {}), + ...polygonRpcHosts.reduce((prevValue, host) => { + prevValue[host.url] = { + timeout: 120_000, + requestsPerUnit: 300, + unit: "second", + }; + return prevValue; + }, {}), "https://arweave.net": { timeout: 120_000, requestsPerUnit: 1000, diff --git a/dist/commands/crawl.js b/dist/commands/crawl.js deleted file mode 100644 index aba3753..0000000 --- a/dist/commands/crawl.js +++ /dev/null @@ -1,223 +0,0 @@ -import ExtractionWorker from "@neume-network/extraction-worker"; -import { toHex, decodeLog } from "eth-fun"; -import { db } from "../database/index.js"; -import { getAllContracts, randomItem } from "../src/utils.js"; -const TRANSFER_EVENT_SELECTOR = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; -const CHAIN_ID = "1"; -// from and to are supposed to be included -export default async function (from, to, recrawl, config, _strategies) { - const allContracts = await getAllContracts(); - const contracts = Object.entries(allContracts).reduce((prevValue, [addr, info]) => { - if (_strategies.filter((s) => s.name === info.name).length) - prevValue = { ...prevValue, ...{ [addr]: info } }; - return prevValue; - }, {}); - const worker = ExtractionWorker(config.worker); - const strategies = _strategies.map((s) => new s(worker, config)); - for (let i = from; i <= to; i += config.crawlStep + 1) { - const fromBlock = i; - const toBlock = Math.min(to, i + config.crawlStep); - await crawl(fromBlock, toBlock, recrawl, config, strategies, contracts, worker); - } - console.log("Exiting from crawl command"); -} -// (from - to) must not be too big -// from and to are supposed to be included -async function crawl(from, to, recrawl, config, strategies, contracts, worker) { - console.log("Crawling from", from, "to", to); - /** - * We start crawling the NFT if the log is for a mint, else if - * the log is for a transfer we wait for all the mints to be - * processed. - * - * This is because to process a transfer we update the owner - * field and not re-fetch the metadata. - */ - /** - * We have processed all the mints if all the promises in this - * array have been resolved. - */ - const mintNFTsPromise = []; - /** - * We store all logs/NFTs beloging to a transfer event in this - * array to process the later that is after all the mints. - */ - const allTransferNFTs = []; - for (let i = from; i <= to; i += config.getLogsBlockSpanSize + 1) { - const fromBlock = i; - const toBlock = Math.min(to, i + config.getLogsBlockSpanSize); - for (let j = 0; j < Object.keys(contracts).length; j += config.getLogsAddressSize) { - const contractsSlice = Object.keys(contracts).slice(j, j + config.getLogsAddressSize); - const rpcHost = randomItem(config.rpc); - const msg = await worker({ - type: "json-rpc", - commissioner: "", - method: "eth_getLogs", - options: { - url: rpcHost.url, - headers: { - ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), - }, - retry: { - retries: 3, - }, - }, - params: [ - { - fromBlock: toHex(fromBlock), - toBlock: toHex(toBlock), - address: contractsSlice, - topics: [TRANSFER_EVENT_SELECTOR], - }, - ], - version: "0.0.1", - }); - if (msg.error) { - console.error(msg); - throw new Error(`Error occured while fetching Transfer events`); - } - const logs = msg.results; - let nfts = logs.map((log) => prepareNFT(contracts, log)); - // Partition NFTs into mints and transfers - const { mintNfts, transferNfts } = nfts.reduce((nfts, nft) => { - if (nft.erc721.transaction.from === "0x0000000000000000000000000000000000000000") { - nfts.mintNfts.push(nft); - } - else { - nfts.transferNfts.push(nft); - } - return nfts; - }, { mintNfts: [], transferNfts: [] }); - const promises = mintNfts.map(async (nft) => { - let nftExists = false; - try { - nftExists = !!(await db.level.get(db.datumToKey({ - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber, - }))); - } - catch (err) { - if (err.code !== "LEVEL_NOT_FOUND") - throw err; - } - if (!recrawl && nftExists) - return; - const strategy = strategies.find((s) => s.constructor.name === nft.platform.name); - if (!strategy) { - throw new Error(`Couldn't find any strategy with the name of ${nft.platform.name} for address ${nft.erc721.address}`); - } - await fetchMetadata(nft, strategy); - }); - mintNFTsPromise.push(...promises); - allTransferNFTs.push(...transferNfts); - } - } - await Promise.all(mintNFTsPromise); - await Promise.all(allTransferNFTs.map(async (nft) => { - let nftExists = false; - try { - nftExists = !!(await db.level.get(db.datumToKey({ - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber, - }))); - } - catch (err) { - if (err.code !== "LEVEL_NOT_FOUND") - throw err; - } - if (!recrawl && nftExists) - return; - const strategy = strategies.find((s) => s.constructor.name === nft.platform.name); - if (!strategy) { - throw new Error(`Couldn't find any strategy with the name of ${nft.platform.name} for address ${nft.erc721.address}`); - } - await updateOwnership(nft, strategy); - })); -} -function prepareNFT(contracts, log) { - if (!log.topics[3] || !log.transactionHash || !log.blockNumber) { - throw new Error(`log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`); - } - const decodedTopics = decodeLog([ - { indexed: true, name: "from", type: "address" }, - { indexed: true, name: "to", type: "address" }, - { indexed: true, name: "tokenId", type: "uint256" }, - ], log.data, log.topics.slice(1)); - return { - platform: { - ...contracts[log.address], - }, - erc721: { - blockNumber: parseInt(log.blockNumber, 16), - address: log.address, - transaction: { - from: decodedTopics[0], - to: decodedTopics[1], - transactionHash: log.transactionHash, - blockNumber: parseInt(log.blockNumber, 16), - }, - token: { - id: BigInt(log.topics[3]).toString(10), - }, - }, - metadata: {}, - }; -} -async function fetchMetadata(nft, strategy) { - let track = null; - try { - track = await strategy?.crawl(nft); - } - catch (err) { - console.log(track); - console.error(`Error occurured while crawling\n`, err, JSON.stringify(nft, null, 2)); - throw err; // Re-throwing to stop the application - } - if (track !== null) { - await db.insert({ - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber, - }, track); - console.log("Found track:", track?.title, track?.platform.version, track?.platform.name, "at", track?.erc721.createdAt); - } -} -async function updateOwnership(nft, strategy) { - let track = undefined; - try { - // Get the last track - track = (await db.getOne({ - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber - 1, - })).value; - } - catch (err) { - if (err.code !== "LEVEL_NOT_FOUND") - throw err; - } - // If the NFT hasn't been crawled before we can't update its owner - if (track) { - track.erc721.transaction = { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }; - // If in future we have to update strategy specific - // ownership data we can call updateOwner function - await db.insert({ - chainId: CHAIN_ID, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - blockNumber: nft.erc721.blockNumber, - }, track); - console.log("Update ownership of", track.title, track?.platform.name, track?.platform.version, "at", nft.erc721.blockNumber, "from", nft.erc721.transaction.from, "to", nft.erc721.transaction.to); - } -} diff --git a/dist/commands/daemon.js b/dist/commands/daemon.js index 8433d86..9e3f21a 100644 --- a/dist/commands/daemon.js +++ b/dist/commands/daemon.js @@ -1,43 +1,47 @@ import { fastify as Fastify } from "fastify"; -import { JSONRPCServer, JSONRPCErrorException } from "json-rpc-2.0"; -import { getLatestBlockNumber, getStrategies, getUserContracts } from "../src/utils.js"; -import crawl from "./crawl.js"; -import filter_contracts from "./filter_contracts.js"; -import { db } from "../database/index.js"; +import { JSONRPCServer } from "json-rpc-2.0"; +import { getLatestBlockNumber, getStrategies } from "../src/utils.js"; import { daemonJsonrpcSchema } from "./daemon/daemon-jsonrpc-schema.js"; import { getLastCrawledBlock, saveLastCrawledBlock } from "../src/state.js"; +import { tracksDB } from "../database/tracks.js"; +import { getLocalStorage } from "../database/localstorage.js"; +import ExtractionWorker from "@neume-network/extraction-worker"; const fastify = Fastify(); -export default async function daemon(_from, crawlFlag, recrawl, port, config, strategyNames) { - const RANGE_FOR_CRAWL = 5000; - let from = _from ?? (await getLastCrawledBlock()); - let to = Math.min(from + RANGE_FOR_CRAWL, await getLatestBlockNumber(config.rpc[0])); - let strategies = getStrategies(strategyNames, from, to); - const task = async () => { - const latestBlockNumber = await getLatestBlockNumber(config.rpc[0]); - to = Math.min(from + RANGE_FOR_CRAWL, latestBlockNumber); - console.log(`\n\n***** Starting a crawl cycle from ${from} to ${to} *****\n`); - strategies = getStrategies(strategyNames, from, to); - await filter_contracts(from, to, recrawl, config, strategies); - await crawl(from, to, recrawl, config, strategies); - await saveLastCrawledBlock(to); - from = to; - const nextTaskWaitTime = to === latestBlockNumber ? config.breatheTimeMS : 0; - setTimeout(task, nextTaskWaitTime); +export default async function daemon(crawlFlag, recrawl, port, config, strategyNames) { + const worker = ExtractionWorker(config.worker); + const allStrategies = getStrategies(strategyNames).map((s) => new s(worker, config)); + const task = async (strategy) => { + const { crawlStep } = config.chain[strategy.chain]; + const latestBlockNumber = await getLatestBlockNumber(config.chain[strategy.chain].rpc[0]); + const from = await getLastCrawledBlock(strategy.constructor.name); + const to = Math.min(from + crawlStep, latestBlockNumber); + if (from >= (strategy.deprecatedAtBlock ?? Number.MAX_VALUE)) { + console.log(`Removing ${strategy.constructor.name} from queue because we have reached deprecatedAtBlock (${strategy.deprecatedAtBlock})`); + return; + } + console.log("Calling strategy", strategy.constructor.name, "from", from, "to", to); + await strategy.crawl(from, to, recrawl); + await saveLastCrawledBlock(strategy.constructor.name, to); + const nextTaskWaitTime = to === latestBlockNumber ? config.breatheTimeMS ?? 1 : 1; + setTimeout(task.bind({}, strategy), nextTaskWaitTime); }; - if (crawlFlag) - task(); + if (crawlFlag) { + allStrategies.forEach((s) => { + task(s); + }); + } await startServer(port); } async function startServer(port) { const server = new JSONRPCServer(); - server.addMethod("getIdsChanged_fill", async ([from, to]) => { - if (to - from > 5000) - return new JSONRPCErrorException("Block range should be less than 5000", -32600); - const res = await db.getIdsChanged_fill(from, to); + server.addMethod("getTracks", async ({ since, platform }) => { + // if (to - from > 5000) + // return new JSONRPCErrorException("Block range should be less than 5000", -32600); + const res = await tracksDB.getTracksChanged(since, platform); return res; }); - server.addMethod("getUserContracts", async () => { - return getUserContracts(); + server.addMethod("getLocalStorage", async ({ platform }) => { + return getLocalStorage(platform); }); fastify.route({ method: "POST", diff --git a/dist/commands/daemon/daemon-jsonrpc-schema.js b/dist/commands/daemon/daemon-jsonrpc-schema.js index 68738fb..888c940 100644 --- a/dist/commands/daemon/daemon-jsonrpc-schema.js +++ b/dist/commands/daemon/daemon-jsonrpc-schema.js @@ -1,91 +1,54 @@ import { compile } from "json-schema-to-typescript"; import { fileURLToPath } from "url"; -const _daemonJsonrpcSchema = { - "type": "object", - "title": 'DaemonJsonrpcType', - "oneOf": [ - { - "$ref": "#/$defs/getIdsChanged_fill", - }, - { - "$ref": "#/$defs/getAllContracts", - }, - ], - "required": ["jsonrpc", "id", "method"], - "$defs": { - "getIdsChanged_fill": { - "type": "object", - "properties": { - "jsonrpc": { "type": "string", "const": "2.0" }, - "id": { "type": "string" }, - "method": { "type": "string", "const": "getIdsChanged_fill" }, - "params": { - "type": "array", - "minItems": 2, - "maxItems": 2, - "items": { - "type": "string" - } - } - }, - "required": ["jsonrpc", "id", "method", "params"] - }, - "getAllContracts": { - "type": "object", - "properties": { - "jsonrpc": { "type": "string", "const": "2.0" }, - "id": { "type": "string" }, - "method": { "type": "string", "const": "getAllContracts" }, - }, - "required": ["jsonrpc", "id", "method"] - } - }, -}; export const daemonJsonrpcSchema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": 'DaemonJsonrpcType', - "oneOf": [ - { "$ref": "#/$defs/request" }, - { "type": "array", "items": { "$ref": "#/$defs/request" } } - ], - "$defs": { - "request": { - "type": "object", - "oneOf": [ + $schema: "http://json-schema.org/draft-07/schema#", + title: "DaemonJsonrpcType", + oneOf: [{ $ref: "#/$defs/request" }, { type: "array", items: { $ref: "#/$defs/request" } }], + $defs: { + request: { + type: "object", + oneOf: [ { - "$ref": "#/$defs/getIdsChanged_fill", + $ref: "#/$defs/getTracks", }, { - "$ref": "#/$defs/getUserContracts", + $ref: "#/$defs/getLocalStorage", }, ], }, - "getIdsChanged_fill": { - "type": "object", - "properties": { - "jsonrpc": { "type": "string", "const": "2.0" }, - "id": { "type": "string" }, - "method": { "type": "string", "const": "getIdsChanged_fill" }, - "params": { - "type": "array", - "minItems": 2, - "maxItems": 2, - "items": { - "type": "number" - } - } + getTracks: { + type: "object", + properties: { + jsonrpc: { type: "string", const: "2.0" }, + id: { type: "string" }, + method: { type: "string", const: "getTracks" }, + params: { + type: "object", + properties: { + since: { type: "number", $comment: "Since is a unix timestamp" }, + platform: { type: "string" }, + }, + required: ["since", "platform"], + }, }, - "required": ["jsonrpc", "id", "method", "params"] + required: ["jsonrpc", "id", "method", "params"], }, - "getUserContracts": { - "type": "object", - "properties": { - "jsonrpc": { "type": "string", "const": "2.0" }, - "id": { "type": "string" }, - "method": { "type": "string", "const": "getUserContracts" }, + getLocalStorage: { + type: "object", + properties: { + jsonrpc: { type: "string", const: "2.0" }, + id: { type: "string" }, + method: { type: "string", const: "getLocalStorage" }, + params: { + type: "object", + properties: { + platform: { type: "string" }, + }, + required: ["platform"], + }, }, - "required": ["jsonrpc", "id", "method"] - } + required: ["jsonrpc", "id", "method", "params"], + }, }, }; // This is called from package.json to compile the above type diff --git a/dist/commands/dump.js b/dist/commands/dump.js deleted file mode 100644 index a509b01..0000000 --- a/dist/commands/dump.js +++ /dev/null @@ -1,38 +0,0 @@ -import fs from "fs"; -import { writeFile, mkdir } from "fs/promises"; -import { canonicalize } from "json-canonicalize"; -import path from "path"; -import { db } from "../database/index.js"; -const DIR = path.resolve("./dump"); -export default async function dump(at) { - if (!fs.existsSync(DIR)) { - fs.mkdirSync(DIR, { recursive: true }); - } - let count = 0; - try { - for await (const { id, value } of db.getMany({ - chainId: "1", - blockNumber: at, - })) { - const track = canonicalize(value); - const outputPath = path.resolve(DIR, `${id.chainId}/${id.address}/${id.tokenId}`); - await mkdir(outputPath, { recursive: true }); - const outputFile = path.resolve(outputPath, `entry.json`); - await writeFile(outputFile, track); - count++; - } - } - catch (err) { - // TOOD: Find a way to not depend on error message - if (err.message === "Couldn't find any items for the given DB query") { - console.log("Nothing to dump. Exiting from dump command."); - process.exit(0); - } - throw err; - } - console.log(`Exiting from dump command; Wrote ${count} tracks.`); - process.exit(0); -} -async function flush(filename, tracks) { - await writeFile(filename, JSON.stringify(tracks, null, 2)); -} diff --git a/dist/commands/filter_contracts.js b/dist/commands/filter_contracts.js deleted file mode 100644 index a7fb35f..0000000 --- a/dist/commands/filter_contracts.js +++ /dev/null @@ -1,29 +0,0 @@ -import ExtractionWorker from "@neume-network/extraction-worker"; -import path from "path"; -import { writeFile } from "fs/promises"; -import { getUserContracts } from "../src/utils.js"; -export default async function (from, to, recrawl, config, _strategies) { - if (!config.rpc.length) - throw new Error("Atleast one RPC host is required"); - const userContracts = await getUserContracts(); - const worker = ExtractionWorker(config.worker); - const strategies = _strategies.map((s) => new s(worker, config)); - for (let i = from; i <= to; i += config.getLogsBlockSpanSize) { - const fromBlock = i; - const toBlock = Math.min(to, i + config.getLogsBlockSpanSize); - console.log("Finding contracts from", fromBlock, toBlock); - await Promise.all(strategies.map(async (strategy) => { - if (!strategy.filterContracts) - return; - const newContracts = await strategy.filterContracts(fromBlock, toBlock); - newContracts.forEach((contract) => { - userContracts[contract.address] = { - name: contract.name, - version: contract.version, - }; - }); - })); - } - await writeFile(path.resolve("./data/contracts.json"), JSON.stringify(userContracts, null, 2)); - console.log("Exiting from filter-contracts command"); -} diff --git a/dist/commands/init.js b/dist/commands/init.js index 98cc8df..6eb35a6 100644 --- a/dist/commands/init.js +++ b/dist/commands/init.js @@ -1,14 +1,12 @@ import path from "path"; import fs from "fs/promises"; import { saveLastCrawledBlock } from "../src/state.js"; -import { CONSTANTS } from "../src/types.js"; +import { CHAINS, CONSTANTS } from "../src/types.js"; +import runMigration from "../database/runMigration.js"; export default async function init() { await fs.copyFile(new URL("../assets/.env-copy", import.meta.url), path.resolve(".env")); await fs.copyFile(new URL("../assets/config.sample.js", import.meta.url), path.resolve("./config.js")); - // Will create file if it does not exist - await fs.writeFile(path.resolve("./data/contracts.json"), "{}", { - flag: "w", - }); // Create the last_crawled_block file in ./data - await saveLastCrawledBlock(CONSTANTS.FIRST_BLOCK); + await saveLastCrawledBlock(CHAINS.eth, CONSTANTS.FIRST_BLOCK[CHAINS.eth]); + await runMigration("up"); } diff --git a/dist/commands/sync.js b/dist/commands/sync.js index d5baeb7..28ee723 100644 --- a/dist/commands/sync.js +++ b/dist/commands/sync.js @@ -1,15 +1,11 @@ import { JSONRPCClient } from "json-rpc-2.0"; import ExtractionWorker from "@neume-network/extraction-worker"; -import fs from "fs/promises"; -import { db } from "../database/index.js"; -import path from "path"; -import { getUserContracts } from "../src/utils.js"; -async function getLastSyncedBlock() { - const lastId = await db.changeIndex.iterator({ reverse: true, limit: 1 }).next(); - return lastId ? parseInt(lastId[0].split("/")[0]) : 15000000; -} -export default async function (from, to, url, config) { +import { localStorage, saveLocalStorage } from "../database/localstorage.js"; +import { tracksDB } from "../database/tracks.js"; +const sync = async function (_since, url, config, strategies) { const worker = ExtractionWorker(config.worker); + const storage = localStorage.sublevel("sync", {}); + const normalizedUrl = new URL(url).host; let client; let id = 0; client = new JSONRPCClient((jsonRPCRequest) => worker({ @@ -32,24 +28,41 @@ export default async function (from, to, url, config) { return Promise.reject(new Error(JSON.stringify(msg.error))); return client.receive(msg.results); }), () => (++id).toString()); - let syncFrom = from ?? (await getLastSyncedBlock()); - console.log("Will sync from", syncFrom, "to", to); - // Sync contracts - const userContractsNew = await client.request("getUserContracts", null); - const userContractsOld = await getUserContracts(); - const userContracts = { ...userContractsNew, ...userContractsOld }; - const userContractsPath = path.resolve("./data/contracts.json"); - await fs.writeFile(userContractsPath, JSON.stringify(userContracts, null, 2)); - console.log("Updated local list of contracts"); - for (let syncedTill = syncFrom; syncedTill <= to; syncedTill += 5000) { - console.log(`Syncing from ${syncedTill} to ${syncedTill + 5000}`); - const returnValues = (await client.request("getIdsChanged_fill", [ - syncedTill, - syncedTill + 5000, - ])); - await Promise.all(returnValues.map(async (r) => { - await db.insert(r.id, r.value); + // Assuming localstorage won't be too big in size. We can + // ask for everything and update our localstorage. + const syncLocalStorage = async (platform) => { + const localStorage = await client.request("getLocalStorage", { + platform, + }); + await saveLocalStorage(platform, localStorage); + }; + const syncTracks = async (platform, since) => { + const { tracks, nextTimestamp } = (await client.request("getTracks", { + since, + platform, })); - console.log(`Wrote ${returnValues.length} entries to database`); - } -} + // This will either create a new track or merge with the existing track + await Promise.all(tracks.map(async (track) => tracksDB.upsertTrack(track))); + console.log(`Upserted ${tracks.length} tracks for ${platform}. startTimestamp=${since} nextTimestamp=${nextTimestamp}`); + if (nextTimestamp) { + await storage.put(`${normalizedUrl}-${platform}`, nextTimestamp); + return nextTimestamp; + } + }; + await Promise.all(strategies.map(async (strategy) => { + let lastSync; + try { + lastSync = await storage.get(`${normalizedUrl}-${strategy.name}`); + } + catch (err) { + if (err.code !== "LEVEL_NOT_FOUND") + throw err; + } + let since = _since ?? lastSync ?? 0; + syncLocalStorage(strategy.name); + do { + since = await syncTracks(strategy.name, since); + } while (since); + })); +}; +export default sync; diff --git a/dist/database/index.test.js b/dist/database/index.test.js deleted file mode 100644 index 226978a..0000000 --- a/dist/database/index.test.js +++ /dev/null @@ -1,248 +0,0 @@ -import test from "ava"; -import { fileURLToPath } from "url"; -import { DB } from "./index.js"; -const db = new DB(fileURLToPath(new URL("./db", import.meta.url))); -const chainId = "1"; -test.serial("should be able insert values", async (t) => { - await t.notThrowsAsync(async () => db.insert({ chainId, address: "0x13", tokenId: "5", blockNumber: 95 }, { test: "data" })); -}); -test.serial("should be able to delete values by id", async (t) => { - await t.notThrowsAsync(async () => db.del({ chainId, address: "0x13", tokenId: "5", blockNumber: 95 })); -}); -test.serial("should be able to get values by id", async (t) => { - const datum = { chainId, address: "0x9b", tokenId: "2", blockNumber: 110 }; - const value = { test: "data" }; - await db.insert(datum, value); - const ret = await db.getOne(datum); - const ids = await db.getIdsChanged(110); - t.deepEqual(ret.id, datum); - t.deepEqual(ret.value, value); - t.deepEqual(ids, [datum]); -}); -test.serial("should throw error if no value with given address and token id", async (t) => { - await db.insert({ chainId, address: "0x9b", tokenId: "2", blockNumber: 110 }, { test: "data" }); - await t.throwsAsync(async () => db.getOne({ chainId, address: "0xab", tokenId: "0" }), { - code: "LEVEL_NOT_FOUND", - }); -}); -test.serial("getOne should get the closest value", async (t) => { - const datum = { chainId, address: "0x9b", tokenId: "2", blockNumber: 110 }; - const value = { test: "data" }; - await db.insert(datum, value); - const ret = await db.getOne({ chainId, address: "0x9b", tokenId: "2", blockNumber: 111 }); - t.deepEqual(ret.id, datum); - t.deepEqual(ret.value, value); -}); -test.serial("should get value at latest block number if no block number is specified", async (t) => { - const values = [ - { - id: { chainId, address: "0x01", tokenId: "1", blockNumber: 9 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0x01", tokenId: "1", blockNumber: 10 }, - value: { test: "updated-data" }, - }, - ]; - await Promise.all(values.map((v) => db.insert(v.id, v.value))); - const { blockNumber, ...datum } = values[0].id; - const ret = await db.getOne(datum); - t.deepEqual(ret.id, values[1].id); - t.deepEqual(ret.value, values[1].value); -}); -test.serial("provided with only chainId and address should get all tokenIds", async (t) => { - const address = "0xb8"; - const values = [ - { - id: { chainId, address, tokenId: "1", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address, tokenId: "2", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address, tokenId: "2", blockNumber: 120 }, - value: { test: "updated-data" }, - }, - { - id: { chainId, address, tokenId: "3", blockNumber: 130 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xc8", tokenId: "3", blockNumber: 110 }, - value: { test: "data" }, - }, - ]; - await Promise.all(values.map((v) => db.insert(v.id, v.value))); - const ret = []; - for await (const r of db.getMany({ chainId, address })) { - ret.push(r); - } - t.deepEqual(ret, [values[0], values[2], values[3]]); -}); -test.serial("provided with chainId, address and a block number should get all tokenIds upto the given block number", async (t) => { - const address = "0xb8"; - const values = [ - { - id: { chainId, address, tokenId: "1", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address, tokenId: "2", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address, tokenId: "2", blockNumber: 120 }, - value: { test: "updated-data" }, - }, - { - id: { chainId, address, tokenId: "3", blockNumber: 130 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xc8", tokenId: "3", blockNumber: 110 }, - value: { test: "data" }, - }, - ]; - await Promise.all(values.map((v) => db.insert(v.id, v.value))); - const ret = []; - for await (const r of db.getMany({ - chainId, - address, - blockNumber: 110, - })) { - ret.push(r); - } - t.deepEqual(ret, [values[0], values[1]]); -}); -test.serial("provided with only chainId should get all tokenIds", async (t) => { - const values = [ - { - id: { chainId, address: "0xb8", tokenId: "1", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xb9", tokenId: "2", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xb9", tokenId: "2", blockNumber: 120 }, - value: { test: "updated-data" }, - }, - { - id: { chainId, address: "0xc8", tokenId: "3", blockNumber: 130 }, - value: { test: "data" }, - }, - ]; - await Promise.all(values.map((v) => db.insert(v.id, v.value))); - const ret = []; - for await (const r of db.getMany({ chainId })) { - ret.push(r); - } - t.deepEqual(ret, [values[0], values[2], values[3]]); -}); -test.serial("provided with chainId and blockNumber should get all tokenIds upto the given block number", async (t) => { - const values = [ - { - id: { chainId, address: "0xb8", tokenId: "1", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xb9", tokenId: "2", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xb9", tokenId: "2", blockNumber: 120 }, - value: { test: "updated-data" }, - }, - { - id: { chainId, address: "0xc8", tokenId: "3", blockNumber: 130 }, - value: { test: "data" }, - }, - ]; - await Promise.all(values.map((v) => db.insert(v.id, v.value))); - const ret = []; - for await (const r of db.getMany({ chainId, blockNumber: 110 })) { - ret.push(r); - } - t.deepEqual(ret, [values[0], values[1]]); -}); -test.serial("should rewrite data if id is same", async (t) => { - const values = [ - { - id: { chainId, address: "0xa0", tokenId: "1", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xa0", tokenId: "1", blockNumber: 110 }, - value: { test: "updated-data" }, - }, - ]; - await Promise.all(values.map((v) => db.insert(v.id, v.value))); - const { blockNumber, ...datum } = values[0].id; - const ret = await db.getOne(datum); - t.deepEqual(ret.id, values[0].id); - t.deepEqual(ret.value, values[1].value); -}); -test.serial("should get empty array if no ids have been changed", async (t) => { - const ids = await db.getIdsChanged(110, 115); - t.deepEqual(ids, []); -}); -test.serial("should get all changed ids given a block range", async (t) => { - const values = [ - { - id: { chainId, address: "0xa0", tokenId: "0", blockNumber: 105 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xa0", tokenId: "1", blockNumber: 110 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xa0", tokenId: "1", blockNumber: 110 }, - value: { test: "updated-data" }, - }, - { - id: { chainId, address: "0xa0", tokenId: "2", blockNumber: 115 }, - value: { test: "data" }, - }, - { - id: { chainId, address: "0xa0", tokenId: "2", blockNumber: 120 }, - value: { test: "updated-data" }, - }, - { - id: { chainId, address: "0xa0", tokenId: "0", blockNumber: 11 }, - value: { test: "data" }, - }, - ]; - await Promise.all(values.map((v) => db.insert(v.id, v.value))); - let ids = await db.getIdsChanged(110, 115); - let returnValues = await db.getIdsChanged_fill(110, 115); - t.deepEqual(ids, [values[2].id, values[3].id]); - t.deepEqual(returnValues, [values[2], values[3]]); - ids = await db.getIdsChanged(105, 115); - returnValues = await db.getIdsChanged_fill(105, 115); - t.deepEqual(ids, [values[0].id, values[2].id, values[3].id]); - t.deepEqual(returnValues, [values[0], values[2], values[3]]); - ids = await db.getIdsChanged(115); - returnValues = await db.getIdsChanged_fill(115); - t.deepEqual(ids, [values[3].id]); - t.deepEqual(returnValues, [values[3]]); - await db.del(values[2].id); - ids = await db.getIdsChanged(105, 115); - returnValues = await db.getIdsChanged_fill(105, 115); - t.deepEqual(ids, [values[0].id, values[3].id]); - t.deepEqual(returnValues, [values[0], values[3]]); - await db.del(values[3].id); - ids = await db.getIdsChanged(105, 120); - returnValues = await db.getIdsChanged_fill(105, 120); - t.deepEqual(ids, [values[0].id, values[4].id]); - t.deepEqual(returnValues, [values[0], values[4]]); -}); -test.beforeEach("clear database", async (t) => { - await Promise.all([db.level.clear(), db.changeIndex.clear()]); -}); -test.after("close database", async (t) => { - await db.level.close(); -}); diff --git a/dist/database/knexfile.js b/dist/database/knexfile.js new file mode 100644 index 0000000..3782a27 --- /dev/null +++ b/dist/database/knexfile.js @@ -0,0 +1,22 @@ +import Knex from "knex"; +import path from "path"; +/** + * @type {Knex.Knex.Config} + */ +const config = { + client: "better-sqlite3", + connection: { + // path.resolve means that a .sqlite3 will be created at the current working directory + filename: path.resolve("./data/neume.sqlite3"), + }, + // Under heavy load acquiring connection may require more time + acquireConnectionTimeout: 120000, + useNullAsDefault: true, + pool: { + afterCreate: function (conn, done) { + conn.pragma("journal_mode = WAL"); + done(); + }, + }, +}; +export default config; diff --git a/dist/database/localstorage.js b/dist/database/localstorage.js new file mode 100644 index 0000000..3ee91a5 --- /dev/null +++ b/dist/database/localstorage.js @@ -0,0 +1,42 @@ +import path from "path"; +import { Level } from "level"; +export class LocalStorage { + constructor(dbPath) { + this.level = new Level(path.resolve(dbPath, "./localstorage"), { + valueEncoding: "json", + }); + } + async insert(key, value, prefix) { + prefix = prefix ?? ""; + return this.level.put(`${prefix}-${key}`, value); + } + async del(key, prefix) { + return this.level.del(`${prefix}-${key}`); + } + async get(key, prefix) { + prefix = prefix ?? ""; + return this.level.get(`${prefix}-${key}`); + } +} +export const localStorage = new Level(path.resolve("./data", "./localstorage"), { + valueEncoding: "json", +}); +export async function getLocalStorage(sublevelName) { + const sublevel = localStorage.sublevel(sublevelName, { valueEncoding: "json" }); + const all = await sublevel.iterator({}).all(); + return all; +} +export async function saveLocalStorage(sublevelName, entries) { + const sublevel = localStorage.sublevel(sublevelName, { valueEncoding: "json" }); + const operations = entries.map((e) => { + return { + type: "put", + key: e[0], + value: e[1], + }; + }); + await sublevel.batch(operations); +} +process.on("exit", async () => { + await localStorage.close(); +}); diff --git a/dist/database/migrations/20230318200321_schema.js b/dist/database/migrations/20230318200321_schema.js new file mode 100644 index 0000000..0d16ebd --- /dev/null +++ b/dist/database/migrations/20230318200321_schema.js @@ -0,0 +1,73 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export const up = async function (knex) { + await knex.schema.createTable("tracks", (table) => { + table.string("version").notNullable(); + table.string("title").notNullable(); + table.string("duration"); + + table.string("artist_version").notNullable(); + table.string("artist_name").notNullable(); + table.string("artist_address").nullable(); + + table.string("platform_name").index().notNullable(); + table.string("platform_version").notNullable(); + table.string("platform_uri").notNullable(); + + table.string("erc721_version").notNullable(); + table.string("erc721_address").notNullable(); + table.string("erc721_uri"); + table.json("erc721_metadata"); + + table.integer("lastUpdatedAt").index().notNullable(); + table.string("uid"); + table.primary("uid"); + }); + + await knex.schema.createTable("manifestations", (table) => { + table.string("version").notNullable(); + table.string("uri").notNullable(); + table.string("mimetype").notNullable(); + + table.string("uid"); + table.foreign(["uid"]).references(["uid"]).on("tracks"); + table.primary(["uid", "uri"]); + }); + + await knex.schema.createTable("tokens", (table) => { + table.string("id").notNullable(); + table.string("uri"); + table.json("metadata"); + + table.string("uid"); + table.foreign(["uid"]).references(["uid"]).on("tracks"); + table.primary(["uid", "id"]); + }); + + await knex.schema.createTable("owners", (table) => { + table.integer("blockNumber").notNullable(); + table.string("from").notNullable(); + table.string("to").notNullable(); + table.string("transactionHash").notNullable(); + table.string("alias"); + + table.string("uid"); + table.string("id"); + + table.foreign(["uid", "id"]).references(["uid", "id"]).on("tokens"); + table.primary(["uid", "id", "transactionHash", "to"]); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +export const down = async function (knex) { + await knex.schema.dropTable("owners"); + await knex.schema.dropTable("tokens"); + await knex.schema.dropTable("manifestations"); + await knex.schema.dropTable("tracks"); +}; diff --git a/dist/database/runMigration.js b/dist/database/runMigration.js new file mode 100644 index 0000000..803e6f7 --- /dev/null +++ b/dist/database/runMigration.js @@ -0,0 +1,18 @@ +import Knex from "knex"; +import path from "path"; +import fs from "fs"; +import { URL } from "url"; +import config from "./knexfile.js"; +// We don't use knex CLI because we need access to __dirname + '/migrations' +export default async function runMigration(type) { + const knex = Knex.default(config); + const dir = path.dirname(config.connection.filename); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + const fn = knex.migrate[type]; + await fn.call(knex.migrate, { + directory: new URL("./migrations", import.meta.url).pathname, + }); + await knex.destroy(); +} diff --git a/dist/database/tracks.js b/dist/database/tracks.js new file mode 100644 index 0000000..4af6174 --- /dev/null +++ b/dist/database/tracks.js @@ -0,0 +1,241 @@ +import knex from "knex"; +import config from "./knexfile.js"; +// Checkout this discussion for alternative Database options https://github.com/orgs/neume-network/discussions/29 +/** + * SQL database to store and retrieve tracks. + */ +export class Tracks { + constructor() { + this.isTrackPresent = async (uid) => { + const rows = await this.db("tracks").select().where("uid", "=", uid); + return Boolean(rows.length); + }; + /** Given a track and it's tokenID return true if the token ID is present in the DB. */ + this.isTokenPresent = async (uid, tokenId) => { + const rows = await this.db("tokens").select().where({ uid, id: tokenId }); + return Boolean(rows.length); + }; + this.upsertTrack = async (track, timestamp = Date.now()) => { + return this.db.transaction(async (trx) => { + await trx("tracks") + .insert({ + version: track.version, + title: track.title, + duration: track.duration, + artist_version: track.artist.version, + artist_name: track.artist.name, + artist_address: track.artist.address, + platform_name: track.platform.name, + platform_version: track.platform.version, + platform_uri: track.platform.uri, + erc721_version: track.erc721.version, + erc721_address: track.erc721.address, + erc721_metadata: track.erc721.metadata, + erc721_uri: track.erc721.uri, + uid: track.uid, + lastUpdatedAt: timestamp, + }) + .onConflict(["uid"]) + .merge(); + await Promise.all(track.manifestations.map((m) => trx("manifestations") + .insert({ + version: m.version, + uri: m.uri, + mimetype: m.mimetype, + uid: track.uid, + }) + .onConflict(["uid", "uri"]) + .merge())); + await Promise.all(track.erc721.tokens.map(async (token) => { + const { owners } = token; + await trx("tokens") + .insert({ + id: token.id, + uri: token.uri, + metadata: token.metadata, + uid: track.uid, + }) + .onConflict(["uid", "id"]) + .merge(); + await Promise.all(owners.map((o) => trx("owners") + .insert({ + blockNumber: o.blockNumber, + from: o.from, + to: o.to, + transactionHash: o.transactionHash, + alias: o.alias, + uid: track.uid, + id: token.id, + }) + .onConflict(["uid", "id", "transactionHash", "to"]) + .merge())); + })); + const inputs = [track, timestamp]; + await this.log?.put(`${track.platform.name}/${this.encodeNumber(timestamp)}/${hashCode(JSON.stringify(inputs))}`, { + operation: "newTrack", + inputs, + }); + }); + }; + this.upsertOwner = async (uid, tokenId, owner, platform, timestamp = Date.now()) => { + const inputs = [uid, tokenId, owner, timestamp]; + await this.db("tracks").update({ lastUpdatedAt: timestamp }).where("uid", "=", uid); + await this.db("owners") + .insert({ + from: owner.from, + to: owner.to, + blockNumber: owner.blockNumber, + transactionHash: owner.transactionHash, + alias: owner.alias, + id: tokenId, + uid, + }) + .onConflict(["uid", "id", "transactionHash", "to"]) + .merge(); + await this.log?.put(`${platform}/${this.encodeNumber(timestamp)}/${hashCode(JSON.stringify(inputs))}`, { + operation: "upsertOwner", + inputs, + }); + }; + this.isOwnerPresent = async (uid, tokenId, owner) => { + const rows = await this.db("owners") + .select("*") + .where({ uid, id: tokenId, transactionHash: owner.transactionHash, to: owner.to }); + return Boolean(rows.length); + }; + this.getTrack = async (uid) => { + const tokensRaw = await this.db("tokens") + .select("*") + .leftJoin("owners", function () { + this.on("tokens.uid", "=", "owners.uid"); + this.on("tokens.id", "=", "owners.id"); + }) + .where("tokens.uid", "=", uid); + const tokens = Object.values(tokensRaw.reduce((tokens, row) => { + if (tokens[row.id]) + tokens[row.id].owners.push({ + from: row.from, + to: row.to, + blockNumber: row.blockNumber, + transactionHash: row.transactionHash, + alias: row.alias, + }); + else + tokens[row.id] = { + id: row.id, + uri: row.uri, + metadata: row.metadata && JSON.parse(row.metadata), + owners: [ + { + from: row.from, + to: row.to, + blockNumber: row.blockNumber, + transactionHash: row.transactionHash, + alias: row.alias, + }, + ], + }; + return tokens; + }, {})); + const manifestations = await this.db("manifestations") + .select("*") + .where("manifestations.uid", "=", uid); + const trackRaw = await this.db("tracks").select("*").where("tracks.uid", "=", uid).limit(1); + const r = trackRaw[0]; + return { + version: r.version, + title: r.title, + uid: r.uid, + duration: r.duration, + artist: { + version: r.artist_version, + name: r.artist_name, + address: r.artist_address, + }, + platform: { + version: r.platform_version, + name: r.platform_name, + uri: r.platform_uri, + }, + erc721: { + version: r.erc721_version, + tokens: tokens, + uri: r.erc721_uri, + address: r.erc721_address, + metadata: r.erc721_metadata && JSON.parse(r.erc721_metadata), + }, + manifestations: manifestations.map((m) => ({ + version: m.version, + uri: m.uri, + mimetype: m.mimetype, + })), + }; + }; + this.getTracksChanged = async (since, platform) => { + const MAX_TRACKS = 500; + const uids = await this.db("tracks") + .select("uid") + .where("lastUpdatedAt", ">=", since) + .andWhere("platform_name", "=", platform) + .orderBy("lastUpdatedAt", "asc") + .limit(MAX_TRACKS); + const tracks = await Promise.all(uids.map(async ({ uid }) => { + return await this.getTrack(uid); + })); + const nextTimestampRaw = await this.db("tracks") + .select("lastUpdatedAt") + .where("lastUpdatedAt", ">=", since) + .andWhere("platform_name", "=", platform) + .orderBy("lastUpdatedAt", "asc") + .offset(MAX_TRACKS) + .limit(1); + const nextTimestamp = nextTimestampRaw[0]?.lastUpdatedAt; + return { tracks, nextTimestamp }; + }; + this.db = knex.default(config); + /** + * The idea behind log is to record all database operations. + * In theory, these operations could be used to replicate the DB's state + * at any given time. + */ + // this.log = new Level(resolve("./data/log"), { + // valueEncoding: "json", + // }); + } + // LevelDB stores keys in lexicographical order. Therefore, + // 1 < 10 < 9. This is unlike natural order where 1 < 9 < 10. + // + // The solution is to pad numbers with zero such that lexicographical + // order is the same as natural order. For example, if we pad + // numbers upto two digits they will become 01 < 09 < 10. + // + // The above solution will only work for postive numbers and + // will break for numbers greater than maximum digits. In the + // above example, the solution will break for numbers greater than 100. + encodeNumber(num) { + const MAX_LENGTH = 20; + if (num.toString().length > MAX_LENGTH) + throw new Error(`Database cannot encode number greater than 10 digits`); + return num.toString().padStart(MAX_LENGTH, "0"); + } + decodeNumber(num) { + return Number(num).toString(); + } + async close() { + return this.db.destroy(); + } +} +function hashCode(str) { + let hash = 0; + for (let i = 0, len = str.length; i < len; i++) { + let chr = str.charCodeAt(i); + hash = (hash << 5) - hash + chr; + hash |= 0; // Convert to 32bit integer + } + return hash; +} +export const tracksDB = new Tracks(); +// console.dir(await tracksDB.isTokenPresent("polygon/13/492", "1823"), { depth: null }); +process.on("exit", async () => { + await tracksDB.close(); +}); diff --git a/dist/neume.js b/dist/neume.js index d4051d7..9f04ffa 100755 --- a/dist/neume.js +++ b/dist/neume.js @@ -3,70 +3,37 @@ import "dotenv/config"; import yargs from "yargs"; import { hideBin } from "yargs/helpers"; import path from "path"; -import crawl from "./commands/crawl.js"; -import dump from "./commands/dump.js"; -import filterContracts from "./commands/filter_contracts.js"; import { getLatestBlockNumber, getStrategies } from "./src/utils.js"; import daemon from "./commands/daemon.js"; import sync from "./commands/sync.js"; import init from "./commands/init.js"; import { db } from "./database/index.js"; +import runMigration from "./database/runMigration.js"; +import { tracksDB } from "./database/tracks.js"; const argv = yargs(hideBin(process.argv)) .usage("Usage: $0 ") .env("NEUME") - .command("crawl", "Find new NFTs from the list of already known contracts", { - from: { - type: "number", - describe: "From block number", - demandOption: true, - }, - to: { - type: "number", - describe: "To block number", - }, - recrawl: { - type: "boolean", - describe: "Re-crawl an NFT if they already exist", - default: false, - }, -}, async (argv) => { - const { config, strategies: strategyNames } = await import(path.resolve("./config.js")); - const from = argv.from; - const to = argv.to ?? (await getLatestBlockNumber(config.rpc[0])); - await crawl(from, to, argv.recrawl, config, getStrategies(strategyNames, from, to)); - process.exit(0); -}) - .command("filter-contracts", "Find new contracts", { - from: { - type: "number", - describe: "From block number", + .command("runMigration", "Run migrations on the database", { + type: { + type: "string", + describe: "Up or Down?", demandOption: true, - }, - to: { - type: "number", - describe: "From block number", - }, - recrawl: { - type: "boolean", - describe: "Re-crawl an NFT if they already exist", - default: false, + choices: ["up", "down"], }, }, async (argv) => { - const { config, strategies: strategyNames } = await import(path.resolve("./config.js")); - const from = argv.from; - const to = argv.to ?? (await getLatestBlockNumber(config.rpc[0])); - await filterContracts(from, to, argv.recrawl, config, getStrategies(strategyNames, from, to)); - process.exit(0); + await runMigration(argv.type); }) - .command("dump", "Export database as JSON", { + .command("dump", "Export database as JSON [Out of date]", { at: { type: "number", describe: "Export database as seen at the given block number", demandOption: true, }, }, async (argv) => { + throw new Error("Not Implemented"); const { config } = await import(path.resolve("./config.js")); const at = argv.at ?? (await getLatestBlockNumber(config.rpc[0])); + // @ts-ignore return dump(at); }) .command("daemon", "Start neume-network daemon", { @@ -95,7 +62,7 @@ const argv = yargs(hideBin(process.argv)) }, }, async (argv) => { const { config, strategies: strategyNames } = await import(path.resolve("./config.js")); - await daemon(argv.from, argv.crawl, argv.recrawl, argv.port, config, strategyNames); + await daemon(argv.crawl, argv.recrawl, argv.port, config, strategyNames); }) .command("sync", "Sync neume-network with another node", { url: { @@ -103,21 +70,15 @@ const argv = yargs(hideBin(process.argv)) describe: "An endpoint that is running the neume-network daemon", demandOption: true, }, - from: { - type: "number", - describe: "From block number", - defaultDescription: "Uses the database to calculate the last synced block", - }, - to: { + since: { type: "number", - describe: "To block number", - defaultDescription: "Syncs to the latest block number", + describe: "`since` is a timestamp. Neume will fetch changes since the provided timestamp.", + defaultDescription: "Uses the database to find the last synced timestamp", }, }, async (argv) => { const { config, strategies: strategyNames } = await import(path.resolve("./config.js")); - const to = argv.to ?? (await getLatestBlockNumber(config.rpc[0])); - await sync(argv.from, to, argv.url, config); - process.exit(0); + await sync(argv.since, argv.url, config, getStrategies(strategyNames)); + await tracksDB.close(); }) .command("create-change-index", "Create change index from primary database", async (argv) => { return db.createChangeIndex(); diff --git a/dist/ownership-history.mjs b/dist/ownership-history.mjs deleted file mode 100644 index 3d66f5c..0000000 --- a/dist/ownership-history.mjs +++ /dev/null @@ -1,56 +0,0 @@ -// This is a script to generate ownership history. -// Incomplete right now. -import { decodeParameters, encodeFunctionCall, toHex } from "eth-fun"; -import { env } from "process"; -import { DB } from "./database/index.js"; -import { messages } from "./extraction-worker/src/api.mjs"; -const BLOCK_NUMBER = parseInt(process.argv[2]); -const db = new DB("./tracks"); -const { route } = messages; -const options = { - url: env.RPC_HTTP_HOST, -}; -if (env.RPC_API_KEY) { - options.headers = { - Authorization: `Bearer ${env.RPC_API_KEY}`, - }; -} -const iterator = db.level.iterator(); -while (true) { - const entries = await iterator.nextv(parseInt(env.EXTRACTION_WORKER_CONCURRENCY)); - if (entries.length === 0) - break; - entries.map(async (e) => { - const [id, value] = e; - console.log(id); - const [chainId, address, tokenId, blockNumber] = id.split("/"); - const data = encodeFunctionCall({ - name: "ownerOf", - type: "function", - inputs: [ - { - name: "tokenId", - type: "uint256", - }, - ], - }, [tokenId]); - const message = await route({ - type: "json-rpc", - options, - version: "0.0.1", - method: "eth_call", - params: [ - { - from: null, - to: address, - data, - }, - toHex(BLOCK_NUMBER), - ], - }); - if (message.error) - console.log(id, message.error); - else - console.log(id, decodeParameters(["address"], message.results)); - }); -} diff --git a/dist/src/components/call-owner.js b/dist/src/components/call-owner.js index bebbc80..9038dc3 100644 --- a/dist/src/components/call-owner.js +++ b/dist/src/components/call-owner.js @@ -1,11 +1,9 @@ import { toHex, encodeFunctionSignature, decodeParameters } from "eth-fun"; import { randomItem } from "../utils.js"; -export async function callOwner(worker, config, to, blockNumber) { - if (!config.rpc.length) - throw new Error("Atleast one RPC host is required"); - const rpc = randomItem(config.rpc); +export async function callOwner(to, blockNumber) { + const rpc = randomItem(this.config.chain[this.chain].rpc); const data = encodeFunctionSignature("owner()"); - const msg = await worker({ + const msg = await this.worker({ type: "json-rpc", commissioner: "", version: "0.0.1", diff --git a/dist/src/components/call-tokenuri.js b/dist/src/components/call-tokenuri.js index 1776f7c..ac7fd0c 100644 --- a/dist/src/components/call-tokenuri.js +++ b/dist/src/components/call-tokenuri.js @@ -1,6 +1,6 @@ import { encodeFunctionCall, decodeParameters, toHex } from "eth-fun"; import { randomItem } from "../utils.js"; -export async function callTokenUri(worker, config, blockNumber, nft, overrideSignature) { +export async function callTokenUri(blockNumber, nft, overrideSignature) { const signature = overrideSignature ?? { name: "tokenURI", type: "function", @@ -11,7 +11,7 @@ export async function callTokenUri(worker, config, blockNumber, nft, overrideSig }, ], }; - const rpc = randomItem(config.rpc); + const rpc = randomItem(this.config.chain[this.chain].rpc); const options = { url: rpc.url, ...(rpc.key && { @@ -40,7 +40,7 @@ export async function callTokenUri(worker, config, blockNumber, nft, overrideSig toHex(blockNumber), ], }; - const ret = await worker(msg); + const ret = await this.worker(msg); if (ret.error) throw new Error(`Error while calling tokenURI on contract: ${JSON.stringify(ret, null, 2)}`); const uri = decodeParameters(["string"], ret.results)[0]; diff --git a/dist/src/components/eth-get-logs.js b/dist/src/components/eth-get-logs.js new file mode 100644 index 0000000..cbee27c --- /dev/null +++ b/dist/src/components/eth-get-logs.js @@ -0,0 +1,32 @@ +import { toHex } from "eth-fun"; +import { randomItem } from "../utils.js"; +export async function ethGetLogs(from, to, topics, address) { + const rpcHost = randomItem(this.config.chain[this.chain].rpc); + const msg = await this.worker({ + type: "json-rpc", + commissioner: "", + method: "eth_getLogs", + options: { + url: rpcHost.url, + headers: { + ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), + }, + retry: { + retries: 3, + }, + }, + params: [ + { + fromBlock: toHex(from), + toBlock: toHex(to), + ...(address && { address: address }), + topics: topics, + }, + ], + version: "0.0.1", + }); + if (msg.error) { + throw new Error(`Error occured while fetching Transfer events: ${JSON.stringify(msg, null, 2)}`); + } + return msg.results; +} diff --git a/dist/src/components/get-ipfs-tokenuri.js b/dist/src/components/get-ipfs-tokenuri.js index fcadc00..a4fc0c4 100644 --- a/dist/src/components/get-ipfs-tokenuri.js +++ b/dist/src/components/get-ipfs-tokenuri.js @@ -1,13 +1,13 @@ -export async function getIpfsTokenUri(uri, worker, config) { - if (!config.ipfs) +export async function getIpfsTokenUri(uri) { + if (!this.config.ipfs) throw new Error(`IPFS configuration is required for getIpfsTokenUri`); - const msg = await worker({ + const msg = await this.worker({ type: "ipfs", version: "0.0.1", commissioner: "", options: { uri: uri, - gateway: config.ipfs.httpsGateway, + gateway: this.config.ipfs.httpsGateway, retry: { retries: 3, }, diff --git a/dist/src/components/handle-transfer.js b/dist/src/components/handle-transfer.js new file mode 100644 index 0000000..cf61567 --- /dev/null +++ b/dist/src/components/handle-transfer.js @@ -0,0 +1,133 @@ +import { decodeLog } from "eth-fun"; +import { ethGetLogs } from "./eth-get-logs.js"; +import { tracksDB } from "../../database/tracks.js"; +const TRANSFER_EVENT_SELECTOR = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; +function debug(fn, name) { + let labelAdded = false; + const timeout = setTimeout(() => { + console.time(name); + labelAdded = true; + }, 30000); + const interval = setInterval(() => { + console.timeLog(name); + }, 60000); + return async function (...args) { + const resp = await fn.call(this, ...args); + clearTimeout(timeout); + clearInterval(interval); + if (labelAdded) + console.timeEnd(name); + return resp; + }; +} +export async function handleTransfer(from, to, recrawl) { + // `from - to` should be smaller than crawlStep but just in case + // it is not, call handleTransfer multiple times + const { crawlStep } = this.config.chain[this.chain]; + for (let i = from; i <= to; i += crawlStep + 1) { + const fromBlock = i; + const toBlock = Math.min(to, i + crawlStep); + await debug(_handleTransfer, `_handleTransfer is hung up for ${this.constructor.name} ${fromBlock}-${toBlock}`).call(this, fromBlock, toBlock, recrawl); + } +} +async function _handleTransfer(from, to, recrawl) { + const contractsStorage = this.contracts; + const { getLogsBlockSpanSize, getLogsAddressSize } = this.config.chain[this.chain]; + const iterator = contractsStorage.iterator(); + const entries = await iterator.all(); + const addresses = entries.map((e) => e[0]); + const mintNFTsPromise = []; + const allTransferNFTs = []; + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) { + const fromBlock = i; + const toBlock = Math.min(to, i + getLogsBlockSpanSize); + for (let j = 0; j < addresses.length; j += getLogsAddressSize) { + // console.log( + // `handle-transfer for ${this.constructor.name} from ${fromBlock} to ${toBlock} [j=${j}]`, + // ); + const addressSlice = addresses.slice(j, j + getLogsAddressSize); + const logs = await debug(ethGetLogs, `eth-getLogs is hung up for ${this.constructor.name} ${fromBlock}-${toBlock}-${j}`).call(this, fromBlock, toBlock, [TRANSFER_EVENT_SELECTOR], addressSlice); + let nfts = logs.map((log) => prepareNFT(log)); + // Partition NFTs into mints and transfers + const { mintNfts, transferNfts } = nfts.reduce((nfts, nft) => { + if (nft.erc721.transaction.from === "0x0000000000000000000000000000000000000000") { + nfts.mintNfts.push(nft); + } + else { + nfts.transferNfts.push(nft); + } + return nfts; + }, { mintNfts: [], transferNfts: [] }); + const promises = mintNfts.map(async (nft) => { + let uid; + if (!recrawl) { + uid = await this.nftToUid(nft); + if (await tracksDB.isTokenPresent(uid, nft.erc721.token.id)) + return; + } + // console.log(`fetching metadata for ${this.constructor.name}`, uid); + const track = await this.fetchMetadata(nft); + if (track) { + console.log("Found new NFT (could be a new track):", track?.title, nft.erc721.token.id, track?.platform.version, track?.platform.name, "at", nft.erc721.blockNumber); + await tracksDB.upsertTrack(track); + } + }); + mintNFTsPromise.push(...promises); + allTransferNFTs.push(...transferNfts); + } + } + await Promise.all(mintNFTsPromise); + await Promise.all(allTransferNFTs.map(async (nft) => { + let alias; + let uid = await this.nftToUid(nft); + let isTrackPresent = await tracksDB.isTrackPresent(uid); + // Track has not been crawled. Most probably we ignored it. + // Makes no sense to record ownership transfer. + if (!isTrackPresent) + return; + const owner = { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: alias ?? undefined, + }; + if (!recrawl && isTrackPresent) { + const isOwnerPresent = await tracksDB.isOwnerPresent(uid, nft.erc721.token.id, owner); + if (isOwnerPresent) + return; + } + await tracksDB.upsertOwner(uid, nft.erc721.token.id, owner, this.constructor.name); + console.log("Update ownership of", nft.erc721.address, "at", nft.erc721.blockNumber, "from", nft.erc721.transaction.from, "to", nft.erc721.transaction.to); + })); +} +function prepareNFT(log) { + if (!log.topics[3] || !log.transactionHash || !log.blockNumber) { + throw new Error(`log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`); + } + const decodedTopics = decodeLog([ + { indexed: true, name: "from", type: "address" }, + { indexed: true, name: "to", type: "address" }, + { indexed: true, name: "tokenId", type: "uint256" }, + ], log.data, log.topics.slice(1)); + return { + platform: { + name: "", + version: "1.0", + }, + erc721: { + blockNumber: parseInt(log.blockNumber, 16), + address: log.address, + transaction: { + from: decodedTopics[0], + to: decodedTopics[1], + transactionHash: log.transactionHash, + blockNumber: parseInt(log.blockNumber, 16), + }, + token: { + id: BigInt(log.topics[3]).toString(10), + }, + }, + metadata: {}, + }; +} diff --git a/dist/src/state.js b/dist/src/state.js index 6068a34..41ca44a 100644 --- a/dist/src/state.js +++ b/dist/src/state.js @@ -4,17 +4,25 @@ import fs from "fs/promises"; import path from "path"; import { CONSTANTS } from "./types.js"; -export async function getLastCrawledBlock() { - const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL); +import { getStrategies } from "./utils.js"; +export async function getLastCrawledBlock(strategy) { + const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL, strategy); + const { createdAtBlock } = getStrategies([strategy])[0]; const fileExists = await fs .access(location, fs.constants.F_OK) .then(() => true) .catch(() => false); if (!fileExists) - await saveLastCrawledBlock(CONSTANTS.FIRST_BLOCK); + await saveLastCrawledBlock(strategy, createdAtBlock); return fs.readFile(location, "utf-8").then(parseInt); } -export async function saveLastCrawledBlock(blockNumber) { - const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL); +export async function saveLastCrawledBlock(strategy, blockNumber) { + const location = path.resolve(CONSTANTS.DATA_DIR, CONSTANTS.STATE.LAST_CRAWL, strategy); + const fileExists = await fs + .access(path.dirname(location), fs.constants.F_OK) + .then(() => true) + .catch(() => false); + if (!fileExists) + await fs.mkdir(path.dirname(location), { recursive: true }); return fs.writeFile(location, blockNumber.toString(), "utf-8"); } diff --git a/dist/src/strategies/catalog.js b/dist/src/strategies/catalog.js deleted file mode 100644 index 9f33bf8..0000000 --- a/dist/src/strategies/catalog.js +++ /dev/null @@ -1,55 +0,0 @@ -// @ts-nocheck -// This file is incomplete -import { callTokenUri } from "../components/call-tokenuri.js"; -import { getIpfsTokenUri } from "../components/get-ipfs-tokenuri.js"; -export async function crawl(nft) { - console.log("crawling catalog"); - await callTokenUri(nft); - await getIpfsTokenUri(nft); - const datum = nft.erc721.token.tokenURIContent; - const version = "2.0.0"; - let duration; - if (datum?.duration) { - duration = `PT${Math.floor(datum.duration / 60)}M${(datum.duration % 60).toFixed(0)}S`; - } - return { - version, - title: datum.title, - duration, - artist: { - version, - name: datum.artist, - }, - platform: { - version, - name: "Catalog", - uri: "https://beta.catalog.works", - }, - erc721: { - // TODO: Stop hard coding this value - owner: "0x489e043540ff11ec22226ca0a6f6f8e3040c7b5a", - version, - createdAt: nft.erc721.createdAt, - tokenId: nft.erc721.token.id, - address: nft.erc721.address, - tokenURI: nft.erc721.token.tokenURI, - metadata: { - ...datum, - name: datum.name, - description: datum.description, - }, - }, - manifestations: [ - { - version, - uri: datum.image, - mimetype: "image", - }, - { - version, - uri: datum.losslessAudio, - mimetype: datum.mimeType, - }, - ], - }; -} diff --git a/dist/src/strategies/catalog_v2.js b/dist/src/strategies/catalog_v2.js index 3ba8016..ee83157 100644 --- a/dist/src/strategies/catalog_v2.js +++ b/dist/src/strategies/catalog_v2.js @@ -5,13 +5,22 @@ import { toHex, encodeFunctionCall, decodeParameters } from "eth-fun"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getIpfsTokenUri } from "../components/get-ipfs-tokenuri.js"; +import { CHAINS } from "../types.js"; import { randomItem } from "../utils.js"; +import { localStorage } from "../../database/localstorage.js"; +import { handleTransfer } from "../components/handle-transfer.js"; export default class CatalogV2 { constructor(worker, config) { - this.crawl = async (nft) => { - nft.erc721.token.uri = await callTokenUri(this.worker, this.config, nft.erc721.blockNumber, nft); + this.createdAtBlock = CatalogV2.createdAtBlock; + this.deprecatedAtBlock = null; + this.chain = CHAINS.eth; + this.crawl = async (from, to, recrawl) => { + await handleTransfer.call(this, from, to, recrawl); + }; + this.fetchMetadata = async (nft) => { + nft.erc721.token.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft); try { - nft.erc721.token.uriContent = await getIpfsTokenUri(nft.erc721.token.uri, this.worker, this.config); + nft.erc721.token.uriContent = (await getIpfsTokenUri.call(this, nft.erc721.token.uri)); } catch (err) { if (err.message.includes("Invalid CID")) { @@ -34,6 +43,7 @@ export default class CatalogV2 { version: CatalogV2.version, title: datum.title, duration, + uid: await this.nftToUid(nft), artist: { version: CatalogV2.version, name: datum.artist, @@ -45,23 +55,30 @@ export default class CatalogV2 { uri: "https://catalog.works", }, erc721: { - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, version: CatalogV2.version, createdAt: nft.erc721.blockNumber, - tokenId: nft.erc721.token.id, address: nft.erc721.address, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.name, - description: datum.description, - image: datum.image, - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.name, + description: datum.description, + image: datum.image, + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -77,8 +94,9 @@ export default class CatalogV2 { ], }; }; + this.nftToUid = async (nft) => `${this.chain}/${CatalogV2.name}/${nft.erc721.address.toLowerCase()}/${nft.erc721.token.id}`; this.callCreator = async (to, blockNumber, tokenId) => { - const rpc = randomItem(this.config.rpc); + const rpc = randomItem(this.config.chain[this.chain].rpc); const data = encodeFunctionCall({ name: "creator", type: "function", @@ -117,9 +135,19 @@ export default class CatalogV2 { }; this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(CatalogV2.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + const CATALOG_NFT_CONTRACT = "0x0bC2A24ce568DAd89691116d5B34DEB6C203F342"; + this.contracts.put(CATALOG_NFT_CONTRACT, { + name: CatalogV2.name, + version: CatalogV2.version, + }); } - updateOwner(nft) { } } CatalogV2.version = "2.0.0"; -CatalogV2.createdAtBlock = 0; -CatalogV2.deprecatedAtBlock = null; +// The Catalog contract was deployed at https://etherscan.io/tx/0x65a0c575267dae42937363299c58cb0d30e35b0a6741ff0dc079ffd927c8e1b2 +CatalogV2.createdAtBlock = 14566826; diff --git a/dist/src/strategies/lens/components.js b/dist/src/strategies/lens/components.js new file mode 100644 index 0000000..6a0febf --- /dev/null +++ b/dist/src/strategies/lens/components.js @@ -0,0 +1,147 @@ +import { toHex, encodeFunctionCall, decodeParameters } from "eth-fun"; +import { randomItem } from "../../utils.js"; +import Lens from "./lens.js"; +export async function getHandle(profileId, blockNumber) { + const rpcHost = randomItem(this.config.chain[this.chain].rpc); + const data = encodeFunctionCall({ + name: "getHandle", + type: "function", + inputs: [ + { + type: "uint256", + name: "", + }, + ], + }, [profileId]); + const msg = await this.worker({ + type: "json-rpc", + commissioner: "", + method: "eth_call", + options: { + url: rpcHost.url, + headers: { + ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), + }, + retry: { + retries: 3, + }, + }, + params: [ + { + to: Lens.LENS_HUB_ADDRESS, + data, + }, + toHex(blockNumber), + ], + version: "0.0.1", + }); + if (msg.error) + throw new Error(`Error while calling getHandle on contract: ${JSON.stringify(msg, null, 2)}`); + const handle = decodeParameters(["string"], msg.results)[0]; + if (typeof handle !== "string") + throw new Error(`Invalid result of getHandle for contract: ${JSON.stringify(msg, null, 2)}`); + return handle; +} +export async function getCollectNFT(profileId, pubId, blockNumber) { + const rpcHost = randomItem(this.config.chain[this.chain].rpc); + const data = encodeFunctionCall({ + name: "getCollectNFT", + type: "function", + inputs: [ + { + type: "uint256", + name: "", + }, + { + type: "uint256", + name: "", + }, + ], + }, [profileId, pubId]); + const msg = await this.worker({ + type: "json-rpc", + commissioner: "", + method: "eth_call", + options: { + url: rpcHost.url, + headers: { + ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), + }, + retry: { + retries: 3, + }, + }, + params: [ + { + to: Lens.LENS_HUB_ADDRESS, + data, + }, + toHex(blockNumber), + ], + version: "0.0.1", + }); + if (msg.error) + throw new Error(`Error while calling getContractNFT on contract: ${JSON.stringify(msg, null, 2)}`); + const address = decodeParameters(["address"], msg.results)[0]; + if (typeof address !== "string") + throw new Error(`invalid result of getContractNFT for contract: ${JSON.stringify(msg, null, 2)}`); + return address; +} +// The address may own multiple profiles but we are currently +// only interested in one of them. Hence, the zero index. +export async function getHandleByAddress(address, blockNumber) { + const rpcHost = randomItem(this.config.chain[this.chain].rpc); + const data = encodeFunctionCall({ + name: "tokenOfOwnerByIndex", + type: "function", + inputs: [ + { + type: "address", + name: "", + }, + { + type: "uint256", + name: "", + }, + ], + }, [address, 0]); + const msg = await this.worker({ + type: "json-rpc", + commissioner: "", + method: "eth_call", + options: { + url: rpcHost.url, + headers: { + ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), + }, + retry: { + retries: 3, + }, + }, + params: [ + { + to: Lens.LENS_HUB_ADDRESS, + data, + }, + toHex(blockNumber), + ], + version: "0.0.1", + }); + if (msg.error) + throw new Error(`Error while calling tokenOwnerByIndex on contract: ${JSON.stringify(msg, null, 2)} \n ${address}`); + const tokenId = parseInt(decodeParameters(["uint256"], msg.results)[0]); + if (typeof tokenId !== "number" || Number.isNaN(tokenId)) + throw new Error(`Invalid result of tokenOwnerByIndex for contract: ${JSON.stringify(msg, null, 2)}`); + const handle = await getHandle.call(this, tokenId, blockNumber); + return handle; +} +export async function getAlias(nft) { + let handle = null; + // Not every address will have an alias. Therefore, ignoring + // failures + try { + handle = await getHandleByAddress.call(this, nft.erc721.transaction.to, nft.erc721.blockNumber); + } + catch (err) { } + return handle; +} diff --git a/dist/src/strategies/lens/lens.js b/dist/src/strategies/lens/lens.js new file mode 100644 index 0000000..39be5cd --- /dev/null +++ b/dist/src/strategies/lens/lens.js @@ -0,0 +1,347 @@ +/** + * Current lens first song - 33474641 + */ +import { decodeLog, encodeParameters } from "eth-fun"; +import { CHAINS, PROTOCOLS } from "../../types.js"; +import { getProtocol } from "../../utils.js"; +import { localStorage } from "../../../database/localstorage.js"; +import { getArweaveTokenUri } from "../../components/get-arweave-tokenuri.js"; +import { getIpfsTokenUri } from "../../components/get-ipfs-tokenuri.js"; +import { fetchTokenUri } from "../../components/fetch-tokenuri.js"; +import { ethGetLogs } from "../../components/eth-get-logs.js"; +import { tracksDB } from "../../../database/tracks.js"; +import { handleTransfer } from "../../components/handle-transfer.js"; +import { getAlias, getCollectNFT, getHandle } from "./components.js"; +import { z } from "zod"; +export default class Lens { + constructor(worker, config) { + this.createdAtBlock = Lens.createdAtBlock; + this.deprecatedAtBlock = null; + this.chain = CHAINS.polygon; + // This is called when a new NFT is minted in Lens + this.fetchMetadata = async (nft) => { + let uid; + try { + uid = await this.addressToId.get(nft.erc721.address); + } + catch (err) { + console.log("Error for", nft); + throw err; + } + const track = await tracksDB.getTrack(uid); + const alias = await getAlias.call(this, nft); + track.erc721.tokens.push({ + id: nft.erc721.token.id, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: alias ?? undefined, + }, + ], + }); + return track; + }; + this.nftToUid = async (nft) => { + return this.addressToId.get(nft.erc721.address); + }; + this.worker = worker; + this.config = config; + this.localStorage = localStorage.sublevel(Lens.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + this.trackedIds = this.localStorage.sublevel("trackedIds", { + valueEncoding: "json", + }); + this.addressToId = this.localStorage.sublevel("addressToId", { + valueEncoding: "json", + }); + this.seenPosts = this.localStorage.sublevel("crawledPosts", { + valueEncoding: "json", + }); + } + async crawl(from, to, recrawl) { + console.time(`${Lens.name} handlePostCreated: ${from}-${to}`); + await this.handlePostCreated(from, to, recrawl); + console.timeEnd(`${Lens.name} handlePostCreated: ${from}-${to}`); + console.time(`${Lens.name} handleCollectNftDeployed: ${from}-${to}`); + await this.handleCollectNftDeployed(from, to, recrawl); + console.timeEnd(`${Lens.name} handleCollectNftDeployed: ${from}-${to}`); + console.time(`${Lens.name} handleTransfer: ${from}-${to}`); + await handleTransfer.call(this, from, to, recrawl); + console.timeEnd(`${Lens.name} handleTransfer: ${from}-${to}`); + } + async handleCollectNftDeployed(from, to, recrawl) { + const promises = []; + const { getLogsBlockSpanSize, getLogsAddressSize } = this.config.chain[this.chain]; + const MAX_TOPICS = getLogsAddressSize; + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) { + const fromBlock = i; + const toBlock = Math.min(to, i + getLogsBlockSpanSize); + const iter = this.trackedIds.iterator(); + const pendingEnteries = []; + while (true) { + const entries = [...pendingEnteries, ...(await iter.nextv(MAX_TOPICS))]; + if (entries.length === 0) { + break; + } + // prepare topics filter. we have to take care of max topics. + const profileIds = new Set(); + const pubIds = new Set(); + for (let j = 0; j < entries.length; j++) { + const [profileId, pubId] = entries[j][0].split("-"); + if (profileIds.size >= MAX_TOPICS - 1 || pubIds.size >= MAX_TOPICS - 1) { + pendingEnteries.push(entries[j]); + } + else { + profileIds.add(profileId); + pubIds.add(pubId); + } + } + const promise = ethGetLogs + .call(this, fromBlock, toBlock, [ + Lens.COLLECT_NFT_DEPLOYED, + Array.from(profileIds).map((i) => encodeParameters(["uint256"], [i])), + Array.from(pubIds).map((i) => encodeParameters(["uint256"], [i])), + ], [Lens.LENS_HUB_ADDRESS]) + .then(async (logs) => { + await Promise.all(logs.map(async (log) => { + if (!log.transactionHash || !log.blockNumber) { + throw new Error(`log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`); + } + const decodedTopics = decodeLog([ + { indexed: true, name: "profileId", type: "uint256" }, + { indexed: true, name: "pubId", type: "uint256" }, + { indexed: true, name: "collectNFT", type: "address" }, + { indexed: false, name: "timestamp", type: "uint256" }, + ], log.data, log.topics.slice(1)); + let { profileId, pubId, collectNFT } = decodedTopics; + collectNFT = collectNFT.toLowerCase(); + try { + await this.trackedIds.get(`${profileId}-${pubId}`); + } + catch (err) { + if (err.code === "LEVEL_NOT_FOUND") { + // We have ignored this particular track but a combination of this + // profileId and pubId is present in topics that is why we are here + return; + } + throw err; + } + await this.contracts.put(collectNFT, { + name: Lens.name, + version: Lens.version, + }); + await this.addressToId.put(collectNFT, `${this.chain}/${profileId}/${pubId}`); + await this.trackedIds.del(`${profileId}-${pubId}`); + const track = await tracksDB.getTrack(`${this.chain}/${profileId}/${pubId}`); + track.erc721.address = collectNFT; // We didn't have erc721.address at the time of crawl. Updating it now. + await tracksDB.upsertTrack(track); + })); + }); + promises.push(promise); + } + } + await Promise.all(promises); + } + async handlePostCreated(from, to, recrawl) { + // We are searching for PostCreatedEvents and not directly for + // CollectedNFTDeployed event because a song maybe posted that + // does not have any collectors + const _handlePostCreated = async (from, to, recrawl) => { + const logs = await ethGetLogs.call(this, from, to, [Lens.POST_CREATED_EVENT_SELECTOR], [Lens.LENS_HUB_ADDRESS]); + const posts = (await Promise.all(logs.map(async (log) => { + if (!log.transactionHash || !log.blockNumber) { + throw new Error(`log doesn't contain the required fields: ${JSON.stringify(log, null, 2)}`); + } + if (Lens.ignoredTransactions.includes(log.transactionHash)) { + return null; + } + const decodedTopics = decodeLog([ + { indexed: true, name: "profileId", type: "uint256" }, + { indexed: true, name: "pubId", type: "uint256" }, + { indexed: false, name: "contentURI", type: "string" }, + { indexed: false, name: "collectModule", type: "address" }, + { indexed: false, name: "collectModuleReturnData", type: "bytes" }, + { indexed: false, name: "referenceModule", type: "address" }, + { indexed: false, name: "referenceModuleReturnData", type: "bytes" }, + { indexed: false, name: "timestamp", type: "uint256" }, + ], log.data, log.topics.slice(1)); + return { + profileId: parseInt(decodedTopics[0]), + pubId: parseInt(decodedTopics[1]), + contentURI: decodedTopics[2], + collectModule: decodedTopics[3], + collectModuleReturnData: decodedTopics[4], + referenceModule: decodedTopics[5], + referenceModuleReturnData: decodedTopics[6], + timestamp: parseInt(decodedTopics[7]), + blockNumber: parseInt(log.blockNumber), + }; + }))).filter((post) => post !== null); + await Promise.all(posts.map(async (post) => { + let trackAlreadyPresent = await this.seenPosts + .get(`${post.profileId}-${post.pubId}`) + .then(() => true) + .catch(() => false); + let track; + try { + if (!recrawl && trackAlreadyPresent) + return; + track = await this.processPost(post); + await this.seenPosts.put(`${post.profileId}-${post.pubId}`, {}); + } + catch (err) { + console.log(post); + throw err; + } + if (track) { + await tracksDB.upsertTrack(track); + await this.trackedIds.put(`${post.profileId}-${post.pubId}`, 1); + console.dir(track, { depth: null }); + } + })); + }; + const { getLogsBlockSpanSize } = this.config.chain[this.chain]; + const promises = []; + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) { + const fromBlock = i; + const toBlock = Math.min(to, i + getLogsBlockSpanSize); + promises.push(_handlePostCreated(fromBlock, toBlock, recrawl)); + } + await Promise.all(promises); + } + async processPost(post) { + // console.log("Processing new post with contentURI", post.contentURI); + if (Lens.ignoredPosts.includes(`${post.profileId}-${post.pubId}`)) { + console.log("This post is ignored; skipping"); + return null; + } + // Regex for valid URIs; from: https://github.com/ajv-validator/ajv-formats/blob/4dd65447575b35d0187c6b125383366969e6267e/src/formats.ts#L229C12 + const URI = /^(?:[a-z][a-z0-9+\-.]*:)(?:\/?\/(?:(?:[a-z0-9\-._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:(?:[0-9a-f]{1,4}:){6}|::(?:[0-9a-f]{1,4}:){5}|(?:[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){4}|(?:(?:[0-9a-f]{1,4}:){0,1}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){3}|(?:(?:[0-9a-f]{1,4}:){0,2}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){2}|(?:(?:[0-9a-f]{1,4}:){0,3}[0-9a-f]{1,4})?::[0-9a-f]{1,4}:|(?:(?:[0-9a-f]{1,4}:){0,4}[0-9a-f]{1,4})?::)(?:[0-9a-f]{1,4}:[0-9a-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))|(?:(?:[0-9a-f]{1,4}:){0,5}[0-9a-f]{1,4})?::[0-9a-f]{1,4}|(?:(?:[0-9a-f]{1,4}:){0,6}[0-9a-f]{1,4})?::)|[Vv][0-9a-f]+\.[a-z0-9\-._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)|(?:[a-z0-9\-._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)(?:\?(?:[a-z0-9\-._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[a-z0-9\-._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; + if (!post.contentURI || !URI.test(post.contentURI)) + return null; + const protocol = getProtocol(post.contentURI); + if (!protocol) { + // console.log("Invalid protocol; skipping", post.contentURI); + return null; + } + let datum; + try { + if (protocol === PROTOCOLS.arweave) { + if (!/ar:\/\/[a-zA-Z0-9-_]{43}.*/.test(post.contentURI)) { + console.log(`Ignoring post id: ${post.profileId}-${post.pubId} because the content URI is invalid:`, post.contentURI); + return null; + } + datum = await getArweaveTokenUri(post.contentURI, this.worker, this.config); + } + else if (protocol === PROTOCOLS.ipfs) { + datum = await getIpfsTokenUri.call(this, post.contentURI); + } + else if (protocol === PROTOCOLS.https) { + datum = await fetchTokenUri(post.contentURI, this.worker); + } + else { + throw new Error(`Invalid Protocol for ${post.contentURI}`); + } + } + catch (err) { + if (err.message.includes("status: 4") || + err.message.includes("Invalid CID") || + err.message.includes("ECONNREFUSED")) { + return null; + } + throw err; + } + if (!datum || !datum.media || datum.version !== "2.0.0") { + // console.log("No media; skipping", datum.media, datum.version); + return null; + } + const media = datum.media.find?.((m) => m?.type?.includes("audio")); + if (!media) { + // console.log("No audio in media; skipping"); + return null; + } + const collectNftAdsress = (await getCollectNFT.call(this, post.profileId, post.pubId, post.blockNumber)).toLowerCase(); + const artistHandle = await getHandle.call(this, post.profileId, post.blockNumber); + try { + const schema = z.object({ + name: z.string(), + content: z.string(), + image: z.string().optional(), + }); + schema.passthrough().parse(datum); + } + catch { + // The required fields are not present in the metadata. Hence, ignoring it. + return null; + } + const track = { + version: Lens.version, + title: datum.name, + uid: `${this.chain}/${post.profileId}/${post.pubId}`, + artist: { + version: Lens.version, + name: artistHandle, + address: post.profileId.toString(), + }, + platform: { + version: Lens.version, + name: Lens.name, + uri: "https://lens.xyz", + }, + erc721: { + version: Lens.version, + address: collectNftAdsress, + tokens: [], + metadata: { + ...datum, + name: datum.name, + description: datum.content, + image: datum.image, + }, + }, + manifestations: [ + { + version: Lens.version, + uri: media.item, + mimetype: "audio", + }, + ], + }; + // datum.image can be undefined + if (datum?.image) + track.manifestations.push({ + version: Lens.version, + uri: datum.image, + mimetype: "image", + }); + return track; + } +} +Lens.version = "1.0.0"; +// The lens hub was created at this block: https://polygonscan.com/tx/0xca69b18b7e2daf4695c6d614e263d6aa9bdee44bee91bee7e0e6e5e5e4262fca +Lens.createdAtBlock = 28384641; +Lens.LENS_HUB_ADDRESS = "0xDb46d1Dc155634FbC732f92E853b10B288AD5a1d"; +Lens.POST_CREATED_EVENT_SELECTOR = "0xc672c38b4d26c3c978228e99164105280410b144af24dd3ed8e4f9d211d96a50"; +Lens.COLLECT_NFT_DEPLOYED = "0x0b227b550ffed48af813b32e246f787e99581ee13206ba8f9d90d63615269b3f"; +Lens.TRANSFER_EVENT_SELECTOR = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; +// `${post.profileId}-${post.pubId}` +Lens.ignoredPosts = [ + "39133-682", + "88863-16", + "18497-28", + "3834-24", + "40863-4", + "40635-1", + "49754-2", +]; +// The following transactions can't be decoded +Lens.ignoredTransactions = [ + "0x52c63367c36eb24a08654c89dc647267a9f1171af3962dafe5cefab31e21293d", +]; diff --git a/dist/src/strategies/mintsongs_v2.js b/dist/src/strategies/mintsongs_v2.js index a35520d..050b921 100644 --- a/dist/src/strategies/mintsongs_v2.js +++ b/dist/src/strategies/mintsongs_v2.js @@ -10,10 +10,19 @@ import { toHex, encodeFunctionCall, decodeParameters } from "eth-fun"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getIpfsTokenUri } from "../components/get-ipfs-tokenuri.js"; +import { CHAINS } from "../types.js"; import { randomItem } from "../utils.js"; +import { handleTransfer } from "../components/handle-transfer.js"; +import { localStorage } from "../../database/localstorage.js"; export default class MintSongsV2 { constructor(worker, config) { - this.crawl = async (nft) => { + this.createdAtBlock = MintSongsV2.createdAtBlock; + this.deprecatedAtBlock = null; + this.chain = CHAINS.eth; + this.crawl = async (from, to, recrawl) => { + await handleTransfer.call(this, from, to, recrawl); + }; + this.fetchMetadata = async (nft) => { // Crawling MintSongs at this block number or higher // because the contract is broken at the block the NFTs // were minted. Contract was upgraded later many times. @@ -22,9 +31,9 @@ export default class MintSongsV2 { console.log(`Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because it is blacklisted`); return null; } - nft.erc721.token.uri = await callTokenUri(this.worker, this.config, Math.max(nft.erc721.blockNumber, BLOCK_NUMBER), nft); + nft.erc721.token.uri = await callTokenUri.call(this, Math.max(nft.erc721.blockNumber, BLOCK_NUMBER), nft); try { - nft.erc721.token.uriContent = await getIpfsTokenUri(nft.erc721.token.uri, this.worker, this.config); + nft.erc721.token.uriContent = (await getIpfsTokenUri.call(this, nft.erc721.token.uri)); } catch (err) { if (err.message.includes("Invalid CID")) { @@ -47,6 +56,7 @@ export default class MintSongsV2 { version: MintSongsV2.version, title: datum.title, duration, + uid: await this.nftToUid(nft), artist: { version: MintSongsV2.version, name: datum.artist, @@ -58,23 +68,30 @@ export default class MintSongsV2 { uri: "https://www.mintsongs.com/", }, erc721: { - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, version: MintSongsV2.version, createdAt: nft.erc721.blockNumber, - tokenId: nft.erc721.token.id, address: nft.erc721.address, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.title, - description: datum.description, - image: datum.image, - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.title, + description: datum.description, + image: datum.image, + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -90,8 +107,9 @@ export default class MintSongsV2 { ], }; }; + this.nftToUid = async (nft) => `${this.chain}/${MintSongsV2.name}/${nft.erc721.address.toLowerCase()}/${nft.erc721.token.id}`; this.callTokenCreator = async (to, blockNumber, tokenId) => { - const rpc = randomItem(this.config.rpc); + const rpc = randomItem(this.config.chain[this.chain].rpc); const data = encodeFunctionCall({ name: "tokenCreator", type: "function", @@ -130,13 +148,22 @@ export default class MintSongsV2 { }; this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(MintSongsV2.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + const MINGSONGS_NFT_CONTRACT = "0x2b5426a5b98a3e366230eba9f95a24f09ae4a584"; + this.contracts.put(MINGSONGS_NFT_CONTRACT, { + name: MintSongsV2.name, + version: MintSongsV2.version, + }); } - updateOwner(nft) { } } MintSongsV2.version = "2.0.0"; // Oldest NFT mint found using OpenSea: https://etherscan.io/tx/0x4dd17de92c1d1ae0a7d17c127c57d99fd509f1b22dd176a483e5587fddf7e0a0 MintSongsV2.createdAtBlock = 14799837; -MintSongsV2.deprecatedAtBlock = null; MintSongsV2.invalidIDs = [ /^0x2b5426a5b98a3e366230eba9f95a24f09ae4a584\/13$/, /^0x2b5426a5b98a3e366230eba9f95a24f09ae4a584\/113$/, // NFT has been burned diff --git a/dist/src/strategies/noizd.js b/dist/src/strategies/noizd.js index c6c43cc..e27ba10 100644 --- a/dist/src/strategies/noizd.js +++ b/dist/src/strategies/noizd.js @@ -7,12 +7,21 @@ */ import { callTokenUri } from "../components/call-tokenuri.js"; import { getIpfsTokenUri } from "../components/get-ipfs-tokenuri.js"; +import { handleTransfer } from "../components/handle-transfer.js"; +import { CHAINS } from "../types.js"; +import { localStorage } from "../../database/localstorage.js"; export default class Noizd { constructor(worker, config) { - this.crawl = async (nft) => { - nft.erc721.token.uri = await callTokenUri(this.worker, this.config, nft.erc721.blockNumber, nft); + this.createdAtBlock = Noizd.createdAtBlock; + this.deprecatedAtBlock = null; + this.chain = CHAINS.eth; + this.crawl = async (from, to, recrawl) => { + await handleTransfer.call(this, from, to, recrawl); + }; + this.fetchMetadata = async (nft) => { + nft.erc721.token.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft); try { - nft.erc721.token.uriContent = await getIpfsTokenUri(nft.erc721.token.uri, this.worker, this.config); + nft.erc721.token.uriContent = (await getIpfsTokenUri.call(this, nft.erc721.token.uri)); } catch (err) { if (err.message.includes("Invalid CID")) { @@ -34,6 +43,7 @@ export default class Noizd { version: Noizd.version, title: datum.name, duration, + uid: await this.nftToUid(nft), artist: { version: Noizd.version, name: datum.artist_name, @@ -45,23 +55,30 @@ export default class Noizd { uri: "https://noizd.com", }, erc721: { - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, version: Noizd.version, createdAt: nft.erc721.blockNumber, - tokenId: nft.erc721.token.id, address: nft.erc721.address, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.name, - description: datum.description, - image: datum.image, - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.name, + description: datum.description, + image: datum.image, + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -77,12 +94,22 @@ export default class Noizd { ], }; }; + this.nftToUid = async (nft) => `${this.chain}/${Noizd.name}/${nft.erc721.address.toLowerCase()}/${nft.erc721.token.id}`; this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(Noizd.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + const NOIZD_NFT_CONTRACT = "0xf5819e27b9bad9f97c177bf007c1f96f26d91ca6"; + this.contracts.put(NOIZD_NFT_CONTRACT, { + name: Noizd.name, + version: Noizd.version, + }); } - updateOwner(nft) { } } Noizd.version = "1.0.0"; // Oldest NFT mint found using OpenSea: https://etherscan.io/tx/0x9cd2b56dadc49a3c6ddb5f130de9c932a78a0ccb21c930ab978ce51cc5819901 Noizd.createdAtBlock = 13493464; -Noizd.deprecatedAtBlock = null; diff --git a/dist/src/strategies/sound.js b/dist/src/strategies/sound.js index 620b833..8a45bcf 100644 --- a/dist/src/strategies/sound.js +++ b/dist/src/strategies/sound.js @@ -5,13 +5,27 @@ import { decodeLog, toHex } from "eth-fun"; import { callTokenUri } from "../components/call-tokenuri.js"; import { fetchTokenUri } from "../components/fetch-tokenuri.js"; import { callOwner } from "../components/call-owner.js"; +import { CHAINS } from "../types.js"; import { randomItem } from "../utils.js"; import { ifIpfsConvertToNativeIpfs } from "ipfs-uri-utils"; +import { localStorage } from "../../database/localstorage.js"; +import { handleTransfer } from "../components/handle-transfer.js"; export default class Sound { constructor(worker, config) { - this.filterContracts = async (from, to) => { + this.createdAtBlock = Sound.createdAtBlock; + this.deprecatedAtBlock = null; + this.chain = CHAINS.eth; + this.crawl = async (from, to, recrawl) => { + const { getLogsBlockSpanSize } = this.config.chain[this.chain]; + const handleArtistCreatedPromises = []; + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) + handleArtistCreatedPromises.push(this.handleArtistCreated(i, i + getLogsBlockSpanSize)); + await Promise.all(handleArtistCreatedPromises); + await handleTransfer.call(this, from, to, recrawl); + }; + this.handleArtistCreated = async (from, to) => { const artistCreatedSelector = "0x23748b43b77f98380e738976c6324996908ffc1989994dd3c68631c87a65a7c0"; - const rpcHost = randomItem(this.config.rpc); + const rpcHost = randomItem(this.config.chain[this.chain].rpc); const options = { url: rpcHost.url, headers: { @@ -69,9 +83,13 @@ export default class Sound { version: Sound.version, }; }); - return contracts; + await Promise.all(contracts.map(async (c) => { + // Save contract address that is to be checked for NFTs in future + await this.contracts.put(c.address, { name: c.name, version: c.version }); + })); }; - this.crawl = async (nft) => { + this.nftToUid = async (nft) => `${this.chain}/${Sound.name}/${nft.erc721.address.toLowerCase()}`; + this.fetchMetadata = async (nft) => { // Instead of querying at the block number soundxyz NFT // was minted, we query at a higher block number because // soundxyz changed their tokenURI and the previous one @@ -85,13 +103,14 @@ export default class Sound { console.log(`Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because it is blacklisted`); return null; } - nft.erc721.token.uri = await callTokenUri(this.worker, this.config, Math.max(nft.erc721.blockNumber, WORKING_AFTER_BLOCK), nft); + nft.erc721.token.uri = (await callTokenUri.call(this, Math.max(nft.erc721.blockNumber, WORKING_AFTER_BLOCK), nft)); nft.erc721.token.uriContent = await fetchTokenUri(nft.erc721.token.uri, this.worker); - nft.creator = await callOwner(this.worker, this.config, nft.erc721.address, nft.erc721.blockNumber); + nft.creator = await callOwner.call(this, nft.erc721.address, nft.erc721.blockNumber); const datum = nft.erc721.token.uriContent; return { version: Sound.version, title: datum.name, + uid: await this.nftToUid(nft), artist: { version: Sound.version, name: datum.artist_name, @@ -105,21 +124,28 @@ export default class Sound { erc721: { version: Sound.version, createdAt: nft.erc721.blockNumber, - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, address: nft.erc721.address, - tokenId: nft.erc721.token.id, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: datum.name, - description: datum.description, - image: datum.image, - }, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: datum.name, + description: datum.description, + image: datum.image, + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], }, manifestations: [ { @@ -142,10 +168,14 @@ export default class Sound { }; this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(Sound.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); } - updateOwner(nft) { } } Sound.version = "1.0.0"; Sound.createdAtBlock = 13725566; -Sound.deprecatedAtBlock = null; Sound.invalidIDs = []; diff --git a/dist/src/strategies/sound_protocol.js b/dist/src/strategies/sound_protocol.js index 8f6973a..8297c01 100644 --- a/dist/src/strategies/sound_protocol.js +++ b/dist/src/strategies/sound_protocol.js @@ -1,43 +1,33 @@ -import { decodeLog, toHex } from "eth-fun"; +import { decodeLog } from "eth-fun"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getArweaveTokenUri } from "../components/get-arweave-tokenuri.js"; import { callOwner } from "../components/call-owner.js"; -import { randomItem } from "../utils.js"; +import { CHAINS } from "../types.js"; +import { ethGetLogs } from "../components/eth-get-logs.js"; +import { localStorage } from "../../database/localstorage.js"; +import { handleTransfer } from "../components/handle-transfer.js"; +import { z } from "zod"; +import { tracksDB } from "../../database/tracks.js"; export default class SoundProtocol { constructor(worker, config) { - this.filterContracts = async (from, to) => { - const editionCreatedSelector = "0x405098db99342b699216d8150e930dbbf2f686f5a43485aed1e69219dafd4935"; - const rpcHost = randomItem(this.config.rpc); - const options = { - url: rpcHost.url, - headers: { - ...(rpcHost.key && { Authorization: `Bearer ${rpcHost.key}` }), - }, - retry: { - retries: 3, - }, - }; - const fromBlock = toHex(from); - const toBlock = toHex(to); - const message = await this.worker({ - type: "json-rpc", - method: "eth_getLogs", - commissioner: SoundProtocol.name, - params: [ - { - fromBlock, - toBlock, - topics: [[editionCreatedSelector]], - }, - ], - version: "0.0.1", - options, - }); - if (message.error) { - throw new Error(`Error occured while filtering ${SoundProtocol.name} contracts: \n${JSON.stringify(message, null, 2)}`); - } - const logs = message.results; - return logs.map((log) => { + this.createdAtBlock = SoundProtocol.createdAtBlock; + this.deprecatedAtBlock = null; + this.chain = SoundProtocol.chain; + this.crawl = async (from, to, recrawl) => { + const { getLogsBlockSpanSize } = this.config.chain[this.chain]; + const handleEditionCreatedPromises = []; + console.time(`${SoundProtocol.name} handleEditionCreated: ${from}-${to}`); + for (let i = from; i <= to; i += getLogsBlockSpanSize + 1) + handleEditionCreatedPromises.push(this.handleEditionCreated(i, i + getLogsBlockSpanSize)); + await Promise.all(handleEditionCreatedPromises); + console.timeEnd(`${SoundProtocol.name} handleEditionCreated: ${from}-${to}`); + console.time(`${SoundProtocol.name} handleTransfer: ${from}-${to}`); + await this.handleTransfer(from, to, recrawl); + console.timeEnd(`${SoundProtocol.name} handleTransfer: ${from}-${to}`); + }; + this.handleEditionCreated = async (from, to) => { + const logs = await ethGetLogs.call(this, from, to, [[SoundProtocol.EDITION_CREATED_SELECTOR]]); + const contracts = logs.map((log) => { const topics = log.topics; topics.shift(); const result = decodeLog([ @@ -54,13 +44,40 @@ export default class SoundProtocol { version: SoundProtocol.version, }; }); + await Promise.all(contracts.map(async (c) => { + console.log("Found a SoundProtocol contract", c.address); + // Save contract address that is to be checked for NFTs in future + await this.contracts.put(c.address, { name: c.name, version: c.version }); + })); }; - this.crawl = async (nft) => { + this.handleTransfer = handleTransfer.bind(this); + this.nftToUid = async (nft) => `${this.chain}/${SoundProtocol.name}/${nft.erc721.address.toLowerCase()}`; + this.fetchMetadata = async (nft) => { if (SoundProtocol.invalidIDs.filter((id) => `${nft.erc721.address}/${nft.erc721.token.id}`.match(id)).length != 0) { - console.log(`Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because it is blacklisted`); + // console.log( + // `Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because it is blacklisted`, + // ); return null; } - nft.erc721.token.uri = await callTokenUri(this.worker, this.config, nft.erc721.blockNumber, nft); + const uid = await this.nftToUid(nft); + if (await tracksDB.isTrackPresent(uid)) { + // Metadata already present, don't fetch from arweave again. + const track = await tracksDB.getTrack(uid); + track.erc721.tokens.push({ + id: nft.erc721.token.id, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }); + return track; + } + nft.erc721.token.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft); if (!nft.erc721.token.uri.includes("ar://")) { console.log(`Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because includes invalid tokenURI`); return null; @@ -75,12 +92,21 @@ export default class SoundProtocol { } throw err; } - nft.creator = await callOwner(this.worker, this.config, nft.erc721.address, nft.erc721.blockNumber); + nft.creator = await callOwner.call(this, nft.erc721.address, nft.erc721.blockNumber); try { const datum = nft.erc721.token.uriContent; + const schema = z.object({ + name: z.string(), + artist: z.string(), + description: z.string(), + image: z.string(), + losslessAudio: z.string(), + }); + schema.passthrough().parse(datum); return { version: SoundProtocol.version, title: datum.name, + uid: await this.nftToUid(nft), artist: { version: SoundProtocol.version, name: datum.artist, @@ -88,21 +114,28 @@ export default class SoundProtocol { }, platform: { version: SoundProtocol.version, - name: "Sound Protocol", + name: SoundProtocol.name, uri: "https://sound.xyz", }, erc721: { version: SoundProtocol.version, createdAt: nft.erc721.blockNumber, - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, address: nft.erc721.address, - tokenId: nft.erc721.token.id, - tokenURI: nft.erc721.token.uri, + tokens: [ + { + id: nft.erc721.token.id, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], + uri: nft.erc721.token.uri, metadata: { ...datum, name: datum.name, @@ -124,19 +157,29 @@ export default class SoundProtocol { ], }; } - catch { + catch (err) { // Failed to transform the track. Most probably the metadata is // incorrectly formatted. Ignoring the track. - console.log(`Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because of incorrect metadata`); + console.log(`Ignoring ${nft.erc721.address}/${nft.erc721.token.id} because of incorrect metadata - ${err.code}`); return null; } }; this.worker = worker; this.config = config; + this.localStorage = localStorage.sublevel(SoundProtocol.name, { + valueEncoding: "json", + }); + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); } - updateOwner(nft) { } } SoundProtocol.version = "2.0.0"; SoundProtocol.createdAtBlock = 15570834; -SoundProtocol.deprecatedAtBlock = null; SoundProtocol.invalidIDs = []; +SoundProtocol.EDITION_CREATED_SELECTOR = "0x405098db99342b699216d8150e930dbbf2f686f5a43485aed1e69219dafd4935"; +SoundProtocol.chain = CHAINS.eth; +// const { config }: { config: Config } = await import(path.resolve("./config.js")); +// const soundProtocol = new SoundProtocol(ExtractionWorker(config.worker), config); +// await soundProtocol.crawl(16_01_0000, 16_05_0000, false); +// process.exit(0); diff --git a/dist/src/strategies/zora.js b/dist/src/strategies/zora.js index 9594ec1..1191bb0 100644 --- a/dist/src/strategies/zora.js +++ b/dist/src/strategies/zora.js @@ -7,11 +7,132 @@ import { toHex, encodeFunctionCall, decodeParameters } from "eth-fun"; import { anyIpfsToNativeIpfs } from "ipfs-uri-utils"; import { callTokenUri } from "../components/call-tokenuri.js"; import { getIpfsTokenUri } from "../components/get-ipfs-tokenuri.js"; +import { CHAINS } from "../types.js"; import { randomItem } from "../utils.js"; +import { handleTransfer } from "../components/handle-transfer.js"; +import { localStorage } from "../../database/localstorage.js"; export default class Zora { constructor(worker, config) { + this.createdAtBlock = Zora.createdAtBlock; // First catalog song: https://etherscan.io/nft/0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7/1678 + // Last song on Zora contract: https://beta.catalog.works/lucalush/velvet-girls + // https://cloud.hasura.io/public/graphiql?endpoint=https%3A%2F%2Fcatalog-prod.hasura.app%2Fv1%2Fgraphql&query=query+MyQuery+%7B%0A++tracks%28%0A++++where%3A+%7Bcontract_address%3A+%7B_iregex%3A+%220xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7%22%7D%7D%0A++++order_by%3A+%7Bcreated_at%3A+desc%7D%0A++%29+%7B%0A++++created_at%0A++++contract_address%0A++++short_url%0A++++title%0A++++nft_id%0A++%7D%0A%7D%0A + this.deprecatedAtBlock = null; + this.chain = CHAINS.eth; + this.crawl = async (from, to, recrawl) => { + await handleTransfer.call(this, from, to, recrawl); + }; + this.fetchMetadata = async (nft) => { + nft.erc721.token.uri = (await callTokenUri.call(this, nft.erc721.blockNumber, nft)); + try { + nft.erc721.token.uri = anyIpfsToNativeIpfs(nft.erc721.token.uri); + } + catch (err) { + console.warn("Invalid tokenURI: Couldn't convert to IPFS URI. Ignoring the given track.", JSON.stringify(nft, null, 2)); + return null; + } + nft.metadata.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft, { + name: "tokenMetadataURI", + type: "function", + inputs: [ + { + name: "tokenId", + type: "uint256", + }, + ], + }); + try { + nft.metadata.uri = anyIpfsToNativeIpfs(nft.metadata.uri); + } + catch (err) { + console.warn("Invalid tokenURI: Couldn't convert to IPFS URI. Ignoring the given track.", JSON.stringify(nft, null, 2)); + return null; + } + try { + nft.metadata.uriContent = await getIpfsTokenUri.call(this, nft.metadata.uri); + } + catch (err) { + if (err.message.includes("Invalid CID")) { + console.warn("Invalid CID: Ignoring the given track.", JSON.stringify(nft, null, 2)); + return null; + } + if (err.message.includes("504") || err.message.includes("AbortError")) { + console.warn("Couldn't find CID on the IPFS network: Ignoring NFT", JSON.stringify(nft, null, 2)); + return null; + } + throw err; + } + // Assumption that is specific to Catalog + if (!nft.metadata.uriContent?.body?.version?.includes("catalog")) { + return null; + } + nft.creator = await this.callTokenCreator(nft.erc721.address, nft.erc721.blockNumber, nft.erc721.token.id); + const datum = nft.metadata.uriContent; + const title = datum?.body?.title || datum?.name; + const artist = datum?.body?.artist; + const description = datum?.body?.notes; + const artwork = datum?.body?.artwork?.info?.uri; + let duration; + if (datum.body && datum.body.duration) { + duration = `PT${Math.floor(datum.body.duration / 60)}M${(datum.body.duration % 60).toFixed(0)}S`; + } + return { + version: Zora.version, + title, + duration, + uid: await this.nftToUid(nft), + artist: { + version: Zora.version, + name: artist, + address: nft.creator, + }, + platform: { + version: Zora.version, + name: "Catalog", + uri: "https://catalog.works", + }, + erc721: { + version: Zora.version, + createdAt: nft.erc721.blockNumber, + address: nft.erc721.address, + tokens: [ + { + id: nft.erc721.token.id, + uri: nft.erc721.token.uri, + metadata: { + ...datum, + name: title, + description, + // TODO: add image here + }, + owners: [ + { + from: nft.erc721.transaction.from, + to: nft.erc721.transaction.to, + blockNumber: nft.erc721.blockNumber, + transactionHash: nft.erc721.transaction.transactionHash, + alias: undefined, + }, + ], + }, + ], + }, + manifestations: [ + { + version: Zora.version, + uri: nft.erc721.token.uri, + mimetype: datum.body.mimeType, + }, + { + version: Zora.version, + uri: artwork, + mimetype: "image", + }, + ], + }; + }; + this.nftToUid = async (nft) => `${this.chain}/${Zora.name}/${nft.erc721.address.toLowerCase()}/${nft.erc721.token.id}`; this.callTokenCreator = async (to, blockNumber, tokenId) => { - const rpc = randomItem(this.config.rpc); + const rpc = randomItem(this.config.chain[this.chain].rpc); const data = encodeFunctionCall({ name: "tokenCreators", type: "function", @@ -50,112 +171,15 @@ export default class Zora { }; this.worker = worker; this.config = config; - } - async crawl(nft) { - nft.erc721.token.uri = await callTokenUri(this.worker, this.config, nft.erc721.blockNumber, nft); - try { - nft.erc721.token.uri = anyIpfsToNativeIpfs(nft.erc721.token.uri); - } - catch (err) { - console.warn("Invalid tokenURI: Couldn't convert to IPFS URI. Ignoring the given track.", JSON.stringify(nft, null, 2)); - return null; - } - nft.metadata.uri = await callTokenUri(this.worker, this.config, nft.erc721.blockNumber, nft, { - name: "tokenMetadataURI", - type: "function", - inputs: [ - { - name: "tokenId", - type: "uint256", - }, - ], + this.localStorage = localStorage.sublevel(Zora.name, { + valueEncoding: "json", }); - try { - nft.metadata.uri = anyIpfsToNativeIpfs(nft.metadata.uri); - } - catch (err) { - console.warn("Invalid tokenURI: Couldn't convert to IPFS URI. Ignoring the given track.", JSON.stringify(nft, null, 2)); - return null; - } - try { - nft.metadata.uriContent = await getIpfsTokenUri(nft.metadata.uri, this.worker, this.config); - } - catch (err) { - if (err.message.includes("Invalid CID")) { - console.warn("Invalid CID: Ignoring the given track.", JSON.stringify(nft, null, 2)); - return null; - } - if (err.message.includes("504") || err.message.includes("AbortError")) { - console.warn("Couldn't find CID on the IPFS network: Ignoring NFT", JSON.stringify(nft, null, 2)); - return null; - } - throw err; - } - // Assumption that is specific to Catalog - if (!nft.metadata.uriContent?.body?.version?.includes("catalog")) { - return null; - } - nft.creator = await this.callTokenCreator(nft.erc721.address, nft.erc721.blockNumber, nft.erc721.token.id); - const datum = nft.metadata.uriContent; - const title = datum?.body?.title || datum?.name; - const artist = datum?.body?.artist; - const description = datum?.body?.notes; - const artwork = datum?.body?.artwork?.info?.uri; - let duration; - if (datum.body && datum.body.duration) { - duration = `PT${Math.floor(datum.body.duration / 60)}M${(datum.body.duration % 60).toFixed(0)}S`; - } - return { - version: Zora.version, - title, - duration, - artist: { - version: Zora.version, - name: artist, - address: nft.creator, - }, - platform: { - version: Zora.version, - name: "Catalog", - uri: "https://catalog.works", - }, - erc721: { - version: Zora.version, - createdAt: nft.erc721.blockNumber, - transaction: { - from: nft.erc721.transaction.from, - to: nft.erc721.transaction.to, - blockNumber: nft.erc721.transaction.blockNumber, - transactionHash: nft.erc721.transaction.transactionHash, - }, - address: nft.erc721.address, - tokenId: nft.erc721.token.id, - tokenURI: nft.erc721.token.uri, - metadata: { - ...datum, - name: title, - description, - // TODO: add image here - }, - }, - manifestations: [ - { - version: Zora.version, - uri: nft.erc721.token.uri, - mimetype: datum.body.mimeType, - }, - { - version: Zora.version, - uri: artwork, - mimetype: "image", - }, - ], - }; + this.contracts = this.localStorage.sublevel("contracts", { + valueEncoding: "json", + }); + const ZORA_NFT_CONTRACT = "0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7"; + this.contracts.put(ZORA_NFT_CONTRACT, { name: Zora.name, version: Zora.version }); } - updateOwner(nft) { } } Zora.version = "1.0.0"; -Zora.createdAtBlock = 11996516; // First catalog song: https://etherscan.io/nft/0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7/1678 -// Last song on Zora contract: https://beta.catalog.works/lucalush/velvet-girls -// https://cloud.hasura.io/public/graphiql?endpoint=https%3A%2F%2Fcatalog-prod.hasura.app%2Fv1%2Fgraphql&query=query+MyQuery+%7B%0A++tracks%28%0A++++where%3A+%7Bcontract_address%3A+%7B_iregex%3A+%220xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7%22%7D%7D%0A++++order_by%3A+%7Bcreated_at%3A+desc%7D%0A++%29+%7B%0A++++created_at%0A++++contract_address%0A++++short_url%0A++++title%0A++++nft_id%0A++%7D%0A%7D%0A -Zora.deprecatedAtBlock = null; +Zora.createdAtBlock = 11996516; diff --git a/dist/src/types.js b/dist/src/types.js index 77041f1..23e2463 100644 --- a/dist/src/types.js +++ b/dist/src/types.js @@ -1,8 +1,22 @@ +export var CHAINS; +(function (CHAINS) { + CHAINS["eth"] = "eth"; + CHAINS["polygon"] = "polygon"; +})(CHAINS = CHAINS || (CHAINS = {})); +export var PROTOCOLS; +(function (PROTOCOLS) { + PROTOCOLS["arweave"] = "arweave"; + PROTOCOLS["https"] = "https"; + PROTOCOLS["ipfs"] = "ipfs"; +})(PROTOCOLS = PROTOCOLS || (PROTOCOLS = {})); export const CONSTANTS = { DATA_DIR: "data", STATE: { LAST_SYNC: "last_synced_block", LAST_CRAWL: "last_crawled_block", }, - FIRST_BLOCK: 11000000, + FIRST_BLOCK: { + [CHAINS.eth]: 11000000, + [CHAINS.polygon]: 11000000, + }, }; diff --git a/dist/src/utils.js b/dist/src/utils.js index ac87cbe..a3e9f06 100644 --- a/dist/src/utils.js +++ b/dist/src/utils.js @@ -1,12 +1,12 @@ -import { readFile } from "fs/promises"; -import path from "path"; import https from "https"; +import { PROTOCOLS } from "./types.js"; import Sound from "./strategies/sound.js"; import SoundProtocol from "./strategies/sound_protocol.js"; import Zora from "./strategies/zora.js"; import CatalogV2 from "./strategies/catalog_v2.js"; import MintSongsV2 from "./strategies/mintsongs_v2.js"; import Noizd from "./strategies/noizd.js"; +import Lens from "./strategies/lens/lens.js"; export function randomItem(arr) { return arr[Math.floor(Math.random() * arr.length)]; } @@ -37,38 +37,27 @@ export function getLatestBlockNumber(rpcHost) { req.end(); }); } -export async function getDefaultContracts() { - const defaultContractsPath = new URL("../assets/contracts.hardcode.json", import.meta.url); - return JSON.parse(await readFile(defaultContractsPath, "utf-8")); -} -export async function getUserContracts() { - const userContractsPath = path.resolve("./data/contracts.json"); - return JSON.parse(await readFile(userContractsPath, "utf-8")); -} -/** - * User's contracts.json contains the new found addresses - * Neume's contracts.hardcode.json contains hardcoded addresses - * This function reads and merge them both. - */ -export async function getAllContracts() { - return { - ...(await getDefaultContracts()), - ...(await getUserContracts()), - }; -} /** * New strategies should be added here. */ -export function getStrategies(strategyNames, from, to) { +export function getStrategies(strategyNames) { const strategies = [ Sound, + Lens, SoundProtocol, Zora, CatalogV2, MintSongsV2, Noizd, ]; - return strategies.filter((s) => s.createdAtBlock <= from && - to <= (s.deprecatedAtBlock ?? Number.MAX_VALUE) && - strategyNames.includes(s.name)); + return strategies.filter((s) => strategyNames.includes(s.name)); +} +export function getProtocol(uri) { + if (uri.startsWith("ar://")) + return PROTOCOLS.arweave; + else if (uri.startsWith("ipfs://")) + return PROTOCOLS.ipfs; + else if (uri.startsWith("http://") || uri.startsWith("https://")) + return PROTOCOLS.https; + return null; } diff --git a/package-lock.json b/package-lock.json index 8192083..9758225 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.1", "license": "GPL-3.0-only", "dependencies": { - "@neume-network/extraction-worker": "github:neume-network/extraction-worker", + "@neume-network/extraction-worker": "github:neume-network/extraction-worker#64e1e5110aa4ccdd503ca88c445654d94d3b8dd3", "@neume-network/schema": "github:neume-network/schema", "ava": "^5.1.0", "better-sqlite3": "^8.2.0", @@ -23,7 +23,8 @@ "knex": "^2.4.2", "level": "^8.0.0", "p-map": "^5.5.0", - "yargs": "^17.6.2" + "yargs": "^17.6.2", + "zod": "^3.21.4" }, "bin": { "neume": "dist/neume.js" @@ -38,22 +39,6 @@ "node": "16" } }, - "../music-os-schema": { - "name": "@neume-network/schema", - "version": "0.8.1", - "extraneous": true, - "license": "LGPL-3.0-only", - "devDependencies": { - "ajv": "8.11.0", - "ajv-formats": "2.1.1", - "ava": "4.2.0", - "husky": "7.0.4", - "json-schema-to-typescript": "11.0.2", - "lint-staged": "12.4.0", - "mime-db": "1.52.0", - "prettier": "2.6.2" - } - }, "node_modules/@bcherny/json-schema-ref-parser": { "version": "10.0.5-fork", "resolved": "https://registry.npmjs.org/@bcherny/json-schema-ref-parser/-/json-schema-ref-parser-10.0.5-fork.tgz", @@ -507,7 +492,8 @@ }, "node_modules/@neume-network/extraction-worker": { "version": "0.7.1", - "resolved": "git+ssh://git@github.com/neume-network/extraction-worker.git#fcd6413f8de6e34617c8e5501568e15bb7eaaaf4", + "resolved": "git+ssh://git@github.com/neume-network/extraction-worker.git#64e1e5110aa4ccdd503ca88c445654d94d3b8dd3", + "integrity": "sha512-Fa6oS3aXbnSKG+EHycJ/pp5R04C1n53b3XvCMzO54lNcO2bbBLnvxXEvjI1BQtNfFYAvDNluDoVDwj+HN8EBRw==", "license": "GPL-3.0-only", "dependencies": { "@neume-network/schema": "github:neume-network/schema", @@ -518,7 +504,7 @@ "cross-fetch": "3.1.5", "debug": "4.3.4", "dotenv": "16.0.0", - "eth-fun": "github:il3ven/eth-fun#fork", + "eth-fun": "github:il3ven/eth-fun#1a57ff48ed7cfa05367bfa4497c51542b345c52b", "fastq": "1.13.0", "limiter": "2.0.1", "multiformats": "9.9.0" @@ -1705,11 +1691,12 @@ }, "node_modules/eth-fun": { "version": "0.9.2", - "resolved": "git+ssh://git@github.com/il3ven/eth-fun.git#bc475c50285de91325e4d94c02610ae77894bdd8", + "resolved": "git+ssh://git@github.com/il3ven/eth-fun.git#1a57ff48ed7cfa05367bfa4497c51542b345c52b", + "integrity": "sha512-Dl51GzWYH5iaaxmrwYDFHyK6qRyvudHWDvJjJpg2kRSrVaw9MFln08bXSLber+F9Z8xXMnNd07IRtlmSqWJZyg==", "license": "GPL-3.0-only", "dependencies": { "async-retry": "1.3.3", - "cross-fetch": "3.1.4", + "cross-fetch": "4.0.0", "web3-eth-abi": "1.4.0" }, "engines": { @@ -1717,19 +1704,30 @@ } }, "node_modules/eth-fun/node_modules/cross-fetch": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.4.tgz", - "integrity": "sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", + "integrity": "sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==", "dependencies": { - "node-fetch": "2.6.1" + "node-fetch": "^2.6.12" } }, "node_modules/eth-fun/node_modules/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/eth-lib": { @@ -4364,6 +4362,14 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.21.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", + "integrity": "sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } }, "dependencies": { @@ -4637,8 +4643,9 @@ "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" }, "@neume-network/extraction-worker": { - "version": "git+ssh://git@github.com/neume-network/extraction-worker.git#fcd6413f8de6e34617c8e5501568e15bb7eaaaf4", - "from": "@neume-network/extraction-worker@github:neume-network/extraction-worker", + "version": "git+ssh://git@github.com/neume-network/extraction-worker.git#64e1e5110aa4ccdd503ca88c445654d94d3b8dd3", + "integrity": "sha512-Fa6oS3aXbnSKG+EHycJ/pp5R04C1n53b3XvCMzO54lNcO2bbBLnvxXEvjI1BQtNfFYAvDNluDoVDwj+HN8EBRw==", + "from": "@neume-network/extraction-worker@https://github.com/neume-network/extraction-worker#64e1e5110aa4ccdd503ca88c445654d94d3b8dd3", "requires": { "@neume-network/schema": "github:neume-network/schema", "abort-controller": "3.0.0", @@ -4648,7 +4655,7 @@ "cross-fetch": "3.1.5", "debug": "4.3.4", "dotenv": "16.0.0", - "eth-fun": "github:il3ven/eth-fun#fork", + "eth-fun": "github:il3ven/eth-fun#1a57ff48ed7cfa05367bfa4497c51542b345c52b", "fastq": "1.13.0", "limiter": "2.0.1", "multiformats": "9.9.0" @@ -5529,26 +5536,30 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" }, "eth-fun": { - "version": "git+ssh://git@github.com/il3ven/eth-fun.git#bc475c50285de91325e4d94c02610ae77894bdd8", + "version": "git+ssh://git@github.com/il3ven/eth-fun.git#1a57ff48ed7cfa05367bfa4497c51542b345c52b", + "integrity": "sha512-Dl51GzWYH5iaaxmrwYDFHyK6qRyvudHWDvJjJpg2kRSrVaw9MFln08bXSLber+F9Z8xXMnNd07IRtlmSqWJZyg==", "from": "eth-fun@^0.9.2", "requires": { "async-retry": "1.3.3", - "cross-fetch": "3.1.4", + "cross-fetch": "4.0.0", "web3-eth-abi": "1.4.0" }, "dependencies": { "cross-fetch": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.4.tgz", - "integrity": "sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", + "integrity": "sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==", "requires": { - "node-fetch": "2.6.1" + "node-fetch": "^2.6.12" } }, "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "requires": { + "whatwg-url": "^5.0.0" + } } } }, @@ -7419,6 +7430,11 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==" + }, + "zod": { + "version": "3.21.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", + "integrity": "sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==" } } } diff --git a/package.json b/package.json index e77ed21..91be6d4 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,8 @@ }, "scripts": { "start": "node ./dist/neume.js", - "build": "npm run compile-schemas && tsc -b", - "postbuild": "cp -r ./assets ./dist", + "build": "npm run compile-schemas && tsc -b --force", + "postbuild": "cp -r ./assets ./dist && cp -r ./database/migrations ./dist/database", "compile-schemas": "node commands/daemon/daemon-jsonrpc-schema.js > commands/daemon/daemon-jsonrpc-type.d.ts", "test": "ava" }, @@ -18,7 +18,7 @@ "author": "", "license": "GPL-3.0-only", "dependencies": { - "@neume-network/extraction-worker": "github:neume-network/extraction-worker", + "@neume-network/extraction-worker": "github:neume-network/extraction-worker#64e1e5110aa4ccdd503ca88c445654d94d3b8dd3", "@neume-network/schema": "github:neume-network/schema", "ava": "^5.1.0", "better-sqlite3": "^8.2.0", @@ -32,7 +32,8 @@ "knex": "^2.4.2", "level": "^8.0.0", "p-map": "^5.5.0", - "yargs": "^17.6.2" + "yargs": "^17.6.2", + "zod": "^3.21.4" }, "engines": { "node": "16" From ec0c4268d0bbcf2296c91a8ec2c27d8072dbdb0b Mon Sep 17 00:00:00 2001 From: il3ven Date: Tue, 1 Aug 2023 16:39:39 +0530 Subject: [PATCH 08/15] make init command safe to call more than once do not create files if they already exists. --- commands/init.ts | 30 +++++++++++++++++++++++------- dist/commands/init.js | 21 +++++++++++++++------ 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/commands/init.ts b/commands/init.ts index 156ed3e..bb3a974 100644 --- a/commands/init.ts +++ b/commands/init.ts @@ -5,12 +5,28 @@ import { CHAINS, CONSTANTS } from "../src/types.js"; import runMigration from "../database/runMigration.js"; export default async function init() { - await fs.copyFile(new URL("../assets/.env-copy", import.meta.url), path.resolve(".env")); - await fs.copyFile( - new URL("../assets/config.sample.js", import.meta.url), - path.resolve("./config.js"), - ); - // Create the last_crawled_block file in ./data - await saveLastCrawledBlock(CHAINS.eth, CONSTANTS.FIRST_BLOCK[CHAINS.eth]); + let fileExists; + + // .env + fileExists = await fs + .access(path.resolve(".env"), fs.constants.F_OK) + .then(() => true) + .catch(() => false); + + if (!fileExists) + await fs.copyFile(new URL("../assets/.env-copy", import.meta.url), path.resolve(".env")); + + // config.js + fileExists = await fs + .access(path.resolve("./config.js"), fs.constants.F_OK) + .then(() => true) + .catch(() => false); + + if (!fileExists) + await fs.copyFile( + new URL("../assets/config.sample.js", import.meta.url), + path.resolve("./config.js"), + ); + await runMigration("up"); } diff --git a/dist/commands/init.js b/dist/commands/init.js index 6eb35a6..93467bd 100644 --- a/dist/commands/init.js +++ b/dist/commands/init.js @@ -1,12 +1,21 @@ import path from "path"; import fs from "fs/promises"; -import { saveLastCrawledBlock } from "../src/state.js"; -import { CHAINS, CONSTANTS } from "../src/types.js"; import runMigration from "../database/runMigration.js"; export default async function init() { - await fs.copyFile(new URL("../assets/.env-copy", import.meta.url), path.resolve(".env")); - await fs.copyFile(new URL("../assets/config.sample.js", import.meta.url), path.resolve("./config.js")); - // Create the last_crawled_block file in ./data - await saveLastCrawledBlock(CHAINS.eth, CONSTANTS.FIRST_BLOCK[CHAINS.eth]); + let fileExists; + // .env + fileExists = await fs + .access(path.resolve(".env"), fs.constants.F_OK) + .then(() => true) + .catch(() => false); + if (!fileExists) + await fs.copyFile(new URL("../assets/.env-copy", import.meta.url), path.resolve(".env")); + // config.js + fileExists = await fs + .access(path.resolve("./config.js"), fs.constants.F_OK) + .then(() => true) + .catch(() => false); + if (!fileExists) + await fs.copyFile(new URL("../assets/config.sample.js", import.meta.url), path.resolve("./config.js")); await runMigration("up"); } From 14dedf822b280c6cd85c2cb77f55f5f26ce00207 Mon Sep 17 00:00:00 2001 From: il3ven Date: Wed, 2 Aug 2023 16:02:55 +0530 Subject: [PATCH 09/15] database speed improvements - Instead of using `upsertTrack` to update token, introduced `upsertToken` function. `upsertTrack` used more bandwidth and required calling `getTrack` before. - Instead of using multiple insert, use one insert for multiple writes. --- database/tracks.test.ts | 38 ++++++++++++++-- database/tracks.ts | 61 +++++++++++++++++++------- dist/database/tracks.js | 34 ++++++++++++-- dist/src/components/handle-transfer.js | 22 +++++++--- dist/src/strategies/lens/lens.js | 6 +-- dist/src/strategies/sound_protocol.js | 6 +-- src/components/handle-transfer.ts | 37 ++++++++++++---- src/strategies/lens/lens.ts | 16 +++---- src/strategies/sound_protocol.ts | 12 ++--- src/strategies/strategy.types.ts | 4 +- 10 files changed, 173 insertions(+), 63 deletions(-) diff --git a/database/tracks.test.ts b/database/tracks.test.ts index 95d8964..061804e 100644 --- a/database/tracks.test.ts +++ b/database/tracks.test.ts @@ -1,8 +1,9 @@ +import { Track } from "@neume-network/schema"; import test from "ava"; import runMigration from "./runMigration.js"; import { tracksDB } from "./tracks.js"; -test.beforeEach(async () => { +test.beforeEach(async (t) => { await runMigration("up"); await runMigration("down"); await runMigration("up"); @@ -227,10 +228,41 @@ test.serial("should be able to get changed tracks", async (t) => { t.is(tracks.length, 1); }); -test("should be able to update track", async (t) => { +test.serial("should be able to update track", async (t) => { await tracksDB.upsertTrack(sample[0], 0); - const newTrack = { ...sample[0], title: "New Title" }; + // adding a new owner to sample[0] + const newTrack: Track = { + ...sample[0], + erc721: { + ...sample[0].erc721, + tokens: [ + ...sample[0].erc721.tokens, + { + ...sample[0].erc721.tokens[0], + id: "2", + }, + ], + }, + }; await tracksDB.upsertTrack(newTrack, 0); const ret = await tracksDB.getTrack(sample[0].uid); t.deepEqual(ret, newTrack); }); + +test.serial("should be able upsert token", async (t) => { + await tracksDB.upsertTrack(sample[0], 0); + const newToken = { + ...sample[0].erc721.tokens[0], + id: "2", + }; + const expectedTrack: Track = { + ...sample[0], + erc721: { + ...sample[0].erc721, + tokens: [...sample[0].erc721.tokens, newToken], + }, + }; + await tracksDB.upsertToken(sample[0].uid, newToken, 1); + const ret = await tracksDB.getTrack(sample[0].uid); + t.deepEqual(ret, expectedTrack); +}); diff --git a/database/tracks.ts b/database/tracks.ts index ad5017a..f615b66 100644 --- a/database/tracks.ts +++ b/database/tracks.ts @@ -96,22 +96,20 @@ export class Tracks { .onConflict(["uid", "id"]) .merge(); - await Promise.all( - owners.map((o) => - trx("owners") - .insert({ - blockNumber: o.blockNumber, - from: o.from, - to: o.to, - transactionHash: o.transactionHash, - alias: o.alias, - uid: track.uid, - id: token.id, - }) - .onConflict(["uid", "id", "transactionHash", "to"]) - .merge(), - ), - ); + await trx("owners") + .insert( + owners.map((o) => ({ + blockNumber: o.blockNumber, + from: o.from, + to: o.to, + transactionHash: o.transactionHash, + alias: o.alias, + uid: track.uid, + id: token.id, + })), + ) + .onConflict(["uid", "id", "transactionHash", "to"]) + .merge(); }), ); @@ -161,6 +159,37 @@ export class Tracks { ); }; + upsertToken = async (uid: string, token: Token, timestamp: number = Date.now()) => { + return this.db.transaction(async (trx) => { + await trx("tracks").update({ lastUpdatedAt: timestamp }).where("uid", "=", uid); + + await trx("tokens") + .insert({ + id: token.id, + uri: token.uri, + metadata: token.metadata, + uid: uid, + }) + .onConflict(["uid", "id"]) + .merge(); + + await trx("owners") + .insert( + token.owners.map((o) => ({ + blockNumber: o.blockNumber, + from: o.from, + to: o.to, + transactionHash: o.transactionHash, + alias: o.alias, + uid: uid, + id: token.id, + })), + ) + .onConflict(["uid", "id", "transactionHash", "to"]) + .merge(); + }); + }; + isOwnerPresent = async (uid: string, tokenId: string, owner: Owner) => { const rows = await this.db("owners") .select("*") diff --git a/dist/database/tracks.js b/dist/database/tracks.js index 4af6174..9f305fb 100644 --- a/dist/database/tracks.js +++ b/dist/database/tracks.js @@ -57,8 +57,8 @@ export class Tracks { }) .onConflict(["uid", "id"]) .merge(); - await Promise.all(owners.map((o) => trx("owners") - .insert({ + await trx("owners") + .insert(owners.map((o) => ({ blockNumber: o.blockNumber, from: o.from, to: o.to, @@ -66,9 +66,9 @@ export class Tracks { alias: o.alias, uid: track.uid, id: token.id, - }) + }))) .onConflict(["uid", "id", "transactionHash", "to"]) - .merge())); + .merge(); })); const inputs = [track, timestamp]; await this.log?.put(`${track.platform.name}/${this.encodeNumber(timestamp)}/${hashCode(JSON.stringify(inputs))}`, { @@ -97,6 +97,32 @@ export class Tracks { inputs, }); }; + this.upsertToken = async (uid, token, timestamp = Date.now()) => { + return this.db.transaction(async (trx) => { + await trx("tracks").update({ lastUpdatedAt: timestamp }).where("uid", "=", uid); + await trx("tokens") + .insert({ + id: token.id, + uri: token.uri, + metadata: token.metadata, + uid: uid, + }) + .onConflict(["uid", "id"]) + .merge(); + await trx("owners") + .insert(token.owners.map((o) => ({ + blockNumber: o.blockNumber, + from: o.from, + to: o.to, + transactionHash: o.transactionHash, + alias: o.alias, + uid: uid, + id: token.id, + }))) + .onConflict(["uid", "id", "transactionHash", "to"]) + .merge(); + }); + }; this.isOwnerPresent = async (uid, tokenId, owner) => { const rows = await this.db("owners") .select("*") diff --git a/dist/src/components/handle-transfer.js b/dist/src/components/handle-transfer.js index cf61567..f880f44 100644 --- a/dist/src/components/handle-transfer.js +++ b/dist/src/components/handle-transfer.js @@ -65,12 +65,19 @@ async function _handleTransfer(from, to, recrawl) { if (await tracksDB.isTokenPresent(uid, nft.erc721.token.id)) return; } - // console.log(`fetching metadata for ${this.constructor.name}`, uid); - const track = await this.fetchMetadata(nft); - if (track) { - console.log("Found new NFT (could be a new track):", track?.title, nft.erc721.token.id, track?.platform.version, track?.platform.name, "at", nft.erc721.blockNumber); - await tracksDB.upsertTrack(track); + const ret = await this.fetchMetadata(nft); + if (!ret) + return; + if (isToken(ret)) { + const token = ret; + const uid = await this.nftToUid(nft); + console.log("Found new NFT for an existing track", uid, nft.erc721.token.id, "at", nft.erc721.blockNumber); + tracksDB.upsertToken(uid, token); + return; } + const track = ret; + console.log("Found new NFT (could be a new track):", track?.title, nft.erc721.token.id, track?.platform.version, track?.platform.name, "at", nft.erc721.blockNumber); + await tracksDB.upsertTrack(track); }); mintNFTsPromise.push(...promises); allTransferNFTs.push(...transferNfts); @@ -131,3 +138,8 @@ function prepareNFT(log) { metadata: {}, }; } +function isToken(data) { + if ("uid" in data) + return false; + return true; +} diff --git a/dist/src/strategies/lens/lens.js b/dist/src/strategies/lens/lens.js index 39be5cd..d5e1549 100644 --- a/dist/src/strategies/lens/lens.js +++ b/dist/src/strategies/lens/lens.js @@ -28,9 +28,8 @@ export default class Lens { console.log("Error for", nft); throw err; } - const track = await tracksDB.getTrack(uid); const alias = await getAlias.call(this, nft); - track.erc721.tokens.push({ + return { id: nft.erc721.token.id, owners: [ { @@ -41,8 +40,7 @@ export default class Lens { alias: alias ?? undefined, }, ], - }); - return track; + }; }; this.nftToUid = async (nft) => { return this.addressToId.get(nft.erc721.address); diff --git a/dist/src/strategies/sound_protocol.js b/dist/src/strategies/sound_protocol.js index 8297c01..0e26cea 100644 --- a/dist/src/strategies/sound_protocol.js +++ b/dist/src/strategies/sound_protocol.js @@ -62,8 +62,7 @@ export default class SoundProtocol { const uid = await this.nftToUid(nft); if (await tracksDB.isTrackPresent(uid)) { // Metadata already present, don't fetch from arweave again. - const track = await tracksDB.getTrack(uid); - track.erc721.tokens.push({ + return { id: nft.erc721.token.id, owners: [ { @@ -74,8 +73,7 @@ export default class SoundProtocol { alias: undefined, }, ], - }); - return track; + }; } nft.erc721.token.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft); if (!nft.erc721.token.uri.includes("ar://")) { diff --git a/src/components/handle-transfer.ts b/src/components/handle-transfer.ts index 1b60b8a..d573c41 100644 --- a/src/components/handle-transfer.ts +++ b/src/components/handle-transfer.ts @@ -4,6 +4,7 @@ import { ethGetLogs } from "./eth-get-logs.js"; import { tracksDB } from "../../database/tracks.js"; import Lens from "../strategies/lens/lens.js"; import { ERC721Strategy } from "../strategies/strategy.types.js"; +import { Token, Track } from "@neume-network/schema"; const TRANSFER_EVENT_SELECTOR = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; @@ -100,22 +101,37 @@ async function _handleTransfer( if (await tracksDB.isTokenPresent(uid, nft.erc721.token.id)) return; } - // console.log(`fetching metadata for ${this.constructor.name}`, uid); - const track = await this.fetchMetadata(nft); + const ret = await this.fetchMetadata(nft); - if (track) { + if (!ret) return; + + if (isToken(ret)) { + const token = ret; + const uid = await this.nftToUid(nft); console.log( - "Found new NFT (could be a new track):", - track?.title, + "Found new NFT for an existing track", + uid, nft.erc721.token.id, - track?.platform.version, - track?.platform.name, "at", nft.erc721.blockNumber, ); - await tracksDB.upsertTrack(track); + tracksDB.upsertToken(uid, token); + + return; } + + const track = ret; + console.log( + "Found new NFT (could be a new track):", + track?.title, + nft.erc721.token.id, + track?.platform.version, + track?.platform.name, + "at", + nft.erc721.blockNumber, + ); + await tracksDB.upsertTrack(track); }); mintNFTsPromise.push(...promises); @@ -200,3 +216,8 @@ function prepareNFT(log: JsonRpcLog): NFT { metadata: {}, }; } + +function isToken(data: Track | Token): data is Token { + if ("uid" in data) return false; + return true; +} diff --git a/src/strategies/lens/lens.ts b/src/strategies/lens/lens.ts index c7d4c07..a514608 100644 --- a/src/strategies/lens/lens.ts +++ b/src/strategies/lens/lens.ts @@ -2,9 +2,9 @@ * Current lens first song - 33474641 */ -import ExtractionWorker, { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; -import { Track } from "@neume-network/schema"; -import { toHex, decodeLog, encodeParameters, decodeParameters } from "eth-fun"; +import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; +import { Token, Track } from "@neume-network/schema"; +import { toHex, decodeLog, encodeParameters } from "eth-fun"; import { CHAINS, Config, Contract, NFT, PROTOCOLS } from "../../types.js"; import { Strategy } from "../strategy.types.js"; @@ -315,7 +315,7 @@ export default class Lens implements Strategy { } // This is called when a new NFT is minted in Lens - fetchMetadata = async (nft: NFT): Promise => { + fetchMetadata = async (nft: NFT): Promise => { let uid; try { uid = await this.addressToId.get(nft.erc721.address); @@ -323,10 +323,10 @@ export default class Lens implements Strategy { console.log("Error for", nft); throw err; } - const track = await tracksDB.getTrack(uid); + const alias = await getAlias.call(this, nft); - track.erc721.tokens.push({ + return { id: nft.erc721.token.id, owners: [ { @@ -337,9 +337,7 @@ export default class Lens implements Strategy { alias: alias ?? undefined, }, ], - }); - - return track; + }; }; async processPost(post: Post): Promise { diff --git a/src/strategies/sound_protocol.ts b/src/strategies/sound_protocol.ts index d783019..c0de61f 100644 --- a/src/strategies/sound_protocol.ts +++ b/src/strategies/sound_protocol.ts @@ -1,5 +1,5 @@ import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; -import { Track } from "@neume-network/schema"; +import { Token, Track } from "@neume-network/schema"; import { Level } from "level"; import { decodeLog } from "eth-fun"; import { AbstractSublevel } from "abstract-level"; @@ -96,7 +96,7 @@ export default class SoundProtocol implements ERC721Strategy { nftToUid = async (nft: NFT) => `${this.chain}/${SoundProtocol.name}/${nft.erc721.address.toLowerCase()}`; - fetchMetadata = async (nft: NFT): Promise => { + fetchMetadata = async (nft: NFT): Promise => { if ( SoundProtocol.invalidIDs.filter((id) => `${nft.erc721.address}/${nft.erc721.token.id}`.match(id), @@ -112,9 +112,7 @@ export default class SoundProtocol implements ERC721Strategy { if (await tracksDB.isTrackPresent(uid)) { // Metadata already present, don't fetch from arweave again. - const track = await tracksDB.getTrack(uid); - - track.erc721.tokens.push({ + return { id: nft.erc721.token.id, owners: [ { @@ -125,9 +123,7 @@ export default class SoundProtocol implements ERC721Strategy { alias: undefined, }, ], - }); - - return track; + }; } nft.erc721.token.uri = await callTokenUri.call(this, nft.erc721.blockNumber, nft); diff --git a/src/strategies/strategy.types.ts b/src/strategies/strategy.types.ts index 75f75ea..f260164 100644 --- a/src/strategies/strategy.types.ts +++ b/src/strategies/strategy.types.ts @@ -1,5 +1,5 @@ import { ExtractionWorkerHandler } from "@neume-network/extraction-worker"; -import { Track } from "@neume-network/schema"; +import { Token, Track } from "@neume-network/schema"; import { AbstractSublevel } from "abstract-level"; import { Level } from "level"; import { CHAINS, Config, Contract, NFT } from "../types.js"; @@ -61,5 +61,5 @@ export declare class ERC721Strategy extends Strategy { **/ contracts: AbstractSublevel; - fetchMetadata: (nft: NFT) => Promise; + fetchMetadata: (nft: NFT) => Promise; } From fb5c269062d385fa69bec4cf58b4a47817454521 Mon Sep 17 00:00:00 2001 From: il3ven Date: Wed, 2 Aug 2023 16:36:10 +0530 Subject: [PATCH 10/15] use shrink_memory pragma --- database/knexfile.js | 1 + database/tracks.ts | 24 +++++++++++------------- dist/database/knexfile.js | 1 + dist/database/tracks.js | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/database/knexfile.js b/database/knexfile.js index a56ef59..d576ae8 100644 --- a/database/knexfile.js +++ b/database/knexfile.js @@ -16,6 +16,7 @@ const config = { pool: { afterCreate: function (conn, done) { conn.pragma("journal_mode = WAL"); + conn.pragma("shrink_memory"); done(); }, }, diff --git a/database/tracks.ts b/database/tracks.ts index f615b66..e4e36d8 100644 --- a/database/tracks.ts +++ b/database/tracks.ts @@ -69,19 +69,17 @@ export class Tracks { .onConflict(["uid"]) .merge(); - await Promise.all( - track.manifestations.map((m) => - trx("manifestations") - .insert({ - version: m.version, - uri: m.uri, - mimetype: m.mimetype, - uid: track.uid, - }) - .onConflict(["uid", "uri"]) - .merge(), - ), - ); + await trx("manifesations") + .insert( + track.manifestations.map((m) => ({ + version: m.version, + uri: m.uri, + mimetype: m.mimetype, + uid: track.uid, + })), + ) + .onConflict(["uid", "uri"]) + .merge(); await Promise.all( track.erc721.tokens.map(async (token) => { diff --git a/dist/database/knexfile.js b/dist/database/knexfile.js index 3782a27..8f465c5 100644 --- a/dist/database/knexfile.js +++ b/dist/database/knexfile.js @@ -15,6 +15,7 @@ const config = { pool: { afterCreate: function (conn, done) { conn.pragma("journal_mode = WAL"); + conn.pragma("shrink_memory"); done(); }, }, diff --git a/dist/database/tracks.js b/dist/database/tracks.js index 9f305fb..5d7ca1d 100644 --- a/dist/database/tracks.js +++ b/dist/database/tracks.js @@ -37,15 +37,15 @@ export class Tracks { }) .onConflict(["uid"]) .merge(); - await Promise.all(track.manifestations.map((m) => trx("manifestations") - .insert({ + await trx("manifesations") + .insert(track.manifestations.map((m) => ({ version: m.version, uri: m.uri, mimetype: m.mimetype, uid: track.uid, - }) + }))) .onConflict(["uid", "uri"]) - .merge())); + .merge(); await Promise.all(track.erc721.tokens.map(async (token) => { const { owners } = token; await trx("tokens") From 972c334d7d4edb1ff8920284b093698a4deea248 Mon Sep 17 00:00:00 2001 From: il3ven Date: Wed, 2 Aug 2023 17:33:05 +0530 Subject: [PATCH 11/15] fix small typo --- database/tracks.ts | 2 +- dist/database/tracks.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/database/tracks.ts b/database/tracks.ts index e4e36d8..f47b3c2 100644 --- a/database/tracks.ts +++ b/database/tracks.ts @@ -69,7 +69,7 @@ export class Tracks { .onConflict(["uid"]) .merge(); - await trx("manifesations") + await trx("manifestations") .insert( track.manifestations.map((m) => ({ version: m.version, diff --git a/dist/database/tracks.js b/dist/database/tracks.js index 5d7ca1d..8aade02 100644 --- a/dist/database/tracks.js +++ b/dist/database/tracks.js @@ -37,7 +37,7 @@ export class Tracks { }) .onConflict(["uid"]) .merge(); - await trx("manifesations") + await trx("manifestations") .insert(track.manifestations.map((m) => ({ version: m.version, uri: m.uri, From f2643a011266f7d110d536000c7fa7cc4af817b8 Mon Sep 17 00:00:00 2001 From: il3ven Date: Wed, 2 Aug 2023 18:51:23 +0530 Subject: [PATCH 12/15] ignore a lens post --- dist/src/strategies/lens/lens.js | 3 ++- src/strategies/lens/lens.ts | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dist/src/strategies/lens/lens.js b/dist/src/strategies/lens/lens.js index d5e1549..0c5b779 100644 --- a/dist/src/strategies/lens/lens.js +++ b/dist/src/strategies/lens/lens.js @@ -222,7 +222,8 @@ export default class Lens { } // Regex for valid URIs; from: https://github.com/ajv-validator/ajv-formats/blob/4dd65447575b35d0187c6b125383366969e6267e/src/formats.ts#L229C12 const URI = /^(?:[a-z][a-z0-9+\-.]*:)(?:\/?\/(?:(?:[a-z0-9\-._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:(?:[0-9a-f]{1,4}:){6}|::(?:[0-9a-f]{1,4}:){5}|(?:[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){4}|(?:(?:[0-9a-f]{1,4}:){0,1}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){3}|(?:(?:[0-9a-f]{1,4}:){0,2}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){2}|(?:(?:[0-9a-f]{1,4}:){0,3}[0-9a-f]{1,4})?::[0-9a-f]{1,4}:|(?:(?:[0-9a-f]{1,4}:){0,4}[0-9a-f]{1,4})?::)(?:[0-9a-f]{1,4}:[0-9a-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))|(?:(?:[0-9a-f]{1,4}:){0,5}[0-9a-f]{1,4})?::[0-9a-f]{1,4}|(?:(?:[0-9a-f]{1,4}:){0,6}[0-9a-f]{1,4})?::)|[Vv][0-9a-f]+\.[a-z0-9\-._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)|(?:[a-z0-9\-._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)(?:\?(?:[a-z0-9\-._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[a-z0-9\-._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; - if (!post.contentURI || !URI.test(post.contentURI)) + // postter.xyz is down + if (!post.contentURI || !URI.test(post.contentURI) || post.contentURI.includes("postter.xyz")) return null; const protocol = getProtocol(post.contentURI); if (!protocol) { diff --git a/src/strategies/lens/lens.ts b/src/strategies/lens/lens.ts index a514608..fae544d 100644 --- a/src/strategies/lens/lens.ts +++ b/src/strategies/lens/lens.ts @@ -350,7 +350,10 @@ export default class Lens implements Strategy { // Regex for valid URIs; from: https://github.com/ajv-validator/ajv-formats/blob/4dd65447575b35d0187c6b125383366969e6267e/src/formats.ts#L229C12 const URI = /^(?:[a-z][a-z0-9+\-.]*:)(?:\/?\/(?:(?:[a-z0-9\-._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:(?:[0-9a-f]{1,4}:){6}|::(?:[0-9a-f]{1,4}:){5}|(?:[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){4}|(?:(?:[0-9a-f]{1,4}:){0,1}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){3}|(?:(?:[0-9a-f]{1,4}:){0,2}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){2}|(?:(?:[0-9a-f]{1,4}:){0,3}[0-9a-f]{1,4})?::[0-9a-f]{1,4}:|(?:(?:[0-9a-f]{1,4}:){0,4}[0-9a-f]{1,4})?::)(?:[0-9a-f]{1,4}:[0-9a-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))|(?:(?:[0-9a-f]{1,4}:){0,5}[0-9a-f]{1,4})?::[0-9a-f]{1,4}|(?:(?:[0-9a-f]{1,4}:){0,6}[0-9a-f]{1,4})?::)|[Vv][0-9a-f]+\.[a-z0-9\-._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)|(?:[a-z0-9\-._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[a-z0-9\-._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)(?:\?(?:[a-z0-9\-._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[a-z0-9\-._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; - if (!post.contentURI || !URI.test(post.contentURI)) return null; + + // postter.xyz is down + if (!post.contentURI || !URI.test(post.contentURI) || post.contentURI.includes("postter.xyz")) + return null; const protocol = getProtocol(post.contentURI); From 84f7daaed052b60a1dd33b11b4f31124695eab35 Mon Sep 17 00:00:00 2001 From: il3ven Date: Wed, 2 Aug 2023 22:53:46 +0530 Subject: [PATCH 13/15] in lens ignore incorrect IPFS URIs --- dist/src/strategies/lens/lens.js | 8 ++++++++ src/strategies/lens/lens.ts | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/dist/src/strategies/lens/lens.js b/dist/src/strategies/lens/lens.js index 0c5b779..0d0fd2f 100644 --- a/dist/src/strategies/lens/lens.js +++ b/dist/src/strategies/lens/lens.js @@ -13,6 +13,7 @@ import { tracksDB } from "../../../database/tracks.js"; import { handleTransfer } from "../../components/handle-transfer.js"; import { getAlias, getCollectNFT, getHandle } from "./components.js"; import { z } from "zod"; +import { breakdownIpfs } from "ipfs-uri-utils"; export default class Lens { constructor(worker, config) { this.createdAtBlock = Lens.createdAtBlock; @@ -240,6 +241,13 @@ export default class Lens { datum = await getArweaveTokenUri(post.contentURI, this.worker, this.config); } else if (protocol === PROTOCOLS.ipfs) { + // Check IPFS URI before fetching. Ignore incorrect URIs. + try { + breakdownIpfs(post.contentURI); + } + catch { + return null; + } datum = await getIpfsTokenUri.call(this, post.contentURI); } else if (protocol === PROTOCOLS.https) { diff --git a/src/strategies/lens/lens.ts b/src/strategies/lens/lens.ts index fae544d..c14eb67 100644 --- a/src/strategies/lens/lens.ts +++ b/src/strategies/lens/lens.ts @@ -20,6 +20,7 @@ import { tracksDB } from "../../../database/tracks.js"; import { handleTransfer } from "../../components/handle-transfer.js"; import { getAlias, getCollectNFT, getHandle } from "./components.js"; import { z } from "zod"; +import { breakdownIpfs } from "ipfs-uri-utils"; // Post from Lens type Post = { @@ -374,6 +375,12 @@ export default class Lens implements Strategy { } datum = await getArweaveTokenUri(post.contentURI, this.worker, this.config); } else if (protocol === PROTOCOLS.ipfs) { + // Check IPFS URI before fetching. Ignore incorrect URIs. + try { + breakdownIpfs(post.contentURI); + } catch { + return null; + } datum = await getIpfsTokenUri.call(this, post.contentURI); } else if (protocol === PROTOCOLS.https) { datum = await fetchTokenUri(post.contentURI, this.worker); From 02cb790d6d9f56ca533628e966f823520f523a7c Mon Sep 17 00:00:00 2001 From: il3ven Date: Thu, 3 Aug 2023 03:22:00 +0530 Subject: [PATCH 14/15] use write-file-atomic --- package-lock.json | 56 ++++++++++++++++++++++++++++++++++------------- package.json | 2 ++ src/state.ts | 3 ++- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9758225..140f4b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "knex": "^2.4.2", "level": "^8.0.0", "p-map": "^5.5.0", + "write-file-atomic": "^5.0.1", "yargs": "^17.6.2", "zod": "^3.21.4" }, @@ -31,6 +32,7 @@ }, "devDependencies": { "@types/node": "^18.11.9", + "@types/write-file-atomic": "^4.0.0", "@types/yargs": "^17.0.14", "ts-node": "^10.9.1", "typescript": "^4.8.4" @@ -613,6 +615,15 @@ "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.2.tgz", "integrity": "sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg==" }, + "node_modules/@types/write-file-atomic": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/write-file-atomic/-/write-file-atomic-4.0.0.tgz", + "integrity": "sha512-piEKt2KKBUtye+feTlfdPjtW7uPFsAaLNX3/f6AJD+Y1T1YPTFwnqtlO9Y+gy9qGshrvxKa/Kay9vqbyVIuhwQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/yargs": { "version": "17.0.24", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", @@ -3563,9 +3574,15 @@ "integrity": "sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==" }, "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } }, "node_modules/simple-concat": { "version": "1.0.1", @@ -4179,12 +4196,12 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/write-file-atomic": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.0.tgz", - "integrity": "sha512-R7NYMnHSlV42K54lwY9lvW6MnSm1HSJqZL3xiSgi9E7//FYaI74r2G0rd+/X6VAMkHEdzxQaU5HUOXWUz5kA/w==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", + "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dependencies": { "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" + "signal-exit": "^4.0.1" }, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" @@ -4645,7 +4662,7 @@ "@neume-network/extraction-worker": { "version": "git+ssh://git@github.com/neume-network/extraction-worker.git#64e1e5110aa4ccdd503ca88c445654d94d3b8dd3", "integrity": "sha512-Fa6oS3aXbnSKG+EHycJ/pp5R04C1n53b3XvCMzO54lNcO2bbBLnvxXEvjI1BQtNfFYAvDNluDoVDwj+HN8EBRw==", - "from": "@neume-network/extraction-worker@https://github.com/neume-network/extraction-worker#64e1e5110aa4ccdd503ca88c445654d94d3b8dd3", + "from": "@neume-network/extraction-worker@github:neume-network/extraction-worker#64e1e5110aa4ccdd503ca88c445654d94d3b8dd3", "requires": { "@neume-network/schema": "github:neume-network/schema", "abort-controller": "3.0.0", @@ -4753,6 +4770,15 @@ "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.2.tgz", "integrity": "sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg==" }, + "@types/write-file-atomic": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/write-file-atomic/-/write-file-atomic-4.0.0.tgz", + "integrity": "sha512-piEKt2KKBUtye+feTlfdPjtW7uPFsAaLNX3/f6AJD+Y1T1YPTFwnqtlO9Y+gy9qGshrvxKa/Kay9vqbyVIuhwQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/yargs": { "version": "17.0.24", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", @@ -6840,9 +6866,9 @@ "integrity": "sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==" }, "signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" }, "simple-concat": { "version": "1.0.1", @@ -7285,12 +7311,12 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "write-file-atomic": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.0.tgz", - "integrity": "sha512-R7NYMnHSlV42K54lwY9lvW6MnSm1HSJqZL3xiSgi9E7//FYaI74r2G0rd+/X6VAMkHEdzxQaU5HUOXWUz5kA/w==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", + "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "requires": { "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" + "signal-exit": "^4.0.1" } }, "xhr": { diff --git a/package.json b/package.json index 91be6d4..58fa4d4 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "knex": "^2.4.2", "level": "^8.0.0", "p-map": "^5.5.0", + "write-file-atomic": "^5.0.1", "yargs": "^17.6.2", "zod": "^3.21.4" }, @@ -40,6 +41,7 @@ }, "devDependencies": { "@types/node": "^18.11.9", + "@types/write-file-atomic": "^4.0.0", "@types/yargs": "^17.0.14", "ts-node": "^10.9.1", "typescript": "^4.8.4" diff --git a/src/state.ts b/src/state.ts index 1dcfad3..42f90e3 100644 --- a/src/state.ts +++ b/src/state.ts @@ -3,6 +3,7 @@ */ import fs from "fs/promises"; +import writeFileAtomic from "write-file-atomic"; import path from "path"; import { CONSTANTS } from "./types.js"; import { getStrategies } from "./utils.js"; @@ -25,5 +26,5 @@ export async function saveLastCrawledBlock(strategy: string, blockNumber: number .then(() => true) .catch(() => false); if (!fileExists) await fs.mkdir(path.dirname(location), { recursive: true }); - return fs.writeFile(location, blockNumber.toString(), "utf-8"); + return writeFileAtomic(location, blockNumber.toString(), "utf-8"); } From 781f3658dace4957e3697daae0f44a29d1dd097e Mon Sep 17 00:00:00 2001 From: il3ven Date: Thu, 3 Aug 2023 03:22:35 +0530 Subject: [PATCH 15/15] ignore all failed calls to contentURI in lesns --- dist/src/state.js | 3 ++- dist/src/strategies/lens/lens.js | 8 ++------ src/strategies/lens/lens.ts | 10 ++-------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/dist/src/state.js b/dist/src/state.js index 41ca44a..9ec07a8 100644 --- a/dist/src/state.js +++ b/dist/src/state.js @@ -2,6 +2,7 @@ * Utility functions to manage state */ import fs from "fs/promises"; +import writeFileAtomic from "write-file-atomic"; import path from "path"; import { CONSTANTS } from "./types.js"; import { getStrategies } from "./utils.js"; @@ -24,5 +25,5 @@ export async function saveLastCrawledBlock(strategy, blockNumber) { .catch(() => false); if (!fileExists) await fs.mkdir(path.dirname(location), { recursive: true }); - return fs.writeFile(location, blockNumber.toString(), "utf-8"); + return writeFileAtomic(location, blockNumber.toString(), "utf-8"); } diff --git a/dist/src/strategies/lens/lens.js b/dist/src/strategies/lens/lens.js index 0d0fd2f..3854f68 100644 --- a/dist/src/strategies/lens/lens.js +++ b/dist/src/strategies/lens/lens.js @@ -258,12 +258,8 @@ export default class Lens { } } catch (err) { - if (err.message.includes("status: 4") || - err.message.includes("Invalid CID") || - err.message.includes("ECONNREFUSED")) { - return null; - } - throw err; + // Assuming that the problem is with the URI endpoint. Therefore, ignoring the track. + return null; } if (!datum || !datum.media || datum.version !== "2.0.0") { // console.log("No media; skipping", datum.media, datum.version); diff --git a/src/strategies/lens/lens.ts b/src/strategies/lens/lens.ts index c14eb67..caef382 100644 --- a/src/strategies/lens/lens.ts +++ b/src/strategies/lens/lens.ts @@ -388,14 +388,8 @@ export default class Lens implements Strategy { throw new Error(`Invalid Protocol for ${post.contentURI}`); } } catch (err: any) { - if ( - err.message.includes("status: 4") || - err.message.includes("Invalid CID") || - err.message.includes("ECONNREFUSED") - ) { - return null; - } - throw err; + // Assuming that the problem is with the URI endpoint. Therefore, ignoring the track. + return null; } if (!datum || !datum.media || datum.version !== "2.0.0") {