diff --git a/api/.gitignore b/api/.gitignore new file mode 100644 index 0000000..8338de9 --- /dev/null +++ b/api/.gitignore @@ -0,0 +1,7 @@ +.DS_Store +node_modules +*.tar.gz +*.zip +*.log +dist/ +build/ diff --git a/api/package.json b/api/package.json index 08690a0..c74624f 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,13 @@ { "name": "yearn-tokenlist-api", - "version": "1.0.0", + "version": "1.0.2", "main": "index.js", + "files": [ "scripts/*", "src/*"], + "repository": { + "type": "git", + "url": "https://github.com/sambacha/yearn-finance-tokenlist.git", + "directory": "api/" + }, "scripts": { "build-list:all": "yarn build-list:erc20-mainnet && yarn build-list:erc721-mainnet && yarn build-list:erc1155-mainnet && yarn scrape-open-graph", "build-list:erc20-mainnet": "ts-node ./scripts/erc20-mainnet.ts && yarn run lint:fix-json", @@ -10,8 +16,8 @@ "scrape-open-graph": "ts-node ./scripts/open-graph-scrapper.ts && yarn run lint:fix-json", "lint:fix-json": "prettier --write \"../index/**/*.json\"" }, - "author": "", - "license": "ISC", + "author": "sam bacha ", + "license": "MIT", "devDependencies": { "@nomiclabs/hardhat-ethers": "2.0.2", "@openzeppelin/contracts": "3.4.1", @@ -21,6 +27,7 @@ "@types/node": "14.14.35", "@types/node-fetch": "2.5.8", "@uniswap/token-lists": "1.0.0-beta.19", + "@0xsequence/collectible-lists": "1.1.0", "ajv": "6.12.2", "ajv-formats": "1.5.1", "cli-progress": "3.9.0", diff --git a/api/scripts/open-graph-scrapper.ts b/api/scripts/open-graph-scrapper.ts index d5eee55..6d2bfb8 100644 --- a/api/scripts/open-graph-scrapper.ts +++ b/api/scripts/open-graph-scrapper.ts @@ -1,83 +1,98 @@ -import * as fs from "fs" -import { YearnInfo, YearnList, schema as yearn_schema } from "@yfi/tokenlist"; -import { TokenInfo, TokenList, schema as token_schema } from "@uniswap/token-lists"; +// @file graphql scrapper +import * as fs from 'fs'; +import { YearnInfo, YearnList, schema as yearn_schema } from 'yearn-tokenlist'; +import { + TokenInfo, + TokenList, + schema as token_schema, +} from '@uniswap/token-lists'; const cliProgress = require('cli-progress'); -const Ajv = require("ajv") +const Ajv = require('ajv'); const ogs = require('open-graph-scraper'); // Progress bar -const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); +const progressBar = new cliProgress.SingleBar( + {}, + cliProgress.Presets.shades_classic, +); const main = async () => { - // Get all token list files - let folders = fs.readdirSync('../index/').filter(folder => !folder.includes('.')) - let files: string[] = folders.reduce((files: string[], folder:string) => { - const currentFiles: string[] = fs.readdirSync('../index/' + folder) - currentFiles.forEach(file => { - files.push('../index/' + folder + '/' + file) - }) - return files - }, []) + let folders = fs + .readdirSync('../index/') + .filter((folder) => !folder.includes('.')); + let files: string[] = folders.reduce((files: string[], folder: string) => { + const currentFiles: string[] = fs.readdirSync('../index/' + folder); + currentFiles.forEach((file) => { + files.push('../index/' + folder + '/' + file); + }); + return files; + }, []); // Iterate over all lists - for (let i=0; i < files.length; i++) { - const f: string = files[i] - console.log('Images for ' + f) - const list_path = f + for (let i = 0; i < files.length; i++) { + const f: string = files[i]; + console.log('Images for ' + f); + const list_path = f; // Load list - let list = f.includes('erc20') ? require('../' + list_path) as TokenList : require('../' + list_path) as YearnList - progressBar.start(list.tokens.length, 0) - for (let j = 0; j < list.tokens.length; j++ ) { - const t: YearnInfo | TokenInfo = list.tokens[j] + let list = f.includes('erc20') + ? (require('../' + list_path) as TokenList) + : (require('../' + list_path) as YearnList); + progressBar.start(list.tokens.length, 0); + for (let j = 0; j < list.tokens.length; j++) { + const t: YearnInfo | TokenInfo = list.tokens[j]; //@ts-ignore if (t.extensions.link) { - try{ + try { //@ts-ignore - const graph = await ogs({'url': t.extensions.link, 'timeout' : 5000}) - const image = graph.result ? (await graph.result).ogImage : null + const graph = await ogs({ url: t.extensions.link, timeout: 5000 }); + const image = graph.result ? (await graph.result).ogImage : null; //@ts-ignore - list.tokens[j].extensions.ogImage = image ? (image.url ? image.url : null) : null + list.tokens[j].extensions.ogImage = image + ? image.url + ? image.url + : null + : null; } catch (e) { //@ts-ignore - list.tokens[j].extensions.ogImage = null + list.tokens[j].extensions.ogImage = null; } } else { //@ts-ignore - list.tokens[j].extensions.ogImage = null + list.tokens[j].extensions.ogImage = null; } - progressBar.update(j+1) + progressBar.update(j + 1); } - progressBar.stop() + progressBar.stop(); // Validate list against schema - const ajv = new Ajv() - const validateList = ajv.compile(f.includes('erc20') ? token_schema : yearn_schema) - + const ajv = new Ajv(); + const validateList = ajv.compile( + f.includes('erc20') ? token_schema : yearn_schema, + ); + if (!validateList(list)) { - console.log("New list has invalid schema: ") - console.log(validateList.errors) + console.log('New list has invalid schema: '); + console.log(validateList.errors); //throw Error("^^^") } fs.writeFile( list_path, JSON.stringify(list), - { flag: "w+" }, + { flag: 'w+' }, function (err) { - if (err) throw err - } - ) + if (err) throw err; + }, + ); } - -} +}; main() .then(() => { - console.log("Finished") + console.log('Finished'); }) .catch((error) => { - console.error(error) - }) - + console.error(error); +}); diff --git a/api/src/utils.ts b/api/src/utils.ts index ef514c0..bd685f0 100644 --- a/api/src/utils.ts +++ b/api/src/utils.ts @@ -1,13 +1,14 @@ -import * as dotenv from 'dotenv' -import * as path from 'path' +// @file utils +import * as dotenv from 'dotenv'; +import * as path from 'path'; export const getEnvConfig = () => { - const envFile = path.resolve(__dirname, '../config/creds.env') - const envLoad = dotenv.config({ path: envFile }) + const envFile = path.resolve(__dirname, '../config/creds.env'); + const envLoad = dotenv.config({ path: envFile }); if (envLoad.error) { - throw new Error(envLoad.error.message) + throw new Error(envLoad.error.message); } - return envLoad.parsed || {} -} \ No newline at end of file + return envLoad.parsed || {}; +}; diff --git a/package.json b/package.json index ac56999..185de93 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,13 @@ { "name": "yearn-tokenlist", - "version": "1.1.2", + "version": "1.1.3", "description": "Yearn Finance tokenlist with schema validation", "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/sambacha/yearn-finance-tokenlist.git", + "directory": "/" + }, "main": "index.mjs", "devDependencies": { "@ethersproject/address": "5.0.11",