From 2988e07d25dccb158520ce4c6a90b6898e644a33 Mon Sep 17 00:00:00 2001 From: liu Date: Mon, 5 Oct 2020 13:02:21 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E5=AE=8C=E6=88=90=E5=B0=8F=E8=AF=B4=E4=B8=8B=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 5 ++ index.js | 74 +++++++++++++++++++++ package-lock.json | 163 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 17 +++++ 4 files changed, 259 insertions(+) create mode 100644 .gitignore create mode 100644 index.js create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5a6e9a5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/node_modules/ + +/dist/ + +novel.txt \ No newline at end of file diff --git a/index.js b/index.js new file mode 100644 index 0000000..a6e8204 --- /dev/null +++ b/index.js @@ -0,0 +1,74 @@ +#!/usr/local/bin/node + +const axios = require('axios') +const cheerio = require('cheerio') +const fs = require('fs') + +const url = 'https://www.soshuw.com' +/* +** 获取小说列表 +*/ +async function getNovelList() { + const $ = cheerio.load((await axios.get(`${url}/ChuanYueZuoBaoJun/`)).data) + const list = $('.novel_list') + const linkArr = [] + + $(list[1]).find('dd').each((index, item) => { + const aLink = $(item).find('a') + linkArr.push({ + name: $(aLink).text(), + href: $(aLink).attr('href') + }) + }); + return linkArr +} + +/* +** 获取小说内容 +*/ + +async function getNovelContent() { + const novelList = await getNovelList() + + for (const item of novelList) { + const novelUrl = url + item.href + + let res = null + try { + res = await axios.get(novelUrl, { timeout: 20000 }) + console.log(`${item.name}下载成功`); + } catch (error) { + console.log(error); + continue + + } + const contentNode = res.data + const $ = cheerio.load(contentNode) + $('p').remove() + const novelContent = $('.content').text() + console.log(`${item.name}解析成功`); + // console.log(novelContent); + const result = await writeNovelData(item.name, novelContent) + console.log(result); + } + + +} + +/* +** 写入小说内容 +*/ +async function writeNovelData(name, data) { + return new Promise((res, rej) => { + const datas = name + '\n\n' + data + fs.writeFile('./novel.txt', datas, { flag: 'a' }, (err) => { + if (err) { + rej(err) + } else { + res(`${name}写入成功`) + } + }) + }) + +} +getNovelContent() \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..d791c5b --- /dev/null +++ b/package-lock.json @@ -0,0 +1,163 @@ +{ + "name": "downloadNeval", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@types/node": { + "version": "14.11.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", + "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==" + }, + "axios": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.20.0.tgz", + "integrity": "sha512-ANA4rr2BDcmmAQLOKft2fufrtuvlqR+cXNNinUmvfeSNCOF98PZL+7M/v1zIdGo7OLjEA9J2gXJL+j4zGsl0bA==", + "requires": { + "follow-redirects": "^1.10.0" + } + }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, + "cheerio": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", + "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.1", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==" + }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + }, + "follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" + }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "requires": { + "boolbase": "~1.0.0" + } + }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..a3a1393 --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "downloadneval", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start":"node ./index.js" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^0.20.0", + "cheerio": "^1.0.0-rc.3" + } +} -- Gitee From 09af7547e5d8141d41ebbcd6931ed046e1389ae7 Mon Sep 17 00:00:00 2001 From: liukaipeng <862843658@qq.com> Date: Tue, 6 Oct 2020 21:43:13 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20README.md=20=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b510dcd..764bf99 100644 --- a/README.md +++ b/README.md @@ -9,15 +9,11 @@ #### 安装教程 -1. xxxx -2. xxxx -3. xxxx +1. 直接运行`node ./index.js` #### 使用说明 -1. xxxx -2. xxxx -3. xxxx +1. 运行`node ./index.js`，小说开始按照列表下载并写入到novel.text文件里面 #### 参与贡献 @@ -25,13 +21,3 @@ 2. 新建 Feat_xxx 分支 3. 提交代码 4. 新建 Pull Request - - -#### 特技 - -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md -2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目，是综合评定出的优秀开源项目 -5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) -- Gitee