master
刘东琪 9 months ago
parent f9114990e0
commit 7ff0826521
  1. 2
      app.js
  2. 2
      enums/AreaNameEnum.js
  3. 1
      package-lock.json
  4. 1
      package.json
  5. 3
      public/hubei/index.html
  6. 220
      routes/index.js
  7. 24
      test/downloadFileTest.js
  8. 63
      utils/JsUtil.js

@ -1,6 +1,6 @@
const express = require('express'); const express = require('express');
// const bodyParser = require('body-parser'); // const bodyParser = require('body-parser');
let IpUtil = require('./utils/IpUtil'); const IpUtil = require('./utils/IpUtil');
const rs = require("./routes"); const rs = require("./routes");
// 读取环境变量 // 读取环境变量

@ -5,12 +5,10 @@ const AreaNameEnum = Object.freeze({
YUN_NAN: Object.freeze({ YUN_NAN: Object.freeze({
AREA_CODE: '5300', AREA_CODE: '5300',
AREA_NAME: 'yunnan', AREA_NAME: 'yunnan',
JS_FILE: path.resolve(__dirname, '../public/yunnan/MO5zzCMcub4d.b4c45da.js'),
}), }),
HU_BEI: Object.freeze({ HU_BEI: Object.freeze({
AREA_CODE: '4200', AREA_CODE: '4200',
AREA_NAME: 'hubei', AREA_NAME: 'hubei',
JS_FILE: path.resolve(__dirname, '../public/hubei/5PXGXoOF7eGJ.ed63b8f.js'),
}), }),
getByAreaCode(areaCode) { getByAreaCode(areaCode) {

1
package-lock.json generated

@ -11,6 +11,7 @@
"dependencies": { "dependencies": {
"body-parser": "^1.20.2", "body-parser": "^1.20.2",
"express": "^4.19.2", "express": "^4.19.2",
"node-fetch": "^2.7.0",
"sdenv": "^0.2.2", "sdenv": "^0.2.2",
"sdenv-extend": "^1.3.1", "sdenv-extend": "^1.3.1",
"sdenv-jsdom": "^1.1.0" "sdenv-jsdom": "^1.1.0"

@ -12,6 +12,7 @@
"dependencies": { "dependencies": {
"body-parser": "^1.20.2", "body-parser": "^1.20.2",
"express": "^4.19.2", "express": "^4.19.2",
"node-fetch": "^2.7.0",
"sdenv": "^0.2.2", "sdenv": "^0.2.2",
"sdenv-extend": "^1.3.1", "sdenv-extend": "^1.3.1",
"sdenv-jsdom": "^1.1.0" "sdenv-jsdom": "^1.1.0"

@ -4,4 +4,5 @@
window.location.href="./ie.html" window.location.href="./ie.html"
}, 1000) }, 1000)
</script> </script>
<![endif]--><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=viewport content="initial-scale=1,user-scalable=yes,minimum-scale=0.2,maximum-scale=5"><link rel=icon href=https://static.tpass.chinatax.gov.cn/favicon.ico><link rel=stylesheet href=https://static.tpass.chinatax.gov.cn/qrcode/index.css><title></title><link href=https://static.tpass.chinatax.gov.cn/static/css/app.03e78ef7.css rel=preload as=style><link href=https://static.tpass.chinatax.gov.cn/static/css/chunk-elementUI.1194f7cb.css rel=preload as=style><link href=https://static.tpass.chinatax.gov.cn/static/css/chunk-libs.3dfb7769.css rel=preload as=style><link href=https://static.tpass.chinatax.gov.cn/static/js/app.36e20af7.js rel=preload as=script><link href=https://static.tpass.chinatax.gov.cn/static/js/chunk-elementUI.e8a89fa6.js rel=preload as=script><link href=https://static.tpass.chinatax.gov.cn/static/js/chunk-libs.94e3d0ee.js rel=preload as=script><link href=https://static.tpass.chinatax.gov.cn/static/css/chunk-elementUI.1194f7cb.css rel=stylesheet><link href=https://static.tpass.chinatax.gov.cn/static/css/chunk-libs.3dfb7769.css rel=stylesheet><link href=https://static.tpass.chinatax.gov.cn/static/css/app.03e78ef7.css rel=stylesheet></head><body><noscript><strong>We're sorry but doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=https://static.tpass.chinatax.gov.cn/static/js/runtime.33a16895.js></script><script src=https://static.tpass.chinatax.gov.cn/static/js/chunk-elementUI.e8a89fa6.js></script><script src=https://static.tpass.chinatax.gov.cn/static/js/chunk-libs.94e3d0ee.js></script><script src=https://static.tpass.chinatax.gov.cn/static/js/app.36e20af7.js></script></body><script src=https://static.tpass.chinatax.gov.cn/qrcode/qrcode.min.js></script><script src=https://static.tpass.chinatax.gov.cn/qrcode/fzKeyboard.js></script><script src=https://static.tpass.chinatax.gov.cn/qrcode/bundle.js></script></html> <![endif]--><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=viewport content="initial-scale=1,user-scalable=yes,minimum-scale=0.2,maximum-scale=5"><link rel=icon href=https://static.tpass.chinatax.gov.cn/favicon.ico><link rel=stylesheet href=https://static.tpass.chinatax.gov.cn/qrcode/index.css><title></title><link href=https://static.tpass.chinatax.gov.cn/static/css/app.03e78ef7.css rel=preload as=style><link href=https://static.tpass.chinatax.gov.cn/static/css/chunk-elementUI.1194f7cb.css rel=preload as=style><link href=https://static.tpass.chinatax.gov.cn/static/css/chunk-libs.3dfb7769.css rel=preload as=style><link href=https://static.tpass.chinatax.gov.cn/static/js/app.36e20af7.js rel=preload as=script><link href=https://static.tpass.chinatax.gov.cn/static/js/chunk-elementUI.e8a89fa6.js rel=preload as=script><link href=https://static.tpass.chinatax.gov.cn/static/js/chunk-libs.94e3d0ee.js rel=preload as=script><link href=https://static.tpass.chinatax.gov.cn/static/css/chunk-elementUI.1194f7cb.css rel=stylesheet><link href=https://static.tpass.chinatax.gov.cn/static/css/chunk-libs.3dfb7769.css rel=stylesheet><link href=https://static.tpass.chinatax.gov.cn/static/css/app.03e78ef7.css rel=stylesheet></head><body><noscript><strong>We're sorry but doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=https://static.tpass.chinatax.gov.cn/static/js/runtime.33a16895.js></script><script src=https://static.tpass.chinatax.gov.cn/static/js/chunk-elementUI.e8a89fa6.js></script><script src=https://static.tpass.chinatax.gov.cn/static/js/chunk-libs.94e3d0ee.js></script><script src=https://static.tpass.chinatax.gov.cn/static/js/app.36e20af7.js></script></body><script src=https://static.tpass.chinatax.gov.cn/qrcode/qrcode.min.js></script><script src=https://static.tpass.chinatax.gov.cn/qrcode/fzKeyboard.js></script>
<script src=https://static.tpass.chinatax.gov.cn/qrcode/bundle.js></script></html>

@ -3,11 +3,12 @@ const router = express.Router();
const {jsdomFromText, browser} = require("sdenv"); const {jsdomFromText, browser} = require("sdenv");
const {Script} = require("node:vm"); const {Script} = require("node:vm");
const fs = require("node:fs");
const crypto = require("node:crypto") const crypto = require("node:crypto")
const AreaNameEnum = require('../enums/AreaNameEnum'); const AreaNameEnum = require('../enums/AreaNameEnum');
const Store = require("../utils/Store"); const Store = require("../utils/Store");
const JsUtil = require('../utils/JsUtil');
let store = new Store(); let store = new Store();
let sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
router.post('/rsCookie', async (req, res) => { router.post('/rsCookie', async (req, res) => {
@ -21,7 +22,6 @@ router.post('/rsCookie', async (req, res) => {
let cookie = req.body['cookieBase64']; let cookie = req.body['cookieBase64'];
let userAgent = req.body['userAgentBase64']; let userAgent = req.body['userAgentBase64'];
console.log(`${uuid};接收到 ${areaName} 请求:${url}`) console.log(`${uuid};接收到 ${areaName} 请求:${url}`)
// fs.writeFileSync(`./back/${uuid}.url`, `${url}\n ${userAgent}\n ${cookie} \n`);
if (url == null || url === '') { if (url == null || url === '') {
return res.status(500).send('error url') return res.status(500).send('error url')
} }
@ -29,28 +29,11 @@ router.post('/rsCookie', async (req, res) => {
return res.status(500).send('error html') return res.status(500).send('error html')
} }
let jsText; let cookies = await handle(url, uuid, areaName,
let loadHtmlJs;
if (jsStr == null || jsStr === "") {
let jsPath = AreaNameEnum.getByAreaName(areaName).JS_FILE
if (jsPath == null) {
console.error('未找到js文件')
return res.send('未找到js文件')
}
jsText = fs.readFileSync(jsPath).toString('utf8');
loadHtmlJs = true;
} else {
jsText = Buffer.from(jsStr, 'base64').toString('utf-8')
loadHtmlJs = false
}
let cookies = await handle(url,
Buffer.from(htmlStr, 'base64').toString('utf-8'), Buffer.from(htmlStr, 'base64').toString('utf-8'),
jsText,
cookie != null && cookie !== "" ? Buffer.from(cookie, 'base64').toString('utf-8') : null, cookie != null && cookie !== "" ? Buffer.from(cookie, 'base64').toString('utf-8') : null,
userAgent != null && userAgent !== "" ? Buffer.from(userAgent, 'base64').toString('utf-8') : null, userAgent != null && userAgent !== "" ? Buffer.from(userAgent, 'base64').toString('utf-8') : null,
uuid, loadHtmlJs) )
console.log(`${uuid};返回cookie ---->`, cookies.split('; ')) console.log(`${uuid};返回cookie ---->`, cookies.split('; '))
@ -64,24 +47,6 @@ router.post('/rsCookie', async (req, res) => {
}) })
function loadJs(window, jsText) {
// 加载js
let js = '';
// 加载 页面上的js
const allScript = window.document.querySelectorAll('script[r="m"]');
for (let i = 0; i < allScript.length; i++) {
const script = allScript[i];
let attr = script.textContent;
if (attr) {
js += attr
} else {
js += jsText
}
js += ";\n"
}
return js;
}
function CookieStr2List(cookies) { function CookieStr2List(cookies) {
let list = [] let list = []
for (let cookie of cookies.trim().split("; ")) { for (let cookie of cookies.trim().split("; ")) {
@ -90,7 +55,7 @@ function CookieStr2List(cookies) {
return list return list
} }
async function handle(url, htmlStr, jsText, cookie, userAgent, uuid, loadHtmlJs) { async function handle(url, uuid, areaName, htmlStr, cookie, userAgent) {
// 获取 origin // 获取 origin
let baseUrl = new URL(url).origin; let baseUrl = new URL(url).origin;
// 初始化 jsDom 和 cookieJar // 初始化 jsDom 和 cookieJar
@ -105,62 +70,173 @@ async function handle(url, htmlStr, jsText, cookie, userAgent, uuid, loadHtmlJs)
if (cookie != null) { if (cookie != null) {
let cookieList = CookieStr2List(cookie); let cookieList = CookieStr2List(cookie);
console.log(`${uuid};cookie 加载长度--->`, cookieList, baseUrl) console.log(`${uuid};cookie 加载长度--->`, cookieList, baseUrl)
// fs.writeFileSync(`./back/${uuid}.cookie`, cookie)
for (let i = 0; i < cookieList.length; i++) { for (let i = 0; i < cookieList.length; i++) {
cookieJar.setCookieSync(cookieList[i], baseUrl); cookieJar.setCookieSync(cookieList[i], baseUrl);
} }
} }
// 设置cookie的回掉
const superSetCookie = cookieJar.setCookie;
cookieJar.setCookie = function (cookie, currentUrl, options, callback) {
console.log(`${uuid};正在设置 Cookie:`, cookie, currentUrl);
// store.set(uuid, cookie)
return superSetCookie.call(this, cookie, currentUrl, options, callback);
};
// 加载dom // 加载dom
let dom = await jsDom(htmlStr); let dom = await jsDom(htmlStr);
// console.log('html 加载长度--->', dom.serialize())
console.log(`${uuid};html 加载长度--->`, dom.serialize().length) console.log(`${uuid};html 加载长度--->`, dom.serialize().length)
// fs.writeFileSync(`./back/${uuid}.html`, dom.serialize())
window = dom.window window = dom.window
// 标志判断cookie是否生成
window[uuid] = false
// 方案1 通过监听cookie 判断cookie是否生成
cookieJar.delete
const superSetCookie = cookieJar.setCookie;
// 设置 setCookie 代理
cookieJar.setCookie = function (cookie, currentUrl, options, callback) {
console.log(`${uuid};正在设置 Cookie:`, cookie, currentUrl);
let call = superSetCookie.call(this, cookie, currentUrl, options, callback);
// 设置标志可取标志
if (cookie.includes('YqQ7a3SgknV8P')) {
window[uuid] = true
}
return call;
};
// js执行成功后会跳转页面 会触发onbeforeunload钩子 // js执行成功后会跳转页面 会触发onbeforeunload钩子
window.onbeforeunload = async (url) => { window.onbeforeunload = async (url) => {
const cookies = cookieJar.getCookieStringSync(baseUrl); console.debug(`${url} 页面回调完成`);
console.debug(`${url} 生成cookie:`, cookies); window[uuid] = true
store.set(uuid, cookies)
// window.close();
} }
// 初始化浏览器 // 初始化浏览器
browser(window, 'chrome'); browser(window, 'chrome');
// 加载js // 加载js
let js; let js = await JsUtil.loadJs(window.document, areaName, cookie);
if (loadHtmlJs) {
js = loadJs(window, jsText);
} else {
js = jsText;
}
console.log(`${uuid};js 加载长度--->`, js.length) console.log(`${uuid};js 加载长度--->`, js.length)
// fs.writeFileSync(`./back/${uuid}.js`, js)
// 执行 js // 执行 js
let script = new Script(js); let script = new Script(js);
let internalVMContext = dom.getInternalVMContext(); let internalVMContext = dom.getInternalVMContext();
script.runInContext(internalVMContext); script.runInContext(internalVMContext, {timeout: 1000});
for (let i = 0; i < 10; i++) {
if (window[uuid]) {
break;
}
await sleep(100)
}
let resCookie = cookieJar.getCookieStringSync(baseUrl);
window.close()
return resCookie;
}
/**
* 方案1
* 利用rs特性 js加载完成后会刷新页面
* 利用 window.onbeforeunload 判断页面是否执行完毕
* @param window
* @param cookieJar
* @param uuid
* @returns {Promise<*|null>}
*/
function scheme1_before(window, cookieJar, uuid) {
window.onbeforeunload = async (url) => {
let baseUrl = new URL(url).origin;
const cookies = cookieJar.getCookieStringSync(baseUrl);
console.debug(`${url} 页面回调生成cookie:`, cookies);
store.set(uuid, cookies)
// window.close();
}
}
/**
* 方案1
* @param window
* @param cookieJar
* @param uuid
* @returns {Promise<*|null>}
*/
async function scheme1_after(window, cookieJar, uuid) {
// 等待 onbeforeunload 钩子触发后的回掉 // 等待 onbeforeunload 钩子触发后的回掉
// let val = await store.waitGetAndDelete(uuid, 100, 10) let val = await store.waitGetAndDelete(uuid, 100, 10)
if (val != null) {
return val;
}
return null;
}
// bug 关闭后 部分参数是需要永久保存在内存中的 比如 cookie 实例 下次调用会报错
// internalVMContext.close()
// window.close()
// dom = null
// if (val != null) { /**
// return val; * 监听cookie出现指定cookie
* @param window
* @param cookieJar
* @param uuid
* @returns {Promise<null|string>}
*/
function scheme2_before(window, cookieJar, uuid) {
// window[uuid] = false;
const superSetCookie = cookieJar.setCookie;
// 设置 setCookie 代理
cookieJar.setCookie = function (cookie, currentUrl, options, callback) {
console.log(`${uuid};正在设置 Cookie:`, cookie, currentUrl);
return superSetCookie.call(this, cookie, currentUrl, options, callback);
// let cookieStringSync = super.getCookieStringSync();
// console.log(cookieStringSync)
// if (cookie.includes(key)) {
// // 设置标志可取标志
// window[uuid] = true
// } // }
// console.log(`${uuid} 执行超时`) // return call;
return cookieJar.getCookieStringSync(baseUrl); };
// throw new Error(`${uuid}执行超时`) }
/**
* 方案2
* @param window
* @param cookieJar
* @param key
* @param uuid
* @returns {Promise<null|string>}
*/
async function scheme2_after(window, cookieJar, key, uuid) {
let sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
for (let i = 0; i < 10; i++) {
let cookieStringSync = cookieJar.getCookieStringSync();
if (cookieStringSync.includes(key)) {
return cookieStringSync;
}
await sleep(100)
}
return null;
}
/**
* 方案3 根据cookie现有的数量判断
* @param window
* @param cookieJar
* @param baseUrl
* @param uuid
*/
function scheme3_before(window, cookieJar, baseUrl, uuid) {
const initCookie = cookieJar.getCookieStringSync(baseUrl);
window[uuid + 'CookieSize'] = initCookie != null ? initCookie.trim().split("; ").length : 0;
}
/**
* 方案3
* @param window
* @param cookieJar
* @param baseUrl
* @param uuid
* @returns {Promise<null|string>}
*/
async function scheme3_after(window, cookieJar, baseUrl, uuid) {
let initCookieLength = window[uuid + 'CookieSize']
let sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
for (let i = 0; i < 10; i++) {
let cookieStringSync = cookieJar.getCookieStringSync(baseUrl);
let cookies = cookieStringSync != null ? cookieStringSync.trim().split("; ").length : 0;
if (cookies > initCookieLength) {
return cookieStringSync;
}
await sleep(100)
}
return null;
} }

@ -0,0 +1,24 @@
const fs = require('fs');
// const fetch = require('node-fetch'); // Install this via npm
async function downloadFile(url, path) {
try {
const response = await fetch(url);
if (!response.ok){
throw new Error(`HTTP error! Status: ${response.status}`);
}
const arrayBuffer = await response.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
fs.writeFileSync(path, buffer);
console.log('File downloaded and saved successfully.');
} catch (error) {
console.error('Error downloading the file:', error);
}
}
const downloadUrl = 'https://app.yunnan.chinatax.gov.cn/GVv7ud1ebech/MO5zzCMcub4d.b4c45da.js'; // Replace with your file URL
const filePath = './downloadedFile.js'; // Desired file path
downloadFile(downloadUrl, filePath);

@ -0,0 +1,63 @@
let fs = require('node:fs');
const path = require("node:path");
class JsUtil {
// 加载 js 文本
static async loadJs(document, areaName, cookie) {
let js = '';
// 加载 页面上的js
const allScript = document.querySelectorAll('script[r="m"]');
for (let i = 0; i < allScript.length; i++) {
const script = allScript[i];
let attr = script.textContent;
if (attr) {
js += attr
} else {
//获取script @src 属性
let jsSrc = script.src;
let url = new URL(jsSrc);
let fileName = areaName + url.pathname.replaceAll('/', '.');
let filePath = path.resolve(__dirname, `../public/static/js/`);
let existDir = fs.existsSync(filePath);
if (!existDir) {
fs.mkdirSync(filePath, {recursive: true});
}
let file =path.resolve(filePath, fileName);
let existFile = fs.existsSync(file);
let jsText;
if (existFile) {
let buffer = fs.readFileSync(file);
jsText = buffer.toString('utf8')
} else {
// 文件下载
jsText = await this.downloadJs(jsSrc, filePath, fileName, cookie)
}
js += jsText
}
js += ";\n"
}
return js;
}
static async downloadJs(downloadUrl, filePath, fileName) {
let url = new URL(downloadUrl);
const response = await fetch(url);
if (!response.ok) {
throw new Error(`HTTP error! Status: ${response.status}`);
}
const arrayBuffer = await response.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
let file = path.resolve(filePath, fileName);
fs.writeFile(file, buffer, {encoding: 'utf8'}, (err) => {
if (err){
console.error(`文件${file}写入失败 ---> ${err}`)
}
})
console.log('File downloaded and saved successfully.');
return buffer.toString('utf8')
}
}
module.exports = JsUtil
Loading…
Cancel
Save