@@ -1,6 +1,8 @@
import { chromium } from 'playwright' ;
import cron from 'node-cron' ;
import fs from 'node:fs' ;
import path from 'node:path' ;
import readline from 'node:readline' ;
import { execSync } from 'node:child_process' ;
import { config , datasets } from './config.js' ;
import { sendLoginAlert } from './notify.js' ;
@@ -17,6 +19,106 @@ import {
const sleep = ( ms ) => new Promise ( ( resolve ) => setTimeout ( resolve , ms ) ) ;
let _context = null ;
let _runtimeController = null ;
const AUTH _PAGE _KEYWORDS = [
'RAM 用户登录' ,
'主账号登录' ,
'钉钉扫码登录' ,
'用户名' ,
'下一步' ,
'登录并使用 RAM' ,
] ;
async function closeContextIfNeeded ( ) {
if ( ! _context ) return ;
await _context . close ( ) ;
_context = null ;
}
function getRuntimeController ( ) {
if ( _runtimeController ) return _runtimeController ;
let paused = false ;
let terminated = false ;
let keypressBound = false ;
const onKeypress = ( _str , key = { } ) => {
if ( key . name === 'f7' ) {
if ( ! paused ) {
paused = true ;
console . log ( '[控制] 已暂停( F7) 。按 F8 继续,按 F9 终止。' ) ;
}
return ;
}
if ( key . name === 'f8' ) {
if ( paused ) {
paused = false ;
console . log ( '[控制] 已继续( F8) 。' ) ;
}
return ;
}
if ( key . name === 'f9' ) {
terminated = true ;
paused = false ;
console . log ( '[控制] 已请求终止( F9) , 将在安全检查点停止。' ) ;
}
} ;
const bind = ( ) => {
if ( keypressBound || ! process . stdin . isTTY ) return ;
readline . emitKeypressEvents ( process . stdin ) ;
if ( typeof process . stdin . setRawMode === 'function' ) {
process . stdin . setRawMode ( true ) ;
}
process . stdin . resume ( ) ;
process . stdin . on ( 'keypress' , onKeypress ) ;
keypressBound = true ;
console . log ( '[控制] 热键已启用: F7 暂停 / F8 继续 / F9 终止' ) ;
} ;
const unbind = ( ) => {
if ( ! keypressBound ) return ;
process . stdin . off ( 'keypress' , onKeypress ) ;
if ( process . stdin . isTTY && typeof process . stdin . setRawMode === 'function' ) {
process . stdin . setRawMode ( false ) ;
}
keypressBound = false ;
} ;
const waitIfPaused = async ( label = '任务' ) => {
if ( terminated ) {
throw new Error ( ` [控制] 已终止: ${ label } ` ) ;
}
while ( paused ) {
await sleep ( 300 ) ;
if ( terminated ) {
throw new Error ( ` [控制] 已终止: ${ label } ` ) ;
}
}
} ;
const throwIfTerminated = ( label = '任务' ) => {
if ( terminated ) {
throw new Error ( ` [控制] 已终止: ${ label } ` ) ;
}
} ;
_runtimeController = {
bind ,
unbind ,
waitIfPaused ,
throwIfTerminated ,
} ;
return _runtimeController ;
}
async function runtimeCheckpoint ( label ) {
const controller = getRuntimeController ( ) ;
controller . throwIfTerminated ( label ) ;
await controller . waitIfPaused ( label ) ;
}
async function getContext ( ) {
if ( _context ) return _context ;
@@ -26,24 +128,103 @@ async function getContext() {
acceptDownloads : true ,
downloadsPath : config . downloadDir ,
} ) ;
await restoreStorageState ( _context ) ;
return _context ;
}
async function restoreStorageState ( context ) {
if ( ! fs . existsSync ( config . storageStateFile ) ) {
return ;
}
try {
const state = JSON . parse ( fs . readFileSync ( config . storageStateFile , 'utf-8' ) ) ;
if ( Array . isArray ( state . cookies ) && state . cookies . length > 0 ) {
await context . addCookies ( state . cookies ) ;
console . log ( ` [storageState] 已恢复 ${ state . cookies . length } 个 cookie ` ) ;
}
} catch ( error ) {
console . warn ( ` [storageState] 恢复失败,继续使用 .browser profile: ${ error . message } ` ) ;
}
}
async function saveStorageState ( context ) {
await context . storageState ( { path : config . storageStateFile } ) ;
console . log ( ` [storageState] 已保存登录态快照: ${ config . storageStateFile } ` ) ;
}
async function getPageBodyPreview ( page ) {
return page
. evaluate ( ( ) => document . body ? . innerText ? . substring ( 0 , 500 ) || '(空)' )
. catch ( ( ) => '(无法获取)' ) ;
}
function isAuthUrl ( url ) {
return /account\.aliyun\.com|signin\.aliyun\.com/ . test ( url )
|| url . includes ( 'login.htm' )
|| url . includes ( '/#/signin' ) ;
}
function hasAuthKeywords ( text ) {
return AUTH _PAGE _KEYWORDS . some ( ( keyword ) => text . includes ( keyword ) ) ;
}
async function detectAuthRedirect ( page ) {
const currentUrl = page . url ( ) ;
const bodyText = await getPageBodyPreview ( page ) ;
return {
currentUrl ,
bodyText ,
isAuthPage : isAuthUrl ( currentUrl ) || hasAuthKeywords ( bodyText ) ,
} ;
}
async function ensureDatasetAccessible ( page , dataset , timeout = 120000 , options = { } ) {
await page . goto ( dataset . url , { waitUntil : 'domcontentloaded' } ) ;
await waitUntilReady ( page , dataset . heading , timeout , options ) ;
}
export async function login ( ) {
const runtimeController = getRuntimeController ( ) ;
runtimeController . bind ( ) ;
const context = await getContext ( ) ;
const cleanupAndExit = async ( signal ) => {
console . log ( ` [login] 收到 ${ signal } ,正在保存登录态并关闭浏览器... ` ) ;
await closeContextIfNeeded ( ) ;
process . exit ( 130 ) ;
} ;
const page = context . pages ( ) [ 0 ] || ( await context . newPage ( ) ) ;
await page . goto ( datasets . customers . url , { waitUntil : 'domcontentloaded' } ) ;
console . log ( '请在打开的浏览器里完成阿里云伙伴中心登录,然后回到终端按 Ctrl+C 结束。' ) ;
await waitUntilReady ( page , datasets . customers . heading , 10 * 60 * 1000 ) ;
console . log ( '登录态已写入 .browser 目录,后续可直接执行 npm run sync。 ') ;
const onSigint = ( ) => {
void cleanupAndExit ( 'SIGINT' ) ;
} ;
const onSigterm = ( ) => {
void cleanupAndExit ( 'SIGTERM ' ) ;
} ;
// 必须正常关闭 context, 否则登录态不会持久化到磁盘
await context . close ( ) ;
_context = null ;
process . once ( 'SIGINT' , onSigint ) ;
process . once ( 'SIGTERM' , onSigterm ) ;
try {
const page = context . pages ( ) [ 0 ] || ( await context . newPage ( ) ) ;
await page . goto ( datasets . customers . url , { waitUntil : 'domcontentloaded' } ) ;
console . log ( '请在打开的浏览器里完成阿里云伙伴中心登录。检测到进入“我的客户”和“账单查询”页面后,脚本会自动保存登录态并关闭浏览器。' ) ;
await waitUntilReady ( page , datasets . customers . heading , 10 * 60 * 1000 , { allowInteractiveAuth : true } ) ;
console . log ( '[login] 我的客户页验证通过,继续验证账单页登录态...' ) ;
await ensureDatasetAccessible ( page , datasets . bills , 60 * 1000 , { allowInteractiveAuth : true } ) ;
await sleep ( 1000 ) ;
await saveStorageState ( context ) ;
console . log ( '登录态已写入 .browser 目录,且已验证“我的客户”和“账单查询”页面可访问,后续可直接执行 npm run sync 或 npm run bills。' ) ;
} finally {
process . off ( 'SIGINT' , onSigint ) ;
process . off ( 'SIGTERM' , onSigterm ) ;
await closeContextIfNeeded ( ) ;
runtimeController . unbind ( ) ;
}
}
export async function syncAll ( ) {
const runtimeController = getRuntimeController ( ) ;
runtimeController . bind ( ) ;
const context = await getContext ( ) ;
try {
@@ -67,11 +248,36 @@ export async function syncAll() {
return summary ;
} finally {
if ( config . closeBrowser ) {
await context. close ( ) ;
_context = null ;
await closeC ontextIfNeeded ( ) ;
} else {
console . log ( '浏览器保持运行' ) ;
}
runtimeController . unbind ( ) ;
}
}
export async function syncBillsOnly ( ) {
const runtimeController = getRuntimeController ( ) ;
runtimeController . bind ( ) ;
const context = await getContext ( ) ;
try {
const summary = { startedAt : new Date ( ) . toISOString ( ) , datasets : { } } ;
const page = context . pages ( ) [ 0 ] || ( await context . newPage ( ) ) ;
summary . datasets . bills = await syncBills ( page ) ;
summary . finishedAt = new Date ( ) . toISOString ( ) ;
const stamp = nowStamp ( ) ;
saveRunSummary ( stamp , summary ) ;
return summary ;
} finally {
if ( config . closeBrowser ) {
await closeContextIfNeeded ( ) ;
} else {
console . log ( '浏览器保持运行' ) ;
}
runtimeController . unbind ( ) ;
}
}
@@ -106,6 +312,7 @@ export async function scheduleSync() {
}
async function syncCustomers ( page ) {
await runtimeCheckpoint ( '同步客户' ) ;
const dataset = datasets . customers ;
await page . goto ( dataset . url , { waitUntil : 'domcontentloaded' } ) ;
await waitUntilReady ( page , dataset . heading ) ;
@@ -115,6 +322,7 @@ async function syncCustomers(page) {
}
async function syncCustomerDetails ( page ) {
await runtimeCheckpoint ( '同步客户详情' ) ;
const dataset = datasets . customerDetails ;
const customersState = loadCurrentState ( 'customers' ) ;
const allAccountIds = collectValidAccountIds ( customersState . records || [ ] ) ;
@@ -130,6 +338,7 @@ async function syncCustomerDetails(page) {
'https://aps.aliyun.com/?spm=5176.12818093.top-nav.ditem-fx.785716d0LKDpKT#/detail/my_customer/~/customer/' ;
for ( let index = 0 ; index < allAccountIds . length ; index += 1 ) {
await runtimeCheckpoint ( ` 客户详情 ${ index + 1 } / ${ allAccountIds . length } ` ) ;
const accountId = allAccountIds [ index ] ;
console . log ( ` [客户详情] ${ index + 1 } / ${ allAccountIds . length } accountId= ${ accountId } ` ) ;
@@ -158,23 +367,20 @@ async function syncCustomerDetails(page) {
}
async function syncOrders ( page ) {
await runtimeCheckpoint ( '同步订单' ) ;
const dataset = datasets . orders ;
let windows ;
if ( config . fullSync ) {
windows = buildMonthlyDateWindows ( config . orderStartDate ) ;
} else {
// 增量模式:只查前一天
const yesterday = new Date ( ) ;
yesterday . setDate ( yesterday . getDate ( ) - 1 ) ;
const dateStr = formatDate ( yesterday ) ;
windows = [ { windowStart : dateStr , windowEnd : dateStr , start : dateStr , end : dateStr } ] ;
console . log ( ` [增量模式] 订单仅查询: ${ dateStr } ` ) ;
windows = buildIncrementalOrderWindows ( ) ;
}
const allRecords = [ ] ;
for ( const window of windows ) {
await runtimeCheckpoint ( ` 订单窗口 ${ window . start } ~ ${ window . end } ` ) ;
await page . goto ( dataset . url , { waitUntil : 'domcontentloaded' } ) ;
await waitUntilReady ( page , dataset . heading ) ;
await setDateRange ( page , window . start , window . end ) ;
@@ -187,7 +393,40 @@ async function syncOrders(page) {
return persistDataset ( dataset , dedupeByHash ( allRecords ) , { } ) ;
}
function buildIncrementalOrderWindows ( ) {
const configuredStartDate = normalizeConfiguredDate ( config . incrementalOrderStartDate ) ;
if ( configuredStartDate ) {
const windows = buildMonthlyDateWindows ( configuredStartDate ) ;
console . log ( ` [增量模式] 订单从指定日期开始查询: ${ configuredStartDate } ` ) ;
return windows ;
}
const yesterday = new Date ( ) ;
yesterday . setDate ( yesterday . getDate ( ) - 1 ) ;
const dateStr = formatDate ( yesterday ) ;
console . log ( ` [增量模式] 订单仅查询: ${ dateStr } ` ) ;
return [ { windowStart : dateStr , windowEnd : dateStr , start : dateStr , end : dateStr } ] ;
}
function normalizeConfiguredDate ( value ) {
const normalized = String ( value || '' ) . trim ( ) ;
if ( ! normalized ) {
return '' ;
}
if ( ! /^\d{4}-\d{2}-\d{2}$/ . test ( normalized ) ) {
throw new Error ( ` ALIYUN_APS_INCREMENTAL_ORDER_START_DATE 格式无效: ${ normalized } ,期望 YYYY-MM-DD ` ) ;
}
const parsed = new Date ( ` ${ normalized } T00:00:00+08:00 ` ) ;
if ( Number . isNaN ( parsed . getTime ( ) ) ) {
throw new Error ( ` ALIYUN_APS_INCREMENTAL_ORDER_START_DATE 不是有效日期: ${ normalized } ` ) ;
}
return normalized ;
}
async function syncBills ( page ) {
await runtimeCheckpoint ( '同步账单' ) ;
const dataset = datasets . bills ;
let months ;
let latestConsumptionDate = null ;
@@ -205,6 +444,7 @@ async function syncBills(page) {
const allRecords = [ ] ;
for ( const month of months ) {
await runtimeCheckpoint ( ` 账单月份 ${ month } ` ) ;
await page . goto ( dataset . url , { waitUntil : 'domcontentloaded' } ) ;
await waitUntilReady ( page , dataset . heading ) ;
await setMonthValue ( page , month ) ;
@@ -247,6 +487,7 @@ function isAfterLatestConsumptionDate(record, latestConsumptionDate) {
}
async function syncOrderDetails ( page , cachedOrderIds ) {
await runtimeCheckpoint ( '同步订单详情' ) ;
const dataset = datasets . orderDetails ;
// 使用传入的 orderId 列表(在 syncOrders 覆盖 orders.json 之前缓存的)
@@ -262,6 +503,7 @@ async function syncOrderDetails(page, cachedOrderIds) {
const detailBaseUrl = 'https://aps.aliyun.com/?spm=5176.12818093.top-nav.ditem-fx.785716d0LKDpKT#/detail/order/~/costCenter/order/detail/' ;
for ( let index = 0 ; index < allOrderIds . length ; index += 1 ) {
await runtimeCheckpoint ( ` 订单详情 ${ index + 1 } / ${ allOrderIds . length } ` ) ;
const orderId = allOrderIds [ index ] ;
console . log ( ` [订单详情] ${ index + 1 } / ${ allOrderIds . length } orderId= ${ orderId } ` ) ;
@@ -304,11 +546,30 @@ function persistDataset(dataset, records, context) {
} ;
}
async function waitUntilReady ( page , heading , timeout = 120000 ) {
async function waitUntilReady ( page , heading , timeout = 120000 , options = { } ) {
await runtimeCheckpoint ( ` 等待页面 ${ heading } ` ) ;
const { allowInteractiveAuth = false } = options ;
await page . waitForLoadState ( 'domcontentloaded' ) ;
console . log ( ` [waitUntilReady] 当前URL: ${ page . url ( ) } ` ) ;
console . log ( ` [waitUntilReady] 等待页面出现: " ${ heading } " ` ) ;
const initialState = await detectAuthRedirect ( page ) ;
if ( initialState . isAuthPage ) {
console . error ( ` [waitUntilReady] 检测到登录页/鉴权页: ${ initialState . currentUrl } ` ) ;
console . error ( ` [waitUntilReady] 页面内容前500字: ${ initialState . bodyText } ` ) ;
if ( ! allowInteractiveAuth && isAuthUrl ( initialState . currentUrl ) ) {
try {
await sendLoginAlert ( initialState . currentUrl ) ;
} catch ( notifyErr ) {
console . error ( '[通知] 发送登录提醒失败:' , notifyErr . message ) ;
}
}
if ( ! allowInteractiveAuth ) {
throw new Error ( ` 当前页面仍处于登录/鉴权页,无法进入「 ${ heading } 」。请重新执行 npm run login, 并确认该账号对该页面有访问权限。 ` ) ;
}
console . log ( ` [waitUntilReady] 允许交互式登录,等待用户完成认证后进入「 ${ heading } 」... ` ) ;
}
try {
await page . waitForFunction (
( text ) => document . body && document . body . innerText . includes ( text ) ,
@@ -317,22 +578,26 @@ async function waitUntilReady(page, heading, timeout = 120000) {
) ;
} catch ( err ) {
// 超时时打印诊断信息
const currentUrl = page . url ( ) ;
const bodyText = await page . evaluate ( ( ) => document . body ? . innerText ? . substring ( 0 , 500 ) || '(空)' ) . catch ( ( ) => '(无法获取)' ) ;
const { currentUrl , bodyText , isAuthPage } = await detectAuthRedirect ( page ) ;
console . error ( ` [waitUntilReady] 超时! 当前URL: ${ currentUrl } ` ) ;
console . error ( ` [waitUntilReady] 页面内容前500字: ${ bodyText } ` ) ;
if ( currentUrl . includes ( 'signin' ) ) {
if ( isAuthPage && ! allowInteractiveAuth ) {
try {
await sendLoginAlert ( ) ;
await sendLoginAlert ( currentUrl ) ;
} catch ( notifyErr ) {
console . error ( '[通知] 发送登录提醒失败:' , notifyErr . message ) ;
}
throw new Error ( ` 当前页面停留在登录/鉴权页,未能进入「 ${ heading } 」。请重新执行 npm run login, 并确认该账号对该页面有访问权限。 ` ) ;
}
if ( isAuthPage && allowInteractiveAuth ) {
throw new Error ( ` 交互式登录超时,仍未进入「 ${ heading } 」。请确认已在浏览器中完成 RAM/阿里云登录,并且当前账号有访问该页面的权限。 ` ) ;
}
throw err ;
}
if ( ( await page . locator ( 'text=登录' ) . count ( ) ) > 0 && page . url ( ) . includes ( 'login' ) ) {
throw new Error ( '当前未登录,请先执行 npm run login' ) ;
const finalState = await detectAuthRedirect ( page ) ;
if ( finalState . isAuthPage && ! allowInteractiveAuth ) {
throw new Error ( ` 当前页面仍处于登录/鉴权页,未成功进入「 ${ heading } 」。请重新执行 npm run login, 并确认该账号对该页面有访问权限。 ` ) ;
}
await sleep ( 1500 ) ;
}
@@ -342,6 +607,7 @@ async function scrapePagedTable(page, dataset, context) {
const visited = new Set ( ) ;
while ( true ) {
await runtimeCheckpoint ( ` 抓取 ${ dataset . name } 分页 ` ) ;
await waitForTableRows ( page ) ;
const pageData = await extractTable ( page ) ;
const pageNum = await currentPageNumber ( page ) ;
@@ -403,6 +669,7 @@ async function extractTable(page) {
}
async function waitForTableRows ( page ) {
await runtimeCheckpoint ( '等待表格加载' ) ;
await page . waitForFunction ( ( ) => document . querySelectorAll ( 'table tbody tr' ) . length > 0 , null , { timeout : 120000 } ) ;
await sleep ( 800 ) ;
}
@@ -414,6 +681,7 @@ async function currentPageNumber(page) {
}
async function gotoNextPage ( page ) {
await runtimeCheckpoint ( '翻页' ) ;
const before = await currentPageNumber ( page ) ;
// 用 Playwright locator 定位"下一页"按钮
@@ -439,6 +707,7 @@ async function gotoNextPage(page) {
}
async function trySetPageSize ( page , pageSize ) {
await runtimeCheckpoint ( ` 设置每页 ${ pageSize } ` ) ;
const input = page . locator ( 'input[aria-label="请选择每页显示几条"]' ) . first ( ) ;
if ( ( await input . count ( ) ) === 0 ) return ;
await input . click ( ) . catch ( ( ) => null ) ;
@@ -453,6 +722,7 @@ async function trySetPageSize(page, pageSize) {
}
async function setDateRange ( page , start , end ) {
await runtimeCheckpoint ( ` 设置订单日期 ${ start } ~ ${ end } ` ) ;
console . log ( ` [订单日期] 设置: ${ start } ~ ${ end } ` ) ;
await _fillDateRange ( page , start , end ) ;
@@ -474,6 +744,7 @@ async function setDateRange(page, start, end) {
}
async function _fillDateRange ( page , start , end , startFirst = false ) {
await runtimeCheckpoint ( '填写订单日期' ) ;
const trigger = page . locator ( 'input[placeholder="结束日期"]' ) ;
await trigger . click ( ) ;
await sleep ( 1000 ) ;
@@ -520,6 +791,7 @@ async function _fillDateRange(page, start, end, startFirst = false) {
}
async function setMonthValue ( page , month ) {
await runtimeCheckpoint ( ` 设置账单月份 ${ month } ` ) ;
// 先尝试按 inputValue 匹配 YYYY-MM 格式
const inputs = page . locator ( 'input' ) ;
const total = await inputs . count ( ) ;
@@ -568,6 +840,7 @@ async function setMonthValue(page, month) {
* 即使面板弹出,快速键入 + Tab 也能在面板滚动前完成提交并关闭。
*/
async function typeIntoDateInput ( locator , value , page ) {
await runtimeCheckpoint ( ` 填写日期输入 ${ value } ` ) ;
// 移除 readonly
await locator . evaluate ( ( node ) => node . removeAttribute ( 'readonly' ) ) ;
@@ -599,6 +872,7 @@ async function typeIntoDateInput(locator, value, page) {
}
async function clickQuery ( page ) {
await runtimeCheckpoint ( '点击查询' ) ;
const button = page . locator ( 'button:has-text("查询")' ) . first ( ) ;
await button . click ( ) ;
await sleep ( 1800 ) ;