项目初始化
This commit is contained in:
5
.idea/.gitignore
generated
vendored
Normal file
5
.idea/.gitignore
generated
vendored
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
# 默认忽略的文件
|
||||||
|
/shelf/
|
||||||
|
/workspace.xml
|
||||||
|
# 基于编辑器的 HTTP 客户端请求
|
||||||
|
/httpRequests/
|
||||||
6
.idea/misc.xml
generated
Normal file
6
.idea/misc.xml
generated
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="ProjectRootManager">
|
||||||
|
<output url="file://$PROJECT_DIR$/out" />
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
8
.idea/modules.xml
generated
Normal file
8
.idea/modules.xml
generated
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="ProjectModuleManager">
|
||||||
|
<modules>
|
||||||
|
<module fileurl="file://$PROJECT_DIR$/.idea/python.iml" filepath="$PROJECT_DIR$/.idea/python.iml" />
|
||||||
|
</modules>
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
9
.idea/python.iml
generated
Normal file
9
.idea/python.iml
generated
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<module type="JAVA_MODULE" version="4">
|
||||||
|
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
||||||
|
<exclude-output />
|
||||||
|
<content url="file://$MODULE_DIR$" />
|
||||||
|
<orderEntry type="inheritedJdk" />
|
||||||
|
<orderEntry type="sourceFolder" forTests="false" />
|
||||||
|
</component>
|
||||||
|
</module>
|
||||||
6
.idea/vcs.xml
generated
Normal file
6
.idea/vcs.xml
generated
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="VcsDirectoryMappings">
|
||||||
|
<mapping directory="" vcs="Git" />
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
6
aliyun-sync/aliyun-aps-sync/.env.example
Normal file
6
aliyun-sync/aliyun-aps-sync/.env.example
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
ALIYUN_APS_BASE_URL=https://aps.aliyun.com
|
||||||
|
ALIYUN_APS_HEADLESS=false
|
||||||
|
ALIYUN_APS_TIMEZONE=Asia/Shanghai
|
||||||
|
ALIYUN_APS_CRON=0 6 * * *
|
||||||
|
ALIYUN_APS_ORDER_START_DATE=2024-01-01
|
||||||
|
ALIYUN_APS_BILL_START_MONTH=2024-01
|
||||||
5
aliyun-sync/aliyun-aps-sync/.gitignore
vendored
Normal file
5
aliyun-sync/aliyun-aps-sync/.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
node_modules/
|
||||||
|
.env
|
||||||
|
.browser/
|
||||||
|
data/
|
||||||
|
downloads/
|
||||||
57
aliyun-sync/aliyun-aps-sync/README.md
Normal file
57
aliyun-sync/aliyun-aps-sync/README.md
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
# aliyun-aps-sync
|
||||||
|
|
||||||
|
用于抓取阿里云伙伴中心里的 `我的客户`、`订单查询`、`账单查询`，并把结果和本地上一次同步结果做增量对比。
|
||||||
|
|
||||||
|
## 功能
|
||||||
|
|
||||||
|
- 首次 `login` 用浏览器手动登录，后续复用本地登录态。
|
||||||
|
- `sync` 会同步 3 个模块的数据。
|
||||||
|
- 同步后会生成：
|
||||||
|
- `data/current/*.json` 当前全量
|
||||||
|
- `data/history/<dataset>/*.json` 每次快照
|
||||||
|
- `data/delta/<dataset>/*.json` 增量变化
|
||||||
|
- `data/runs/*.json` 每次任务汇总
|
||||||
|
- `schedule` 支持常驻进程方式按 cron 表达式每天自动同步。
|
||||||
|
|
||||||
|
## 安装
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /Users/qiangredhad/aliyun-aps-sync
|
||||||
|
npm install
|
||||||
|
cp .env.example .env
|
||||||
|
```
|
||||||
|
|
||||||
|
## 配置
|
||||||
|
|
||||||
|
`.env` 里最重要的两个时间范围：
|
||||||
|
|
||||||
|
- `ALIYUN_APS_ORDER_START_DATE`: 订单查询的起始日期，会按月滚动抓取直到今天。
|
||||||
|
- `ALIYUN_APS_BILL_START_MONTH`: 账单查询的起始佣金月份，会按月滚动抓取直到当前月。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
1. 首次登录并保存会话
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run login
|
||||||
|
```
|
||||||
|
|
||||||
|
2. 手动执行一次同步
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run sync
|
||||||
|
```
|
||||||
|
|
||||||
|
3. 常驻定时同步
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run schedule
|
||||||
|
```
|
||||||
|
|
||||||
|
默认 cron 是每天早上 6 点，可在 `.env` 里改 `ALIYUN_APS_CRON`。
|
||||||
|
|
||||||
|
## 注意
|
||||||
|
|
||||||
|
- 脚本现在基于页面表格 DOM 抓取，如果阿里云伙伴中心页面结构改版，需要调整 `src/sync.js` 里的表格和筛选器选择逻辑。
|
||||||
|
- 订单和账单的日期输入框是通过页面已有日期值自动识别的，所以首次跑之前建议先在页面确认默认筛选存在。
|
||||||
|
- 如果登录态过期，重新执行 `npm run login` 即可。
|
||||||
92
aliyun-sync/aliyun-aps-sync/package-lock.json
generated
Normal file
92
aliyun-sync/aliyun-aps-sync/package-lock.json
generated
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
{
|
||||||
|
"name": "aliyun-aps-sync",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"lockfileVersion": 3,
|
||||||
|
"requires": true,
|
||||||
|
"packages": {
|
||||||
|
"": {
|
||||||
|
"name": "aliyun-aps-sync",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"dependencies": {
|
||||||
|
"dotenv": "^16.6.1",
|
||||||
|
"node-cron": "^4.2.1",
|
||||||
|
"nodemailer": "^6.10.1",
|
||||||
|
"playwright": "^1.58.2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/dotenv": {
|
||||||
|
"version": "16.6.1",
|
||||||
|
"resolved": "https://registry.npmmirror.com/dotenv/-/dotenv-16.6.1.tgz",
|
||||||
|
"integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==",
|
||||||
|
"license": "BSD-2-Clause",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://dotenvx.com"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/fsevents": {
|
||||||
|
"version": "2.3.2",
|
||||||
|
"resolved": "https://registry.npmmirror.com/fsevents/-/fsevents-2.3.2.tgz",
|
||||||
|
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||||
|
"hasInstallScript": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/node-cron": {
|
||||||
|
"version": "4.2.1",
|
||||||
|
"resolved": "https://registry.npmmirror.com/node-cron/-/node-cron-4.2.1.tgz",
|
||||||
|
"integrity": "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg==",
|
||||||
|
"license": "ISC",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=6.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/nodemailer": {
|
||||||
|
"version": "6.10.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.10.1.tgz",
|
||||||
|
"integrity": "sha512-Z+iLaBGVaSjbIzQ4pX6XV41HrooLsQ10ZWPUehGmuantvzWoDVBnmsdUcOIDM1t+yPor5pDhVlDESgOMEGxhHA==",
|
||||||
|
"license": "MIT-0",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=6.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/playwright": {
|
||||||
|
"version": "1.58.2",
|
||||||
|
"resolved": "https://registry.npmmirror.com/playwright/-/playwright-1.58.2.tgz",
|
||||||
|
"integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"playwright-core": "1.58.2"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"playwright": "cli.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"fsevents": "2.3.2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/playwright-core": {
|
||||||
|
"version": "1.58.2",
|
||||||
|
"resolved": "https://registry.npmmirror.com/playwright-core/-/playwright-core-1.58.2.tgz",
|
||||||
|
"integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"bin": {
|
||||||
|
"playwright-core": "cli.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
18
aliyun-sync/aliyun-aps-sync/package.json
Normal file
18
aliyun-sync/aliyun-aps-sync/package.json
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"name": "aliyun-aps-sync",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"login": "node src/index.js login",
|
||||||
|
"sync": "node src/index.js sync",
|
||||||
|
"bills": "node src/index.js bills",
|
||||||
|
"schedule": "node src/index.js schedule"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"dotenv": "^16.6.1",
|
||||||
|
"nodemailer": "^6.10.1",
|
||||||
|
"node-cron": "^4.2.1",
|
||||||
|
"playwright": "^1.58.2"
|
||||||
|
}
|
||||||
|
}
|
||||||
188
aliyun-sync/aliyun-aps-sync/src/config.js
Normal file
188
aliyun-sync/aliyun-aps-sync/src/config.js
Normal file
@@ -0,0 +1,188 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import process from 'node:process';
|
||||||
|
import dotenv from 'dotenv';
|
||||||
|
|
||||||
|
const rootDir = path.resolve(process.cwd());
|
||||||
|
dotenv.config({ path: path.join(rootDir, '.env') });
|
||||||
|
|
||||||
|
const toBool = (value, fallback) => {
|
||||||
|
if (value == null) return fallback;
|
||||||
|
return ['1', 'true', 'yes', 'y', 'on'].includes(String(value).trim().toLowerCase());
|
||||||
|
};
|
||||||
|
|
||||||
|
const ensureDir = (dirPath) => {
|
||||||
|
fs.mkdirSync(dirPath, { recursive: true });
|
||||||
|
return dirPath;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const config = {
|
||||||
|
rootDir,
|
||||||
|
baseUrl: process.env.ALIYUN_APS_BASE_URL || 'https://aps.aliyun.com',
|
||||||
|
headless: toBool(process.env.ALIYUN_APS_HEADLESS, false),
|
||||||
|
timezone: process.env.ALIYUN_APS_TIMEZONE || 'Asia/Shanghai',
|
||||||
|
cron: process.env.ALIYUN_APS_CRON || '0 6 * * *',
|
||||||
|
orderStartDate: process.env.ALIYUN_APS_ORDER_START_DATE || '2024-01-01',
|
||||||
|
billStartMonth: process.env.ALIYUN_APS_BILL_START_MONTH || '2024-01',
|
||||||
|
smtp: {
|
||||||
|
host: process.env.ALIYUN_APS_SMTP_HOST || 'smtp.qq.com',
|
||||||
|
port: parseInt(process.env.ALIYUN_APS_SMTP_PORT || '465', 10),
|
||||||
|
secure: toBool(process.env.ALIYUN_APS_SMTP_SECURE, true),
|
||||||
|
user: process.env.ALIYUN_APS_SMTP_USER || '',
|
||||||
|
pass: process.env.ALIYUN_APS_SMTP_PASS || '',
|
||||||
|
},
|
||||||
|
notifyEmail: process.env.ALIYUN_APS_NOTIFY_EMAIL || '',
|
||||||
|
closeBrowser: toBool(process.env.ALIYUN_APS_CLOSE_BROWSER, true),
|
||||||
|
fullSync: toBool(process.env.ALIYUN_APS_FULL_SYNC, true),
|
||||||
|
resumeBillMonth: process.env.ALIYUN_APS_RESUME_BILL_MONTH || '',
|
||||||
|
resumeBillPage: Math.max(1, Number.parseInt(process.env.ALIYUN_APS_RESUME_BILL_PAGE || '1', 10) || 1),
|
||||||
|
dbSyncScript: process.env.ALIYUN_APS_DB_SYNC_SCRIPT || '../aps-aliyun-sync/aps_db_sync.py',
|
||||||
|
userDataDir: ensureDir(path.join(rootDir, '.browser')),
|
||||||
|
dataDir: ensureDir(path.join(rootDir, 'data')),
|
||||||
|
downloadDir: ensureDir(path.join(rootDir, 'downloads')),
|
||||||
|
};
|
||||||
|
|
||||||
|
export const datasets = {
|
||||||
|
customers: {
|
||||||
|
name: 'customers',
|
||||||
|
url: `${config.baseUrl}/#/detail/my_customer/~/customer/list`,
|
||||||
|
heading: '我的客户',
|
||||||
|
pageSize: 100,
|
||||||
|
uniqueKey: (record) => record.accountId || record.loginName || record.__hash,
|
||||||
|
normalize: (record) => {
|
||||||
|
const loginAndUid = record['登录名称/账号ID'] || '';
|
||||||
|
const [loginName = '', accountId = ''] = splitLines(loginAndUid);
|
||||||
|
return {
|
||||||
|
loginName: loginName.replace(/\s+/g, ''),
|
||||||
|
accountId,
|
||||||
|
realName: record['UID实名认证名称'] || '',
|
||||||
|
reportSource: record['报备来源'] || '',
|
||||||
|
reportType: record['报备类型'] || '',
|
||||||
|
tradeMode: record['交易模式'] || '',
|
||||||
|
authStatus: record['实名认证状态'] || '',
|
||||||
|
relationTime: record['关联日期'] || '',
|
||||||
|
owner: record['跟进员工'] || '',
|
||||||
|
cashBalanceCny: record['账户现金余额(CNY)'] || '',
|
||||||
|
invoicePendingCny: record['待开票金额(CNY)'] || '',
|
||||||
|
lastMonthConsumptionCny: record['上月消费金额(CNY)'] || '',
|
||||||
|
thisMonthConsumptionCny: record['本月消费金额(CNY)'] || '',
|
||||||
|
paymentNoticeStatus: record['代为支付告知状态'] || '',
|
||||||
|
inviteType: record['邀约注册类型'] || '',
|
||||||
|
isNewCustomer: record['是否新客户'] || '',
|
||||||
|
isPerformanceQualified: record['是否达成业绩起算点'] || '',
|
||||||
|
customerCategory: record['客户分类'] || '',
|
||||||
|
remark: record['备注'] || '',
|
||||||
|
inactiveMonths: record['客户无消费月数'] || '',
|
||||||
|
releasePlanTime: record['计划释放时间'] || '',
|
||||||
|
releasePlanReason: record['计划释放原因'] || '',
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
orders: {
|
||||||
|
name: 'orders',
|
||||||
|
url: `${config.baseUrl}/#/detail/order/~/costCenter/order`,
|
||||||
|
heading: '订单查询',
|
||||||
|
pageSize: 100,
|
||||||
|
uniqueKey: (record) => record.orderId || record.__hash,
|
||||||
|
normalize: (record, context) => ({
|
||||||
|
orderId: record['订单号'] || '',
|
||||||
|
customerAccount: (record['客户账号'] || '').replace(/\s+/g, ''),
|
||||||
|
customerCategory: record['客户分类'] || '',
|
||||||
|
orderType: record['订单类型'] || '',
|
||||||
|
orderOriginalPriceCny: record['订单原价 (CNY)'] || '',
|
||||||
|
actualPaidCny: record['实付金额 (CNY)'] || '',
|
||||||
|
orderStatus: record['订单状态'] || '',
|
||||||
|
createdAt: record['下单时间'] || '',
|
||||||
|
windowStart: context.windowStart || '',
|
||||||
|
windowEnd: context.windowEnd || '',
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
orderDetails: {
|
||||||
|
name: 'orderDetails',
|
||||||
|
url: `${config.baseUrl}/#/detail/order/~/costCenter/order`,
|
||||||
|
heading: '订单详情',
|
||||||
|
pageSize: 100,
|
||||||
|
uniqueKey: (record) => record.orderId || record.__hash,
|
||||||
|
normalize: (record, context) => ({
|
||||||
|
orderId: record.orderId || '',
|
||||||
|
orderType: record.orderType || '',
|
||||||
|
status: record.status || '',
|
||||||
|
tradeType: record.tradeType || '',
|
||||||
|
customerCategory: record.customerCategory || '',
|
||||||
|
dealerName: record.dealerName || '',
|
||||||
|
dealerUid: record.dealerUid || '',
|
||||||
|
customerType: record.customerType || '',
|
||||||
|
opportunityId: record.opportunityId || '',
|
||||||
|
paymentTime: record.paymentTime || '',
|
||||||
|
orderTime: record.orderTime || '',
|
||||||
|
productName: record.productName || '',
|
||||||
|
productCode: record.productCode || '',
|
||||||
|
originalPriceCny: record.originalPriceCny || '',
|
||||||
|
paidAmountCny: record.paidAmountCny || '',
|
||||||
|
discount: record.discount || '',
|
||||||
|
payableAmountCny: record.payableAmountCny || '',
|
||||||
|
couponAmountCny: record.couponAmountCny || '',
|
||||||
|
windowStart: context.windowStart || '',
|
||||||
|
windowEnd: context.windowEnd || '',
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
customerDetails: {
|
||||||
|
name: 'customerDetails',
|
||||||
|
url: `${config.baseUrl}/#/detail/my_customer/~/customer/list`,
|
||||||
|
heading: '详情',
|
||||||
|
pageSize: 100,
|
||||||
|
uniqueKey: (record) => record.accountId || record.__hash,
|
||||||
|
normalize: (record, context) => ({
|
||||||
|
accountId: context.accountId || '',
|
||||||
|
customerAccount: record.customerAccount || '',
|
||||||
|
customerName: record.customerName || '',
|
||||||
|
customerType: record.customerType || '',
|
||||||
|
tradeMode: record.tradeMode || '',
|
||||||
|
customerSource: record.customerSource || '',
|
||||||
|
realNameStatus: record.realNameStatus || '',
|
||||||
|
email: record.email || '',
|
||||||
|
relationDate: record.relationDate || '',
|
||||||
|
phone: record.phone || '',
|
||||||
|
remark: record.remark || '',
|
||||||
|
paymentNoticeStatus: record.paymentNoticeStatus || '',
|
||||||
|
department: record.department || '',
|
||||||
|
lastMonthPayableTotalCny: record.lastMonthPayableTotalCny || '',
|
||||||
|
lastMonthPrepayCny: record.lastMonthPrepayCny || '',
|
||||||
|
lastMonthPostpayCny: record.lastMonthPostpayCny || '',
|
||||||
|
currentMonthPayableTotalCny: record.currentMonthPayableTotalCny || '',
|
||||||
|
currentMonthPrepayCny: record.currentMonthPrepayCny || '',
|
||||||
|
currentMonthPostpayCny: record.currentMonthPostpayCny || '',
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
bills: {
|
||||||
|
name: 'bills',
|
||||||
|
url: `${config.baseUrl}/#/detail/bill/~/costCenter/bill`,
|
||||||
|
heading: '账单查询',
|
||||||
|
pageSize: 100,
|
||||||
|
uniqueKey: (record) => record.__hash,
|
||||||
|
normalize: (record, context) => ({
|
||||||
|
billingMonth: record['账期'] || '',
|
||||||
|
consumeDate: record['消费时间'] || '',
|
||||||
|
customerAccount: (record['客户账号'] || '').replace(/\s+/g, ''),
|
||||||
|
customerCategory: record['客户分类'] || '',
|
||||||
|
productCategory: record['产品分类'] || '',
|
||||||
|
productName: record['产品名称'] || '',
|
||||||
|
originalPriceCny: record['原价 (CNY)'] || '',
|
||||||
|
customerPayableCny: record['客户应付金额 (CNY)'] || '',
|
||||||
|
billType: record['账单类型'] || '',
|
||||||
|
countsForPerformance: record['是否计入业绩'] || '',
|
||||||
|
commissionable: record['是否返佣'] || '',
|
||||||
|
commissionMonth: record['佣金月份'] || context.month || '',
|
||||||
|
inviteType: record['邀约注册类型'] || '',
|
||||||
|
serviceStartAt: record['服务开始时间'] || '',
|
||||||
|
serviceEndAt: record['服务结束时间'] || '',
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
function splitLines(value) {
|
||||||
|
return String(value)
|
||||||
|
.split('\n')
|
||||||
|
.map((part) => part.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
91
aliyun-sync/aliyun-aps-sync/src/debug-inputs.js
Normal file
91
aliyun-sync/aliyun-aps-sync/src/debug-inputs.js
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
import { chromium } from 'playwright';
|
||||||
|
import { config, datasets } from './config.js';
|
||||||
|
|
||||||
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||||
|
|
||||||
|
const target = process.argv[2] || 'orders';
|
||||||
|
const dataset = datasets[target];
|
||||||
|
|
||||||
|
console.log(`正在打开: ${dataset.heading}`);
|
||||||
|
|
||||||
|
const context = await chromium.launchPersistentContext(config.userDataDir, {
|
||||||
|
channel: 'chrome',
|
||||||
|
headless: false,
|
||||||
|
acceptDownloads: true,
|
||||||
|
downloadsPath: config.downloadDir,
|
||||||
|
});
|
||||||
|
|
||||||
|
const page = context.pages()[0] || (await context.newPage());
|
||||||
|
await page.goto(dataset.url, { waitUntil: 'domcontentloaded' });
|
||||||
|
|
||||||
|
// 等页面加载完
|
||||||
|
await page.waitForFunction(
|
||||||
|
(text) => document.body && document.body.innerText.includes(text),
|
||||||
|
dataset.heading,
|
||||||
|
{ timeout: 60000 },
|
||||||
|
);
|
||||||
|
await sleep(3000);
|
||||||
|
|
||||||
|
// 探测所有 input 元素的详细信息
|
||||||
|
const inputInfo = await page.evaluate(() => {
|
||||||
|
const inputs = document.querySelectorAll('input');
|
||||||
|
return Array.from(inputs).map((el, i) => {
|
||||||
|
// 找到最近的日期选择器父组件
|
||||||
|
const pickerParent = el.closest(
|
||||||
|
'[class*="date-picker"], [class*="month-picker"], [class*="range-picker"], [class*="calendar"]'
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
index: i,
|
||||||
|
value: el.value,
|
||||||
|
placeholder: el.placeholder,
|
||||||
|
type: el.type,
|
||||||
|
readOnly: el.readOnly,
|
||||||
|
className: el.className.substring(0, 120),
|
||||||
|
parentClass: el.parentElement?.className?.substring(0, 120) || '',
|
||||||
|
pickerClass: pickerParent?.className?.substring(0, 150) || '(无)',
|
||||||
|
ariaLabel: el.getAttribute('aria-label') || '',
|
||||||
|
role: el.getAttribute('role') || '',
|
||||||
|
};
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('\n=== 页面所有 input 元素 ===');
|
||||||
|
for (const info of inputInfo) {
|
||||||
|
console.log(`\n[input ${info.index}]`);
|
||||||
|
console.log(` value: "${info.value}"`);
|
||||||
|
console.log(` placeholder: "${info.placeholder}"`);
|
||||||
|
console.log(` type: ${info.type}`);
|
||||||
|
console.log(` readOnly: ${info.readOnly}`);
|
||||||
|
console.log(` className: ${info.className}`);
|
||||||
|
console.log(` parentClass: ${info.parentClass}`);
|
||||||
|
console.log(` pickerClass: ${info.pickerClass}`);
|
||||||
|
console.log(` ariaLabel: "${info.ariaLabel}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 探测 RangePicker 结构
|
||||||
|
const rangeInfo = await page.evaluate(() => {
|
||||||
|
const rangePickers = document.querySelectorAll('[class*="range-picker"], [class*="date-picker2"]');
|
||||||
|
return Array.from(rangePickers).map((el, i) => ({
|
||||||
|
index: i,
|
||||||
|
className: el.className.substring(0, 200),
|
||||||
|
inputCount: el.querySelectorAll('input').length,
|
||||||
|
inputs: Array.from(el.querySelectorAll('input')).map(inp => ({
|
||||||
|
value: inp.value,
|
||||||
|
placeholder: inp.placeholder,
|
||||||
|
readOnly: inp.readOnly,
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('\n=== RangePicker / DatePicker2 组件 ===');
|
||||||
|
for (const rp of rangeInfo) {
|
||||||
|
console.log(`\n[picker ${rp.index}]`);
|
||||||
|
console.log(` className: ${rp.className}`);
|
||||||
|
console.log(` inputs (${rp.inputCount}):`);
|
||||||
|
for (const inp of rp.inputs) {
|
||||||
|
console.log(` value="${inp.value}" placeholder="${inp.placeholder}" readOnly=${inp.readOnly}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await context.close();
|
||||||
|
process.exit(0);
|
||||||
135
aliyun-sync/aliyun-aps-sync/src/debug-pagination.js
Normal file
135
aliyun-sync/aliyun-aps-sync/src/debug-pagination.js
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
import { chromium } from 'playwright';
|
||||||
|
import { config, datasets } from './config.js';
|
||||||
|
|
||||||
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||||
|
|
||||||
|
console.log('打开订单页面，设置2024-11月，探测分页结构...');
|
||||||
|
|
||||||
|
const context = await chromium.launchPersistentContext(config.userDataDir, {
|
||||||
|
channel: 'chrome',
|
||||||
|
headless: false,
|
||||||
|
acceptDownloads: true,
|
||||||
|
downloadsPath: config.downloadDir,
|
||||||
|
});
|
||||||
|
|
||||||
|
const page = context.pages()[0] || (await context.newPage());
|
||||||
|
await page.goto(datasets.orders.url, { waitUntil: 'domcontentloaded' });
|
||||||
|
|
||||||
|
await page.waitForFunction(
|
||||||
|
(text) => document.body && document.body.innerText.includes(text),
|
||||||
|
datasets.orders.heading,
|
||||||
|
{ timeout: 60000 },
|
||||||
|
);
|
||||||
|
await sleep(3000);
|
||||||
|
|
||||||
|
// 设置日期到2024-11月（数据多的月份）
|
||||||
|
const trigger = page.locator('input[placeholder="结束日期"]');
|
||||||
|
await trigger.click();
|
||||||
|
await sleep(1000);
|
||||||
|
const panelEndInput = page.locator('.next-range-picker-panel-input-end-date input');
|
||||||
|
await panelEndInput.click();
|
||||||
|
await sleep(100);
|
||||||
|
await page.keyboard.press('Control+A');
|
||||||
|
await page.keyboard.type('2024-11-30', { delay: 30 });
|
||||||
|
await sleep(300);
|
||||||
|
const panelStartInput = page.locator('.next-range-picker-panel-input-start-date input');
|
||||||
|
await panelStartInput.click();
|
||||||
|
await sleep(100);
|
||||||
|
await page.keyboard.press('Control+A');
|
||||||
|
await page.keyboard.type('2024-11-01', { delay: 30 });
|
||||||
|
await sleep(300);
|
||||||
|
await page.keyboard.press('Enter');
|
||||||
|
await sleep(500);
|
||||||
|
await page.mouse.click(0, 0);
|
||||||
|
await sleep(300);
|
||||||
|
await page.keyboard.press('Escape');
|
||||||
|
await sleep(1000);
|
||||||
|
|
||||||
|
// 点查询
|
||||||
|
const queryBtn = page.locator('button:has-text("查询")').first();
|
||||||
|
await queryBtn.click();
|
||||||
|
await sleep(3000);
|
||||||
|
|
||||||
|
// 探测分页组件
|
||||||
|
const paginationInfo = await page.evaluate(() => {
|
||||||
|
const result = {
|
||||||
|
// 找所有 pagination 相关的元素
|
||||||
|
paginationContainers: [],
|
||||||
|
allButtons: [],
|
||||||
|
nextItems: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
// 所有 pagination 容器
|
||||||
|
const containers = document.querySelectorAll('[class*="pagination"]');
|
||||||
|
containers.forEach((el, i) => {
|
||||||
|
result.paginationContainers.push({
|
||||||
|
index: i,
|
||||||
|
tag: el.tagName,
|
||||||
|
className: el.className.substring(0, 200),
|
||||||
|
childCount: el.children.length,
|
||||||
|
innerHTML: el.innerHTML.substring(0, 500),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// 所有带 next-btn 类名的按钮（在 pagination 内）
|
||||||
|
const btns = document.querySelectorAll('[class*="pagination"] button, [class*="pagination"] [role="button"], [class*="pagination"] .next-btn');
|
||||||
|
btns.forEach((el, i) => {
|
||||||
|
result.allButtons.push({
|
||||||
|
index: i,
|
||||||
|
tag: el.tagName,
|
||||||
|
text: el.innerText?.trim()?.substring(0, 50) || '',
|
||||||
|
className: el.className.substring(0, 200),
|
||||||
|
disabled: el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true',
|
||||||
|
ariaLabel: el.getAttribute('aria-label') || '',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// 找 next-next 类
|
||||||
|
const nextNextEls = document.querySelectorAll('.next-next, [class*="next-next"]');
|
||||||
|
nextNextEls.forEach((el, i) => {
|
||||||
|
result.nextItems.push({
|
||||||
|
index: i,
|
||||||
|
tag: el.tagName,
|
||||||
|
text: el.innerText?.trim()?.substring(0, 50) || '',
|
||||||
|
className: el.className.substring(0, 200),
|
||||||
|
disabled: el.hasAttribute('disabled'),
|
||||||
|
parentClass: el.parentElement?.className?.substring(0, 100) || '',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return result;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('\n=== Pagination 容器 ===');
|
||||||
|
for (const c of paginationInfo.paginationContainers) {
|
||||||
|
console.log(`\n[容器 ${c.index}] <${c.tag}> children=${c.childCount}`);
|
||||||
|
console.log(` class: ${c.className}`);
|
||||||
|
console.log(` html: ${c.innerHTML.substring(0, 300)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\n=== Pagination 内的按钮 ===');
|
||||||
|
for (const b of paginationInfo.allButtons) {
|
||||||
|
console.log(`[btn ${b.index}] <${b.tag}> text="${b.text}" disabled=${b.disabled} aria="${b.ariaLabel}" class="${b.className}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\n=== next-next 元素 ===');
|
||||||
|
for (const n of paginationInfo.nextItems) {
|
||||||
|
console.log(`[next ${n.index}] <${n.tag}> text="${n.text}" disabled=${n.disabled} class="${n.className}" parent="${n.parentClass}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 再看看当前页码
|
||||||
|
const currentPage = await page.evaluate(() => {
|
||||||
|
const active = document.querySelector('.next-pagination-item.current, .next-pagination-list .current');
|
||||||
|
return active ? { text: active.innerText, class: active.className } : null;
|
||||||
|
});
|
||||||
|
console.log('\n当前页码:', currentPage);
|
||||||
|
|
||||||
|
// 看看总共多少条记录的提示
|
||||||
|
const totalInfo = await page.evaluate(() => {
|
||||||
|
const el = document.querySelector('[class*="pagination"]');
|
||||||
|
return el ? el.innerText : '(无)';
|
||||||
|
});
|
||||||
|
console.log('分页文本:', totalInfo);
|
||||||
|
|
||||||
|
await context.close();
|
||||||
|
process.exit(0);
|
||||||
91
aliyun-sync/aliyun-aps-sync/src/debug-panel.js
Normal file
91
aliyun-sync/aliyun-aps-sync/src/debug-panel.js
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
import { chromium } from 'playwright';
|
||||||
|
import { config, datasets } from './config.js';
|
||||||
|
|
||||||
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||||
|
|
||||||
|
console.log('打开订单查询页面并探测 RangePicker 面板...');
|
||||||
|
|
||||||
|
const context = await chromium.launchPersistentContext(config.userDataDir, {
|
||||||
|
channel: 'chrome',
|
||||||
|
headless: false,
|
||||||
|
acceptDownloads: true,
|
||||||
|
downloadsPath: config.downloadDir,
|
||||||
|
});
|
||||||
|
|
||||||
|
const page = context.pages()[0] || (await context.newPage());
|
||||||
|
await page.goto(datasets.orders.url, { waitUntil: 'domcontentloaded' });
|
||||||
|
|
||||||
|
await page.waitForFunction(
|
||||||
|
(text) => document.body && document.body.innerText.includes(text),
|
||||||
|
datasets.orders.heading,
|
||||||
|
{ timeout: 60000 },
|
||||||
|
);
|
||||||
|
await sleep(3000);
|
||||||
|
|
||||||
|
// 点击起始日期 trigger 打开面板
|
||||||
|
console.log('--- 点击起始日期触发器 ---');
|
||||||
|
const startTrigger = page.locator('input[placeholder="起始日期"]');
|
||||||
|
await startTrigger.click();
|
||||||
|
await sleep(2000);
|
||||||
|
|
||||||
|
// 探测弹出面板里的 input
|
||||||
|
const panelInfo = await page.evaluate(() => {
|
||||||
|
// 查找面板（弹出层里的日期输入）
|
||||||
|
const allInputs = document.querySelectorAll('input');
|
||||||
|
const results = [];
|
||||||
|
for (let i = 0; i < allInputs.length; i++) {
|
||||||
|
const el = allInputs[i];
|
||||||
|
// 找 panel 里的 input（不是 trigger 里的）
|
||||||
|
const inPanel = el.closest('.next-range-picker-panel, .next-date-picker-panel, .next-range-picker-body, [class*="overlay"], [class*="popup"]');
|
||||||
|
results.push({
|
||||||
|
index: i,
|
||||||
|
value: el.value,
|
||||||
|
placeholder: el.placeholder,
|
||||||
|
readOnly: el.readOnly,
|
||||||
|
parentClass: el.parentElement?.className?.substring(0, 150) || '',
|
||||||
|
inPanel: !!inPanel,
|
||||||
|
panelClass: inPanel?.className?.substring(0, 150) || '(不在面板内)',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('\n=== 面板打开后所有 input ===');
|
||||||
|
for (const info of panelInfo) {
|
||||||
|
const marker = info.inPanel ? '📌 面板内' : ' 普通';
|
||||||
|
console.log(`${marker} [input ${info.index}] value="${info.value}" placeholder="${info.placeholder}" readOnly=${info.readOnly}`);
|
||||||
|
if (info.inPanel) {
|
||||||
|
console.log(` parentClass: ${info.parentClass}`);
|
||||||
|
console.log(` panelClass: ${info.panelClass}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 再看看有没有 panel 级别的面板结构
|
||||||
|
const panelStructure = await page.evaluate(() => {
|
||||||
|
const overlays = document.querySelectorAll('.next-overlay-wrapper, .next-overlay-inner, [class*="range-picker-panel"], [class*="range-picker-body"]');
|
||||||
|
return Array.from(overlays).map((el, i) => ({
|
||||||
|
index: i,
|
||||||
|
tag: el.tagName,
|
||||||
|
className: el.className.substring(0, 200),
|
||||||
|
inputCount: el.querySelectorAll('input').length,
|
||||||
|
inputs: Array.from(el.querySelectorAll('input')).map(inp => ({
|
||||||
|
value: inp.value,
|
||||||
|
placeholder: inp.placeholder,
|
||||||
|
readOnly: inp.readOnly,
|
||||||
|
parentClass: inp.parentElement?.className?.substring(0, 120) || '',
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('\n=== 弹出面板结构 ===');
|
||||||
|
for (const p of panelStructure) {
|
||||||
|
console.log(`\n[overlay ${p.index}] <${p.tag}>`);
|
||||||
|
console.log(` className: ${p.className}`);
|
||||||
|
console.log(` inputs (${p.inputCount}):`);
|
||||||
|
for (const inp of p.inputs) {
|
||||||
|
console.log(` value="${inp.value}" placeholder="${inp.placeholder}" readOnly=${inp.readOnly} parent="${inp.parentClass}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await context.close();
|
||||||
|
process.exit(0);
|
||||||
27
aliyun-sync/aliyun-aps-sync/src/index.js
Normal file
27
aliyun-sync/aliyun-aps-sync/src/index.js
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
import { login, scheduleSync, syncAll, syncBillsOnly } from './sync.js';
|
||||||
|
|
||||||
|
const command = process.argv[2] || 'sync';
|
||||||
|
|
||||||
|
if (command === 'login') {
|
||||||
|
await login();
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (command === 'sync') {
|
||||||
|
const summary = await syncAll();
|
||||||
|
console.log(JSON.stringify(summary, null, 2));
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (command === 'bills') {
|
||||||
|
const summary = await syncBillsOnly();
|
||||||
|
console.log(JSON.stringify(summary, null, 2));
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (command === 'schedule') {
|
||||||
|
await scheduleSync();
|
||||||
|
} else {
|
||||||
|
console.error(`不支持的命令: ${command}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
56
aliyun-sync/aliyun-aps-sync/src/notify.js
Normal file
56
aliyun-sync/aliyun-aps-sync/src/notify.js
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
import nodemailer from 'nodemailer';
|
||||||
|
import { config } from './config.js';
|
||||||
|
|
||||||
|
let lastSentAt = 0;
|
||||||
|
const ONE_HOUR_MS = 60 * 60 * 1000;
|
||||||
|
|
||||||
|
export async function sendLoginAlert() {
|
||||||
|
const now = Date.now();
|
||||||
|
if (now - lastSentAt < ONE_HOUR_MS) {
|
||||||
|
console.log('[通知] 登录提醒1小时内已发送过，跳过重复发送');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { smtp, notifyEmail, baseUrl } = config;
|
||||||
|
|
||||||
|
if (!smtp.user || !smtp.pass) {
|
||||||
|
console.warn('[通知] SMTP 用户名或密码为空，跳过发送登录提醒邮件');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!notifyEmail) {
|
||||||
|
console.warn('[通知] 未配置 ALIYUN_APS_NOTIFY_EMAIL，跳过发送登录提醒邮件');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const transporter = nodemailer.createTransport({
|
||||||
|
host: smtp.host,
|
||||||
|
port: smtp.port,
|
||||||
|
secure: smtp.secure,
|
||||||
|
auth: {
|
||||||
|
user: smtp.user,
|
||||||
|
pass: smtp.pass,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const url = `${baseUrl}/#/signin`;
|
||||||
|
const timestamp = new Date().toISOString();
|
||||||
|
const subject = '[APS同步] 登录态已过期，请手动登录';
|
||||||
|
const text = [
|
||||||
|
'检测到 APS 页面处于登录页，登录态可能已过期。',
|
||||||
|
`时间: ${timestamp}`,
|
||||||
|
`地址: ${url}`,
|
||||||
|
'',
|
||||||
|
'请尽快手动执行登录流程（npm run login）以恢复同步。',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
await transporter.sendMail({
|
||||||
|
from: smtp.user,
|
||||||
|
to: notifyEmail,
|
||||||
|
subject,
|
||||||
|
text,
|
||||||
|
});
|
||||||
|
|
||||||
|
lastSentAt = now;
|
||||||
|
console.log(`[通知] 登录提醒邮件已发送至 ${notifyEmail}`);
|
||||||
|
}
|
||||||
25
aliyun-sync/aliyun-aps-sync/src/open-page.js
Normal file
25
aliyun-sync/aliyun-aps-sync/src/open-page.js
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
import { chromium } from 'playwright';
|
||||||
|
import { config, datasets } from './config.js';
|
||||||
|
|
||||||
|
const target = process.argv[2] || 'orders'; // orders | bills | customers
|
||||||
|
const dataset = datasets[target];
|
||||||
|
if (!dataset) {
|
||||||
|
console.error(`未知页面: ${target}，可选: orders, bills, customers`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`正在打开: ${dataset.heading} (${dataset.url})`);
|
||||||
|
|
||||||
|
const context = await chromium.launchPersistentContext(config.userDataDir, {
|
||||||
|
channel: 'chrome',
|
||||||
|
headless: false,
|
||||||
|
acceptDownloads: true,
|
||||||
|
downloadsPath: config.downloadDir,
|
||||||
|
});
|
||||||
|
|
||||||
|
const page = context.pages()[0] || (await context.newPage());
|
||||||
|
await page.goto(dataset.url, { waitUntil: 'domcontentloaded' });
|
||||||
|
console.log('页面已打开，浏览器保持运行。按 Ctrl+C 关闭。');
|
||||||
|
|
||||||
|
// 保持进程运行
|
||||||
|
await new Promise(() => {});
|
||||||
80
aliyun-sync/aliyun-aps-sync/src/storage.js
Normal file
80
aliyun-sync/aliyun-aps-sync/src/storage.js
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
import crypto from 'node:crypto';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { config } from './config.js';
|
||||||
|
|
||||||
|
const ensureDir = (dirPath) => {
|
||||||
|
fs.mkdirSync(dirPath, { recursive: true });
|
||||||
|
return dirPath;
|
||||||
|
};
|
||||||
|
|
||||||
|
const readJson = (filePath, fallback) => {
|
||||||
|
if (!fs.existsSync(filePath)) return fallback;
|
||||||
|
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
||||||
|
};
|
||||||
|
|
||||||
|
const writeJson = (filePath, value) => {
|
||||||
|
ensureDir(path.dirname(filePath));
|
||||||
|
fs.writeFileSync(filePath, JSON.stringify(value, null, 2));
|
||||||
|
};
|
||||||
|
|
||||||
|
export const nowStamp = () => new Date().toISOString().replace(/[:.]/g, '-');
|
||||||
|
|
||||||
|
export const withHash = (record) => ({
|
||||||
|
...record,
|
||||||
|
__hash: crypto.createHash('sha256').update(JSON.stringify(record)).digest('hex'),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const loadCurrentState = (dataset) =>
|
||||||
|
readJson(path.join(config.dataDir, 'current', `${dataset}.json`), { records: [], index: {} });
|
||||||
|
|
||||||
|
export function saveDatasetRun(dataset, payload) {
|
||||||
|
const stamp = nowStamp();
|
||||||
|
writeJson(path.join(config.dataDir, 'history', dataset, `${stamp}.json`), payload);
|
||||||
|
writeJson(path.join(config.dataDir, 'current', `${dataset}.json`), payload);
|
||||||
|
return stamp;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function saveDelta(dataset, stamp, delta) {
|
||||||
|
writeJson(path.join(config.dataDir, 'delta', dataset, `${stamp}.json`), delta);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function saveRunSummary(stamp, summary) {
|
||||||
|
writeJson(path.join(config.dataDir, 'runs', `${stamp}.json`), summary);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function diffRecords(previousState, records, uniqueKey) {
|
||||||
|
const previousIndex = previousState.index || {};
|
||||||
|
const nextIndex = {};
|
||||||
|
const added = [];
|
||||||
|
const updated = [];
|
||||||
|
|
||||||
|
for (const record of records) {
|
||||||
|
const key = uniqueKey(record);
|
||||||
|
if (!key) continue;
|
||||||
|
nextIndex[key] = record;
|
||||||
|
if (!previousIndex[key]) {
|
||||||
|
added.push(record);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (previousIndex[key].__hash !== record.__hash) {
|
||||||
|
updated.push({ before: previousIndex[key], after: record });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const removed = Object.keys(previousIndex)
|
||||||
|
.filter((key) => !nextIndex[key])
|
||||||
|
.map((key) => previousIndex[key]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
records,
|
||||||
|
index: nextIndex,
|
||||||
|
stats: {
|
||||||
|
total: records.length,
|
||||||
|
added: added.length,
|
||||||
|
updated: updated.length,
|
||||||
|
removed: removed.length,
|
||||||
|
},
|
||||||
|
delta: { added, updated, removed },
|
||||||
|
};
|
||||||
|
}
|
||||||
824
aliyun-sync/aliyun-aps-sync/src/sync.js
Normal file
824
aliyun-sync/aliyun-aps-sync/src/sync.js
Normal file
@@ -0,0 +1,824 @@
|
|||||||
|
import { chromium } from 'playwright';
|
||||||
|
import cron from 'node-cron';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { execSync } from 'node:child_process';
|
||||||
|
import { config, datasets } from './config.js';
|
||||||
|
import { sendLoginAlert } from './notify.js';
|
||||||
|
import {
|
||||||
|
diffRecords,
|
||||||
|
loadCurrentState,
|
||||||
|
nowStamp,
|
||||||
|
saveDatasetRun,
|
||||||
|
saveDelta,
|
||||||
|
saveRunSummary,
|
||||||
|
withHash,
|
||||||
|
} from './storage.js';
|
||||||
|
|
||||||
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
|
||||||
|
let _context = null;
|
||||||
|
|
||||||
|
async function getContext() {
|
||||||
|
if (_context) return _context;
|
||||||
|
_context = await chromium.launchPersistentContext(config.userDataDir, {
|
||||||
|
channel: 'chrome',
|
||||||
|
headless: config.headless,
|
||||||
|
acceptDownloads: true,
|
||||||
|
downloadsPath: config.downloadDir,
|
||||||
|
});
|
||||||
|
return _context;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function login() {
|
||||||
|
const context = await getContext();
|
||||||
|
|
||||||
|
const page = context.pages()[0] || (await context.newPage());
|
||||||
|
await page.goto(datasets.customers.url, { waitUntil: 'domcontentloaded' });
|
||||||
|
console.log('请在打开的浏览器里完成阿里云伙伴中心登录，然后回到终端按 Ctrl+C 结束。');
|
||||||
|
await waitUntilReady(page, datasets.customers.heading, 10 * 60 * 1000);
|
||||||
|
console.log('登录态已写入 .browser 目录，后续可直接执行 npm run sync。');
|
||||||
|
|
||||||
|
// 必须正常关闭 context，否则登录态不会持久化到磁盘
|
||||||
|
await context.close();
|
||||||
|
_context = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function syncAll() {
|
||||||
|
const context = await getContext();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const summary = { startedAt: new Date().toISOString(), datasets: {} };
|
||||||
|
const page = context.pages()[0] || (await context.newPage());
|
||||||
|
|
||||||
|
summary.datasets.customers = await syncCustomers(page);
|
||||||
|
summary.datasets.customerDetails = await syncCustomerDetails(page);
|
||||||
|
summary.datasets.orders = await syncOrders(page);
|
||||||
|
|
||||||
|
// syncOrders 完成后，从最新的 orders.json 读取 orderId 列表
|
||||||
|
const latestOrders = loadCurrentState('orders');
|
||||||
|
const orderIdsForDetail = collectValidOrderIds(latestOrders.records || []);
|
||||||
|
|
||||||
|
summary.datasets.orderDetails = await syncOrderDetails(page, orderIdsForDetail);
|
||||||
|
summary.datasets.bills = await syncBills(page);
|
||||||
|
summary.finishedAt = new Date().toISOString();
|
||||||
|
|
||||||
|
const stamp = nowStamp();
|
||||||
|
saveRunSummary(stamp, summary);
|
||||||
|
return summary;
|
||||||
|
} finally {
|
||||||
|
if (config.closeBrowser) {
|
||||||
|
await context.close();
|
||||||
|
_context = null;
|
||||||
|
} else {
|
||||||
|
console.log('浏览器保持运行');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function scheduleSync() {
|
||||||
|
console.log(`定时任务已启动: ${config.cron} (${config.timezone})`);
|
||||||
|
cron.schedule(
|
||||||
|
config.cron,
|
||||||
|
async () => {
|
||||||
|
try {
|
||||||
|
console.log(`[${new Date().toISOString()}] 开始执行同步`);
|
||||||
|
const summary = await syncAll();
|
||||||
|
console.log(`[${new Date().toISOString()}] 同步完成`, JSON.stringify(summary, null, 2));
|
||||||
|
try {
|
||||||
|
const scriptPath = path.resolve(config.rootDir, config.dbSyncScript);
|
||||||
|
const incrementalFlag = config.fullSync ? '' : ' --incremental';
|
||||||
|
console.log(`[入库] 执行 ${scriptPath}${incrementalFlag ? ' (增量模式)' : ''}`);
|
||||||
|
const output = execSync(`python "${scriptPath}"${incrementalFlag}`, {
|
||||||
|
cwd: path.dirname(scriptPath),
|
||||||
|
encoding: 'utf-8',
|
||||||
|
timeout: 120000,
|
||||||
|
});
|
||||||
|
console.log(output);
|
||||||
|
} catch (e) {
|
||||||
|
console.error('[入库] 失败:', e.message);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`[${new Date().toISOString()}] 同步失败`, error);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{ timezone: config.timezone },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncCustomers(page) {
|
||||||
|
const dataset = datasets.customers;
|
||||||
|
await page.goto(dataset.url, { waitUntil: 'domcontentloaded' });
|
||||||
|
await waitUntilReady(page, dataset.heading);
|
||||||
|
await trySetPageSize(page, dataset.pageSize);
|
||||||
|
const records = await scrapePagedTable(page, dataset, {});
|
||||||
|
return persistDataset(dataset, records, {});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncCustomerDetails(page) {
|
||||||
|
const dataset = datasets.customerDetails;
|
||||||
|
const customersState = loadCurrentState('customers');
|
||||||
|
const allAccountIds = collectValidAccountIds(customersState.records || []);
|
||||||
|
|
||||||
|
if (allAccountIds.length === 0) {
|
||||||
|
console.log('[客户详情] 本地无有效客户 accountId，跳过');
|
||||||
|
return persistDataset(dataset, [], {});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[客户详情] 共 ${allAccountIds.length} 个客户需要获取详情`);
|
||||||
|
const allDetails = [];
|
||||||
|
const detailBaseUrl =
|
||||||
|
'https://aps.aliyun.com/?spm=5176.12818093.top-nav.ditem-fx.785716d0LKDpKT#/detail/my_customer/~/customer/';
|
||||||
|
|
||||||
|
for (let index = 0; index < allAccountIds.length; index += 1) {
|
||||||
|
const accountId = allAccountIds[index];
|
||||||
|
console.log(`[客户详情] ${index + 1}/${allAccountIds.length} accountId=${accountId}`);
|
||||||
|
|
||||||
|
// 先跳 about:blank 再跳详情URL（强制 SPA 完整重新加载）
|
||||||
|
await page.goto('about:blank');
|
||||||
|
await sleep(300);
|
||||||
|
await page.goto(`${detailBaseUrl}${accountId}`, { waitUntil: 'domcontentloaded' });
|
||||||
|
|
||||||
|
try {
|
||||||
|
await page.waitForFunction(
|
||||||
|
(text) => document.body && document.body.innerText.includes(text),
|
||||||
|
'详情',
|
||||||
|
{ timeout: 15000 },
|
||||||
|
);
|
||||||
|
await sleep(1000);
|
||||||
|
} catch {
|
||||||
|
console.warn(`[客户详情] ${accountId} 详情页加载超时，跳过`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const detail = await extractCustomerDetail(page);
|
||||||
|
allDetails.push({ ...detail, __context: { accountId } });
|
||||||
|
}
|
||||||
|
|
||||||
|
return persistDataset(dataset, dedupeByHash(allDetails), {});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncOrders(page) {
|
||||||
|
const dataset = datasets.orders;
|
||||||
|
let windows;
|
||||||
|
|
||||||
|
if (config.fullSync) {
|
||||||
|
windows = buildMonthlyDateWindows(config.orderStartDate);
|
||||||
|
} else {
|
||||||
|
// 增量模式：只查前一天
|
||||||
|
const yesterday = new Date();
|
||||||
|
yesterday.setDate(yesterday.getDate() - 1);
|
||||||
|
const dateStr = formatDate(yesterday);
|
||||||
|
windows = [{ windowStart: dateStr, windowEnd: dateStr, start: dateStr, end: dateStr }];
|
||||||
|
console.log(`[增量模式] 订单仅查询: ${dateStr}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const allRecords = [];
|
||||||
|
|
||||||
|
for (const window of windows) {
|
||||||
|
await page.goto(dataset.url, { waitUntil: 'domcontentloaded' });
|
||||||
|
await waitUntilReady(page, dataset.heading);
|
||||||
|
await setDateRange(page, window.start, window.end);
|
||||||
|
await clickQuery(page);
|
||||||
|
await trySetPageSize(page, dataset.pageSize);
|
||||||
|
const records = await scrapePagedTable(page, dataset, window);
|
||||||
|
allRecords.push(...records);
|
||||||
|
}
|
||||||
|
|
||||||
|
return persistDataset(dataset, dedupeByHash(allRecords), {});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncBills(page) {
|
||||||
|
const dataset = datasets.bills;
|
||||||
|
let months;
|
||||||
|
let latestConsumptionDate = null;
|
||||||
|
|
||||||
|
if (config.fullSync) {
|
||||||
|
months = buildMonthList(config.billStartMonth);
|
||||||
|
} else {
|
||||||
|
latestConsumptionDate = getLatestBillConsumptionDate();
|
||||||
|
const incrementalMonth = latestConsumptionDate?.slice(0, 7)
|
||||||
|
|| `${new Date().getFullYear()}-${String(new Date().getMonth() + 1).padStart(2, '0')}`;
|
||||||
|
months = [incrementalMonth];
|
||||||
|
console.log(`[增量模式] 账单仅查询: ${incrementalMonth}${latestConsumptionDate ? `, 数据库最新消费时间: ${latestConsumptionDate}` : ''}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const allRecords = [];
|
||||||
|
|
||||||
|
for (const month of months) {
|
||||||
|
await page.goto(dataset.url, { waitUntil: 'domcontentloaded' });
|
||||||
|
await waitUntilReady(page, dataset.heading);
|
||||||
|
await setMonthValue(page, month);
|
||||||
|
await clickQuery(page);
|
||||||
|
await trySetPageSize(page, dataset.pageSize);
|
||||||
|
let records = await scrapePagedTable(page, dataset, { month });
|
||||||
|
if (latestConsumptionDate) {
|
||||||
|
const before = records.length;
|
||||||
|
records = records.filter((record) => isAfterLatestConsumptionDate(record, latestConsumptionDate));
|
||||||
|
console.log(`[增量模式] 账单按消费时间过滤: ${before} -> ${records.length}`);
|
||||||
|
}
|
||||||
|
allRecords.push(...records);
|
||||||
|
}
|
||||||
|
|
||||||
|
return persistDataset(dataset, dedupeByHash(allRecords), {});
|
||||||
|
}
|
||||||
|
|
||||||
|
function getLatestBillConsumptionDate() {
|
||||||
|
const scriptPath = path.resolve(config.rootDir, config.dbSyncScript);
|
||||||
|
try {
|
||||||
|
const output = execSync(`python "${scriptPath}" --latest-bill-consumption-time`, {
|
||||||
|
cwd: path.dirname(scriptPath),
|
||||||
|
encoding: 'utf-8',
|
||||||
|
timeout: 120000,
|
||||||
|
}).trim();
|
||||||
|
const latest = output.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).at(-1) || '';
|
||||||
|
return /^\d{4}-\d{2}-\d{2}/.test(latest) ? latest.slice(0, 10) : null;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[增量模式] 查询数据库最新账单消费时间失败:', error.message);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAfterLatestConsumptionDate(record, latestConsumptionDate) {
|
||||||
|
const consumeDate = String(record['消费时间'] || record.consumeDate || '').trim().slice(0, 10);
|
||||||
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(consumeDate)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return consumeDate > latestConsumptionDate;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncOrderDetails(page, cachedOrderIds) {
|
||||||
|
const dataset = datasets.orderDetails;
|
||||||
|
|
||||||
|
// 使用传入的 orderId 列表（在 syncOrders 覆盖 orders.json 之前缓存的）
|
||||||
|
const allOrderIds = cachedOrderIds || [];
|
||||||
|
|
||||||
|
if (allOrderIds.length === 0) {
|
||||||
|
console.log('[订单详情] 本地无订单数据，跳过');
|
||||||
|
return persistDataset(dataset, [], {});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[订单详情] 共 ${allOrderIds.length} 个订单需要获取详情`);
|
||||||
|
const allDetails = [];
|
||||||
|
const detailBaseUrl = 'https://aps.aliyun.com/?spm=5176.12818093.top-nav.ditem-fx.785716d0LKDpKT#/detail/order/~/costCenter/order/detail/';
|
||||||
|
|
||||||
|
for (let index = 0; index < allOrderIds.length; index += 1) {
|
||||||
|
const orderId = allOrderIds[index];
|
||||||
|
console.log(`[订单详情] ${index + 1}/${allOrderIds.length} orderId=${orderId}`);
|
||||||
|
|
||||||
|
// 先跳 about:blank 再跳详情URL（强制 SPA 完整重新加载）
|
||||||
|
await page.goto('about:blank');
|
||||||
|
await sleep(300);
|
||||||
|
await page.goto(`${detailBaseUrl}${orderId}?projectId=`, { waitUntil: 'domcontentloaded' });
|
||||||
|
|
||||||
|
try {
|
||||||
|
await page.waitForFunction(
|
||||||
|
(text) => document.body && document.body.innerText.includes(text),
|
||||||
|
'订单详情',
|
||||||
|
{ timeout: 15000 },
|
||||||
|
);
|
||||||
|
await sleep(1000);
|
||||||
|
} catch {
|
||||||
|
console.warn(`[订单详情] ${orderId} 详情页加载超时，跳过`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const detail = await extractOrderDetail(page);
|
||||||
|
if (!isValidOrderId(detail.orderId)) {
|
||||||
|
detail.orderId = orderId;
|
||||||
|
}
|
||||||
|
allDetails.push({ ...detail, __context: {} });
|
||||||
|
}
|
||||||
|
|
||||||
|
return persistDataset(dataset, dedupeByHash(allDetails), {});
|
||||||
|
}
|
||||||
|
|
||||||
|
function persistDataset(dataset, records, context) {
|
||||||
|
const normalized = records.map((record) => withHash(dataset.normalize(record, record.__context || context)));
|
||||||
|
const previousState = loadCurrentState(dataset.name);
|
||||||
|
const nextState = diffRecords(previousState, normalized, dataset.uniqueKey);
|
||||||
|
const stamp = saveDatasetRun(dataset.name, nextState);
|
||||||
|
saveDelta(dataset.name, stamp, nextState.delta);
|
||||||
|
return {
|
||||||
|
stamp,
|
||||||
|
stats: nextState.stats,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitUntilReady(page, heading, timeout = 120000) {
|
||||||
|
await page.waitForLoadState('domcontentloaded');
|
||||||
|
console.log(`[waitUntilReady] 当前URL: ${page.url()}`);
|
||||||
|
console.log(`[waitUntilReady] 等待页面出现: "${heading}"`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await page.waitForFunction(
|
||||||
|
(text) => document.body && document.body.innerText.includes(text),
|
||||||
|
heading,
|
||||||
|
{ timeout },
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
// 超时时打印诊断信息
|
||||||
|
const currentUrl = page.url();
|
||||||
|
const bodyText = await page.evaluate(() => document.body?.innerText?.substring(0, 500) || '(空)').catch(() => '(无法获取)');
|
||||||
|
console.error(`[waitUntilReady] 超时！当前URL: ${currentUrl}`);
|
||||||
|
console.error(`[waitUntilReady] 页面内容前500字: ${bodyText}`);
|
||||||
|
if (currentUrl.includes('signin')) {
|
||||||
|
try {
|
||||||
|
await sendLoginAlert();
|
||||||
|
} catch (notifyErr) {
|
||||||
|
console.error('[通知] 发送登录提醒失败:', notifyErr.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((await page.locator('text=登录').count()) > 0 && page.url().includes('login')) {
|
||||||
|
throw new Error('当前未登录，请先执行 npm run login');
|
||||||
|
}
|
||||||
|
await sleep(1500);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function scrapePagedTable(page, dataset, context) {
|
||||||
|
const pages = [];
|
||||||
|
const visited = new Set();
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
await waitForTableRows(page);
|
||||||
|
const pageData = await extractTable(page);
|
||||||
|
const pageNum = await currentPageNumber(page);
|
||||||
|
const pageKey = `${pageNum}-${pageData.rows.length}`;
|
||||||
|
console.log(`[抓取] 第${pageNum}页, ${pageData.rows.length}行, key="${pageKey}"`);
|
||||||
|
if (visited.has(pageKey)) {
|
||||||
|
console.log(`[抓取] 重复页面key，停止翻页`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
visited.add(pageKey);
|
||||||
|
pages.push(...pageData.rows.map((row) => ({ ...row, __context: context })));
|
||||||
|
|
||||||
|
const moved = await gotoNextPage(page);
|
||||||
|
if (!moved) {
|
||||||
|
console.log(`[抓取] 翻页失败或已到最后一页，停止`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[抓取] 共采集 ${pages.length} 条记录`);
|
||||||
|
return pages;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function extractTable(page) {
|
||||||
|
return page.evaluate(() => {
|
||||||
|
const normalize = (value) =>
|
||||||
|
String(value || '')
|
||||||
|
.replace(/\u00a0/g, ' ')
|
||||||
|
.replace(/\s+\n/g, '\n')
|
||||||
|
.replace(/\n\s+/g, '\n')
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
const headerTables = Array.from(document.querySelectorAll('table')).filter((table) => table.querySelectorAll('thead th').length > 1);
|
||||||
|
const headerTable = headerTables.sort((a, b) => b.querySelectorAll('thead th').length - a.querySelectorAll('thead th').length)[0];
|
||||||
|
if (!headerTable) return { headers: [], rows: [] };
|
||||||
|
|
||||||
|
const headers = Array.from(headerTable.querySelectorAll('thead th')).map((cell) => normalize(cell.textContent));
|
||||||
|
const bodyTables = Array.from(document.querySelectorAll('table')).filter((table) => table.querySelectorAll('tbody tr').length > 0);
|
||||||
|
const bodyTable = bodyTables.sort((a, b) => {
|
||||||
|
const aSize = Math.max(...Array.from(a.querySelectorAll('tbody tr')).map((row) => row.querySelectorAll('td').length), 0);
|
||||||
|
const bSize = Math.max(...Array.from(b.querySelectorAll('tbody tr')).map((row) => row.querySelectorAll('td').length), 0);
|
||||||
|
return bSize - aSize;
|
||||||
|
})[0];
|
||||||
|
if (!bodyTable) return { headers, rows: [] };
|
||||||
|
|
||||||
|
const rows = Array.from(bodyTable.querySelectorAll('tbody tr'))
|
||||||
|
.map((row) => Array.from(row.querySelectorAll('td')).map((cell) => normalize(cell.innerText || cell.textContent)))
|
||||||
|
.filter((cells) => cells.some(Boolean))
|
||||||
|
.map((cells) => {
|
||||||
|
const record = {};
|
||||||
|
headers.forEach((header, index) => {
|
||||||
|
record[header || `column_${index + 1}`] = cells[index] || '';
|
||||||
|
});
|
||||||
|
return record;
|
||||||
|
});
|
||||||
|
|
||||||
|
return { headers, rows };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForTableRows(page) {
|
||||||
|
await page.waitForFunction(() => document.querySelectorAll('table tbody tr').length > 0, null, { timeout: 120000 });
|
||||||
|
await sleep(800);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function currentPageNumber(page) {
|
||||||
|
const active = page.locator('.next-pagination-item.next-current');
|
||||||
|
if ((await active.count()) === 0) return 1;
|
||||||
|
return Number.parseInt((await active.first().innerText()).trim(), 10) || 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function gotoNextPage(page) {
|
||||||
|
const before = await currentPageNumber(page);
|
||||||
|
|
||||||
|
// 用 Playwright locator 定位"下一页"按钮
|
||||||
|
const nextBtn = page.locator('button.next-pagination-item.next-next');
|
||||||
|
if ((await nextBtn.count()) === 0) {
|
||||||
|
console.log('[翻页] 未找到下一页按钮');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const disabled = (await nextBtn.getAttribute('disabled')) != null;
|
||||||
|
if (disabled) {
|
||||||
|
console.log('[翻页] 下一页按钮已禁用');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 用 Playwright click（而非 DOM click），确保 React 事件正常触发
|
||||||
|
await nextBtn.click();
|
||||||
|
await sleep(2000);
|
||||||
|
|
||||||
|
const after = await currentPageNumber(page);
|
||||||
|
console.log(`[翻页] ${before} -> ${after}`);
|
||||||
|
return before !== after;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function trySetPageSize(page, pageSize) {
|
||||||
|
const input = page.locator('input[aria-label="请选择每页显示几条"]').first();
|
||||||
|
if ((await input.count()) === 0) return;
|
||||||
|
await input.click().catch(() => null);
|
||||||
|
await sleep(300);
|
||||||
|
const option = page.locator(`text=${pageSize}`).last();
|
||||||
|
if ((await option.count()) === 0) {
|
||||||
|
await page.keyboard.press('Escape').catch(() => null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await option.click().catch(() => null);
|
||||||
|
await sleep(1200);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setDateRange(page, start, end) {
|
||||||
|
console.log(`[订单日期] 设置: ${start} ~ ${end}`);
|
||||||
|
|
||||||
|
await _fillDateRange(page, start, end);
|
||||||
|
|
||||||
|
// 验证
|
||||||
|
const startActual = await page.locator('input[placeholder="起始日期"]').inputValue().catch(() => '');
|
||||||
|
const endActual = await page.locator('input[placeholder="结束日期"]').inputValue().catch(() => '');
|
||||||
|
|
||||||
|
// 如果结果不对，用反向顺序重试（先填开始再填结束）
|
||||||
|
if (startActual !== start || endActual !== end) {
|
||||||
|
console.log(`[订单日期] 首次结果不对: "${startActual}" ~ "${endActual}"，反向重试`);
|
||||||
|
await _fillDateRange(page, start, end, true);
|
||||||
|
const s2 = await page.locator('input[placeholder="起始日期"]').inputValue().catch(() => '');
|
||||||
|
const e2 = await page.locator('input[placeholder="结束日期"]').inputValue().catch(() => '');
|
||||||
|
console.log(`[订单日期] 重试结果: "${s2}" ~ "${e2}"`);
|
||||||
|
} else {
|
||||||
|
console.log(`[订单日期] 结果: "${startActual}" ~ "${endActual}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function _fillDateRange(page, start, end, startFirst = false) {
|
||||||
|
const trigger = page.locator('input[placeholder="结束日期"]');
|
||||||
|
await trigger.click();
|
||||||
|
await sleep(1000);
|
||||||
|
|
||||||
|
const panelStartInput = page.locator('.next-range-picker-panel-input-start-date input');
|
||||||
|
const panelEndInput = page.locator('.next-range-picker-panel-input-end-date input');
|
||||||
|
|
||||||
|
if (startFirst) {
|
||||||
|
// 先填开始日期
|
||||||
|
await panelStartInput.click();
|
||||||
|
await sleep(100);
|
||||||
|
await page.keyboard.press('Control+A');
|
||||||
|
await page.keyboard.type(start, { delay: 30 });
|
||||||
|
await sleep(300);
|
||||||
|
// 再填结束日期
|
||||||
|
await panelEndInput.click();
|
||||||
|
await sleep(100);
|
||||||
|
await page.keyboard.press('Control+A');
|
||||||
|
await page.keyboard.type(end, { delay: 30 });
|
||||||
|
await sleep(300);
|
||||||
|
} else {
|
||||||
|
// 先填结束日期（默认）
|
||||||
|
await panelEndInput.click();
|
||||||
|
await sleep(100);
|
||||||
|
await page.keyboard.press('Control+A');
|
||||||
|
await page.keyboard.type(end, { delay: 30 });
|
||||||
|
await sleep(300);
|
||||||
|
// 再填开始日期
|
||||||
|
await panelStartInput.click();
|
||||||
|
await sleep(100);
|
||||||
|
await page.keyboard.press('Control+A');
|
||||||
|
await page.keyboard.type(start, { delay: 30 });
|
||||||
|
await sleep(300);
|
||||||
|
}
|
||||||
|
|
||||||
|
await page.keyboard.press('Enter');
|
||||||
|
await sleep(500);
|
||||||
|
await page.mouse.click(0, 0);
|
||||||
|
await sleep(300);
|
||||||
|
await page.keyboard.press('Escape');
|
||||||
|
await sleep(300);
|
||||||
|
await page.locator('.next-overlay-wrapper.opened').waitFor({ state: 'hidden', timeout: 3000 }).catch(() => null);
|
||||||
|
await sleep(300);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setMonthValue(page, month) {
|
||||||
|
// 先尝试按 inputValue 匹配 YYYY-MM 格式
|
||||||
|
const inputs = page.locator('input');
|
||||||
|
const total = await inputs.count();
|
||||||
|
const allValues = [];
|
||||||
|
|
||||||
|
for (let index = 0; index < total; index += 1) {
|
||||||
|
const input = inputs.nth(index);
|
||||||
|
const value = await input.inputValue().catch(() => '');
|
||||||
|
const placeholder = await input.getAttribute('placeholder').catch(() => '');
|
||||||
|
allValues.push({ index, value, placeholder });
|
||||||
|
|
||||||
|
if (/^\d{4}-\d{2}$/.test(value)) {
|
||||||
|
console.log(`[账单月份] 通过 value 匹配到 input[${index}], 设置: ${month}`);
|
||||||
|
await typeIntoDateInput(input, month, page);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果 value 为空，尝试按 placeholder 匹配月份选择器
|
||||||
|
for (const item of allValues) {
|
||||||
|
if (item.placeholder && /月/.test(item.placeholder)) {
|
||||||
|
console.log(`[账单月份] 通过 placeholder 匹配到 input[${item.index}], 设置: ${month}`);
|
||||||
|
await typeIntoDateInput(inputs.nth(item.index), month, page);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 兜底：找任何看起来像日期/月份选择器的 input（排除搜索框等）
|
||||||
|
for (const item of allValues) {
|
||||||
|
const input = inputs.nth(item.index);
|
||||||
|
const cls = await input.evaluate((el) => el.closest('[class*="date-picker"], [class*="month-picker"], [class*="range-picker"]')?.className || '').catch(() => '');
|
||||||
|
if (cls) {
|
||||||
|
console.log(`[账单月份] 通过父级 class 匹配到 input[${item.index}] (${cls}), 设置: ${month}`);
|
||||||
|
await typeIntoDateInput(input, month, page);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error('[DEBUG] 账单页面所有 input:', JSON.stringify(allValues, null, 2));
|
||||||
|
throw new Error('未识别到账单佣金月份输入框，请打开页面确认结构是否变化。');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 用键盘输入日期值。
|
||||||
|
* 策略：focus → 全选 → 快速键入 → Tab 移开焦点（触发 blur 提交，但不会像 click 那样打开面板）。
|
||||||
|
* 即使面板弹出，快速键入 + Tab 也能在面板滚动前完成提交并关闭。
|
||||||
|
*/
|
||||||
|
async function typeIntoDateInput(locator, value, page) {
|
||||||
|
// 移除 readonly
|
||||||
|
await locator.evaluate((node) => node.removeAttribute('readonly'));
|
||||||
|
|
||||||
|
// focus 并全选当前内容
|
||||||
|
await locator.focus();
|
||||||
|
await sleep(100);
|
||||||
|
await page.keyboard.press('Control+A');
|
||||||
|
await sleep(100);
|
||||||
|
|
||||||
|
// 快速逐字符输入新值
|
||||||
|
await page.keyboard.type(value, { delay: 30 });
|
||||||
|
await sleep(200);
|
||||||
|
|
||||||
|
// Tab 移开焦点 → 触发 onBlur 提交值 + 关闭面板
|
||||||
|
await page.keyboard.press('Tab');
|
||||||
|
await sleep(300);
|
||||||
|
|
||||||
|
// 如果面板还在，Escape 兜底关闭
|
||||||
|
await page.keyboard.press('Escape');
|
||||||
|
await sleep(300);
|
||||||
|
|
||||||
|
// 验证
|
||||||
|
const actual = await locator.inputValue().catch(() => '');
|
||||||
|
if (actual !== value) {
|
||||||
|
console.warn(`[WARN] typeIntoDateInput: 期望 "${value}"，实际 "${actual}"`);
|
||||||
|
} else {
|
||||||
|
console.log(`[日期设置] 成功: "${value}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function clickQuery(page) {
|
||||||
|
const button = page.locator('button:has-text("查询")').first();
|
||||||
|
await button.click();
|
||||||
|
await sleep(1800);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildMonthlyDateWindows(startDate) {
|
||||||
|
const start = new Date(`${startDate}T00:00:00+08:00`);
|
||||||
|
const end = new Date();
|
||||||
|
const windows = [];
|
||||||
|
const cursor = new Date(start.getFullYear(), start.getMonth(), 1);
|
||||||
|
|
||||||
|
while (cursor <= end) {
|
||||||
|
const windowStart = new Date(cursor);
|
||||||
|
const windowEnd = new Date(cursor.getFullYear(), cursor.getMonth() + 1, 0);
|
||||||
|
const actualEnd = windowEnd > end ? end : windowEnd;
|
||||||
|
windows.push({
|
||||||
|
windowStart: formatDate(windowStart),
|
||||||
|
windowEnd: formatDate(actualEnd),
|
||||||
|
start: formatDate(windowStart),
|
||||||
|
end: formatDate(actualEnd),
|
||||||
|
});
|
||||||
|
cursor.setMonth(cursor.getMonth() + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return windows;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildMonthList(startMonth) {
|
||||||
|
const [year, month] = startMonth.split('-').map(Number);
|
||||||
|
const cursor = new Date(year, month - 1, 1);
|
||||||
|
const end = new Date();
|
||||||
|
const months = [];
|
||||||
|
|
||||||
|
while (cursor <= end) {
|
||||||
|
months.push(`${cursor.getFullYear()}-${String(cursor.getMonth() + 1).padStart(2, '0')}`);
|
||||||
|
cursor.setMonth(cursor.getMonth() + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return months;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDate(date) {
|
||||||
|
return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function dedupeByHash(records) {
|
||||||
|
const seen = new Set();
|
||||||
|
return records.filter((record) => {
|
||||||
|
const key = JSON.stringify(record);
|
||||||
|
if (seen.has(key)) return false;
|
||||||
|
seen.add(key);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectValidOrderIds(records) {
|
||||||
|
const ids = [];
|
||||||
|
const seen = new Set();
|
||||||
|
for (const record of records) {
|
||||||
|
// 支持两种字段名：normalized 后的 orderId 和原始的 订单号
|
||||||
|
const rawOrderId = String(record.orderId || record['订单号'] || '').trim();
|
||||||
|
if (!rawOrderId || rawOrderId.includes('没有数据')) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!isValidOrderId(rawOrderId)) {
|
||||||
|
console.log(`[订单详情] 跳过无效订单号: ${rawOrderId}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (seen.has(rawOrderId)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seen.add(rawOrderId);
|
||||||
|
ids.push(rawOrderId);
|
||||||
|
}
|
||||||
|
return ids;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectValidAccountIds(records) {
|
||||||
|
const ids = [];
|
||||||
|
const seen = new Set();
|
||||||
|
for (const record of records) {
|
||||||
|
const rawAccountId = String(record.accountId || '').trim();
|
||||||
|
if (!rawAccountId || rawAccountId.includes('没有数据')) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!isValidAccountId(rawAccountId)) {
|
||||||
|
console.log(`[客户详情] 跳过无效 accountId: ${rawAccountId}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (seen.has(rawAccountId)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seen.add(rawAccountId);
|
||||||
|
ids.push(rawAccountId);
|
||||||
|
}
|
||||||
|
return ids;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isValidOrderId(orderId) {
|
||||||
|
const value = String(orderId || '').trim();
|
||||||
|
if (!value) return false;
|
||||||
|
if (value.includes('<27>')) return false;
|
||||||
|
return /^\d+$/.test(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isValidAccountId(accountId) {
|
||||||
|
const value = String(accountId || '').trim();
|
||||||
|
if (!value) return false;
|
||||||
|
if (value.includes('<27>')) return false;
|
||||||
|
return /^\d+$/.test(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function extractOrderDetail(page) {
|
||||||
|
return page.evaluate(() => {
|
||||||
|
const text = document.body?.innerText || '';
|
||||||
|
|
||||||
|
const extract = (label) => {
|
||||||
|
const lineBreakPattern = new RegExp(`${label}\\s*(?:\\r?\\n)+\\s*([^\\r\\n]+)`);
|
||||||
|
const lineBreakMatch = text.match(lineBreakPattern);
|
||||||
|
if (lineBreakMatch) return lineBreakMatch[1].trim();
|
||||||
|
|
||||||
|
const inlinePattern = new RegExp(`${label}\\s*[：:]?\\s*([^\\r\\n]+)`);
|
||||||
|
const inlineMatch = text.match(inlinePattern);
|
||||||
|
return inlineMatch ? inlineMatch[1].trim() : '';
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
orderId: extract('订单号'),
|
||||||
|
orderType: extract('订单类型'),
|
||||||
|
status: extract('状态'),
|
||||||
|
tradeType: extract('交易类型'),
|
||||||
|
customerCategory: extract('客户分类'),
|
||||||
|
dealerName: extract('二级经销商名称'),
|
||||||
|
dealerUid: extract('二级经销商UID'),
|
||||||
|
customerType: extract('客户类型'),
|
||||||
|
opportunityId: extract('商机ID'),
|
||||||
|
paymentTime: extract('支付时间'),
|
||||||
|
orderTime: extract('下单时间'),
|
||||||
|
productName: extract('产品名称'),
|
||||||
|
productCode: extract('产品code'),
|
||||||
|
originalPriceCny: extract('订单原价\\(CNY\\)'),
|
||||||
|
paidAmountCny: extract('实付金额\\(CNY\\)'),
|
||||||
|
discount: extract('订单折扣'),
|
||||||
|
payableAmountCny: extract('应付金额（实付\\+代金券）\\(CNY\\)'),
|
||||||
|
couponAmountCny: extract('代金券金额\\(CNY\\)'),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function extractCustomerDetail(page) {
|
||||||
|
return page.evaluate(() => {
|
||||||
|
const normalize = (value) =>
|
||||||
|
String(value || '')
|
||||||
|
.replace(/\u00a0/g, ' ')
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
const text = normalize(document.body?.innerText || '').replace(/\r/g, '');
|
||||||
|
|
||||||
|
const extract = (label, sourceText = text) => {
|
||||||
|
const lineBreakPattern = new RegExp(`${label}\\s*(?:\\n)+\\s*([^\\n]+)`);
|
||||||
|
const lineBreakMatch = sourceText.match(lineBreakPattern);
|
||||||
|
if (lineBreakMatch) return normalize(lineBreakMatch[1]);
|
||||||
|
|
||||||
|
const inlinePattern = new RegExp(`${label}\\s*[：:]?\\s*([^\\n]+)`);
|
||||||
|
const inlineMatch = sourceText.match(inlinePattern);
|
||||||
|
return inlineMatch ? normalize(inlineMatch[1]) : '';
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalizeAmount = (value) => normalize(value).replace(/[￥,]/g, '').trim();
|
||||||
|
|
||||||
|
const buildSection = (startLabel, endLabel = '') => {
|
||||||
|
const start = text.indexOf(startLabel);
|
||||||
|
if (start < 0) return '';
|
||||||
|
const end = endLabel ? text.indexOf(endLabel, start + startLabel.length) : -1;
|
||||||
|
if (end > start) return text.slice(start, end);
|
||||||
|
return text.slice(start);
|
||||||
|
};
|
||||||
|
|
||||||
|
const lastMonthSection = buildSection('上月应付总金额（CNY）', '本月应付总金额（CNY）');
|
||||||
|
const currentMonthSection = buildSection('本月应付总金额（CNY）');
|
||||||
|
|
||||||
|
const extractAmountFromSection = (sectionText, label) => normalizeAmount(extract(label, sectionText));
|
||||||
|
|
||||||
|
let department = '';
|
||||||
|
const table = Array.from(document.querySelectorAll('table')).find((node) =>
|
||||||
|
(node.innerText || '').includes('所属部门'),
|
||||||
|
);
|
||||||
|
if (table) {
|
||||||
|
const rows = table.querySelectorAll('tbody tr');
|
||||||
|
for (const row of rows) {
|
||||||
|
const cells = row.querySelectorAll('td');
|
||||||
|
if (cells.length >= 2) {
|
||||||
|
const value = normalize(cells[1]?.innerText || cells[1]?.textContent || '');
|
||||||
|
if (value) {
|
||||||
|
department = value;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
customerAccount: extract('客户账号'),
|
||||||
|
customerName: extract('客户名称'),
|
||||||
|
customerType: extract('客户类型'),
|
||||||
|
tradeMode: extract('交易模式'),
|
||||||
|
customerSource: extract('客户来源'),
|
||||||
|
realNameStatus: extract('实名状态'),
|
||||||
|
email: extract('邮箱'),
|
||||||
|
relationDate: extract('关联日期'),
|
||||||
|
phone: extract('手机号'),
|
||||||
|
remark: extract('备注'),
|
||||||
|
paymentNoticeStatus: extract('代为支付告知状态'),
|
||||||
|
department,
|
||||||
|
lastMonthPayableTotalCny: extractAmountFromSection(lastMonthSection, '上月应付总金额（CNY）'),
|
||||||
|
lastMonthPrepayCny: extractAmountFromSection(lastMonthSection, '预付费金额'),
|
||||||
|
lastMonthPostpayCny: extractAmountFromSection(lastMonthSection, '后付费金额'),
|
||||||
|
currentMonthPayableTotalCny: extractAmountFromSection(currentMonthSection, '本月应付总金额（CNY）'),
|
||||||
|
currentMonthPrepayCny: extractAmountFromSection(currentMonthSection, '预付费金额'),
|
||||||
|
currentMonthPostpayCny: extractAmountFromSection(currentMonthSection, '后付费金额'),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
1073
aliyun-sync/aps-aliyun-sync/aps_db_sync.py
Normal file
1073
aliyun-sync/aps-aliyun-sync/aps_db_sync.py
Normal file
File diff suppressed because it is too large
Load Diff
130
aliyun-sync/aps-aliyun-sync/aps_scheduler.py
Normal file
130
aliyun-sync/aps-aliyun-sync/aps_scheduler.py
Normal file
@@ -0,0 +1,130 @@
|
|||||||
|
"""
|
||||||
|
Long-running scheduler — watches for new JSON files and syncs to MySQL.
|
||||||
|
|
||||||
|
Modes:
|
||||||
|
--watch : File watcher, syncs immediately when new JSON appears (default)
|
||||||
|
--cron : One-shot sync, meant to be called by system cron/launchd
|
||||||
|
--daemon : Combined: runs initial sync + watches for changes
|
||||||
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
import sys
|
||||||
|
import signal
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from aps_db_sync import APSSyncer, DB_CONFIG, JSON_DIR
|
||||||
|
|
||||||
|
LOG_FORMAT = "%(asctime)s [%(levelname)s] %(message)s"
|
||||||
|
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
|
||||||
|
logger = logging.getLogger("aps_scheduler")
|
||||||
|
|
||||||
|
WATCH_INTERVAL_SECONDS = 30
|
||||||
|
PROCESSED_MARKER_DIR = JSON_DIR / ".aps_sync_processed"
|
||||||
|
|
||||||
|
|
||||||
|
def _update_watch_interval(value: int):
|
||||||
|
global WATCH_INTERVAL_SECONDS
|
||||||
|
WATCH_INTERVAL_SECONDS = value
|
||||||
|
|
||||||
|
|
||||||
|
class SyncScheduler:
|
||||||
|
def __init__(self, db_config: dict = None):
|
||||||
|
self.db_config = db_config or DB_CONFIG
|
||||||
|
self.running = True
|
||||||
|
PROCESSED_MARKER_DIR.mkdir(exist_ok=True)
|
||||||
|
signal.signal(signal.SIGINT, self._shutdown)
|
||||||
|
signal.signal(signal.SIGTERM, self._shutdown)
|
||||||
|
|
||||||
|
def _shutdown(self, signum, frame):
|
||||||
|
logger.info("Shutdown signal received, stopping...")
|
||||||
|
self.running = False
|
||||||
|
|
||||||
|
def _marker_path(self, json_path: Path) -> Path:
|
||||||
|
return PROCESSED_MARKER_DIR / f"{json_path.stem}.synced"
|
||||||
|
|
||||||
|
def _is_processed(self, json_path: Path) -> bool:
|
||||||
|
marker = self._marker_path(json_path)
|
||||||
|
if not marker.exists():
|
||||||
|
return False
|
||||||
|
marker_mtime = marker.stat().st_mtime
|
||||||
|
json_mtime = json_path.stat().st_mtime
|
||||||
|
return marker_mtime >= json_mtime
|
||||||
|
|
||||||
|
def _mark_processed(self, json_path: Path):
|
||||||
|
marker = self._marker_path(json_path)
|
||||||
|
marker.write_text(datetime.now().isoformat())
|
||||||
|
|
||||||
|
def find_unprocessed_files(self) -> list[Path]:
|
||||||
|
pattern = "aps_aliyun_customers_with_bills_*.json"
|
||||||
|
all_files = sorted(JSON_DIR.glob(pattern), key=lambda p: p.stat().st_mtime)
|
||||||
|
return [f for f in all_files if not self._is_processed(f)]
|
||||||
|
|
||||||
|
def sync_file(self, json_path: Path) -> bool:
|
||||||
|
logger.info("Syncing: %s", json_path.name)
|
||||||
|
try:
|
||||||
|
syncer = APSSyncer(db_config=self.db_config)
|
||||||
|
syncer.sync_from_json(str(json_path))
|
||||||
|
self._mark_processed(json_path)
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Sync failed for %s: %s", json_path.name, e)
|
||||||
|
return False
|
||||||
|
|
||||||
|
def run_once(self):
|
||||||
|
unprocessed = self.find_unprocessed_files()
|
||||||
|
if not unprocessed:
|
||||||
|
logger.info("No unprocessed JSON files found")
|
||||||
|
return 0
|
||||||
|
count = 0
|
||||||
|
for f in unprocessed:
|
||||||
|
if self.sync_file(f):
|
||||||
|
count += 1
|
||||||
|
logger.info("Processed %d/%d files", count, len(unprocessed))
|
||||||
|
return count
|
||||||
|
|
||||||
|
def run_watch(self):
|
||||||
|
logger.info("Watching %s for new JSON files (interval=%ds)", JSON_DIR, WATCH_INTERVAL_SECONDS)
|
||||||
|
self.run_once()
|
||||||
|
while self.running:
|
||||||
|
time.sleep(WATCH_INTERVAL_SECONDS)
|
||||||
|
unprocessed = self.find_unprocessed_files()
|
||||||
|
for f in unprocessed:
|
||||||
|
if not self.running:
|
||||||
|
break
|
||||||
|
self.sync_file(f)
|
||||||
|
logger.info("Watcher stopped")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="APS Sync Scheduler")
|
||||||
|
parser.add_argument("--mode", choices=["watch", "cron", "daemon"], default="watch",
|
||||||
|
help="watch=file watcher, cron=one-shot, daemon=watch with initial sync")
|
||||||
|
parser.add_argument("--host", default=DB_CONFIG["host"])
|
||||||
|
parser.add_argument("--port", type=int, default=DB_CONFIG["port"])
|
||||||
|
parser.add_argument("--user", default=DB_CONFIG["user"])
|
||||||
|
parser.add_argument("--password", default=DB_CONFIG["password"])
|
||||||
|
parser.add_argument("--database", default=DB_CONFIG["database"])
|
||||||
|
parser.add_argument("--interval", type=int, default=WATCH_INTERVAL_SECONDS,
|
||||||
|
help="Watch interval in seconds")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
_update_watch_interval(args.interval)
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"host": args.host, "port": args.port, "user": args.user,
|
||||||
|
"password": args.password, "database": args.database, "charset": "utf8mb4",
|
||||||
|
}
|
||||||
|
scheduler = SyncScheduler(db_config=config)
|
||||||
|
|
||||||
|
if args.mode == "cron":
|
||||||
|
count = scheduler.run_once()
|
||||||
|
sys.exit(0 if count >= 0 else 1)
|
||||||
|
else:
|
||||||
|
scheduler.run_watch()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
27
aliyun-sync/aps-aliyun-sync/check_db.py
Normal file
27
aliyun-sync/aps-aliyun-sync/check_db.py
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
import pymysql
|
||||||
|
|
||||||
|
conn = pymysql.connect(
|
||||||
|
host='8.156.34.195',
|
||||||
|
port=23149,
|
||||||
|
user='ray',
|
||||||
|
password='GV0C$ErephgQO7RQc7b6',
|
||||||
|
database='goonseek-dev',
|
||||||
|
charset='utf8mb4',
|
||||||
|
)
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
# 检查客户详情字段
|
||||||
|
cur.execute('SELECT account_id, customer_name, phone, department FROM aps_customer LIMIT 5')
|
||||||
|
rows = cur.fetchall()
|
||||||
|
print(f'aps_customer ({len(rows)} rows):')
|
||||||
|
for r in rows:
|
||||||
|
print(f' {r}')
|
||||||
|
|
||||||
|
# 检查快照金额字段
|
||||||
|
cur.execute('SELECT account_id, snapshot_month, last_month_payable_total_cny, current_month_payable_total_cny FROM aps_customer_snapshot LIMIT 5')
|
||||||
|
rows2 = cur.fetchall()
|
||||||
|
print(f'\naps_customer_snapshot ({len(rows2)} rows):')
|
||||||
|
for r in rows2:
|
||||||
|
print(f' {r}')
|
||||||
|
|
||||||
|
conn.close()
|
||||||
Reference in New Issue
Block a user