Compare commits
10 Commits
d14ed90597
...
e5d4b027b2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e5d4b027b2 | ||
| 87f629ac29 | |||
| f4854a2630 | |||
| 3a7f91419e | |||
| a06cdc70f1 | |||
| bbba33cab0 | |||
| f8d6274674 | |||
| 5355d8b7d2 | |||
| c4e7f8dd58 | |||
| 1c92478867 |
@@ -31,7 +31,7 @@ npm run login
|
|||||||
npm run sync
|
npm run sync
|
||||||
```
|
```
|
||||||
|
|
||||||
如果要从已有 checkpoint 继续全量流程(当前主要覆盖 orders + bills):
|
如果要从已有 checkpoint 继续全量流程(覆盖 customers / customerDetails / orders / orderDetails / bills / messages):
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
npm run sync -- --resume
|
npm run sync -- --resume
|
||||||
@@ -84,6 +84,8 @@ npm run messages
|
|||||||
npm run orders
|
npm run orders
|
||||||
```
|
```
|
||||||
|
|
||||||
|
说明:会同时抓取订单列表与订单详情。
|
||||||
|
|
||||||
订单增量:
|
订单增量:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ ALIYUN_APS_BROWSER_EXECUTABLE_PATH=
|
|||||||
ALIYUN_APS_CDP_URL=http://127.0.0.1:9222
|
ALIYUN_APS_CDP_URL=http://127.0.0.1:9222
|
||||||
ALIYUN_APS_TIMEZONE=Asia/Shanghai
|
ALIYUN_APS_TIMEZONE=Asia/Shanghai
|
||||||
ALIYUN_APS_CRON=0 6 * * *
|
ALIYUN_APS_CRON=0 6 * * *
|
||||||
|
ALIYUN_APS_HOT_CRON=*/5 * * * *
|
||||||
ALIYUN_APS_SCHEDULE_MODE=incremental
|
ALIYUN_APS_SCHEDULE_MODE=incremental
|
||||||
ALIYUN_APS_CLOSE_BROWSER=true
|
ALIYUN_APS_CLOSE_BROWSER=true
|
||||||
ALIYUN_APS_FULL_SYNC=true
|
ALIYUN_APS_FULL_SYNC=true
|
||||||
@@ -15,6 +16,13 @@ ALIYUN_APS_BILL_START_MONTH=2024-01
|
|||||||
ALIYUN_APS_ORDER_INCREMENTAL_OVERLAP_DAYS=2
|
ALIYUN_APS_ORDER_INCREMENTAL_OVERLAP_DAYS=2
|
||||||
ALIYUN_APS_BILL_INCREMENTAL_OVERLAP_DAYS=7
|
ALIYUN_APS_BILL_INCREMENTAL_OVERLAP_DAYS=7
|
||||||
ALIYUN_APS_MESSAGE_INCREMENTAL_OVERLAP_DAYS=7
|
ALIYUN_APS_MESSAGE_INCREMENTAL_OVERLAP_DAYS=7
|
||||||
|
ALIYUN_APS_HOT_MESSAGE_OVERLAP_MINUTES=15
|
||||||
|
ALIYUN_APS_HOT_ORDER_STABLE_THRESHOLD=100
|
||||||
|
ALIYUN_APS_HOT_ORDER_STABLE_PAGE_THRESHOLD=2
|
||||||
|
ALIYUN_APS_HOT_ORDER_MAX_PAGES=20
|
||||||
|
ALIYUN_APS_HOT_MESSAGE_MAX_PAGES=10
|
||||||
|
ALIYUN_APS_HOT_ORDER_DETAIL_REFRESH_MINUTES=30
|
||||||
|
ALIYUN_APS_HOT_FINAL_STATUSES=已完成,已关闭,已取消,已退款完成
|
||||||
ALIYUN_APS_DB_HOST=
|
ALIYUN_APS_DB_HOST=
|
||||||
ALIYUN_APS_DB_PORT=3306
|
ALIYUN_APS_DB_PORT=3306
|
||||||
ALIYUN_APS_DB_USER=
|
ALIYUN_APS_DB_USER=
|
||||||
@@ -22,6 +30,7 @@ ALIYUN_APS_DB_PASSWORD=
|
|||||||
ALIYUN_APS_DB_NAME=
|
ALIYUN_APS_DB_NAME=
|
||||||
ALIYUN_APS_DB_CHARSET=utf8mb4
|
ALIYUN_APS_DB_CHARSET=utf8mb4
|
||||||
ALIYUN_APS_DB_CONNECTION_LIMIT=5
|
ALIYUN_APS_DB_CONNECTION_LIMIT=5
|
||||||
|
ALIYUN_APS_DB_CONNECT_TIMEOUT=20000
|
||||||
ALIYUN_APS_SMTP_HOST=smtp.163.com
|
ALIYUN_APS_SMTP_HOST=smtp.163.com
|
||||||
ALIYUN_APS_SMTP_PORT=465
|
ALIYUN_APS_SMTP_PORT=465
|
||||||
ALIYUN_APS_SMTP_SECURE=true
|
ALIYUN_APS_SMTP_SECURE=true
|
||||||
|
|||||||
27
aliyun-sync/aliyun-aps-sync/.github/workflows/playwright.yml
vendored
Normal file
27
aliyun-sync/aliyun-aps-sync/.github/workflows/playwright.yml
vendored
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
name: Playwright Tests
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ main, master ]
|
||||||
|
pull_request:
|
||||||
|
branches: [ main, master ]
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
timeout-minutes: 60
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: lts/*
|
||||||
|
- name: Install dependencies
|
||||||
|
run: npm ci
|
||||||
|
- name: Install Playwright Browsers
|
||||||
|
run: npx playwright install --with-deps
|
||||||
|
- name: Run Playwright tests
|
||||||
|
run: npx playwright test
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
if: ${{ !cancelled() }}
|
||||||
|
with:
|
||||||
|
name: playwright-report
|
||||||
|
path: playwright-report/
|
||||||
|
retention-days: 30
|
||||||
8
aliyun-sync/aliyun-aps-sync/.gitignore
vendored
8
aliyun-sync/aliyun-aps-sync/.gitignore
vendored
@@ -3,3 +3,11 @@ node_modules/
|
|||||||
.browser/
|
.browser/
|
||||||
data/
|
data/
|
||||||
downloads/
|
downloads/
|
||||||
|
.sisyphus/
|
||||||
|
|
||||||
|
# Playwright
|
||||||
|
/test-results/
|
||||||
|
/playwright-report/
|
||||||
|
/blob-report/
|
||||||
|
/playwright/.cache/
|
||||||
|
/playwright/.auth/
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ Node 版阿里云 APS 同步工具。
|
|||||||
npm run sync
|
npm run sync
|
||||||
```
|
```
|
||||||
|
|
||||||
如果要让 full sync 从已有 checkpoint 继续(当前主要覆盖 orders + bills):
|
如果要让 full sync 从已有 checkpoint 继续(覆盖 customers / customerDetails / orders / orderDetails / bills / messages):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npm run sync -- --resume
|
npm run sync -- --resume
|
||||||
@@ -54,6 +54,28 @@ npm run incremental
|
|||||||
- 抓 orders / orderDetails / bills / messages
|
- 抓 orders / orderDetails / bills / messages
|
||||||
- 以数据库 watermark + overlap 为增量窗口
|
- 以数据库 watermark + overlap 为增量窗口
|
||||||
|
|
||||||
|
### Hot 模式
|
||||||
|
|
||||||
|
执行:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run hot
|
||||||
|
```
|
||||||
|
|
||||||
|
行为:
|
||||||
|
|
||||||
|
- 每次只抓**当天订单**
|
||||||
|
- 从订单第一页开始扫描
|
||||||
|
- 订单列表按“连续稳定行 / 连续稳定页 / 最大页数”提前停止
|
||||||
|
- 订单详情只抓:新增订单、列表有变化订单、缺失详情订单、非终态且到达兜底刷新时间的订单
|
||||||
|
- 消息按数据库最新时间回退分钟 overlap 后抓取,并在旧页提前停止
|
||||||
|
|
||||||
|
适用场景:
|
||||||
|
|
||||||
|
- 白天高频追当天订单
|
||||||
|
- 订单量较大,不希望每 5 分钟重复扫完整个当天分页
|
||||||
|
- 需要兼顾详情完整性和抓取效率
|
||||||
|
|
||||||
## 登录
|
## 登录
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -92,6 +114,11 @@ npm run bills -- --resume
|
|||||||
npm run orders
|
npm run orders
|
||||||
```
|
```
|
||||||
|
|
||||||
|
说明:该命令会同时抓取:
|
||||||
|
|
||||||
|
- orders(订单列表)
|
||||||
|
- orderDetails(订单详情)
|
||||||
|
|
||||||
订单增量:
|
订单增量:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -112,6 +139,26 @@ npm run orders -- --resume
|
|||||||
npm run messages
|
npm run messages
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## 高频同步
|
||||||
|
|
||||||
|
手动执行一次高频同步:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run hot
|
||||||
|
```
|
||||||
|
|
||||||
|
如果 PowerShell 禁止 `npm.ps1`,可以直接执行:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
node src/index.js hot
|
||||||
|
```
|
||||||
|
|
||||||
|
说明:
|
||||||
|
|
||||||
|
- `hot` 只覆盖当天订单、订单详情、消息
|
||||||
|
- 不抓 customer / customerDetails / bills
|
||||||
|
- 适合作为工作时间内的高频轮询任务
|
||||||
|
|
||||||
## 定时任务
|
## 定时任务
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -126,6 +173,26 @@ ALIYUN_APS_SCHEDULE_MODE=incremental
|
|||||||
|
|
||||||
执行日增量。
|
执行日增量。
|
||||||
|
|
||||||
|
如果要执行 5 分钟高频同步,可以设置:
|
||||||
|
|
||||||
|
```env
|
||||||
|
ALIYUN_APS_SCHEDULE_MODE=hot
|
||||||
|
ALIYUN_APS_HOT_CRON=*/5 * * * *
|
||||||
|
```
|
||||||
|
|
||||||
|
然后执行:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run schedule
|
||||||
|
```
|
||||||
|
|
||||||
|
说明:
|
||||||
|
|
||||||
|
- `incremental`:按现有增量策略抓 orders / orderDetails / bills / messages
|
||||||
|
- `full`:按全量策略执行
|
||||||
|
- `hot`:每轮只抓当天 orders / orderDetails / messages
|
||||||
|
- hot 模式内置任务锁;如果上一轮还没结束,会跳过下一轮,避免重叠执行
|
||||||
|
|
||||||
## 增量窗口
|
## 增量窗口
|
||||||
|
|
||||||
### orders
|
### orders
|
||||||
@@ -152,6 +219,41 @@ ALIYUN_APS_BILL_INCREMENTAL_OVERLAP_DAYS=7
|
|||||||
ALIYUN_APS_MESSAGE_INCREMENTAL_OVERLAP_DAYS=7
|
ALIYUN_APS_MESSAGE_INCREMENTAL_OVERLAP_DAYS=7
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## 高频模式配置
|
||||||
|
|
||||||
|
推荐配置:
|
||||||
|
|
||||||
|
```env
|
||||||
|
ALIYUN_APS_SCHEDULE_MODE=hot
|
||||||
|
ALIYUN_APS_HOT_CRON=*/5 * * * *
|
||||||
|
ALIYUN_APS_HOT_MESSAGE_OVERLAP_MINUTES=15
|
||||||
|
ALIYUN_APS_HOT_ORDER_STABLE_THRESHOLD=100
|
||||||
|
ALIYUN_APS_HOT_ORDER_STABLE_PAGE_THRESHOLD=2
|
||||||
|
ALIYUN_APS_HOT_ORDER_MAX_PAGES=20
|
||||||
|
ALIYUN_APS_HOT_MESSAGE_MAX_PAGES=10
|
||||||
|
ALIYUN_APS_HOT_ORDER_DETAIL_REFRESH_MINUTES=30
|
||||||
|
ALIYUN_APS_HOT_FINAL_STATUSES=已完成,已关闭,已取消,已退款完成
|
||||||
|
```
|
||||||
|
|
||||||
|
含义:
|
||||||
|
|
||||||
|
- `ALIYUN_APS_HOT_CRON`:高频任务 cron,默认每 5 分钟一次
|
||||||
|
- `ALIYUN_APS_HOT_MESSAGE_OVERLAP_MINUTES`:消息高频模式的回扫分钟数
|
||||||
|
- `ALIYUN_APS_HOT_ORDER_STABLE_THRESHOLD`:订单扫描中连续多少条稳定记录后停止
|
||||||
|
- `ALIYUN_APS_HOT_ORDER_STABLE_PAGE_THRESHOLD`:订单扫描中连续多少页无新增/变更后停止
|
||||||
|
- `ALIYUN_APS_HOT_ORDER_MAX_PAGES`:订单每轮最多扫描页数,防止高峰期跑太久
|
||||||
|
- `ALIYUN_APS_HOT_MESSAGE_MAX_PAGES`:消息每轮最多扫描页数
|
||||||
|
- `ALIYUN_APS_HOT_ORDER_DETAIL_REFRESH_MINUTES`:非终态订单详情兜底刷新间隔
|
||||||
|
- `ALIYUN_APS_HOT_FINAL_STATUSES`:视为终态的订单状态,终态订单在无变化时会尽量跳过详情抓取
|
||||||
|
|
||||||
|
默认策略:
|
||||||
|
|
||||||
|
- 订单按最新到最旧扫描
|
||||||
|
- 新订单或列表字段变化的订单会进入详情抓取
|
||||||
|
- 已抓过且无变化的终态订单会直接跳过详情
|
||||||
|
- 非终态订单会按兜底刷新时间周期性重抓详情
|
||||||
|
- 消息使用 watermark + overlap,避免 5 分钟轮询时漏边界消息
|
||||||
|
|
||||||
## 数据库配置
|
## 数据库配置
|
||||||
|
|
||||||
`.env` 需要配置:
|
`.env` 需要配置:
|
||||||
|
|||||||
46
aliyun-sync/aliyun-aps-sync/package-lock.json
generated
46
aliyun-sync/aliyun-aps-sync/package-lock.json
generated
@@ -13,14 +13,31 @@
|
|||||||
"node-cron": "^4.2.1",
|
"node-cron": "^4.2.1",
|
||||||
"nodemailer": "^6.10.1",
|
"nodemailer": "^6.10.1",
|
||||||
"playwright": "^1.58.2"
|
"playwright": "^1.58.2"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@playwright/test": "^1.59.1",
|
||||||
|
"@types/node": "^25.6.2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@playwright/test": {
|
||||||
|
"version": "1.59.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz",
|
||||||
|
"integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==",
|
||||||
|
"dev": true,
|
||||||
|
"dependencies": {
|
||||||
|
"playwright": "1.59.1"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"playwright": "cli.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@types/node": {
|
"node_modules/@types/node": {
|
||||||
"version": "25.6.0",
|
"version": "25.6.2",
|
||||||
"resolved": "https://registry.npmmirror.com/@types/node/-/node-25.6.0.tgz",
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.2.tgz",
|
||||||
"integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==",
|
"integrity": "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw==",
|
||||||
"license": "MIT",
|
|
||||||
"peer": true,
|
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"undici-types": "~7.19.0"
|
"undici-types": "~7.19.0"
|
||||||
}
|
}
|
||||||
@@ -174,12 +191,11 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/playwright": {
|
"node_modules/playwright": {
|
||||||
"version": "1.58.2",
|
"version": "1.59.1",
|
||||||
"resolved": "https://registry.npmmirror.com/playwright/-/playwright-1.58.2.tgz",
|
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz",
|
||||||
"integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==",
|
"integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==",
|
||||||
"license": "Apache-2.0",
|
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"playwright-core": "1.58.2"
|
"playwright-core": "1.59.1"
|
||||||
},
|
},
|
||||||
"bin": {
|
"bin": {
|
||||||
"playwright": "cli.js"
|
"playwright": "cli.js"
|
||||||
@@ -192,10 +208,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/playwright-core": {
|
"node_modules/playwright-core": {
|
||||||
"version": "1.58.2",
|
"version": "1.59.1",
|
||||||
"resolved": "https://registry.npmmirror.com/playwright-core/-/playwright-core-1.58.2.tgz",
|
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz",
|
||||||
"integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==",
|
"integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==",
|
||||||
"license": "Apache-2.0",
|
|
||||||
"bin": {
|
"bin": {
|
||||||
"playwright-core": "cli.js"
|
"playwright-core": "cli.js"
|
||||||
},
|
},
|
||||||
@@ -228,8 +243,7 @@
|
|||||||
"version": "7.19.2",
|
"version": "7.19.2",
|
||||||
"resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-7.19.2.tgz",
|
"resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-7.19.2.tgz",
|
||||||
"integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==",
|
"integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==",
|
||||||
"license": "MIT",
|
"license": "MIT"
|
||||||
"peer": true
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
"login": "node src/index.js login",
|
"login": "node src/index.js login",
|
||||||
"sync": "node src/index.js sync",
|
"sync": "node src/index.js sync",
|
||||||
"incremental": "node src/index.js incremental",
|
"incremental": "node src/index.js incremental",
|
||||||
|
"hot": "node src/index.js hot",
|
||||||
"bills": "node src/index.js bills",
|
"bills": "node src/index.js bills",
|
||||||
"orders": "node src/index.js orders",
|
"orders": "node src/index.js orders",
|
||||||
"messages": "node src/index.js messages",
|
"messages": "node src/index.js messages",
|
||||||
@@ -15,8 +16,12 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"dotenv": "^16.6.1",
|
"dotenv": "^16.6.1",
|
||||||
"mysql2": "^3.15.2",
|
"mysql2": "^3.15.2",
|
||||||
"nodemailer": "^6.10.1",
|
|
||||||
"node-cron": "^4.2.1",
|
"node-cron": "^4.2.1",
|
||||||
|
"nodemailer": "^6.10.1",
|
||||||
"playwright": "^1.58.2"
|
"playwright": "^1.58.2"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@playwright/test": "^1.59.1",
|
||||||
|
"@types/node": "^25.6.2"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
79
aliyun-sync/aliyun-aps-sync/playwright.config.ts
Normal file
79
aliyun-sync/aliyun-aps-sync/playwright.config.ts
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
import { defineConfig, devices } from '@playwright/test';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read environment variables from file.
|
||||||
|
* https://github.com/motdotla/dotenv
|
||||||
|
*/
|
||||||
|
// import dotenv from 'dotenv';
|
||||||
|
// import path from 'path';
|
||||||
|
// dotenv.config({ path: path.resolve(__dirname, '.env') });
|
||||||
|
|
||||||
|
/**
|
||||||
|
* See https://playwright.dev/docs/test-configuration.
|
||||||
|
*/
|
||||||
|
export default defineConfig({
|
||||||
|
testDir: './tests',
|
||||||
|
/* Run tests in files in parallel */
|
||||||
|
fullyParallel: true,
|
||||||
|
/* Fail the build on CI if you accidentally left test.only in the source code. */
|
||||||
|
forbidOnly: !!process.env.CI,
|
||||||
|
/* Retry on CI only */
|
||||||
|
retries: process.env.CI ? 2 : 0,
|
||||||
|
/* Opt out of parallel tests on CI. */
|
||||||
|
workers: process.env.CI ? 1 : undefined,
|
||||||
|
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
|
||||||
|
reporter: 'html',
|
||||||
|
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
|
||||||
|
use: {
|
||||||
|
/* Base URL to use in actions like `await page.goto('')`. */
|
||||||
|
// baseURL: 'http://localhost:3000',
|
||||||
|
|
||||||
|
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
|
||||||
|
trace: 'on-first-retry',
|
||||||
|
},
|
||||||
|
|
||||||
|
/* Configure projects for major browsers */
|
||||||
|
projects: [
|
||||||
|
{
|
||||||
|
name: 'chromium',
|
||||||
|
use: { ...devices['Desktop Chrome'] },
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: 'firefox',
|
||||||
|
use: { ...devices['Desktop Firefox'] },
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
name: 'webkit',
|
||||||
|
use: { ...devices['Desktop Safari'] },
|
||||||
|
},
|
||||||
|
|
||||||
|
/* Test against mobile viewports. */
|
||||||
|
// {
|
||||||
|
// name: 'Mobile Chrome',
|
||||||
|
// use: { ...devices['Pixel 5'] },
|
||||||
|
// },
|
||||||
|
// {
|
||||||
|
// name: 'Mobile Safari',
|
||||||
|
// use: { ...devices['iPhone 12'] },
|
||||||
|
// },
|
||||||
|
|
||||||
|
/* Test against branded browsers. */
|
||||||
|
// {
|
||||||
|
// name: 'Microsoft Edge',
|
||||||
|
// use: { ...devices['Desktop Edge'], channel: 'msedge' },
|
||||||
|
// },
|
||||||
|
// {
|
||||||
|
// name: 'Google Chrome',
|
||||||
|
// use: { ...devices['Desktop Chrome'], channel: 'chrome' },
|
||||||
|
// },
|
||||||
|
],
|
||||||
|
|
||||||
|
/* Run your local dev server before starting the tests */
|
||||||
|
// webServer: {
|
||||||
|
// command: 'npm run start',
|
||||||
|
// url: 'http://localhost:3000',
|
||||||
|
// reuseExistingServer: !process.env.CI,
|
||||||
|
// },
|
||||||
|
});
|
||||||
@@ -11,6 +11,24 @@ const toBool = (value, fallback) => {
|
|||||||
return ['1', 'true', 'yes', 'y', 'on'].includes(String(value).trim().toLowerCase());
|
return ['1', 'true', 'yes', 'y', 'on'].includes(String(value).trim().toLowerCase());
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const toInt = (value, fallback, min = Number.MIN_SAFE_INTEGER) => {
|
||||||
|
const parsed = Number.parseInt(String(value ?? ''), 10);
|
||||||
|
if (Number.isNaN(parsed)) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
return Math.max(min, parsed);
|
||||||
|
};
|
||||||
|
|
||||||
|
const toList = (value, fallback = []) => {
|
||||||
|
if (value == null || String(value).trim() === '') {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
return String(value)
|
||||||
|
.split(',')
|
||||||
|
.map((item) => item.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
};
|
||||||
|
|
||||||
const ensureDir = (dirPath) => {
|
const ensureDir = (dirPath) => {
|
||||||
fs.mkdirSync(dirPath, { recursive: true });
|
fs.mkdirSync(dirPath, { recursive: true });
|
||||||
return dirPath;
|
return dirPath;
|
||||||
@@ -26,12 +44,20 @@ export const config = {
|
|||||||
cdpUrl: (process.env.ALIYUN_APS_CDP_URL || 'http://127.0.0.1:9222').trim(),
|
cdpUrl: (process.env.ALIYUN_APS_CDP_URL || 'http://127.0.0.1:9222').trim(),
|
||||||
timezone: process.env.ALIYUN_APS_TIMEZONE || 'Asia/Shanghai',
|
timezone: process.env.ALIYUN_APS_TIMEZONE || 'Asia/Shanghai',
|
||||||
cron: process.env.ALIYUN_APS_CRON || '0 6 * * *',
|
cron: process.env.ALIYUN_APS_CRON || '0 6 * * *',
|
||||||
|
hotCron: process.env.ALIYUN_APS_HOT_CRON || '*/5 * * * *',
|
||||||
orderStartDate: process.env.ALIYUN_APS_ORDER_START_DATE || '2024-01-01',
|
orderStartDate: process.env.ALIYUN_APS_ORDER_START_DATE || '2024-01-01',
|
||||||
incrementalOrderStartDate: process.env.ALIYUN_APS_INCREMENTAL_ORDER_START_DATE || '',
|
incrementalOrderStartDate: process.env.ALIYUN_APS_INCREMENTAL_ORDER_START_DATE || '',
|
||||||
billStartMonth: process.env.ALIYUN_APS_BILL_START_MONTH || '2024-01',
|
billStartMonth: process.env.ALIYUN_APS_BILL_START_MONTH || '2024-01',
|
||||||
orderIncrementalOverlapDays: Math.max(0, Number.parseInt(process.env.ALIYUN_APS_ORDER_INCREMENTAL_OVERLAP_DAYS || '2', 10) || 2),
|
orderIncrementalOverlapDays: toInt(process.env.ALIYUN_APS_ORDER_INCREMENTAL_OVERLAP_DAYS || '2', 2, 0),
|
||||||
billIncrementalOverlapDays: Math.max(0, Number.parseInt(process.env.ALIYUN_APS_BILL_INCREMENTAL_OVERLAP_DAYS || '7', 10) || 7),
|
billIncrementalOverlapDays: toInt(process.env.ALIYUN_APS_BILL_INCREMENTAL_OVERLAP_DAYS || '7', 7, 0),
|
||||||
messageIncrementalOverlapDays: Math.max(0, Number.parseInt(process.env.ALIYUN_APS_MESSAGE_INCREMENTAL_OVERLAP_DAYS || '7', 10) || 7),
|
messageIncrementalOverlapDays: toInt(process.env.ALIYUN_APS_MESSAGE_INCREMENTAL_OVERLAP_DAYS || '7', 7, 0),
|
||||||
|
hotMessageOverlapMinutes: toInt(process.env.ALIYUN_APS_HOT_MESSAGE_OVERLAP_MINUTES || '15', 15, 0),
|
||||||
|
hotOrderStableThreshold: toInt(process.env.ALIYUN_APS_HOT_ORDER_STABLE_THRESHOLD || '100', 100, 1),
|
||||||
|
hotOrderStablePageThreshold: toInt(process.env.ALIYUN_APS_HOT_ORDER_STABLE_PAGE_THRESHOLD || '2', 2, 1),
|
||||||
|
hotOrderMaxPagesPerRun: toInt(process.env.ALIYUN_APS_HOT_ORDER_MAX_PAGES || '20', 20, 1),
|
||||||
|
hotMessageMaxPagesPerRun: toInt(process.env.ALIYUN_APS_HOT_MESSAGE_MAX_PAGES || '10', 10, 1),
|
||||||
|
hotOrderDetailRefreshMinutes: toInt(process.env.ALIYUN_APS_HOT_ORDER_DETAIL_REFRESH_MINUTES || '30', 30, 1),
|
||||||
|
hotFinalStatuses: toList(process.env.ALIYUN_APS_HOT_FINAL_STATUSES, ['已完成', '已关闭', '已取消', '已退款完成']),
|
||||||
scheduleMode: process.env.ALIYUN_APS_SCHEDULE_MODE || 'incremental',
|
scheduleMode: process.env.ALIYUN_APS_SCHEDULE_MODE || 'incremental',
|
||||||
smtp: {
|
smtp: {
|
||||||
host: process.env.ALIYUN_APS_SMTP_HOST || 'smtp.qq.com',
|
host: process.env.ALIYUN_APS_SMTP_HOST || 'smtp.qq.com',
|
||||||
@@ -58,6 +84,7 @@ export const config = {
|
|||||||
database: process.env.ALIYUN_APS_DB_NAME || '',
|
database: process.env.ALIYUN_APS_DB_NAME || '',
|
||||||
charset: process.env.ALIYUN_APS_DB_CHARSET || 'utf8mb4',
|
charset: process.env.ALIYUN_APS_DB_CHARSET || 'utf8mb4',
|
||||||
connectionLimit: Math.max(1, Number.parseInt(process.env.ALIYUN_APS_DB_CONNECTION_LIMIT || '5', 10) || 5),
|
connectionLimit: Math.max(1, Number.parseInt(process.env.ALIYUN_APS_DB_CONNECTION_LIMIT || '5', 10) || 5),
|
||||||
|
connectTimeout: Math.max(1000, Number.parseInt(process.env.ALIYUN_APS_DB_CONNECT_TIMEOUT || '20000', 10) || 20000),
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -68,12 +95,13 @@ export const datasets = {
|
|||||||
heading: '我的客户',
|
heading: '我的客户',
|
||||||
pageSize: 20,
|
pageSize: 20,
|
||||||
uniqueKey: (record) => record.accountId || record.loginName || record.__hash,
|
uniqueKey: (record) => record.accountId || record.loginName || record.__hash,
|
||||||
normalize: (record) => {
|
normalize: (record, context = {}) => {
|
||||||
const loginAndUid = record['登录名称/账号ID'] || '';
|
const loginAndUid = record['登录名称/账号ID'] || '';
|
||||||
const [loginName = '', accountId = ''] = splitLines(loginAndUid);
|
const [loginName = '', accountId = ''] = splitLines(loginAndUid);
|
||||||
return {
|
return {
|
||||||
loginName: loginName.replace(/\s+/g, ''),
|
loginName: loginName.replace(/\s+/g, ''),
|
||||||
accountId,
|
accountId,
|
||||||
|
listPageNum: context.pageNum || '',
|
||||||
realName: record['UID实名认证名称'] || '',
|
realName: record['UID实名认证名称'] || '',
|
||||||
reportSource: record['报备来源'] || '',
|
reportSource: record['报备来源'] || '',
|
||||||
reportType: record['报备类型'] || '',
|
reportType: record['报备类型'] || '',
|
||||||
@@ -101,10 +129,11 @@ export const datasets = {
|
|||||||
name: 'orders',
|
name: 'orders',
|
||||||
url: `${config.baseUrl}/#/detail/order/~/costCenter/order`,
|
url: `${config.baseUrl}/#/detail/order/~/costCenter/order`,
|
||||||
heading: '订单查询',
|
heading: '订单查询',
|
||||||
pageSize: 100,
|
pageSize: 20,
|
||||||
uniqueKey: (record) => record.orderId || record.__hash,
|
uniqueKey: (record) => record.orderId || record.__hash,
|
||||||
normalize: (record, context) => ({
|
normalize: (record, context) => ({
|
||||||
orderId: record['订单号'] || '',
|
orderId: record['订单号'] || '',
|
||||||
|
listPageNum: context.pageNum || '',
|
||||||
customerAccount: (record['客户账号'] || '').replace(/\s+/g, ''),
|
customerAccount: (record['客户账号'] || '').replace(/\s+/g, ''),
|
||||||
customerCategory: record['客户分类'] || '',
|
customerCategory: record['客户分类'] || '',
|
||||||
orderType: record['订单类型'] || '',
|
orderType: record['订单类型'] || '',
|
||||||
@@ -143,6 +172,7 @@ export const datasets = {
|
|||||||
couponAmountCny: record.couponAmountCny || '',
|
couponAmountCny: record.couponAmountCny || '',
|
||||||
windowStart: context.windowStart || '',
|
windowStart: context.windowStart || '',
|
||||||
windowEnd: context.windowEnd || '',
|
windowEnd: context.windowEnd || '',
|
||||||
|
detailSyncedAt: context.detailSyncedAt || record.detailSyncedAt || '',
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
customerDetails: {
|
customerDetails: {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import mysql from 'mysql2/promise';
|
|||||||
import { config } from './config.js';
|
import { config } from './config.js';
|
||||||
|
|
||||||
let pool = null;
|
let pool = null;
|
||||||
|
let customerMapCache = null;
|
||||||
|
|
||||||
const MESSAGE_TABLE_DDL = `
|
const MESSAGE_TABLE_DDL = `
|
||||||
CREATE TABLE IF NOT EXISTS aliyun_aps_messages (
|
CREATE TABLE IF NOT EXISTS aliyun_aps_messages (
|
||||||
@@ -52,6 +53,7 @@ function getPool() {
|
|||||||
database: config.db.database,
|
database: config.db.database,
|
||||||
charset: config.db.charset,
|
charset: config.db.charset,
|
||||||
connectionLimit: config.db.connectionLimit,
|
connectionLimit: config.db.connectionLimit,
|
||||||
|
connectTimeout: config.db.connectTimeout,
|
||||||
waitForConnections: true,
|
waitForConnections: true,
|
||||||
});
|
});
|
||||||
return pool;
|
return pool;
|
||||||
@@ -130,6 +132,9 @@ function safeDateTime(value) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function getCustomerMap() {
|
async function getCustomerMap() {
|
||||||
|
if (customerMapCache) {
|
||||||
|
return customerMapCache;
|
||||||
|
}
|
||||||
const [rows] = await getPool().query('SELECT account_id, login_name FROM aps_customer');
|
const [rows] = await getPool().query('SELECT account_id, login_name FROM aps_customer');
|
||||||
const map = new Map();
|
const map = new Map();
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
@@ -141,7 +146,8 @@ async function getCustomerMap() {
|
|||||||
map.set(loginName, accountId);
|
map.set(loginName, accountId);
|
||||||
map.set(loginName.replace(/\s+/g, ''), accountId);
|
map.set(loginName.replace(/\s+/g, ''), accountId);
|
||||||
}
|
}
|
||||||
return map;
|
customerMapCache = map;
|
||||||
|
return customerMapCache;
|
||||||
}
|
}
|
||||||
|
|
||||||
function resolveCustomerAccountId(customerMap, customerAccount) {
|
function resolveCustomerAccountId(customerMap, customerAccount) {
|
||||||
@@ -186,6 +192,7 @@ export async function closeDbPool() {
|
|||||||
}
|
}
|
||||||
await pool.end();
|
await pool.end();
|
||||||
pool = null;
|
pool = null;
|
||||||
|
customerMapCache = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function ensureMessagesTable() {
|
export async function ensureMessagesTable() {
|
||||||
@@ -469,8 +476,8 @@ export async function upsertOrderDetails(records) {
|
|||||||
safeString(record.dealerUid),
|
safeString(record.dealerUid),
|
||||||
safeString(record.customerType),
|
safeString(record.customerType),
|
||||||
safeString(record.opportunityId),
|
safeString(record.opportunityId),
|
||||||
safeString(record.paymentTime),
|
safeDateTime(record.paymentTime),
|
||||||
safeString(record.orderTime),
|
safeDateTime(record.orderTime),
|
||||||
safeString(record.productName),
|
safeString(record.productName),
|
||||||
safeString(record.productCode),
|
safeString(record.productCode),
|
||||||
safeNumber(record.originalPriceCny),
|
safeNumber(record.originalPriceCny),
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ const command = args[0] || 'sync';
|
|||||||
const extraArgs = args.slice(1);
|
const extraArgs = args.slice(1);
|
||||||
const billsResume = extraArgs.includes('--resume');
|
const billsResume = extraArgs.includes('--resume');
|
||||||
const ordersIncremental = extraArgs.includes('--incremental');
|
const ordersIncremental = extraArgs.includes('--incremental');
|
||||||
|
const messagesResume = extraArgs.includes('--resume');
|
||||||
|
const hotResume = extraArgs.includes('--resume');
|
||||||
|
|
||||||
for (const arg of extraArgs) {
|
for (const arg of extraArgs) {
|
||||||
if (arg.startsWith('--incremental-order-start-date=')) {
|
if (arg.startsWith('--incremental-order-start-date=')) {
|
||||||
@@ -12,7 +14,7 @@ for (const arg of extraArgs) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const { login, scheduleSync, syncAll, syncAllIncremental, syncBillsOnly, syncMessagesOnly, syncOrdersOnly } = await import('./sync.js');
|
const { login, scheduleSync, syncAll, syncAllIncremental, syncBillsOnly, syncMessagesOnly, syncOrdersOnly, syncHot } = await import('./sync.js');
|
||||||
|
|
||||||
if (command === 'login') {
|
if (command === 'login') {
|
||||||
await login();
|
await login();
|
||||||
@@ -44,7 +46,13 @@ if (command === 'orders') {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (command === 'messages') {
|
if (command === 'messages') {
|
||||||
const summary = await syncMessagesOnly({ incremental: config.scheduleMode === 'incremental' });
|
const summary = await syncMessagesOnly({ incremental: config.scheduleMode === 'incremental', resume: messagesResume });
|
||||||
|
console.log(JSON.stringify(summary, null, 2));
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (command === 'hot') {
|
||||||
|
const summary = await syncHot({ resume: hotResume });
|
||||||
console.log(JSON.stringify(summary, null, 2));
|
console.log(JSON.stringify(summary, null, 2));
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
18
aliyun-sync/aliyun-aps-sync/tests/example.spec.ts
Normal file
18
aliyun-sync/aliyun-aps-sync/tests/example.spec.ts
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
import { test, expect } from '@playwright/test';
|
||||||
|
|
||||||
|
test('has title', async ({ page }) => {
|
||||||
|
await page.goto('https://playwright.dev/');
|
||||||
|
|
||||||
|
// Expect a title "to contain" a substring.
|
||||||
|
await expect(page).toHaveTitle(/Playwright/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('get started link', async ({ page }) => {
|
||||||
|
await page.goto('https://playwright.dev/');
|
||||||
|
|
||||||
|
// Click the get started link.
|
||||||
|
await page.getByRole('link', { name: 'Get started' }).click();
|
||||||
|
|
||||||
|
// Expects page to have a heading with the name of Installation.
|
||||||
|
await expect(page.getByRole('heading', { name: 'Installation' })).toBeVisible();
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user