|
@@ -12,17 +12,34 @@ use yii\console\ExitCode;
|
|
|
/**
|
|
/**
|
|
|
* 补充历史订单 salt 字段
|
|
* 补充历史订单 salt 字段
|
|
|
*
|
|
*
|
|
|
|
|
+ * 大表安全说明:
|
|
|
|
|
+ * - 不做 COUNT(*),避免全表统计
|
|
|
|
|
+ * - 默认 empty 模式只查 salt='' 的记录,补完后即停止,不会扫全表
|
|
|
|
|
+ * - 查询均为普通 SELECT(MVCC 快照读),不会锁表;UPDATE 仅锁单行
|
|
|
|
|
+ * - 大表建议先加索引: ALTER TABLE xhGhsOrder ADD INDEX idx_salt_id (salt, id);
|
|
|
|
|
+ *
|
|
|
* 用法:
|
|
* 用法:
|
|
|
* php yii salt/fill
|
|
* php yii salt/fill
|
|
|
* php yii salt/fill --dryRun=1
|
|
* php yii salt/fill --dryRun=1
|
|
|
* php yii salt/fill --batchSize=50 --sleepMs=200
|
|
* php yii salt/fill --batchSize=50 --sleepMs=200
|
|
|
|
|
+ * php yii salt/fill --mode=pk --scanSize=2000
|
|
|
* php yii salt/fill-ghs-order
|
|
* php yii salt/fill-ghs-order
|
|
|
* php yii salt/fill-cg
|
|
* php yii salt/fill-cg
|
|
|
*/
|
|
*/
|
|
|
class SaltController extends Controller
|
|
class SaltController extends Controller
|
|
|
{
|
|
{
|
|
|
/** @var int 每批处理条数 */
|
|
/** @var int 每批处理条数 */
|
|
|
- public $batchSize = 100;
|
|
|
|
|
|
|
+ public $batchSize = 5000;
|
|
|
|
|
+
|
|
|
|
|
+ /** @var int pk 模式下每次主键游标扫描行数 */
|
|
|
|
|
+ public $scanSize = 5000;
|
|
|
|
|
+
|
|
|
|
|
+ /**
|
|
|
|
|
+ * 扫描模式:
|
|
|
|
|
+ * - empty: 只查 salt='',适合待补数据量较少(默认)
|
|
|
|
|
+ * - pk: 按主键分段扫描,适合 salt 列无索引且 empty 模式执行计划不佳时
|
|
|
|
|
+ */
|
|
|
|
|
+ public $mode = 'empty';
|
|
|
|
|
|
|
|
/** @var int 每批之间的休眠毫秒数,降低白天执行时的数据库压力 */
|
|
/** @var int 每批之间的休眠毫秒数,降低白天执行时的数据库压力 */
|
|
|
public $sleepMs = 100;
|
|
public $sleepMs = 100;
|
|
@@ -35,7 +52,14 @@ class SaltController extends Controller
|
|
|
|
|
|
|
|
public function options($actionID)
|
|
public function options($actionID)
|
|
|
{
|
|
{
|
|
|
- return array_merge(parent::options($actionID), ['batchSize', 'sleepMs', 'limit', 'dryRun']);
|
|
|
|
|
|
|
+ return array_merge(parent::options($actionID), [
|
|
|
|
|
+ 'batchSize',
|
|
|
|
|
+ 'scanSize',
|
|
|
|
|
+ 'mode',
|
|
|
|
|
+ 'sleepMs',
|
|
|
|
|
+ 'limit',
|
|
|
|
|
+ 'dryRun',
|
|
|
|
|
+ ]);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -78,6 +102,14 @@ class SaltController extends Controller
|
|
|
$this->stderr("batchSize 必须大于 0\n");
|
|
$this->stderr("batchSize 必须大于 0\n");
|
|
|
return ExitCode::UNSPECIFIED_ERROR;
|
|
return ExitCode::UNSPECIFIED_ERROR;
|
|
|
}
|
|
}
|
|
|
|
|
+ if ($this->scanSize < 1) {
|
|
|
|
|
+ $this->stderr("scanSize 必须大于 0\n");
|
|
|
|
|
+ return ExitCode::UNSPECIFIED_ERROR;
|
|
|
|
|
+ }
|
|
|
|
|
+ if (!in_array($this->mode, ['empty', 'pk'], true)) {
|
|
|
|
|
+ $this->stderr("mode 仅支持 empty 或 pk\n");
|
|
|
|
|
+ return ExitCode::UNSPECIFIED_ERROR;
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
$exitCode = ExitCode::OK;
|
|
$exitCode = ExitCode::OK;
|
|
|
foreach ($tables as $label => $tableName) {
|
|
foreach ($tables as $label => $tableName) {
|
|
@@ -93,21 +125,34 @@ class SaltController extends Controller
|
|
|
private function fillTable($label, $tableName)
|
|
private function fillTable($label, $tableName)
|
|
|
{
|
|
{
|
|
|
$db = Yii::$app->db;
|
|
$db = Yii::$app->db;
|
|
|
- $remaining = (int)$db->createCommand(
|
|
|
|
|
- "SELECT COUNT(*) FROM {$tableName} WHERE salt = ''"
|
|
|
|
|
- )->queryScalar();
|
|
|
|
|
|
|
+ $db->createCommand('SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED')->execute();
|
|
|
|
|
|
|
|
- $this->stdout("{$label}({$tableName}) 待补充: {$remaining}\n");
|
|
|
|
|
- if ($remaining === 0) {
|
|
|
|
|
- return ExitCode::OK;
|
|
|
|
|
|
|
+ $this->stdout("{$label}({$tableName}) 开始补充 salt,模式: {$this->mode}\n");
|
|
|
|
|
+ if ($this->mode === 'empty') {
|
|
|
|
|
+ $processed = $this->fillByEmptySalt($label, $tableName);
|
|
|
|
|
+ } else {
|
|
|
|
|
+ $processed = $this->fillByPrimaryKeyScan($label, $tableName);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ $this->stdout("{$label} 完成,本次处理 {$processed} 条\n");
|
|
|
|
|
+ return ExitCode::OK;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ /**
|
|
|
|
|
+ * 只查 salt='' 的记录,补完即停,不会扫描已有 salt 的上千万数据
|
|
|
|
|
+ */
|
|
|
|
|
+ private function fillByEmptySalt($label, $tableName)
|
|
|
|
|
+ {
|
|
|
|
|
+ $db = Yii::$app->db;
|
|
|
$processed = 0;
|
|
$processed = 0;
|
|
|
- $maxProcess = $this->limit > 0 ? min($this->limit, $remaining) : $remaining;
|
|
|
|
|
$lastId = 0;
|
|
$lastId = 0;
|
|
|
|
|
|
|
|
- while ($processed < $maxProcess) {
|
|
|
|
|
- $batchLimit = min($this->batchSize, $maxProcess - $processed);
|
|
|
|
|
|
|
+ while (true) {
|
|
|
|
|
+ if ($this->limit > 0 && $processed >= $this->limit) {
|
|
|
|
|
+ break;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ $batchLimit = $this->limit > 0 ? min($this->batchSize, $this->limit - $processed) : $this->batchSize;
|
|
|
$rows = $db->createCommand(
|
|
$rows = $db->createCommand(
|
|
|
"SELECT id FROM {$tableName} WHERE salt = '' AND id > :lastId ORDER BY id ASC LIMIT {$batchLimit}"
|
|
"SELECT id FROM {$tableName} WHERE salt = '' AND id > :lastId ORDER BY id ASC LIMIT {$batchLimit}"
|
|
|
)->bindValue(':lastId', $lastId)->queryColumn();
|
|
)->bindValue(':lastId', $lastId)->queryColumn();
|
|
@@ -116,41 +161,100 @@ class SaltController extends Controller
|
|
|
break;
|
|
break;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- foreach ($rows as $id) {
|
|
|
|
|
- $id = (int)$id;
|
|
|
|
|
- $lastId = $id;
|
|
|
|
|
|
|
+ $processed += $this->updateIds($label, $tableName, $rows);
|
|
|
|
|
+ $lastId = (int)end($rows);
|
|
|
|
|
|
|
|
- if ($this->dryRun) {
|
|
|
|
|
- $this->stdout("[dry-run] {$label} id={$id}\n");
|
|
|
|
|
- $processed++;
|
|
|
|
|
- continue;
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ $this->stdout("{$label} 已处理 {$processed} 条,当前 id <= {$lastId}\n");
|
|
|
|
|
+ $this->sleepBetweenBatch();
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
- $salt = stringUtil::charsShuffleLowerCase(10);
|
|
|
|
|
- $affected = $db->createCommand(
|
|
|
|
|
- "UPDATE {$tableName} SET salt = :salt WHERE id = :id AND salt = ''"
|
|
|
|
|
- )->bindValues([
|
|
|
|
|
- ':salt' => $salt,
|
|
|
|
|
- ':id' => $id,
|
|
|
|
|
- ])->execute();
|
|
|
|
|
|
|
+ return $processed;
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
- if ($affected > 0) {
|
|
|
|
|
- $processed++;
|
|
|
|
|
|
|
+ /**
|
|
|
|
|
+ * 按主键游标分段扫描,查询只走主键索引;适合 empty 模式执行计划不佳时使用
|
|
|
|
|
+ */
|
|
|
|
|
+ private function fillByPrimaryKeyScan($label, $tableName)
|
|
|
|
|
+ {
|
|
|
|
|
+ $db = Yii::$app->db;
|
|
|
|
|
+ $processed = 0;
|
|
|
|
|
+ $lastId = 0;
|
|
|
|
|
+
|
|
|
|
|
+ while (true) {
|
|
|
|
|
+ if ($this->limit > 0 && $processed >= $this->limit) {
|
|
|
|
|
+ break;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ $ids = $db->createCommand(
|
|
|
|
|
+ "SELECT id FROM {$tableName} WHERE id > :lastId ORDER BY id ASC LIMIT {$this->scanSize}"
|
|
|
|
|
+ )->bindValue(':lastId', $lastId)->queryColumn();
|
|
|
|
|
+
|
|
|
|
|
+ if (empty($ids)) {
|
|
|
|
|
+ break;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ $lastId = (int)end($ids);
|
|
|
|
|
+ $minId = (int)$ids[0];
|
|
|
|
|
+ $maxId = $lastId;
|
|
|
|
|
+
|
|
|
|
|
+ $emptyIds = $db->createCommand(
|
|
|
|
|
+ "SELECT id FROM {$tableName} WHERE id >= :minId AND id <= :maxId AND salt = '' ORDER BY id ASC"
|
|
|
|
|
+ )->bindValues([
|
|
|
|
|
+ ':minId' => $minId,
|
|
|
|
|
+ ':maxId' => $maxId,
|
|
|
|
|
+ ])->queryColumn();
|
|
|
|
|
+
|
|
|
|
|
+ if (!empty($emptyIds)) {
|
|
|
|
|
+ if ($this->limit > 0) {
|
|
|
|
|
+ $emptyIds = array_slice($emptyIds, 0, $this->limit - $processed);
|
|
|
}
|
|
}
|
|
|
|
|
+ $processed += $this->updateIds($label, $tableName, $emptyIds);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- $this->stdout("{$label} 已处理 {$processed}/{$maxProcess}\n");
|
|
|
|
|
|
|
+ $this->stdout("{$label} 已处理 {$processed} 条,已扫描到 id={$lastId}\n");
|
|
|
|
|
+ $this->sleepBetweenBatch();
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
- if ($this->sleepMs > 0 && $processed < $maxProcess) {
|
|
|
|
|
- usleep($this->sleepMs * 1000);
|
|
|
|
|
|
|
+ return $processed;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private function updateIds($label, $tableName, array $ids)
|
|
|
|
|
+ {
|
|
|
|
|
+ $db = Yii::$app->db;
|
|
|
|
|
+ $updated = 0;
|
|
|
|
|
+
|
|
|
|
|
+ foreach ($ids as $id) {
|
|
|
|
|
+ $id = (int)$id;
|
|
|
|
|
+ if ($id <= 0) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if ($this->dryRun) {
|
|
|
|
|
+ $this->stdout("[dry-run] {$label} id={$id}\n");
|
|
|
|
|
+ $updated++;
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ $salt = stringUtil::charsShuffleLowerCase(10);
|
|
|
|
|
+ $affected = $db->createCommand(
|
|
|
|
|
+ "UPDATE {$tableName} SET salt = :salt WHERE id = :id AND salt = ''"
|
|
|
|
|
+ )->bindValues([
|
|
|
|
|
+ ':salt' => $salt,
|
|
|
|
|
+ ':id' => $id,
|
|
|
|
|
+ ])->execute();
|
|
|
|
|
+
|
|
|
|
|
+ if ($affected > 0) {
|
|
|
|
|
+ $updated++;
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- $left = (int)$db->createCommand(
|
|
|
|
|
- "SELECT COUNT(*) FROM {$tableName} WHERE salt = ''"
|
|
|
|
|
- )->queryScalar();
|
|
|
|
|
- $this->stdout("{$label} 完成,本次处理 {$processed} 条,剩余 {$left} 条\n");
|
|
|
|
|
|
|
+ return $updated;
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
- return ExitCode::OK;
|
|
|
|
|
|
|
+ private function sleepBetweenBatch()
|
|
|
|
|
+ {
|
|
|
|
|
+ if ($this->sleepMs > 0) {
|
|
|
|
|
+ usleep($this->sleepMs * 1000);
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|