chore: init commit
This commit is contained in:
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
node_modules/
|
||||
28
package.json
Normal file
28
package.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "migrations",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "src/index.js",
|
||||
"scripts": {
|
||||
"start": "node src/index.js",
|
||||
"start:dry-run": "DRY_RUN=true node src/index.js"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"devEngines": {
|
||||
"packageManager": {
|
||||
"name": "pnpm",
|
||||
"version": "^11.5.1",
|
||||
"onFail": "download"
|
||||
}
|
||||
},
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.645.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.940.0",
|
||||
"mysql2": "^3.14.4",
|
||||
"reflect-metadata": "^0.2.2",
|
||||
"typeorm": "0.3.20"
|
||||
}
|
||||
}
|
||||
2523
pnpm-lock.yaml
generated
Normal file
2523
pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
18
src/data-source.js
Normal file
18
src/data-source.js
Normal file
@@ -0,0 +1,18 @@
|
||||
import "reflect-metadata";
|
||||
import { DataSource } from "typeorm";
|
||||
import { AssetEntity } from "./entity/Assets.js";
|
||||
import { BrandEntity } from "./entity/Brand.js";
|
||||
import { BrandAssetEntity } from "./entity/BrandAsset.js";
|
||||
|
||||
export const AppDataSource = new DataSource({
|
||||
type: "mysql",
|
||||
host: "95.156.253.15",
|
||||
port: 3306,
|
||||
username: "root",
|
||||
password: "Aa9923994970",
|
||||
database: "temporary",
|
||||
logging: false,
|
||||
entities: [AssetEntity, BrandEntity, BrandAssetEntity],
|
||||
migrations: [],
|
||||
subscribers: [],
|
||||
});
|
||||
51
src/entity/Assets.js
Normal file
51
src/entity/Assets.js
Normal file
@@ -0,0 +1,51 @@
|
||||
import { EntitySchema } from "typeorm";
|
||||
|
||||
export const AssetEntity = new EntitySchema({
|
||||
name: "AssetEntity",
|
||||
tableName: "assets",
|
||||
columns: {
|
||||
id: {
|
||||
type: Number,
|
||||
primary: true,
|
||||
generated: "increment",
|
||||
},
|
||||
name: {
|
||||
type: String,
|
||||
},
|
||||
type: {
|
||||
type: String,
|
||||
},
|
||||
size: {
|
||||
type: Number,
|
||||
},
|
||||
description: {
|
||||
type: "text",
|
||||
},
|
||||
path: {
|
||||
type: String,
|
||||
},
|
||||
isVisible: {
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
parentId: {
|
||||
type: Number,
|
||||
nullable: true,
|
||||
},
|
||||
createdBy: {
|
||||
type: String,
|
||||
},
|
||||
createdAt: {
|
||||
type: Date,
|
||||
},
|
||||
lastModifiedBy: {
|
||||
type: String,
|
||||
},
|
||||
lastModifiedAt: {
|
||||
type: Date,
|
||||
},
|
||||
deleted: {
|
||||
type: Boolean,
|
||||
},
|
||||
},
|
||||
});
|
||||
47
src/entity/Brand.js
Normal file
47
src/entity/Brand.js
Normal file
@@ -0,0 +1,47 @@
|
||||
import { EntitySchema } from "typeorm";
|
||||
|
||||
export const BrandEntity = new EntitySchema({
|
||||
name: "BrandEntity",
|
||||
tableName: "brand",
|
||||
columns: {
|
||||
id: {
|
||||
type: Number,
|
||||
primary: true,
|
||||
generated: true,
|
||||
},
|
||||
groupId: {
|
||||
type: String,
|
||||
},
|
||||
createdBy: {
|
||||
type: String,
|
||||
},
|
||||
createdAt: {
|
||||
type: Date,
|
||||
},
|
||||
lastModifiedBy: {
|
||||
type: String,
|
||||
},
|
||||
lastModifiedAt: {
|
||||
type: Date,
|
||||
},
|
||||
registerCode: {
|
||||
type: String,
|
||||
nullable: true,
|
||||
},
|
||||
typeId: {
|
||||
type: Number,
|
||||
nullable: true,
|
||||
},
|
||||
logoAssetId: {
|
||||
type: String,
|
||||
nullable: true,
|
||||
},
|
||||
urlName: {
|
||||
type: String,
|
||||
nullable: true,
|
||||
},
|
||||
deleted: {
|
||||
type: Boolean,
|
||||
},
|
||||
},
|
||||
});
|
||||
43
src/entity/BrandAsset.js
Normal file
43
src/entity/BrandAsset.js
Normal file
@@ -0,0 +1,43 @@
|
||||
import { EntitySchema } from "typeorm";
|
||||
|
||||
export const BrandAssetEntity = new EntitySchema({
|
||||
name: "BrandAssetEntity",
|
||||
tableName: "brand_asset",
|
||||
columns: {
|
||||
id: {
|
||||
type: Number,
|
||||
primary: true,
|
||||
generated: true,
|
||||
},
|
||||
groupId: {
|
||||
type: String,
|
||||
},
|
||||
assetId: {
|
||||
type: Number,
|
||||
},
|
||||
hash: {
|
||||
type: String,
|
||||
},
|
||||
createdBy: {
|
||||
type: String,
|
||||
},
|
||||
createdAt: {
|
||||
type: Date,
|
||||
},
|
||||
lastModifiedBy: {
|
||||
type: String,
|
||||
},
|
||||
lastModifiedAt: {
|
||||
type: Date,
|
||||
},
|
||||
color: {
|
||||
type: String,
|
||||
},
|
||||
general: {
|
||||
type: Boolean,
|
||||
},
|
||||
deleted: {
|
||||
type: Boolean,
|
||||
},
|
||||
},
|
||||
});
|
||||
21
src/entity/User.js
Normal file
21
src/entity/User.js
Normal file
@@ -0,0 +1,21 @@
|
||||
import { EntitySchema } from "typeorm";
|
||||
|
||||
export const User = new EntitySchema({
|
||||
name: "User",
|
||||
columns: {
|
||||
id: {
|
||||
type: Number,
|
||||
primary: true,
|
||||
generated: true,
|
||||
},
|
||||
firstName: {
|
||||
type: String,
|
||||
},
|
||||
lastName: {
|
||||
type: String,
|
||||
},
|
||||
age: {
|
||||
type: Number,
|
||||
},
|
||||
},
|
||||
});
|
||||
352
src/index.js
Normal file
352
src/index.js
Normal file
@@ -0,0 +1,352 @@
|
||||
import {
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { AppDataSource } from "./data-source.js";
|
||||
import { AssetEntity } from "./entity/Assets.js";
|
||||
import { BrandEntity } from "./entity/Brand.js";
|
||||
import { BrandAssetEntity } from "./entity/BrandAsset.js";
|
||||
|
||||
const AVATAR_PREFIX = "avatar";
|
||||
const THUMBNAIL_PREFIX = "lg";
|
||||
const DRY_RUN = process.env.DRY_RUN === "true";
|
||||
|
||||
const SOURCE_S3 = {
|
||||
endpoint: process.env.SOURCE_S3_ENDPOINT ?? "http://95.156.253.15:9001/",
|
||||
region: process.env.SOURCE_S3_REGION ?? "default",
|
||||
accessKeyId: process.env.SOURCE_S3_ACCESS_KEY ?? "sirCxqNKw85GdnlLfzko",
|
||||
secretAccessKey:
|
||||
process.env.SOURCE_S3_SECRET_KEY ??
|
||||
"qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w",
|
||||
bucket: process.env.SOURCE_S3_BUCKET ?? "test",
|
||||
};
|
||||
|
||||
const DEST_S3 = {
|
||||
endpoint:
|
||||
process.env.DEST_S3_ENDPOINT ??
|
||||
process.env.SOURCE_S3_ENDPOINT ??
|
||||
"http://95.156.253.15:9001/",
|
||||
region:
|
||||
process.env.DEST_S3_REGION ?? process.env.SOURCE_S3_REGION ?? "default",
|
||||
accessKeyId:
|
||||
process.env.DEST_S3_ACCESS_KEY ??
|
||||
process.env.SOURCE_S3_ACCESS_KEY ??
|
||||
"sirCxqNKw85GdnlLfzko",
|
||||
secretAccessKey:
|
||||
process.env.DEST_S3_SECRET_KEY ??
|
||||
process.env.SOURCE_S3_SECRET_KEY ??
|
||||
"qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w",
|
||||
bucket: process.env.DEST_S3_BUCKET ?? "migrations",
|
||||
};
|
||||
|
||||
function createS3Client(config) {
|
||||
return new S3Client({
|
||||
region: config.region,
|
||||
endpoint: config.endpoint,
|
||||
forcePathStyle: true,
|
||||
credentials: {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function getAssetById(id) {
|
||||
const assetRepository = AppDataSource.getRepository(AssetEntity);
|
||||
return assetRepository.findOneBy({ id });
|
||||
}
|
||||
|
||||
async function getBrandAssetsByGroupId(groupId) {
|
||||
const brandAssetRepo = AppDataSource.getRepository(BrandAssetEntity);
|
||||
return brandAssetRepo.find({
|
||||
where: {
|
||||
groupId,
|
||||
deleted: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function getBrandUrlNames() {
|
||||
const brandRepo = AppDataSource.getRepository(BrandEntity);
|
||||
const brands = await brandRepo.find({
|
||||
where: {
|
||||
deleted: false,
|
||||
},
|
||||
});
|
||||
|
||||
const seen = new Set();
|
||||
const result = [];
|
||||
|
||||
for (const brand of brands) {
|
||||
if (!brand.urlName) continue;
|
||||
|
||||
const key = `${brand.groupId}:${brand.urlName.toLowerCase()}`;
|
||||
if (seen.has(key)) continue;
|
||||
|
||||
seen.add(key);
|
||||
result.push({
|
||||
name: brand.urlName,
|
||||
groupId: brand.groupId,
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function resolveAssetSourceKey(asset) {
|
||||
return asset.id.toString();
|
||||
}
|
||||
|
||||
async function objectExists(client, bucket, key) {
|
||||
try {
|
||||
await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
|
||||
return true;
|
||||
} catch (error) {
|
||||
const statusCode =
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
"$metadata" in error &&
|
||||
typeof error.$metadata === "object" &&
|
||||
error.$metadata !== null &&
|
||||
"httpStatusCode" in error.$metadata
|
||||
? error.$metadata.httpStatusCode
|
||||
: undefined;
|
||||
|
||||
if (statusCode === 404) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function copyFileCrossClient(
|
||||
sourceClient,
|
||||
destClient,
|
||||
sourceBucket,
|
||||
destBucket,
|
||||
sourceKey,
|
||||
destKey,
|
||||
) {
|
||||
const response = await sourceClient.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: sourceBucket,
|
||||
Key: sourceKey,
|
||||
}),
|
||||
);
|
||||
|
||||
if (!response.Body) {
|
||||
throw new Error(`Empty response body for ${sourceBucket}/${sourceKey}`);
|
||||
}
|
||||
|
||||
await destClient.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: destBucket,
|
||||
Key: destKey,
|
||||
Body: response.Body,
|
||||
...(response.ContentType ? { ContentType: response.ContentType } : {}),
|
||||
...(response.ContentLength !== undefined
|
||||
? { ContentLength: response.ContentLength }
|
||||
: {}),
|
||||
...(response.Metadata ? { Metadata: response.Metadata } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function copyIfNeeded(
|
||||
sourceClient,
|
||||
destClient,
|
||||
sourceBucket,
|
||||
destBucket,
|
||||
sourceKey,
|
||||
destKey,
|
||||
dryRun,
|
||||
type = "file",
|
||||
) {
|
||||
if (type !== "directory") {
|
||||
const sourceExists = await objectExists(
|
||||
sourceClient,
|
||||
sourceBucket,
|
||||
sourceKey,
|
||||
);
|
||||
if (!sourceExists) return "missing";
|
||||
}
|
||||
|
||||
const destExists = await objectExists(destClient, destBucket, destKey);
|
||||
if (destExists) return "skipped";
|
||||
|
||||
if (dryRun) {
|
||||
console.log(
|
||||
`[dry-run] copy ${sourceBucket}/${sourceKey} -> ${destBucket}/${destKey}`,
|
||||
);
|
||||
return "copied";
|
||||
}
|
||||
|
||||
if (type === "file") {
|
||||
await copyFileCrossClient(
|
||||
sourceClient,
|
||||
destClient,
|
||||
sourceBucket,
|
||||
destBucket,
|
||||
sourceKey,
|
||||
destKey,
|
||||
);
|
||||
} else {
|
||||
await destClient.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: destBucket,
|
||||
Key: destKey,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return "copied";
|
||||
}
|
||||
|
||||
async function createBrandsFolders(sourceClient, destClient, brands, stats) {
|
||||
for (const brand of brands) {
|
||||
const folderKey = `${brand.name.toLowerCase()}/`;
|
||||
const result = await copyIfNeeded(
|
||||
sourceClient,
|
||||
destClient,
|
||||
SOURCE_S3.bucket,
|
||||
DEST_S3.bucket,
|
||||
"",
|
||||
folderKey,
|
||||
DRY_RUN,
|
||||
"directory",
|
||||
);
|
||||
|
||||
if (result === "copied") stats.foldersCreated++;
|
||||
if (result === "skipped") stats.foldersSkipped++;
|
||||
}
|
||||
}
|
||||
|
||||
async function copyAssetVariant(
|
||||
sourceClient,
|
||||
destClient,
|
||||
sourceKey,
|
||||
destKey,
|
||||
stats,
|
||||
) {
|
||||
const result = await copyIfNeeded(
|
||||
sourceClient,
|
||||
destClient,
|
||||
SOURCE_S3.bucket,
|
||||
DEST_S3.bucket,
|
||||
sourceKey,
|
||||
destKey,
|
||||
DRY_RUN,
|
||||
"file",
|
||||
);
|
||||
|
||||
if (result === "copied") stats.assetsCopied++;
|
||||
if (result === "skipped") stats.assetsSkipped++;
|
||||
if (result === "missing") stats.assetsMissing++;
|
||||
}
|
||||
|
||||
async function copyBrandAssets(sourceClient, destClient, brands, stats) {
|
||||
for (const brand of brands) {
|
||||
const brandAssets = await getBrandAssetsByGroupId(brand.groupId);
|
||||
const brandFolder = brand.name.toLowerCase();
|
||||
|
||||
console.log(
|
||||
`Processing brand "${brand.name}" (groupId=${brand.groupId}, assets=${brandAssets.length})`,
|
||||
);
|
||||
|
||||
for (const brandAsset of brandAssets) {
|
||||
const asset = await getAssetById(brandAsset.assetId);
|
||||
if (!asset || asset.deleted) {
|
||||
console.warn(
|
||||
`Skipping brand_asset ${brandAsset.id}: asset ${brandAsset.assetId} not found or deleted`,
|
||||
);
|
||||
stats.assetsMissing++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const sourceKey = resolveAssetSourceKey(asset);
|
||||
const destBase = `${brandFolder}/${asset.id}`;
|
||||
|
||||
try {
|
||||
await copyAssetVariant(
|
||||
sourceClient,
|
||||
destClient,
|
||||
sourceKey,
|
||||
destBase,
|
||||
stats,
|
||||
);
|
||||
// await copyAssetVariant(
|
||||
// sourceClient,
|
||||
// destClient,
|
||||
// `${AVATAR_PREFIX}/${sourceKey}`,
|
||||
// `${destBase}/${AVATAR_PREFIX}`,
|
||||
// stats,
|
||||
// );
|
||||
// await copyAssetVariant(
|
||||
// sourceClient,
|
||||
// destClient,
|
||||
// `${THUMBNAIL_PREFIX}/${sourceKey}`,
|
||||
// `${destBase}/${THUMBNAIL_PREFIX}`,
|
||||
// stats,
|
||||
// );
|
||||
} catch (error) {
|
||||
stats.errors++;
|
||||
console.error(
|
||||
`Failed to copy asset ${asset.id} for brand "${brand.name}":`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function printSummary(stats) {
|
||||
console.log("\nMigration summary:");
|
||||
console.log(` Folders created: ${stats.foldersCreated}`);
|
||||
console.log(` Folders skipped: ${stats.foldersSkipped}`);
|
||||
console.log(` Assets copied: ${stats.assetsCopied}`);
|
||||
console.log(` Assets skipped: ${stats.assetsSkipped}`);
|
||||
console.log(` Assets missing: ${stats.assetsMissing}`);
|
||||
console.log(` Errors: ${stats.errors}`);
|
||||
if (DRY_RUN) console.log(" (dry-run mode — no objects were written)");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const sourceClient = createS3Client(SOURCE_S3);
|
||||
const destClient = createS3Client(DEST_S3);
|
||||
const stats = {
|
||||
foldersCreated: 0,
|
||||
foldersSkipped: 0,
|
||||
assetsCopied: 0,
|
||||
assetsSkipped: 0,
|
||||
assetsMissing: 0,
|
||||
errors: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
await AppDataSource.initialize();
|
||||
console.log("Data source initialized");
|
||||
console.log(`Source S3: ${SOURCE_S3.endpoint} (${SOURCE_S3.bucket})`);
|
||||
console.log(`Dest S3: ${DEST_S3.endpoint} (${DEST_S3.bucket})`);
|
||||
|
||||
const brands = await getBrandUrlNames();
|
||||
console.log(`Found ${brands.length} brands to migrate`);
|
||||
|
||||
await createBrandsFolders(sourceClient, destClient, brands, stats);
|
||||
await copyBrandAssets(sourceClient, destClient, brands, stats);
|
||||
|
||||
printSummary(stats);
|
||||
|
||||
if (stats.errors > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Migration failed:", err);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
if (AppDataSource.isInitialized) {
|
||||
await AppDataSource.destroy();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user