refactor: adding logger and catch blocks
This commit is contained in:
11
.env.example
11
.env.example
@@ -4,8 +4,9 @@ SOURCE_S3_ACCESS_KEY="sirCxqNKw85GdnlLfzko"
|
||||
SOURCE_S3_SECRET_KEY="qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w"
|
||||
SOURCE_S3_BUCKET="test"
|
||||
|
||||
DEST_S3_ENDPOINT="http://95.156.253.15:9001/"
|
||||
DEST_S3_REGION="default"
|
||||
DEST_S3_ACCESS_KEY="sirCxqNKw85GdnlLfzko"
|
||||
DEST_S3_SECRET_KEY="qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w"
|
||||
DEST_S3_BUCKET="test"
|
||||
# ParsPack: endpoint is https://<bucket-id>.parspack.net and bucket name is the same id (e.g. c530168)
|
||||
DEST_S3_ENDPOINT="https://c530168.parspack.net"
|
||||
DEST_S3_REGION="us-west-2"
|
||||
DEST_S3_ACCESS_KEY="your-access-key"
|
||||
DEST_S3_SECRET_KEY="your-secret-key"
|
||||
DEST_S3_BUCKET="c530168"
|
||||
@@ -3,16 +3,26 @@ import { DataSource } from "typeorm";
|
||||
import { AssetEntity } from "./entity/Assets.js";
|
||||
import { BrandEntity } from "./entity/Brand.js";
|
||||
import { BrandAssetEntity } from "./entity/BrandAsset.js";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
export const AppDataSource = new DataSource({
|
||||
const dbConfig = {
|
||||
type: "mysql",
|
||||
host: "95.156.253.15",
|
||||
port: 3306,
|
||||
username: "root",
|
||||
password: "Aa9923994970",
|
||||
database: "temporary",
|
||||
logging: false,
|
||||
host: "31.214.255.158",
|
||||
port: 3301,
|
||||
username: "brandprod",
|
||||
password: "cdcc9b7152905d21baa7758586fed2aa",
|
||||
database: "brandifa_production",
|
||||
logging: process.env.DB_LOGGING === "true",
|
||||
entities: [AssetEntity, BrandEntity, BrandAssetEntity],
|
||||
migrations: [],
|
||||
subscribers: [],
|
||||
};
|
||||
|
||||
logger.info("data-source", "creating AppDataSource", {
|
||||
host: dbConfig.host,
|
||||
port: dbConfig.port,
|
||||
database: dbConfig.database,
|
||||
logging: dbConfig.logging,
|
||||
});
|
||||
|
||||
export const AppDataSource = new DataSource(dbConfig);
|
||||
|
||||
592
src/index.js
592
src/index.js
@@ -1,4 +1,4 @@
|
||||
import "dotenv/config"
|
||||
import "dotenv/config";
|
||||
import {
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
@@ -9,11 +9,32 @@ import { AppDataSource } from "./data-source.js";
|
||||
import { AssetEntity } from "./entity/Assets.js";
|
||||
import { BrandEntity } from "./entity/Brand.js";
|
||||
import { BrandAssetEntity } from "./entity/BrandAsset.js";
|
||||
import { logger, runStep } from "./logger.js";
|
||||
|
||||
const AVATAR_PREFIX = "avatar";
|
||||
const THUMBNAIL_PREFIX = "lg";
|
||||
const DRY_RUN = process.env.DRY_RUN === "true";
|
||||
|
||||
function resolveParsPackBucket(endpoint, bucket) {
|
||||
const match = endpoint?.match(/https?:\/\/(c\d+)\.parspack\.net/i);
|
||||
if (!match) return bucket;
|
||||
|
||||
const bucketId = match[1];
|
||||
if (bucket !== bucketId) {
|
||||
logger.warn(
|
||||
"resolveParsPackBucket",
|
||||
`ParsPack bucket name must match the endpoint id; using "${bucketId}" instead of "${bucket}"`,
|
||||
);
|
||||
}
|
||||
|
||||
return bucketId;
|
||||
}
|
||||
|
||||
function normalizeParsPackEndpoint(endpoint) {
|
||||
if (!endpoint?.includes("parspack.net")) return endpoint;
|
||||
return endpoint.replace(/^http:/i, "https:").replace(/\/$/, "");
|
||||
}
|
||||
|
||||
const SOURCE_S3 = {
|
||||
endpoint: process.env.SOURCE_S3_ENDPOINT ?? "http://95.156.253.15:9001/",
|
||||
region: process.env.SOURCE_S3_REGION ?? "default",
|
||||
@@ -24,13 +45,21 @@ const SOURCE_S3 = {
|
||||
bucket: process.env.SOURCE_S3_BUCKET ?? "test",
|
||||
};
|
||||
|
||||
const DEST_S3 = {
|
||||
endpoint:
|
||||
process.env.DEST_S3_ENDPOINT ??
|
||||
const destEndpoint = normalizeParsPackEndpoint(
|
||||
process.env.DEST_S3_ENDPOINT ??
|
||||
process.env.SOURCE_S3_ENDPOINT ??
|
||||
"http://95.156.253.15:9001/",
|
||||
);
|
||||
const destBucket = resolveParsPackBucket(
|
||||
destEndpoint,
|
||||
process.env.DEST_S3_BUCKET ?? "migrations",
|
||||
);
|
||||
|
||||
const DEST_S3 = {
|
||||
endpoint: destEndpoint,
|
||||
region:
|
||||
process.env.DEST_S3_REGION ?? process.env.SOURCE_S3_REGION ?? "default",
|
||||
process.env.DEST_S3_REGION ??
|
||||
(destEndpoint.includes("parspack.net") ? "us-west-2" : process.env.SOURCE_S3_REGION ?? "default"),
|
||||
accessKeyId:
|
||||
process.env.DEST_S3_ACCESS_KEY ??
|
||||
process.env.SOURCE_S3_ACCESS_KEY ??
|
||||
@@ -39,61 +68,105 @@ const DEST_S3 = {
|
||||
process.env.DEST_S3_SECRET_KEY ??
|
||||
process.env.SOURCE_S3_SECRET_KEY ??
|
||||
"qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w",
|
||||
bucket: process.env.DEST_S3_BUCKET ?? "migrations",
|
||||
bucket: destBucket,
|
||||
};
|
||||
|
||||
function createS3Client(config) {
|
||||
return new S3Client({
|
||||
region: config.region,
|
||||
function createS3Client(config, label) {
|
||||
const step = `createS3Client:${label}`;
|
||||
logger.info(step, "creating S3 client", {
|
||||
endpoint: config.endpoint,
|
||||
forcePathStyle: true,
|
||||
credentials: {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
},
|
||||
region: config.region,
|
||||
bucket: config.bucket,
|
||||
});
|
||||
|
||||
try {
|
||||
const client = new S3Client({
|
||||
region: config.region,
|
||||
endpoint: config.endpoint,
|
||||
forcePathStyle: true,
|
||||
credentials: {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
},
|
||||
});
|
||||
logger.info(step, "S3 client created");
|
||||
return client;
|
||||
} catch (error) {
|
||||
logger.error(step, "failed to create S3 client", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function getAssetById(id) {
|
||||
const assetRepository = AppDataSource.getRepository(AssetEntity);
|
||||
return assetRepository.findOneBy({ id });
|
||||
const step = `getAssetById:${id}`;
|
||||
try {
|
||||
const assetRepository = AppDataSource.getRepository(AssetEntity);
|
||||
const asset = await assetRepository.findOneBy({ id });
|
||||
logger.debug(step, asset ? "asset found" : "asset not found");
|
||||
return asset;
|
||||
} catch (error) {
|
||||
logger.error(step, "database query failed", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function getBrandAssetsByGroupId(groupId) {
|
||||
const brandAssetRepo = AppDataSource.getRepository(BrandAssetEntity);
|
||||
return brandAssetRepo.find({
|
||||
where: {
|
||||
groupId,
|
||||
deleted: false,
|
||||
},
|
||||
});
|
||||
const step = `getBrandAssetsByGroupId:${groupId}`;
|
||||
try {
|
||||
const brandAssetRepo = AppDataSource.getRepository(BrandAssetEntity);
|
||||
const brandAssets = await brandAssetRepo.find({
|
||||
where: {
|
||||
groupId,
|
||||
deleted: false,
|
||||
},
|
||||
});
|
||||
logger.debug(step, `found ${brandAssets.length} brand assets`);
|
||||
return brandAssets;
|
||||
} catch (error) {
|
||||
logger.error(step, "database query failed", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function getBrandUrlNames() {
|
||||
const brandRepo = AppDataSource.getRepository(BrandEntity);
|
||||
const brands = await brandRepo.find({
|
||||
where: {
|
||||
deleted: false,
|
||||
},
|
||||
});
|
||||
|
||||
const seen = new Set();
|
||||
const result = [];
|
||||
|
||||
for (const brand of brands) {
|
||||
if (!brand.urlName) continue;
|
||||
|
||||
const key = `${brand.groupId}:${brand.urlName.toLowerCase()}`;
|
||||
if (seen.has(key)) continue;
|
||||
|
||||
seen.add(key);
|
||||
result.push({
|
||||
name: brand.urlName,
|
||||
groupId: brand.groupId,
|
||||
const step = "getBrandUrlNames";
|
||||
try {
|
||||
const brandRepo = AppDataSource.getRepository(BrandEntity);
|
||||
const brands = await brandRepo.find({
|
||||
where: {
|
||||
deleted: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
logger.info(step, `loaded ${brands.length} brands from database`);
|
||||
|
||||
return result;
|
||||
const seen = new Set();
|
||||
const result = [];
|
||||
|
||||
for (const brand of brands) {
|
||||
if (!brand.urlName) {
|
||||
logger.debug(step, `skipping brand id=${brand.id}: missing urlName`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const key = `${brand.groupId}:${brand.urlName.toLowerCase()}`;
|
||||
if (seen.has(key)) {
|
||||
logger.debug(step, `skipping duplicate brand urlName=${brand.urlName}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
seen.add(key);
|
||||
result.push({
|
||||
name: brand.urlName,
|
||||
groupId: brand.groupId,
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(step, `resolved ${result.length} unique brands`);
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error(step, "failed to load brands", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveAssetSourceKey(asset) {
|
||||
@@ -101,8 +174,10 @@ function resolveAssetSourceKey(asset) {
|
||||
}
|
||||
|
||||
async function objectExists(client, bucket, key) {
|
||||
const step = `objectExists:${bucket}/${key}`;
|
||||
try {
|
||||
await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
|
||||
logger.debug(step, "object exists");
|
||||
return true;
|
||||
} catch (error) {
|
||||
const statusCode =
|
||||
@@ -115,7 +190,12 @@ async function objectExists(client, bucket, key) {
|
||||
? error.$metadata.httpStatusCode
|
||||
: undefined;
|
||||
|
||||
if (statusCode === 404) return false;
|
||||
if (statusCode === 404) {
|
||||
logger.debug(step, "object not found");
|
||||
return false;
|
||||
}
|
||||
|
||||
logger.error(step, "head object failed", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -128,29 +208,43 @@ async function copyFileCrossClient(
|
||||
sourceKey,
|
||||
destKey,
|
||||
) {
|
||||
const response = await sourceClient.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: sourceBucket,
|
||||
Key: sourceKey,
|
||||
}),
|
||||
);
|
||||
const step = `copyFileCrossClient:${sourceBucket}/${sourceKey}->${destBucket}/${destKey}`;
|
||||
try {
|
||||
logger.debug(step, "fetching source object");
|
||||
const response = await sourceClient.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: sourceBucket,
|
||||
Key: sourceKey,
|
||||
}),
|
||||
);
|
||||
|
||||
if (!response.Body) {
|
||||
throw new Error(`Empty response body for ${sourceBucket}/${sourceKey}`);
|
||||
if (!response.Body) {
|
||||
throw new Error(`Empty response body for ${sourceBucket}/${sourceKey}`);
|
||||
}
|
||||
|
||||
logger.debug(step, "uploading to destination", {
|
||||
contentType: response.ContentType,
|
||||
contentLength: response.ContentLength,
|
||||
});
|
||||
|
||||
await destClient.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: destBucket,
|
||||
Key: destKey,
|
||||
Body: response.Body,
|
||||
...(response.ContentType ? { ContentType: response.ContentType } : {}),
|
||||
...(response.ContentLength !== undefined
|
||||
? { ContentLength: response.ContentLength }
|
||||
: {}),
|
||||
...(response.Metadata ? { Metadata: response.Metadata } : {}),
|
||||
}),
|
||||
);
|
||||
|
||||
logger.info(step, "copy completed");
|
||||
} catch (error) {
|
||||
logger.error(step, "copy failed", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
|
||||
await destClient.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: destBucket,
|
||||
Key: destKey,
|
||||
Body: response.Body,
|
||||
...(response.ContentType ? { ContentType: response.ContentType } : {}),
|
||||
...(response.ContentLength !== undefined
|
||||
? { ContentLength: response.ContentLength }
|
||||
: {}),
|
||||
...(response.Metadata ? { Metadata: response.Metadata } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function copyIfNeeded(
|
||||
@@ -163,62 +257,93 @@ async function copyIfNeeded(
|
||||
dryRun,
|
||||
type = "file",
|
||||
) {
|
||||
if (type !== "directory") {
|
||||
const sourceExists = await objectExists(
|
||||
sourceClient,
|
||||
sourceBucket,
|
||||
sourceKey,
|
||||
);
|
||||
if (!sourceExists) return "missing";
|
||||
}
|
||||
const step = `copyIfNeeded:${type}:${destBucket}/${destKey}`;
|
||||
try {
|
||||
if (type !== "directory") {
|
||||
const sourceExists = await objectExists(
|
||||
sourceClient,
|
||||
sourceBucket,
|
||||
sourceKey,
|
||||
);
|
||||
if (!sourceExists) {
|
||||
logger.warn(step, "source object missing", {
|
||||
sourceBucket,
|
||||
sourceKey,
|
||||
});
|
||||
return "missing";
|
||||
}
|
||||
}
|
||||
|
||||
const destExists = await objectExists(destClient, destBucket, destKey);
|
||||
if (destExists) return "skipped";
|
||||
const destExists = await objectExists(destClient, destBucket, destKey);
|
||||
if (destExists) {
|
||||
logger.debug(step, "destination already exists, skipping");
|
||||
return "skipped";
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
console.log(
|
||||
`[dry-run] copy ${sourceBucket}/${sourceKey} -> ${destBucket}/${destKey}`,
|
||||
);
|
||||
if (dryRun) {
|
||||
logger.info(step, "[dry-run] would copy object", {
|
||||
source: `${sourceBucket}/${sourceKey}`,
|
||||
destination: `${destBucket}/${destKey}`,
|
||||
});
|
||||
return "copied";
|
||||
}
|
||||
|
||||
if (type === "file") {
|
||||
await copyFileCrossClient(
|
||||
sourceClient,
|
||||
destClient,
|
||||
sourceBucket,
|
||||
destBucket,
|
||||
sourceKey,
|
||||
destKey,
|
||||
);
|
||||
} else {
|
||||
logger.debug(step, "creating directory marker");
|
||||
await destClient.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: destBucket,
|
||||
Key: destKey,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
logger.info(step, "copy action completed", { result: "copied" });
|
||||
return "copied";
|
||||
} catch (error) {
|
||||
logger.error(step, "copyIfNeeded failed", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (type === "file") {
|
||||
await copyFileCrossClient(
|
||||
sourceClient,
|
||||
destClient,
|
||||
sourceBucket,
|
||||
destBucket,
|
||||
sourceKey,
|
||||
destKey,
|
||||
);
|
||||
} else {
|
||||
await destClient.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: destBucket,
|
||||
Key: destKey,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return "copied";
|
||||
}
|
||||
|
||||
async function createBrandsFolders(sourceClient, destClient, brands, stats) {
|
||||
for (const brand of brands) {
|
||||
const folderKey = `${brand.name.toLowerCase()}/`;
|
||||
const result = await copyIfNeeded(
|
||||
sourceClient,
|
||||
destClient,
|
||||
SOURCE_S3.bucket,
|
||||
DEST_S3.bucket,
|
||||
"",
|
||||
folderKey,
|
||||
DRY_RUN,
|
||||
"directory",
|
||||
);
|
||||
const step = "createBrandsFolders";
|
||||
logger.info(step, `creating folders for ${brands.length} brands`);
|
||||
|
||||
if (result === "copied") stats.foldersCreated++;
|
||||
if (result === "skipped") stats.foldersSkipped++;
|
||||
for (const brand of brands) {
|
||||
const brandStep = `${step}:${brand.name}`;
|
||||
try {
|
||||
const folderKey = `${brand.name.toLowerCase()}/`;
|
||||
logger.debug(brandStep, "creating folder", { folderKey });
|
||||
|
||||
const result = await copyIfNeeded(
|
||||
sourceClient,
|
||||
destClient,
|
||||
SOURCE_S3.bucket,
|
||||
DEST_S3.bucket,
|
||||
"",
|
||||
folderKey,
|
||||
DRY_RUN,
|
||||
"directory",
|
||||
);
|
||||
|
||||
if (result === "copied") stats.foldersCreated++;
|
||||
if (result === "skipped") stats.foldersSkipped++;
|
||||
|
||||
logger.info(brandStep, "folder step completed", { result });
|
||||
} catch (error) {
|
||||
stats.errors++;
|
||||
logger.error(brandStep, "folder creation failed", logger.formatError(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,91 +354,144 @@ async function copyAssetVariant(
|
||||
destKey,
|
||||
stats,
|
||||
) {
|
||||
const result = await copyIfNeeded(
|
||||
sourceClient,
|
||||
destClient,
|
||||
SOURCE_S3.bucket,
|
||||
DEST_S3.bucket,
|
||||
sourceKey,
|
||||
destKey,
|
||||
DRY_RUN,
|
||||
"file",
|
||||
);
|
||||
const step = `copyAssetVariant:${destKey}`;
|
||||
try {
|
||||
const result = await copyIfNeeded(
|
||||
sourceClient,
|
||||
destClient,
|
||||
SOURCE_S3.bucket,
|
||||
DEST_S3.bucket,
|
||||
sourceKey,
|
||||
destKey,
|
||||
DRY_RUN,
|
||||
"file",
|
||||
);
|
||||
|
||||
if (result === "copied") stats.assetsCopied++;
|
||||
if (result === "skipped") stats.assetsSkipped++;
|
||||
if (result === "missing") stats.assetsMissing++;
|
||||
if (result === "copied") stats.assetsCopied++;
|
||||
if (result === "skipped") stats.assetsSkipped++;
|
||||
if (result === "missing") stats.assetsMissing++;
|
||||
|
||||
logger.debug(step, "variant copy completed", { result });
|
||||
} catch (error) {
|
||||
logger.error(step, "variant copy failed", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function copyBrandAssets(sourceClient, destClient, brands, stats) {
|
||||
const step = "copyBrandAssets";
|
||||
logger.info(step, `copying assets for ${brands.length} brands`);
|
||||
|
||||
for (const brand of brands) {
|
||||
const brandAssets = await getBrandAssetsByGroupId(brand.groupId);
|
||||
const brandFolder = brand.name.toLowerCase();
|
||||
const brandStep = `${step}:${brand.name}`;
|
||||
try {
|
||||
const brandAssets = await getBrandAssetsByGroupId(brand.groupId);
|
||||
const brandFolder = brand.name.toLowerCase();
|
||||
|
||||
console.log(
|
||||
`Processing brand "${brand.name}" (groupId=${brand.groupId}, assets=${brandAssets.length})`,
|
||||
);
|
||||
logger.info(brandStep, "processing brand assets", {
|
||||
groupId: brand.groupId,
|
||||
assetCount: brandAssets.length,
|
||||
});
|
||||
|
||||
for (const brandAsset of brandAssets) {
|
||||
const asset = await getAssetById(brandAsset.assetId);
|
||||
if (!asset || asset.deleted) {
|
||||
console.warn(
|
||||
`Skipping brand_asset ${brandAsset.id}: asset ${brandAsset.assetId} not found or deleted`,
|
||||
);
|
||||
stats.assetsMissing++;
|
||||
continue;
|
||||
for (const brandAsset of brandAssets) {
|
||||
const assetStep = `${brandStep}:asset:${brandAsset.assetId}`;
|
||||
try {
|
||||
const asset = await getAssetById(brandAsset.assetId);
|
||||
if (!asset || asset.deleted) {
|
||||
logger.warn(assetStep, "asset not found or deleted", {
|
||||
brandAssetId: brandAsset.id,
|
||||
assetId: brandAsset.assetId,
|
||||
});
|
||||
stats.assetsMissing++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const sourceKey = resolveAssetSourceKey(asset);
|
||||
const destBase = `${brandFolder}/${asset.id}`;
|
||||
|
||||
await copyAssetVariant(
|
||||
sourceClient,
|
||||
destClient,
|
||||
sourceKey,
|
||||
destBase,
|
||||
stats,
|
||||
);
|
||||
// await copyAssetVariant(
|
||||
// sourceClient,
|
||||
// destClient,
|
||||
// `${AVATAR_PREFIX}/${sourceKey}`,
|
||||
// `${destBase}/${AVATAR_PREFIX}`,
|
||||
// stats,
|
||||
// );
|
||||
// await copyAssetVariant(
|
||||
// sourceClient,
|
||||
// destClient,
|
||||
// `${THUMBNAIL_PREFIX}/${sourceKey}`,
|
||||
// `${destBase}/${THUMBNAIL_PREFIX}`,
|
||||
// stats,
|
||||
// );
|
||||
} catch (error) {
|
||||
stats.errors++;
|
||||
logger.error(assetStep, "asset copy failed", logger.formatError(error));
|
||||
}
|
||||
}
|
||||
|
||||
const sourceKey = resolveAssetSourceKey(asset);
|
||||
const destBase = `${brandFolder}/${asset.id}`;
|
||||
|
||||
try {
|
||||
await copyAssetVariant(
|
||||
sourceClient,
|
||||
destClient,
|
||||
sourceKey,
|
||||
destBase,
|
||||
stats,
|
||||
);
|
||||
// await copyAssetVariant(
|
||||
// sourceClient,
|
||||
// destClient,
|
||||
// `${AVATAR_PREFIX}/${sourceKey}`,
|
||||
// `${destBase}/${AVATAR_PREFIX}`,
|
||||
// stats,
|
||||
// );
|
||||
// await copyAssetVariant(
|
||||
// sourceClient,
|
||||
// destClient,
|
||||
// `${THUMBNAIL_PREFIX}/${sourceKey}`,
|
||||
// `${destBase}/${THUMBNAIL_PREFIX}`,
|
||||
// stats,
|
||||
// );
|
||||
} catch (error) {
|
||||
stats.errors++;
|
||||
console.error(
|
||||
`Failed to copy asset ${asset.id} for brand "${brand.name}":`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
logger.info(brandStep, "brand assets completed");
|
||||
} catch (error) {
|
||||
stats.errors++;
|
||||
logger.error(brandStep, "brand processing failed", logger.formatError(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function printSummary(stats) {
|
||||
console.log("\nMigration summary:");
|
||||
console.log(` Folders created: ${stats.foldersCreated}`);
|
||||
console.log(` Folders skipped: ${stats.foldersSkipped}`);
|
||||
console.log(` Assets copied: ${stats.assetsCopied}`);
|
||||
console.log(` Assets skipped: ${stats.assetsSkipped}`);
|
||||
console.log(` Assets missing: ${stats.assetsMissing}`);
|
||||
console.log(` Errors: ${stats.errors}`);
|
||||
if (DRY_RUN) console.log(" (dry-run mode — no objects were written)");
|
||||
const step = "printSummary";
|
||||
logger.info(step, "migration summary", {
|
||||
foldersCreated: stats.foldersCreated,
|
||||
foldersSkipped: stats.foldersSkipped,
|
||||
assetsCopied: stats.assetsCopied,
|
||||
assetsSkipped: stats.assetsSkipped,
|
||||
assetsMissing: stats.assetsMissing,
|
||||
errors: stats.errors,
|
||||
dryRun: DRY_RUN,
|
||||
});
|
||||
}
|
||||
|
||||
async function initializeDataSource() {
|
||||
const step = "initializeDataSource";
|
||||
try {
|
||||
await AppDataSource.initialize();
|
||||
logger.info(step, "data source initialized", {
|
||||
host: AppDataSource.options.host,
|
||||
port: AppDataSource.options.port,
|
||||
database: AppDataSource.options.database,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(step, "data source initialization failed", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function destroyDataSource() {
|
||||
const step = "destroyDataSource";
|
||||
if (!AppDataSource.isInitialized) {
|
||||
logger.debug(step, "data source not initialized, skipping destroy");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await AppDataSource.destroy();
|
||||
logger.info(step, "data source destroyed");
|
||||
} catch (error) {
|
||||
logger.error(step, "data source destroy failed", logger.formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const sourceClient = createS3Client(SOURCE_S3);
|
||||
const destClient = createS3Client(DEST_S3);
|
||||
const step = "main";
|
||||
logger.info(step, "migration started", { dryRun: DRY_RUN });
|
||||
|
||||
const stats = {
|
||||
foldersCreated: 0,
|
||||
foldersSkipped: 0,
|
||||
@@ -324,30 +502,52 @@ async function main() {
|
||||
};
|
||||
|
||||
try {
|
||||
await AppDataSource.initialize();
|
||||
console.log("Data source initialized");
|
||||
console.log(`Source S3: ${SOURCE_S3.endpoint} (${SOURCE_S3.bucket})`);
|
||||
console.log(`Dest S3: ${DEST_S3.endpoint} (${DEST_S3.bucket})`);
|
||||
const sourceClient = await runStep("createSourceS3Client", async () =>
|
||||
createS3Client(SOURCE_S3, "source"),
|
||||
);
|
||||
const destClient = await runStep("createDestS3Client", async () =>
|
||||
createS3Client(DEST_S3, "dest"),
|
||||
);
|
||||
|
||||
const brands = await getBrandUrlNames();
|
||||
console.log(`Found ${brands.length} brands to migrate`);
|
||||
logger.info(step, "S3 configuration", {
|
||||
source: { endpoint: SOURCE_S3.endpoint, bucket: SOURCE_S3.bucket },
|
||||
dest: { endpoint: DEST_S3.endpoint, bucket: DEST_S3.bucket },
|
||||
});
|
||||
|
||||
await createBrandsFolders(sourceClient, destClient, brands, stats);
|
||||
await copyBrandAssets(sourceClient, destClient, brands, stats);
|
||||
await runStep("initializeDataSource", initializeDataSource);
|
||||
|
||||
const brands = await runStep("getBrandUrlNames", getBrandUrlNames);
|
||||
|
||||
await runStep("createBrandsFolders", () =>
|
||||
createBrandsFolders(sourceClient, destClient, brands, stats),
|
||||
);
|
||||
|
||||
await runStep("copyBrandAssets", () =>
|
||||
copyBrandAssets(sourceClient, destClient, brands, stats),
|
||||
);
|
||||
|
||||
printSummary(stats);
|
||||
|
||||
if (stats.errors > 0) {
|
||||
logger.warn(step, `migration finished with ${stats.errors} error(s)`);
|
||||
process.exitCode = 1;
|
||||
} else {
|
||||
logger.info(step, "migration completed successfully");
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Migration failed:", err);
|
||||
} catch (error) {
|
||||
logger.error(step, "migration failed", logger.formatError(error));
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
if (AppDataSource.isInitialized) {
|
||||
await AppDataSource.destroy();
|
||||
try {
|
||||
await destroyDataSource();
|
||||
} catch (error) {
|
||||
logger.error(step, "cleanup failed", logger.formatError(error));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
main().catch((error) => {
|
||||
logger.error("unhandledRejection", "fatal error in main()", logger.formatError(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
||||
65
src/logger.js
Normal file
65
src/logger.js
Normal file
@@ -0,0 +1,65 @@
|
||||
function timestamp() {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function formatError(error) {
|
||||
if (error instanceof Error) {
|
||||
return {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
...(typeof error === "object" && error !== null && "$metadata" in error
|
||||
? { metadata: error.$metadata }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
return { value: String(error) };
|
||||
}
|
||||
|
||||
const consoleMethods = {
|
||||
info: console.log,
|
||||
warn: console.warn,
|
||||
error: console.error,
|
||||
debug: console.log,
|
||||
};
|
||||
|
||||
function log(level, step, message, details) {
|
||||
const prefix = `[${timestamp()}] [${level}] [${step}]`;
|
||||
const write = consoleMethods[level] ?? console.log;
|
||||
|
||||
if (details !== undefined) {
|
||||
write(`${prefix} ${message}`, details);
|
||||
return;
|
||||
}
|
||||
|
||||
write(`${prefix} ${message}`);
|
||||
}
|
||||
|
||||
export const logger = {
|
||||
info(step, message, details) {
|
||||
log("info", step, message, details);
|
||||
},
|
||||
warn(step, message, details) {
|
||||
log("warn", step, message, details);
|
||||
},
|
||||
error(step, message, details) {
|
||||
log("error", step, message, details);
|
||||
},
|
||||
debug(step, message, details) {
|
||||
log("debug", step, message, details);
|
||||
},
|
||||
formatError,
|
||||
};
|
||||
|
||||
export async function runStep(step, fn) {
|
||||
logger.info(step, "starting");
|
||||
try {
|
||||
const result = await fn();
|
||||
logger.info(step, "completed");
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error(step, "failed", formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user