refactor: adding logger and catch blocks

This commit is contained in:
2026-06-12 20:19:25 +03:30
parent 2565da621f
commit 6111016245
4 changed files with 484 additions and 208 deletions

View File

@@ -4,8 +4,9 @@ SOURCE_S3_ACCESS_KEY="sirCxqNKw85GdnlLfzko"
SOURCE_S3_SECRET_KEY="qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w"
SOURCE_S3_BUCKET="test"
DEST_S3_ENDPOINT="http://95.156.253.15:9001/"
DEST_S3_REGION="default"
DEST_S3_ACCESS_KEY="sirCxqNKw85GdnlLfzko"
DEST_S3_SECRET_KEY="qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w"
DEST_S3_BUCKET="test"
# ParsPack: endpoint is https://<bucket-id>.parspack.net and bucket name is the same id (e.g. c530168)
DEST_S3_ENDPOINT="https://c530168.parspack.net"
DEST_S3_REGION="us-west-2"
DEST_S3_ACCESS_KEY="your-access-key"
DEST_S3_SECRET_KEY="your-secret-key"
DEST_S3_BUCKET="c530168"

View File

@@ -3,16 +3,26 @@ import { DataSource } from "typeorm";
import { AssetEntity } from "./entity/Assets.js";
import { BrandEntity } from "./entity/Brand.js";
import { BrandAssetEntity } from "./entity/BrandAsset.js";
import { logger } from "./logger.js";
export const AppDataSource = new DataSource({
const dbConfig = {
type: "mysql",
host: "95.156.253.15",
port: 3306,
username: "root",
password: "Aa9923994970",
database: "temporary",
logging: false,
host: "31.214.255.158",
port: 3301,
username: "brandprod",
password: "cdcc9b7152905d21baa7758586fed2aa",
database: "brandifa_production",
logging: process.env.DB_LOGGING === "true",
entities: [AssetEntity, BrandEntity, BrandAssetEntity],
migrations: [],
subscribers: [],
};
logger.info("data-source", "creating AppDataSource", {
host: dbConfig.host,
port: dbConfig.port,
database: dbConfig.database,
logging: dbConfig.logging,
});
export const AppDataSource = new DataSource(dbConfig);

View File

@@ -1,4 +1,4 @@
import "dotenv/config"
import "dotenv/config";
import {
GetObjectCommand,
HeadObjectCommand,
@@ -9,11 +9,32 @@ import { AppDataSource } from "./data-source.js";
import { AssetEntity } from "./entity/Assets.js";
import { BrandEntity } from "./entity/Brand.js";
import { BrandAssetEntity } from "./entity/BrandAsset.js";
import { logger, runStep } from "./logger.js";
const AVATAR_PREFIX = "avatar";
const THUMBNAIL_PREFIX = "lg";
const DRY_RUN = process.env.DRY_RUN === "true";
function resolveParsPackBucket(endpoint, bucket) {
const match = endpoint?.match(/https?:\/\/(c\d+)\.parspack\.net/i);
if (!match) return bucket;
const bucketId = match[1];
if (bucket !== bucketId) {
logger.warn(
"resolveParsPackBucket",
`ParsPack bucket name must match the endpoint id; using "${bucketId}" instead of "${bucket}"`,
);
}
return bucketId;
}
function normalizeParsPackEndpoint(endpoint) {
if (!endpoint?.includes("parspack.net")) return endpoint;
return endpoint.replace(/^http:/i, "https:").replace(/\/$/, "");
}
const SOURCE_S3 = {
endpoint: process.env.SOURCE_S3_ENDPOINT ?? "http://95.156.253.15:9001/",
region: process.env.SOURCE_S3_REGION ?? "default",
@@ -24,13 +45,21 @@ const SOURCE_S3 = {
bucket: process.env.SOURCE_S3_BUCKET ?? "test",
};
const DEST_S3 = {
endpoint:
process.env.DEST_S3_ENDPOINT ??
const destEndpoint = normalizeParsPackEndpoint(
process.env.DEST_S3_ENDPOINT ??
process.env.SOURCE_S3_ENDPOINT ??
"http://95.156.253.15:9001/",
);
const destBucket = resolveParsPackBucket(
destEndpoint,
process.env.DEST_S3_BUCKET ?? "migrations",
);
const DEST_S3 = {
endpoint: destEndpoint,
region:
process.env.DEST_S3_REGION ?? process.env.SOURCE_S3_REGION ?? "default",
process.env.DEST_S3_REGION ??
(destEndpoint.includes("parspack.net") ? "us-west-2" : process.env.SOURCE_S3_REGION ?? "default"),
accessKeyId:
process.env.DEST_S3_ACCESS_KEY ??
process.env.SOURCE_S3_ACCESS_KEY ??
@@ -39,61 +68,105 @@ const DEST_S3 = {
process.env.DEST_S3_SECRET_KEY ??
process.env.SOURCE_S3_SECRET_KEY ??
"qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w",
bucket: process.env.DEST_S3_BUCKET ?? "migrations",
bucket: destBucket,
};
function createS3Client(config) {
return new S3Client({
region: config.region,
function createS3Client(config, label) {
const step = `createS3Client:${label}`;
logger.info(step, "creating S3 client", {
endpoint: config.endpoint,
forcePathStyle: true,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
},
region: config.region,
bucket: config.bucket,
});
try {
const client = new S3Client({
region: config.region,
endpoint: config.endpoint,
forcePathStyle: true,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
},
});
logger.info(step, "S3 client created");
return client;
} catch (error) {
logger.error(step, "failed to create S3 client", logger.formatError(error));
throw error;
}
}
async function getAssetById(id) {
const assetRepository = AppDataSource.getRepository(AssetEntity);
return assetRepository.findOneBy({ id });
const step = `getAssetById:${id}`;
try {
const assetRepository = AppDataSource.getRepository(AssetEntity);
const asset = await assetRepository.findOneBy({ id });
logger.debug(step, asset ? "asset found" : "asset not found");
return asset;
} catch (error) {
logger.error(step, "database query failed", logger.formatError(error));
throw error;
}
}
async function getBrandAssetsByGroupId(groupId) {
const brandAssetRepo = AppDataSource.getRepository(BrandAssetEntity);
return brandAssetRepo.find({
where: {
groupId,
deleted: false,
},
});
const step = `getBrandAssetsByGroupId:${groupId}`;
try {
const brandAssetRepo = AppDataSource.getRepository(BrandAssetEntity);
const brandAssets = await brandAssetRepo.find({
where: {
groupId,
deleted: false,
},
});
logger.debug(step, `found ${brandAssets.length} brand assets`);
return brandAssets;
} catch (error) {
logger.error(step, "database query failed", logger.formatError(error));
throw error;
}
}
async function getBrandUrlNames() {
const brandRepo = AppDataSource.getRepository(BrandEntity);
const brands = await brandRepo.find({
where: {
deleted: false,
},
});
const seen = new Set();
const result = [];
for (const brand of brands) {
if (!brand.urlName) continue;
const key = `${brand.groupId}:${brand.urlName.toLowerCase()}`;
if (seen.has(key)) continue;
seen.add(key);
result.push({
name: brand.urlName,
groupId: brand.groupId,
const step = "getBrandUrlNames";
try {
const brandRepo = AppDataSource.getRepository(BrandEntity);
const brands = await brandRepo.find({
where: {
deleted: false,
},
});
}
logger.info(step, `loaded ${brands.length} brands from database`);
return result;
const seen = new Set();
const result = [];
for (const brand of brands) {
if (!brand.urlName) {
logger.debug(step, `skipping brand id=${brand.id}: missing urlName`);
continue;
}
const key = `${brand.groupId}:${brand.urlName.toLowerCase()}`;
if (seen.has(key)) {
logger.debug(step, `skipping duplicate brand urlName=${brand.urlName}`);
continue;
}
seen.add(key);
result.push({
name: brand.urlName,
groupId: brand.groupId,
});
}
logger.info(step, `resolved ${result.length} unique brands`);
return result;
} catch (error) {
logger.error(step, "failed to load brands", logger.formatError(error));
throw error;
}
}
function resolveAssetSourceKey(asset) {
@@ -101,8 +174,10 @@ function resolveAssetSourceKey(asset) {
}
async function objectExists(client, bucket, key) {
const step = `objectExists:${bucket}/${key}`;
try {
await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
logger.debug(step, "object exists");
return true;
} catch (error) {
const statusCode =
@@ -115,7 +190,12 @@ async function objectExists(client, bucket, key) {
? error.$metadata.httpStatusCode
: undefined;
if (statusCode === 404) return false;
if (statusCode === 404) {
logger.debug(step, "object not found");
return false;
}
logger.error(step, "head object failed", logger.formatError(error));
throw error;
}
}
@@ -128,29 +208,43 @@ async function copyFileCrossClient(
sourceKey,
destKey,
) {
const response = await sourceClient.send(
new GetObjectCommand({
Bucket: sourceBucket,
Key: sourceKey,
}),
);
const step = `copyFileCrossClient:${sourceBucket}/${sourceKey}->${destBucket}/${destKey}`;
try {
logger.debug(step, "fetching source object");
const response = await sourceClient.send(
new GetObjectCommand({
Bucket: sourceBucket,
Key: sourceKey,
}),
);
if (!response.Body) {
throw new Error(`Empty response body for ${sourceBucket}/${sourceKey}`);
if (!response.Body) {
throw new Error(`Empty response body for ${sourceBucket}/${sourceKey}`);
}
logger.debug(step, "uploading to destination", {
contentType: response.ContentType,
contentLength: response.ContentLength,
});
await destClient.send(
new PutObjectCommand({
Bucket: destBucket,
Key: destKey,
Body: response.Body,
...(response.ContentType ? { ContentType: response.ContentType } : {}),
...(response.ContentLength !== undefined
? { ContentLength: response.ContentLength }
: {}),
...(response.Metadata ? { Metadata: response.Metadata } : {}),
}),
);
logger.info(step, "copy completed");
} catch (error) {
logger.error(step, "copy failed", logger.formatError(error));
throw error;
}
await destClient.send(
new PutObjectCommand({
Bucket: destBucket,
Key: destKey,
Body: response.Body,
...(response.ContentType ? { ContentType: response.ContentType } : {}),
...(response.ContentLength !== undefined
? { ContentLength: response.ContentLength }
: {}),
...(response.Metadata ? { Metadata: response.Metadata } : {}),
}),
);
}
async function copyIfNeeded(
@@ -163,62 +257,93 @@ async function copyIfNeeded(
dryRun,
type = "file",
) {
if (type !== "directory") {
const sourceExists = await objectExists(
sourceClient,
sourceBucket,
sourceKey,
);
if (!sourceExists) return "missing";
}
const step = `copyIfNeeded:${type}:${destBucket}/${destKey}`;
try {
if (type !== "directory") {
const sourceExists = await objectExists(
sourceClient,
sourceBucket,
sourceKey,
);
if (!sourceExists) {
logger.warn(step, "source object missing", {
sourceBucket,
sourceKey,
});
return "missing";
}
}
const destExists = await objectExists(destClient, destBucket, destKey);
if (destExists) return "skipped";
const destExists = await objectExists(destClient, destBucket, destKey);
if (destExists) {
logger.debug(step, "destination already exists, skipping");
return "skipped";
}
if (dryRun) {
console.log(
`[dry-run] copy ${sourceBucket}/${sourceKey} -> ${destBucket}/${destKey}`,
);
if (dryRun) {
logger.info(step, "[dry-run] would copy object", {
source: `${sourceBucket}/${sourceKey}`,
destination: `${destBucket}/${destKey}`,
});
return "copied";
}
if (type === "file") {
await copyFileCrossClient(
sourceClient,
destClient,
sourceBucket,
destBucket,
sourceKey,
destKey,
);
} else {
logger.debug(step, "creating directory marker");
await destClient.send(
new PutObjectCommand({
Bucket: destBucket,
Key: destKey,
}),
);
}
logger.info(step, "copy action completed", { result: "copied" });
return "copied";
} catch (error) {
logger.error(step, "copyIfNeeded failed", logger.formatError(error));
throw error;
}
if (type === "file") {
await copyFileCrossClient(
sourceClient,
destClient,
sourceBucket,
destBucket,
sourceKey,
destKey,
);
} else {
await destClient.send(
new PutObjectCommand({
Bucket: destBucket,
Key: destKey,
}),
);
}
return "copied";
}
async function createBrandsFolders(sourceClient, destClient, brands, stats) {
for (const brand of brands) {
const folderKey = `${brand.name.toLowerCase()}/`;
const result = await copyIfNeeded(
sourceClient,
destClient,
SOURCE_S3.bucket,
DEST_S3.bucket,
"",
folderKey,
DRY_RUN,
"directory",
);
const step = "createBrandsFolders";
logger.info(step, `creating folders for ${brands.length} brands`);
if (result === "copied") stats.foldersCreated++;
if (result === "skipped") stats.foldersSkipped++;
for (const brand of brands) {
const brandStep = `${step}:${brand.name}`;
try {
const folderKey = `${brand.name.toLowerCase()}/`;
logger.debug(brandStep, "creating folder", { folderKey });
const result = await copyIfNeeded(
sourceClient,
destClient,
SOURCE_S3.bucket,
DEST_S3.bucket,
"",
folderKey,
DRY_RUN,
"directory",
);
if (result === "copied") stats.foldersCreated++;
if (result === "skipped") stats.foldersSkipped++;
logger.info(brandStep, "folder step completed", { result });
} catch (error) {
stats.errors++;
logger.error(brandStep, "folder creation failed", logger.formatError(error));
}
}
}
@@ -229,91 +354,144 @@ async function copyAssetVariant(
destKey,
stats,
) {
const result = await copyIfNeeded(
sourceClient,
destClient,
SOURCE_S3.bucket,
DEST_S3.bucket,
sourceKey,
destKey,
DRY_RUN,
"file",
);
const step = `copyAssetVariant:${destKey}`;
try {
const result = await copyIfNeeded(
sourceClient,
destClient,
SOURCE_S3.bucket,
DEST_S3.bucket,
sourceKey,
destKey,
DRY_RUN,
"file",
);
if (result === "copied") stats.assetsCopied++;
if (result === "skipped") stats.assetsSkipped++;
if (result === "missing") stats.assetsMissing++;
if (result === "copied") stats.assetsCopied++;
if (result === "skipped") stats.assetsSkipped++;
if (result === "missing") stats.assetsMissing++;
logger.debug(step, "variant copy completed", { result });
} catch (error) {
logger.error(step, "variant copy failed", logger.formatError(error));
throw error;
}
}
async function copyBrandAssets(sourceClient, destClient, brands, stats) {
const step = "copyBrandAssets";
logger.info(step, `copying assets for ${brands.length} brands`);
for (const brand of brands) {
const brandAssets = await getBrandAssetsByGroupId(brand.groupId);
const brandFolder = brand.name.toLowerCase();
const brandStep = `${step}:${brand.name}`;
try {
const brandAssets = await getBrandAssetsByGroupId(brand.groupId);
const brandFolder = brand.name.toLowerCase();
console.log(
`Processing brand "${brand.name}" (groupId=${brand.groupId}, assets=${brandAssets.length})`,
);
logger.info(brandStep, "processing brand assets", {
groupId: brand.groupId,
assetCount: brandAssets.length,
});
for (const brandAsset of brandAssets) {
const asset = await getAssetById(brandAsset.assetId);
if (!asset || asset.deleted) {
console.warn(
`Skipping brand_asset ${brandAsset.id}: asset ${brandAsset.assetId} not found or deleted`,
);
stats.assetsMissing++;
continue;
for (const brandAsset of brandAssets) {
const assetStep = `${brandStep}:asset:${brandAsset.assetId}`;
try {
const asset = await getAssetById(brandAsset.assetId);
if (!asset || asset.deleted) {
logger.warn(assetStep, "asset not found or deleted", {
brandAssetId: brandAsset.id,
assetId: brandAsset.assetId,
});
stats.assetsMissing++;
continue;
}
const sourceKey = resolveAssetSourceKey(asset);
const destBase = `${brandFolder}/${asset.id}`;
await copyAssetVariant(
sourceClient,
destClient,
sourceKey,
destBase,
stats,
);
// await copyAssetVariant(
// sourceClient,
// destClient,
// `${AVATAR_PREFIX}/${sourceKey}`,
// `${destBase}/${AVATAR_PREFIX}`,
// stats,
// );
// await copyAssetVariant(
// sourceClient,
// destClient,
// `${THUMBNAIL_PREFIX}/${sourceKey}`,
// `${destBase}/${THUMBNAIL_PREFIX}`,
// stats,
// );
} catch (error) {
stats.errors++;
logger.error(assetStep, "asset copy failed", logger.formatError(error));
}
}
const sourceKey = resolveAssetSourceKey(asset);
const destBase = `${brandFolder}/${asset.id}`;
try {
await copyAssetVariant(
sourceClient,
destClient,
sourceKey,
destBase,
stats,
);
// await copyAssetVariant(
// sourceClient,
// destClient,
// `${AVATAR_PREFIX}/${sourceKey}`,
// `${destBase}/${AVATAR_PREFIX}`,
// stats,
// );
// await copyAssetVariant(
// sourceClient,
// destClient,
// `${THUMBNAIL_PREFIX}/${sourceKey}`,
// `${destBase}/${THUMBNAIL_PREFIX}`,
// stats,
// );
} catch (error) {
stats.errors++;
console.error(
`Failed to copy asset ${asset.id} for brand "${brand.name}":`,
error,
);
}
logger.info(brandStep, "brand assets completed");
} catch (error) {
stats.errors++;
logger.error(brandStep, "brand processing failed", logger.formatError(error));
}
}
}
function printSummary(stats) {
console.log("\nMigration summary:");
console.log(` Folders created: ${stats.foldersCreated}`);
console.log(` Folders skipped: ${stats.foldersSkipped}`);
console.log(` Assets copied: ${stats.assetsCopied}`);
console.log(` Assets skipped: ${stats.assetsSkipped}`);
console.log(` Assets missing: ${stats.assetsMissing}`);
console.log(` Errors: ${stats.errors}`);
if (DRY_RUN) console.log(" (dry-run mode — no objects were written)");
const step = "printSummary";
logger.info(step, "migration summary", {
foldersCreated: stats.foldersCreated,
foldersSkipped: stats.foldersSkipped,
assetsCopied: stats.assetsCopied,
assetsSkipped: stats.assetsSkipped,
assetsMissing: stats.assetsMissing,
errors: stats.errors,
dryRun: DRY_RUN,
});
}
async function initializeDataSource() {
const step = "initializeDataSource";
try {
await AppDataSource.initialize();
logger.info(step, "data source initialized", {
host: AppDataSource.options.host,
port: AppDataSource.options.port,
database: AppDataSource.options.database,
});
} catch (error) {
logger.error(step, "data source initialization failed", logger.formatError(error));
throw error;
}
}
async function destroyDataSource() {
const step = "destroyDataSource";
if (!AppDataSource.isInitialized) {
logger.debug(step, "data source not initialized, skipping destroy");
return;
}
try {
await AppDataSource.destroy();
logger.info(step, "data source destroyed");
} catch (error) {
logger.error(step, "data source destroy failed", logger.formatError(error));
throw error;
}
}
async function main() {
const sourceClient = createS3Client(SOURCE_S3);
const destClient = createS3Client(DEST_S3);
const step = "main";
logger.info(step, "migration started", { dryRun: DRY_RUN });
const stats = {
foldersCreated: 0,
foldersSkipped: 0,
@@ -324,30 +502,52 @@ async function main() {
};
try {
await AppDataSource.initialize();
console.log("Data source initialized");
console.log(`Source S3: ${SOURCE_S3.endpoint} (${SOURCE_S3.bucket})`);
console.log(`Dest S3: ${DEST_S3.endpoint} (${DEST_S3.bucket})`);
const sourceClient = await runStep("createSourceS3Client", async () =>
createS3Client(SOURCE_S3, "source"),
);
const destClient = await runStep("createDestS3Client", async () =>
createS3Client(DEST_S3, "dest"),
);
const brands = await getBrandUrlNames();
console.log(`Found ${brands.length} brands to migrate`);
logger.info(step, "S3 configuration", {
source: { endpoint: SOURCE_S3.endpoint, bucket: SOURCE_S3.bucket },
dest: { endpoint: DEST_S3.endpoint, bucket: DEST_S3.bucket },
});
await createBrandsFolders(sourceClient, destClient, brands, stats);
await copyBrandAssets(sourceClient, destClient, brands, stats);
await runStep("initializeDataSource", initializeDataSource);
const brands = await runStep("getBrandUrlNames", getBrandUrlNames);
await runStep("createBrandsFolders", () =>
createBrandsFolders(sourceClient, destClient, brands, stats),
);
await runStep("copyBrandAssets", () =>
copyBrandAssets(sourceClient, destClient, brands, stats),
);
printSummary(stats);
if (stats.errors > 0) {
logger.warn(step, `migration finished with ${stats.errors} error(s)`);
process.exitCode = 1;
} else {
logger.info(step, "migration completed successfully");
}
} catch (err) {
console.error("Migration failed:", err);
} catch (error) {
logger.error(step, "migration failed", logger.formatError(error));
process.exitCode = 1;
} finally {
if (AppDataSource.isInitialized) {
await AppDataSource.destroy();
try {
await destroyDataSource();
} catch (error) {
logger.error(step, "cleanup failed", logger.formatError(error));
process.exitCode = 1;
}
}
}
main();
main().catch((error) => {
logger.error("unhandledRejection", "fatal error in main()", logger.formatError(error));
process.exitCode = 1;
});

65
src/logger.js Normal file
View File

@@ -0,0 +1,65 @@
function timestamp() {
return new Date().toISOString();
}
function formatError(error) {
if (error instanceof Error) {
return {
name: error.name,
message: error.message,
stack: error.stack,
...(typeof error === "object" && error !== null && "$metadata" in error
? { metadata: error.$metadata }
: {}),
};
}
return { value: String(error) };
}
const consoleMethods = {
info: console.log,
warn: console.warn,
error: console.error,
debug: console.log,
};
function log(level, step, message, details) {
const prefix = `[${timestamp()}] [${level}] [${step}]`;
const write = consoleMethods[level] ?? console.log;
if (details !== undefined) {
write(`${prefix} ${message}`, details);
return;
}
write(`${prefix} ${message}`);
}
export const logger = {
info(step, message, details) {
log("info", step, message, details);
},
warn(step, message, details) {
log("warn", step, message, details);
},
error(step, message, details) {
log("error", step, message, details);
},
debug(step, message, details) {
log("debug", step, message, details);
},
formatError,
};
export async function runStep(step, fn) {
logger.info(step, "starting");
try {
const result = await fn();
logger.info(step, "completed");
return result;
} catch (error) {
logger.error(step, "failed", formatError(error));
throw error;
}
}