diff --git a/.env.example b/.env.example index ff23d9c..874bd01 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,9 @@ SOURCE_S3_ACCESS_KEY="sirCxqNKw85GdnlLfzko" SOURCE_S3_SECRET_KEY="qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w" SOURCE_S3_BUCKET="test" -DEST_S3_ENDPOINT="http://95.156.253.15:9001/" -DEST_S3_REGION="default" -DEST_S3_ACCESS_KEY="sirCxqNKw85GdnlLfzko" -DEST_S3_SECRET_KEY="qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w" -DEST_S3_BUCKET="test" \ No newline at end of file +# ParsPack: endpoint is https://.parspack.net and bucket name is the same id (e.g. c530168) +DEST_S3_ENDPOINT="https://c530168.parspack.net" +DEST_S3_REGION="us-west-2" +DEST_S3_ACCESS_KEY="your-access-key" +DEST_S3_SECRET_KEY="your-secret-key" +DEST_S3_BUCKET="c530168" \ No newline at end of file diff --git a/src/data-source.js b/src/data-source.js index 28aee31..784bb27 100644 --- a/src/data-source.js +++ b/src/data-source.js @@ -3,16 +3,26 @@ import { DataSource } from "typeorm"; import { AssetEntity } from "./entity/Assets.js"; import { BrandEntity } from "./entity/Brand.js"; import { BrandAssetEntity } from "./entity/BrandAsset.js"; +import { logger } from "./logger.js"; -export const AppDataSource = new DataSource({ +const dbConfig = { type: "mysql", - host: "95.156.253.15", - port: 3306, - username: "root", - password: "Aa9923994970", - database: "temporary", - logging: false, + host: "31.214.255.158", + port: 3301, + username: "brandprod", + password: "cdcc9b7152905d21baa7758586fed2aa", + database: "brandifa_production", + logging: process.env.DB_LOGGING === "true", entities: [AssetEntity, BrandEntity, BrandAssetEntity], migrations: [], subscribers: [], +}; + +logger.info("data-source", "creating AppDataSource", { + host: dbConfig.host, + port: dbConfig.port, + database: dbConfig.database, + logging: dbConfig.logging, }); + +export const AppDataSource = new DataSource(dbConfig); diff --git a/src/index.js b/src/index.js index bbfd422..dcd77cf 100644 --- a/src/index.js +++ b/src/index.js @@ -1,4 +1,4 @@ -import "dotenv/config" +import "dotenv/config"; import { GetObjectCommand, HeadObjectCommand, @@ -9,11 +9,32 @@ import { AppDataSource } from "./data-source.js"; import { AssetEntity } from "./entity/Assets.js"; import { BrandEntity } from "./entity/Brand.js"; import { BrandAssetEntity } from "./entity/BrandAsset.js"; +import { logger, runStep } from "./logger.js"; const AVATAR_PREFIX = "avatar"; const THUMBNAIL_PREFIX = "lg"; const DRY_RUN = process.env.DRY_RUN === "true"; +function resolveParsPackBucket(endpoint, bucket) { + const match = endpoint?.match(/https?:\/\/(c\d+)\.parspack\.net/i); + if (!match) return bucket; + + const bucketId = match[1]; + if (bucket !== bucketId) { + logger.warn( + "resolveParsPackBucket", + `ParsPack bucket name must match the endpoint id; using "${bucketId}" instead of "${bucket}"`, + ); + } + + return bucketId; +} + +function normalizeParsPackEndpoint(endpoint) { + if (!endpoint?.includes("parspack.net")) return endpoint; + return endpoint.replace(/^http:/i, "https:").replace(/\/$/, ""); +} + const SOURCE_S3 = { endpoint: process.env.SOURCE_S3_ENDPOINT ?? "http://95.156.253.15:9001/", region: process.env.SOURCE_S3_REGION ?? "default", @@ -24,13 +45,21 @@ const SOURCE_S3 = { bucket: process.env.SOURCE_S3_BUCKET ?? "test", }; -const DEST_S3 = { - endpoint: - process.env.DEST_S3_ENDPOINT ?? +const destEndpoint = normalizeParsPackEndpoint( + process.env.DEST_S3_ENDPOINT ?? process.env.SOURCE_S3_ENDPOINT ?? "http://95.156.253.15:9001/", +); +const destBucket = resolveParsPackBucket( + destEndpoint, + process.env.DEST_S3_BUCKET ?? "migrations", +); + +const DEST_S3 = { + endpoint: destEndpoint, region: - process.env.DEST_S3_REGION ?? process.env.SOURCE_S3_REGION ?? "default", + process.env.DEST_S3_REGION ?? + (destEndpoint.includes("parspack.net") ? "us-west-2" : process.env.SOURCE_S3_REGION ?? "default"), accessKeyId: process.env.DEST_S3_ACCESS_KEY ?? process.env.SOURCE_S3_ACCESS_KEY ?? @@ -39,61 +68,105 @@ const DEST_S3 = { process.env.DEST_S3_SECRET_KEY ?? process.env.SOURCE_S3_SECRET_KEY ?? "qbyQPjiCd5RcfIFpA3l2sG7EW9nT1gtVYkKJN08w", - bucket: process.env.DEST_S3_BUCKET ?? "migrations", + bucket: destBucket, }; -function createS3Client(config) { - return new S3Client({ - region: config.region, +function createS3Client(config, label) { + const step = `createS3Client:${label}`; + logger.info(step, "creating S3 client", { endpoint: config.endpoint, - forcePathStyle: true, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, + region: config.region, + bucket: config.bucket, }); + + try { + const client = new S3Client({ + region: config.region, + endpoint: config.endpoint, + forcePathStyle: true, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }); + logger.info(step, "S3 client created"); + return client; + } catch (error) { + logger.error(step, "failed to create S3 client", logger.formatError(error)); + throw error; + } } async function getAssetById(id) { - const assetRepository = AppDataSource.getRepository(AssetEntity); - return assetRepository.findOneBy({ id }); + const step = `getAssetById:${id}`; + try { + const assetRepository = AppDataSource.getRepository(AssetEntity); + const asset = await assetRepository.findOneBy({ id }); + logger.debug(step, asset ? "asset found" : "asset not found"); + return asset; + } catch (error) { + logger.error(step, "database query failed", logger.formatError(error)); + throw error; + } } async function getBrandAssetsByGroupId(groupId) { - const brandAssetRepo = AppDataSource.getRepository(BrandAssetEntity); - return brandAssetRepo.find({ - where: { - groupId, - deleted: false, - }, - }); + const step = `getBrandAssetsByGroupId:${groupId}`; + try { + const brandAssetRepo = AppDataSource.getRepository(BrandAssetEntity); + const brandAssets = await brandAssetRepo.find({ + where: { + groupId, + deleted: false, + }, + }); + logger.debug(step, `found ${brandAssets.length} brand assets`); + return brandAssets; + } catch (error) { + logger.error(step, "database query failed", logger.formatError(error)); + throw error; + } } async function getBrandUrlNames() { - const brandRepo = AppDataSource.getRepository(BrandEntity); - const brands = await brandRepo.find({ - where: { - deleted: false, - }, - }); - - const seen = new Set(); - const result = []; - - for (const brand of brands) { - if (!brand.urlName) continue; - - const key = `${brand.groupId}:${brand.urlName.toLowerCase()}`; - if (seen.has(key)) continue; - - seen.add(key); - result.push({ - name: brand.urlName, - groupId: brand.groupId, + const step = "getBrandUrlNames"; + try { + const brandRepo = AppDataSource.getRepository(BrandEntity); + const brands = await brandRepo.find({ + where: { + deleted: false, + }, }); - } + logger.info(step, `loaded ${brands.length} brands from database`); - return result; + const seen = new Set(); + const result = []; + + for (const brand of brands) { + if (!brand.urlName) { + logger.debug(step, `skipping brand id=${brand.id}: missing urlName`); + continue; + } + + const key = `${brand.groupId}:${brand.urlName.toLowerCase()}`; + if (seen.has(key)) { + logger.debug(step, `skipping duplicate brand urlName=${brand.urlName}`); + continue; + } + + seen.add(key); + result.push({ + name: brand.urlName, + groupId: brand.groupId, + }); + } + + logger.info(step, `resolved ${result.length} unique brands`); + return result; + } catch (error) { + logger.error(step, "failed to load brands", logger.formatError(error)); + throw error; + } } function resolveAssetSourceKey(asset) { @@ -101,8 +174,10 @@ function resolveAssetSourceKey(asset) { } async function objectExists(client, bucket, key) { + const step = `objectExists:${bucket}/${key}`; try { await client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); + logger.debug(step, "object exists"); return true; } catch (error) { const statusCode = @@ -115,7 +190,12 @@ async function objectExists(client, bucket, key) { ? error.$metadata.httpStatusCode : undefined; - if (statusCode === 404) return false; + if (statusCode === 404) { + logger.debug(step, "object not found"); + return false; + } + + logger.error(step, "head object failed", logger.formatError(error)); throw error; } } @@ -128,29 +208,43 @@ async function copyFileCrossClient( sourceKey, destKey, ) { - const response = await sourceClient.send( - new GetObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - }), - ); + const step = `copyFileCrossClient:${sourceBucket}/${sourceKey}->${destBucket}/${destKey}`; + try { + logger.debug(step, "fetching source object"); + const response = await sourceClient.send( + new GetObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + }), + ); - if (!response.Body) { - throw new Error(`Empty response body for ${sourceBucket}/${sourceKey}`); + if (!response.Body) { + throw new Error(`Empty response body for ${sourceBucket}/${sourceKey}`); + } + + logger.debug(step, "uploading to destination", { + contentType: response.ContentType, + contentLength: response.ContentLength, + }); + + await destClient.send( + new PutObjectCommand({ + Bucket: destBucket, + Key: destKey, + Body: response.Body, + ...(response.ContentType ? { ContentType: response.ContentType } : {}), + ...(response.ContentLength !== undefined + ? { ContentLength: response.ContentLength } + : {}), + ...(response.Metadata ? { Metadata: response.Metadata } : {}), + }), + ); + + logger.info(step, "copy completed"); + } catch (error) { + logger.error(step, "copy failed", logger.formatError(error)); + throw error; } - - await destClient.send( - new PutObjectCommand({ - Bucket: destBucket, - Key: destKey, - Body: response.Body, - ...(response.ContentType ? { ContentType: response.ContentType } : {}), - ...(response.ContentLength !== undefined - ? { ContentLength: response.ContentLength } - : {}), - ...(response.Metadata ? { Metadata: response.Metadata } : {}), - }), - ); } async function copyIfNeeded( @@ -163,62 +257,93 @@ async function copyIfNeeded( dryRun, type = "file", ) { - if (type !== "directory") { - const sourceExists = await objectExists( - sourceClient, - sourceBucket, - sourceKey, - ); - if (!sourceExists) return "missing"; - } + const step = `copyIfNeeded:${type}:${destBucket}/${destKey}`; + try { + if (type !== "directory") { + const sourceExists = await objectExists( + sourceClient, + sourceBucket, + sourceKey, + ); + if (!sourceExists) { + logger.warn(step, "source object missing", { + sourceBucket, + sourceKey, + }); + return "missing"; + } + } - const destExists = await objectExists(destClient, destBucket, destKey); - if (destExists) return "skipped"; + const destExists = await objectExists(destClient, destBucket, destKey); + if (destExists) { + logger.debug(step, "destination already exists, skipping"); + return "skipped"; + } - if (dryRun) { - console.log( - `[dry-run] copy ${sourceBucket}/${sourceKey} -> ${destBucket}/${destKey}`, - ); + if (dryRun) { + logger.info(step, "[dry-run] would copy object", { + source: `${sourceBucket}/${sourceKey}`, + destination: `${destBucket}/${destKey}`, + }); + return "copied"; + } + + if (type === "file") { + await copyFileCrossClient( + sourceClient, + destClient, + sourceBucket, + destBucket, + sourceKey, + destKey, + ); + } else { + logger.debug(step, "creating directory marker"); + await destClient.send( + new PutObjectCommand({ + Bucket: destBucket, + Key: destKey, + }), + ); + } + + logger.info(step, "copy action completed", { result: "copied" }); return "copied"; + } catch (error) { + logger.error(step, "copyIfNeeded failed", logger.formatError(error)); + throw error; } - - if (type === "file") { - await copyFileCrossClient( - sourceClient, - destClient, - sourceBucket, - destBucket, - sourceKey, - destKey, - ); - } else { - await destClient.send( - new PutObjectCommand({ - Bucket: destBucket, - Key: destKey, - }), - ); - } - - return "copied"; } async function createBrandsFolders(sourceClient, destClient, brands, stats) { - for (const brand of brands) { - const folderKey = `${brand.name.toLowerCase()}/`; - const result = await copyIfNeeded( - sourceClient, - destClient, - SOURCE_S3.bucket, - DEST_S3.bucket, - "", - folderKey, - DRY_RUN, - "directory", - ); + const step = "createBrandsFolders"; + logger.info(step, `creating folders for ${brands.length} brands`); - if (result === "copied") stats.foldersCreated++; - if (result === "skipped") stats.foldersSkipped++; + for (const brand of brands) { + const brandStep = `${step}:${brand.name}`; + try { + const folderKey = `${brand.name.toLowerCase()}/`; + logger.debug(brandStep, "creating folder", { folderKey }); + + const result = await copyIfNeeded( + sourceClient, + destClient, + SOURCE_S3.bucket, + DEST_S3.bucket, + "", + folderKey, + DRY_RUN, + "directory", + ); + + if (result === "copied") stats.foldersCreated++; + if (result === "skipped") stats.foldersSkipped++; + + logger.info(brandStep, "folder step completed", { result }); + } catch (error) { + stats.errors++; + logger.error(brandStep, "folder creation failed", logger.formatError(error)); + } } } @@ -229,91 +354,144 @@ async function copyAssetVariant( destKey, stats, ) { - const result = await copyIfNeeded( - sourceClient, - destClient, - SOURCE_S3.bucket, - DEST_S3.bucket, - sourceKey, - destKey, - DRY_RUN, - "file", - ); + const step = `copyAssetVariant:${destKey}`; + try { + const result = await copyIfNeeded( + sourceClient, + destClient, + SOURCE_S3.bucket, + DEST_S3.bucket, + sourceKey, + destKey, + DRY_RUN, + "file", + ); - if (result === "copied") stats.assetsCopied++; - if (result === "skipped") stats.assetsSkipped++; - if (result === "missing") stats.assetsMissing++; + if (result === "copied") stats.assetsCopied++; + if (result === "skipped") stats.assetsSkipped++; + if (result === "missing") stats.assetsMissing++; + + logger.debug(step, "variant copy completed", { result }); + } catch (error) { + logger.error(step, "variant copy failed", logger.formatError(error)); + throw error; + } } async function copyBrandAssets(sourceClient, destClient, brands, stats) { + const step = "copyBrandAssets"; + logger.info(step, `copying assets for ${brands.length} brands`); + for (const brand of brands) { - const brandAssets = await getBrandAssetsByGroupId(brand.groupId); - const brandFolder = brand.name.toLowerCase(); + const brandStep = `${step}:${brand.name}`; + try { + const brandAssets = await getBrandAssetsByGroupId(brand.groupId); + const brandFolder = brand.name.toLowerCase(); - console.log( - `Processing brand "${brand.name}" (groupId=${brand.groupId}, assets=${brandAssets.length})`, - ); + logger.info(brandStep, "processing brand assets", { + groupId: brand.groupId, + assetCount: brandAssets.length, + }); - for (const brandAsset of brandAssets) { - const asset = await getAssetById(brandAsset.assetId); - if (!asset || asset.deleted) { - console.warn( - `Skipping brand_asset ${brandAsset.id}: asset ${brandAsset.assetId} not found or deleted`, - ); - stats.assetsMissing++; - continue; + for (const brandAsset of brandAssets) { + const assetStep = `${brandStep}:asset:${brandAsset.assetId}`; + try { + const asset = await getAssetById(brandAsset.assetId); + if (!asset || asset.deleted) { + logger.warn(assetStep, "asset not found or deleted", { + brandAssetId: brandAsset.id, + assetId: brandAsset.assetId, + }); + stats.assetsMissing++; + continue; + } + + const sourceKey = resolveAssetSourceKey(asset); + const destBase = `${brandFolder}/${asset.id}`; + + await copyAssetVariant( + sourceClient, + destClient, + sourceKey, + destBase, + stats, + ); + // await copyAssetVariant( + // sourceClient, + // destClient, + // `${AVATAR_PREFIX}/${sourceKey}`, + // `${destBase}/${AVATAR_PREFIX}`, + // stats, + // ); + // await copyAssetVariant( + // sourceClient, + // destClient, + // `${THUMBNAIL_PREFIX}/${sourceKey}`, + // `${destBase}/${THUMBNAIL_PREFIX}`, + // stats, + // ); + } catch (error) { + stats.errors++; + logger.error(assetStep, "asset copy failed", logger.formatError(error)); + } } - const sourceKey = resolveAssetSourceKey(asset); - const destBase = `${brandFolder}/${asset.id}`; - - try { - await copyAssetVariant( - sourceClient, - destClient, - sourceKey, - destBase, - stats, - ); - // await copyAssetVariant( - // sourceClient, - // destClient, - // `${AVATAR_PREFIX}/${sourceKey}`, - // `${destBase}/${AVATAR_PREFIX}`, - // stats, - // ); - // await copyAssetVariant( - // sourceClient, - // destClient, - // `${THUMBNAIL_PREFIX}/${sourceKey}`, - // `${destBase}/${THUMBNAIL_PREFIX}`, - // stats, - // ); - } catch (error) { - stats.errors++; - console.error( - `Failed to copy asset ${asset.id} for brand "${brand.name}":`, - error, - ); - } + logger.info(brandStep, "brand assets completed"); + } catch (error) { + stats.errors++; + logger.error(brandStep, "brand processing failed", logger.formatError(error)); } } } function printSummary(stats) { - console.log("\nMigration summary:"); - console.log(` Folders created: ${stats.foldersCreated}`); - console.log(` Folders skipped: ${stats.foldersSkipped}`); - console.log(` Assets copied: ${stats.assetsCopied}`); - console.log(` Assets skipped: ${stats.assetsSkipped}`); - console.log(` Assets missing: ${stats.assetsMissing}`); - console.log(` Errors: ${stats.errors}`); - if (DRY_RUN) console.log(" (dry-run mode — no objects were written)"); + const step = "printSummary"; + logger.info(step, "migration summary", { + foldersCreated: stats.foldersCreated, + foldersSkipped: stats.foldersSkipped, + assetsCopied: stats.assetsCopied, + assetsSkipped: stats.assetsSkipped, + assetsMissing: stats.assetsMissing, + errors: stats.errors, + dryRun: DRY_RUN, + }); +} + +async function initializeDataSource() { + const step = "initializeDataSource"; + try { + await AppDataSource.initialize(); + logger.info(step, "data source initialized", { + host: AppDataSource.options.host, + port: AppDataSource.options.port, + database: AppDataSource.options.database, + }); + } catch (error) { + logger.error(step, "data source initialization failed", logger.formatError(error)); + throw error; + } +} + +async function destroyDataSource() { + const step = "destroyDataSource"; + if (!AppDataSource.isInitialized) { + logger.debug(step, "data source not initialized, skipping destroy"); + return; + } + + try { + await AppDataSource.destroy(); + logger.info(step, "data source destroyed"); + } catch (error) { + logger.error(step, "data source destroy failed", logger.formatError(error)); + throw error; + } } async function main() { - const sourceClient = createS3Client(SOURCE_S3); - const destClient = createS3Client(DEST_S3); + const step = "main"; + logger.info(step, "migration started", { dryRun: DRY_RUN }); + const stats = { foldersCreated: 0, foldersSkipped: 0, @@ -324,30 +502,52 @@ async function main() { }; try { - await AppDataSource.initialize(); - console.log("Data source initialized"); - console.log(`Source S3: ${SOURCE_S3.endpoint} (${SOURCE_S3.bucket})`); - console.log(`Dest S3: ${DEST_S3.endpoint} (${DEST_S3.bucket})`); + const sourceClient = await runStep("createSourceS3Client", async () => + createS3Client(SOURCE_S3, "source"), + ); + const destClient = await runStep("createDestS3Client", async () => + createS3Client(DEST_S3, "dest"), + ); - const brands = await getBrandUrlNames(); - console.log(`Found ${brands.length} brands to migrate`); + logger.info(step, "S3 configuration", { + source: { endpoint: SOURCE_S3.endpoint, bucket: SOURCE_S3.bucket }, + dest: { endpoint: DEST_S3.endpoint, bucket: DEST_S3.bucket }, + }); - await createBrandsFolders(sourceClient, destClient, brands, stats); - await copyBrandAssets(sourceClient, destClient, brands, stats); + await runStep("initializeDataSource", initializeDataSource); + + const brands = await runStep("getBrandUrlNames", getBrandUrlNames); + + await runStep("createBrandsFolders", () => + createBrandsFolders(sourceClient, destClient, brands, stats), + ); + + await runStep("copyBrandAssets", () => + copyBrandAssets(sourceClient, destClient, brands, stats), + ); printSummary(stats); if (stats.errors > 0) { + logger.warn(step, `migration finished with ${stats.errors} error(s)`); process.exitCode = 1; + } else { + logger.info(step, "migration completed successfully"); } - } catch (err) { - console.error("Migration failed:", err); + } catch (error) { + logger.error(step, "migration failed", logger.formatError(error)); process.exitCode = 1; } finally { - if (AppDataSource.isInitialized) { - await AppDataSource.destroy(); + try { + await destroyDataSource(); + } catch (error) { + logger.error(step, "cleanup failed", logger.formatError(error)); + process.exitCode = 1; } } } -main(); +main().catch((error) => { + logger.error("unhandledRejection", "fatal error in main()", logger.formatError(error)); + process.exitCode = 1; +}); diff --git a/src/logger.js b/src/logger.js new file mode 100644 index 0000000..015fd84 --- /dev/null +++ b/src/logger.js @@ -0,0 +1,65 @@ +function timestamp() { + return new Date().toISOString(); +} + +function formatError(error) { + if (error instanceof Error) { + return { + name: error.name, + message: error.message, + stack: error.stack, + ...(typeof error === "object" && error !== null && "$metadata" in error + ? { metadata: error.$metadata } + : {}), + }; + } + + return { value: String(error) }; +} + +const consoleMethods = { + info: console.log, + warn: console.warn, + error: console.error, + debug: console.log, +}; + +function log(level, step, message, details) { + const prefix = `[${timestamp()}] [${level}] [${step}]`; + const write = consoleMethods[level] ?? console.log; + + if (details !== undefined) { + write(`${prefix} ${message}`, details); + return; + } + + write(`${prefix} ${message}`); +} + +export const logger = { + info(step, message, details) { + log("info", step, message, details); + }, + warn(step, message, details) { + log("warn", step, message, details); + }, + error(step, message, details) { + log("error", step, message, details); + }, + debug(step, message, details) { + log("debug", step, message, details); + }, + formatError, +}; + +export async function runStep(step, fn) { + logger.info(step, "starting"); + try { + const result = await fn(); + logger.info(step, "completed"); + return result; + } catch (error) { + logger.error(step, "failed", formatError(error)); + throw error; + } +}