diff --git a/src/index.js b/src/index.js index 27e164b..cf89065 100644 --- a/src/index.js +++ b/src/index.js @@ -3,6 +3,7 @@ import { GetObjectCommand, HeadObjectCommand, PutObjectCommand, + CopyObjectCommand, S3Client, } from "@aws-sdk/client-s3"; import { AppDataSource } from "./data-source.js"; @@ -114,7 +115,9 @@ const DEST_S3 = { endpoint: destEndpoint, region: process.env.DEST_S3_REGION ?? - (destEndpoint.includes("parspack.net") ? "us-west-2" : process.env.SOURCE_S3_REGION ?? "default"), + (destEndpoint.includes("parspack.net") + ? "us-west-2" + : (process.env.SOURCE_S3_REGION ?? "default")), accessKeyId: process.env.DEST_S3_ACCESS_KEY ?? process.env.SOURCE_S3_ACCESS_KEY ?? @@ -221,6 +224,7 @@ async function getBrandUrlNames() { } logger.info(step, `resolved ${result.length} unique brands`); + logger.info(step, `${JSON.stringify(result)} gonna return`); return result; } catch (error) { logger.error(step, "failed to load brands", logger.formatError(error)); @@ -284,6 +288,7 @@ async function copyFileCrossClient( contentLength: response.ContentLength, }); + logger.info(step, `dest key is: ${destKey}`); await sendS3Command( destClient, new PutObjectCommand({ @@ -324,6 +329,7 @@ async function copyIfNeeded( sourceBucket, sourceKey, ); + logger.info(step, "Key is:", sourceKey); if (!sourceExists) { logger.warn(step, "source object missing", { sourceBucket, @@ -403,7 +409,11 @@ async function createBrandsFolders(sourceClient, destClient, brands, stats) { logger.info(brandStep, "folder step completed", { result }); } catch (error) { stats.errors++; - logger.error(brandStep, "folder creation failed", logger.formatError(error)); + logger.error( + brandStep, + "folder creation failed", + logger.formatError(error), + ); } } } @@ -467,7 +477,8 @@ async function copyBrandAssets(sourceClient, destClient, brands, stats) { continue; } - const sourceKey = resolveAssetSourceKey(asset); + // const sourceKey = resolveAssetSourceKey(asset); + const sourceKey = `${brandFolder}/${asset.id.toString()}`; const destBase = `${brandFolder}/${asset.id}`; await copyAssetVariant( @@ -493,7 +504,11 @@ async function copyBrandAssets(sourceClient, destClient, brands, stats) { // ); } catch (error) { stats.errors++; - logger.error(assetStep, "asset copy failed", logger.formatError(error)); + logger.error( + assetStep, + "asset copy failed", + logger.formatError(error), + ); } if (S3_REQUEST_DELAY_MS > 0) { @@ -504,7 +519,11 @@ async function copyBrandAssets(sourceClient, destClient, brands, stats) { logger.info(brandStep, "brand assets completed"); } catch (error) { stats.errors++; - logger.error(brandStep, "brand processing failed", logger.formatError(error)); + logger.error( + brandStep, + "brand processing failed", + logger.formatError(error), + ); } } } @@ -515,6 +534,7 @@ function printSummary(stats) { foldersCreated: stats.foldersCreated, foldersSkipped: stats.foldersSkipped, assetsCopied: stats.assetsCopied, + assetsReplaced: stats.replaced, assetsSkipped: stats.assetsSkipped, assetsMissing: stats.assetsMissing, errors: stats.errors, @@ -532,7 +552,11 @@ async function initializeDataSource() { database: AppDataSource.options.database, }); } catch (error) { - logger.error(step, "data source initialization failed", logger.formatError(error)); + logger.error( + step, + "data source initialization failed", + logger.formatError(error), + ); throw error; } } @@ -553,6 +577,166 @@ async function destroyDataSource() { } } +async function inspectObject(client, bucket, key, filename) { + const head = await client.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + + console.log("BEFORE:", { + key, + filename, + contentType: head.ContentType, + contentDisposition: head.ContentDisposition, + contentLength: head.ContentLength, + etag: head.ETag, + lastModified: head.LastModified, + metadata: head.Metadata, + }); + + return head; +} + +async function copyWithNewMetadata(client, bucket, key, filename) { + const newKey = `${key}-test-copy`; + + const head = await client.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + + const result = await client.send( + new CopyObjectCommand({ + Bucket: bucket, + Key: key, + CopySource: `${bucket}/${key}`, + MetadataDirective: "REPLACE", + ContentDisposition: `attachment; filename="${filename}"`, + + // Preserve existing headers + ContentType: head.ContentType, + CacheControl: head.CacheControl, + ContentEncoding: head.ContentEncoding, + ContentLanguage: head.ContentLanguage, + Metadata: head.Metadata, + }), + ); + + console.log("COPY RESULT:", result); + + return newKey; +} +async function inspectAfterCopy(client, bucket, key, filename) { + const head = await client.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + + console.log("AFTER:", { + key, + filename, + contentType: head.ContentType, + contentDisposition: head.ContentDisposition, + contentLength: head.ContentLength, + etag: head.ETag, + lastModified: head.LastModified, + metadata: head.Metadata, + }); + + return head; +} +async function replaceMetadata(sourceClient, sourceBucket, key, filename) { + const step = `replaceMetadata:${sourceBucket}/${key}`; + try { + await inspectObject(sourceClient, sourceBucket, key, filename); + await copyWithNewMetadata(sourceClient, sourceBucket, key, filename); + await inspectAfterCopy(sourceClient, sourceBucket, key, filename); + + // const sourceExists = await objectExists(sourceClient, sourceBucket, key); + // if (!sourceExists) { + // logger.warn(step, "source object missing", { + // sourceBucket, + // sourceKey, + // }); + // return "missing"; + // } + // logger.info("The existing file is:", sourceExists); + + // await sourceClient.send( + // new CopyObjectCommand({ + // Bucket: sourceBucket, + // Key: key, + // CopySource: `${sourceBucket}/${key}`, + + // MetadataDirective: "REPLACE", + + // ContentDisposition: 'attachment; filename="filan.pdf"', + // // ContentType: "application/pdf", // preserve if needed + // }), + // ); + + // logger.info(step, "replace action completed", { result: "replaced" }); + return "replaced"; + } catch (error) { + logger.error(step, "replace metadata failed", logger.formatError(error)); + throw error; + } +} + +async function replaceAssetsMetadata(sourceClient, brands, stats) { + const step = "replaceAssetsMetadata"; + logger.info(step, `replacing existed files with a new metadata`); + + for (const brand of brands) { + const brandStep = `${step}:${brand.name}`; + const brandAssets = await getBrandAssetsByGroupId(brand.groupId); + for (const brandAsset of brandAssets) { + const asset = await getAssetById(brandAsset.assetId); + try { + const key = `${brand.name.toLowerCase()}/${brandAsset.assetId}`; + logger.debug(brandStep, "Replacing metadata of:", { key }); + + // const result = await copyIfNeeded( + // sourceClient, + // destClient, + // SOURCE_S3.bucket, + // DEST_S3.bucket, + // "", + // folderKey, + // DRY_RUN, + // "directory", + // ); + + const result = await replaceMetadata( + sourceClient, + DEST_S3.bucket, + key, + asset.name, + ); + + // if (result === "copied") stats.foldersCreated++; + // if (result === "skipped") stats.foldersSkipped++; + if (result === "replaced") stats.replaced++; + + logger.info("folder step completed", { result }); + } catch (error) { + stats.errors++; + logger.error( + // brandStep, + "replacing files failed", + logger.formatError(error), + ); + } + } + } +} + async function main() { const step = "main"; logger.info(step, "migration started", { @@ -568,6 +752,7 @@ async function main() { foldersCreated: 0, foldersSkipped: 0, assetsCopied: 0, + replaced: 0, assetsSkipped: 0, assetsMissing: 0, errors: 0, @@ -590,14 +775,18 @@ async function main() { const brands = await runStep("getBrandUrlNames", getBrandUrlNames); - await runStep("createBrandsFolders", () => - createBrandsFolders(sourceClient, destClient, brands, stats), - ); + // await runStep("createBrandsFolders", () => + // createBrandsFolders(sourceClient, destClient, brands, stats), + // ); await runStep("copyBrandAssets", () => copyBrandAssets(sourceClient, destClient, brands, stats), ); + await runStep("replaceAssetsMetadata", () => + replaceAssetsMetadata(destClient, brands, stats), + ); + printSummary(stats); if (stats.errors > 0) { @@ -620,6 +809,10 @@ async function main() { } main().catch((error) => { - logger.error("unhandledRejection", "fatal error in main()", logger.formatError(error)); + logger.error( + "unhandledRejection", + "fatal error in main()", + logger.formatError(error), + ); process.exitCode = 1; });