agp/migration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: change a few things for a new migration version

Browse Source
This commit is contained in:
agp
2026-06-22 11:47:00 +03:30
parent f51845aa7c
commit 03416595c0
1 changed files with 203 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index.js
+203 -10
View File
@@ -3,6 +3,7 @@ import {
GetObjectCommand,
HeadObjectCommand,
PutObjectCommand,
CopyObjectCommand,
S3Client,
} from "@aws-sdk/client-s3";
import { AppDataSource } from "./data-source.js";
@@ -114,7 +115,9 @@ const DEST_S3 = {
endpoint: destEndpoint,
region:
process.env.DEST_S3_REGION ??
(destEndpoint.includes("parspack.net") ? "us-west-2" : process.env.SOURCE_S3_REGION ?? "default"),
(destEndpoint.includes("parspack.net")
? "us-west-2"
: (process.env.SOURCE_S3_REGION ?? "default")),
accessKeyId:
process.env.DEST_S3_ACCESS_KEY ??
process.env.SOURCE_S3_ACCESS_KEY ??
@@ -221,6 +224,7 @@ async function getBrandUrlNames() {
}
logger.info(step, `resolved ${result.length} unique brands`);
logger.info(step, `${JSON.stringify(result)} gonna return`);
return result;
} catch (error) {
logger.error(step, "failed to load brands", logger.formatError(error));
@@ -284,6 +288,7 @@ async function copyFileCrossClient(
contentLength: response.ContentLength,
});
logger.info(step, `dest key is: ${destKey}`);
await sendS3Command(
destClient,
new PutObjectCommand({
@@ -324,6 +329,7 @@ async function copyIfNeeded(
sourceBucket,
sourceKey,
);
logger.info(step, "Key is:", sourceKey);
if (!sourceExists) {
logger.warn(step, "source object missing", {
sourceBucket,
@@ -403,7 +409,11 @@ async function createBrandsFolders(sourceClient, destClient, brands, stats) {
logger.info(brandStep, "folder step completed", { result });
} catch (error) {
stats.errors++;
logger.error(brandStep, "folder creation failed", logger.formatError(error));
logger.error(
brandStep,
"folder creation failed",
logger.formatError(error),
);
}
}
}
@@ -467,7 +477,8 @@ async function copyBrandAssets(sourceClient, destClient, brands, stats) {
continue;
}
const sourceKey = resolveAssetSourceKey(asset);
// const sourceKey = resolveAssetSourceKey(asset);
const sourceKey = `${brandFolder}/${asset.id.toString()}`;
const destBase = `${brandFolder}/${asset.id}`;
await copyAssetVariant(
@@ -493,7 +504,11 @@ async function copyBrandAssets(sourceClient, destClient, brands, stats) {
// );
} catch (error) {
stats.errors++;
logger.error(assetStep, "asset copy failed", logger.formatError(error));
logger.error(
assetStep,
"asset copy failed",
logger.formatError(error),
);
}
if (S3_REQUEST_DELAY_MS > 0) {
@@ -504,7 +519,11 @@ async function copyBrandAssets(sourceClient, destClient, brands, stats) {
logger.info(brandStep, "brand assets completed");
} catch (error) {
stats.errors++;
logger.error(brandStep, "brand processing failed", logger.formatError(error));
logger.error(
brandStep,
"brand processing failed",
logger.formatError(error),
);
}
}
}
@@ -515,6 +534,7 @@ function printSummary(stats) {
foldersCreated: stats.foldersCreated,
foldersSkipped: stats.foldersSkipped,
assetsCopied: stats.assetsCopied,
assetsReplaced: stats.replaced,
assetsSkipped: stats.assetsSkipped,
assetsMissing: stats.assetsMissing,
errors: stats.errors,
@@ -532,7 +552,11 @@ async function initializeDataSource() {
database: AppDataSource.options.database,
});
} catch (error) {
logger.error(step, "data source initialization failed", logger.formatError(error));
logger.error(
step,
"data source initialization failed",
logger.formatError(error),
);
throw error;
}
}
@@ -553,6 +577,166 @@ async function destroyDataSource() {
}
}
async function inspectObject(client, bucket, key, filename) {
const head = await client.send(
new HeadObjectCommand({
Bucket: bucket,
Key: key,
}),
);
console.log("BEFORE:", {
key,
filename,
contentType: head.ContentType,
contentDisposition: head.ContentDisposition,
contentLength: head.ContentLength,
etag: head.ETag,
lastModified: head.LastModified,
metadata: head.Metadata,
});
return head;
}
async function copyWithNewMetadata(client, bucket, key, filename) {
const newKey = `${key}-test-copy`;
const head = await client.send(
new HeadObjectCommand({
Bucket: bucket,
Key: key,
}),
);
const result = await client.send(
new CopyObjectCommand({
Bucket: bucket,
Key: key,
CopySource: `${bucket}/${key}`,
MetadataDirective: "REPLACE",
ContentDisposition: `attachment; filename="${filename}"`,
// Preserve existing headers
ContentType: head.ContentType,
CacheControl: head.CacheControl,
ContentEncoding: head.ContentEncoding,
ContentLanguage: head.ContentLanguage,
Metadata: head.Metadata,
}),
);
console.log("COPY RESULT:", result);
return newKey;
}
async function inspectAfterCopy(client, bucket, key, filename) {
const head = await client.send(
new HeadObjectCommand({
Bucket: bucket,
Key: key,
}),
);
console.log("AFTER:", {
key,
filename,
contentType: head.ContentType,
contentDisposition: head.ContentDisposition,
contentLength: head.ContentLength,
etag: head.ETag,
lastModified: head.LastModified,
metadata: head.Metadata,
});
return head;
}
async function replaceMetadata(sourceClient, sourceBucket, key, filename) {
const step = `replaceMetadata:${sourceBucket}/${key}`;
try {
await inspectObject(sourceClient, sourceBucket, key, filename);
await copyWithNewMetadata(sourceClient, sourceBucket, key, filename);
await inspectAfterCopy(sourceClient, sourceBucket, key, filename);
// const sourceExists = await objectExists(sourceClient, sourceBucket, key);
// if (!sourceExists) {
// logger.warn(step, "source object missing", {
// sourceBucket,
// sourceKey,
// });
// return "missing";
// }
// logger.info("The existing file is:", sourceExists);
// await sourceClient.send(
// new CopyObjectCommand({
// Bucket: sourceBucket,
// Key: key,
// CopySource: `${sourceBucket}/${key}`,
// MetadataDirective: "REPLACE",
// ContentDisposition: 'attachment; filename="filan.pdf"',
// // ContentType: "application/pdf", // preserve if needed
// }),
// );
// logger.info(step, "replace action completed", { result: "replaced" });
return "replaced";
} catch (error) {
logger.error(step, "replace metadata failed", logger.formatError(error));
throw error;
}
}
async function replaceAssetsMetadata(sourceClient, brands, stats) {
const step = "replaceAssetsMetadata";
logger.info(step, `replacing existed files with a new metadata`);
for (const brand of brands) {
const brandStep = `${step}:${brand.name}`;
const brandAssets = await getBrandAssetsByGroupId(brand.groupId);
for (const brandAsset of brandAssets) {
const asset = await getAssetById(brandAsset.assetId);
try {
const key = `${brand.name.toLowerCase()}/${brandAsset.assetId}`;
logger.debug(brandStep, "Replacing metadata of:", { key });
// const result = await copyIfNeeded(
// sourceClient,
// destClient,
// SOURCE_S3.bucket,
// DEST_S3.bucket,
// "",
// folderKey,
// DRY_RUN,
// "directory",
// );
const result = await replaceMetadata(
sourceClient,
DEST_S3.bucket,
key,
asset.name,
);
// if (result === "copied") stats.foldersCreated++;
// if (result === "skipped") stats.foldersSkipped++;
if (result === "replaced") stats.replaced++;
logger.info("folder step completed", { result });
} catch (error) {
stats.errors++;
logger.error(
// brandStep,
"replacing files failed",
logger.formatError(error),
);
}
}
}
}
async function main() {
const step = "main";
logger.info(step, "migration started", {
@@ -568,6 +752,7 @@ async function main() {
foldersCreated: 0,
foldersSkipped: 0,
assetsCopied: 0,
replaced: 0,
assetsSkipped: 0,
assetsMissing: 0,
errors: 0,
@@ -590,14 +775,18 @@ async function main() {
const brands = await runStep("getBrandUrlNames", getBrandUrlNames);
await runStep("createBrandsFolders", () =>
createBrandsFolders(sourceClient, destClient, brands, stats),
);
// await runStep("createBrandsFolders", () =>
// createBrandsFolders(sourceClient, destClient, brands, stats),
// );
await runStep("copyBrandAssets", () =>
copyBrandAssets(sourceClient, destClient, brands, stats),
);
await runStep("replaceAssetsMetadata", () =>
replaceAssetsMetadata(destClient, brands, stats),
);
printSummary(stats);
if (stats.errors > 0) {
@@ -620,6 +809,10 @@ async function main() {
}
main().catch((error) => {
logger.error("unhandledRejection", "fatal error in main()", logger.formatError(error));
logger.error(
"unhandledRejection",
"fatal error in main()",
logger.formatError(error),
);
process.exitCode = 1;
});