passtive warnings & errors piped to global log file, unified batch pr…

…oc handler added, resusable progress bar, duplicated artifacts filter added #15, progress bar added to all ops #15,
cresteem · Dec 23, 2024 · 1ee2891 · 1ee2891
1 parent ef89786
commit 1ee2891
Show file tree

Hide file tree

Showing 14 changed files with 356 additions and 235 deletions.
diff --git a/lib/core/image.ts b/lib/core/image.ts
@@ -5,7 +5,6 @@ import { basename, dirname, extname, join, relative } from "node:path";
 import { ImagePool } from "remige";
 import { optimize } from "svgo";
 
-import { Presets, SingleBar } from "cli-progress";
 import {
 	AvifEncodeOptions,
 	ConfigurationOptions,
@@ -14,7 +13,14 @@ import {
 	SvgOptions,
 	WebpEncodeOptions,
 } from "../types";
-import { currentTime, logWriter, makeDirf, terminate } from "../utils";
+import {
+	batchProcess,
+	currentTime,
+	initProgressBar,
+	logWriter,
+	makeDirf,
+	terminate,
+} from "../utils";
 
 class _SVGWorker {
 	#cpuAllocation: number;
@@ -34,33 +40,20 @@ class _SVGWorker {
 	async #_svgBatchHandler(
 		outputPromises: (() => Promise<void>)[],
 	): Promise<void> {
-		const promiseBatches: (() => Promise<void>)[][] = [];
-
 		const batchSize: number = this.#cpuAllocation * 4;
 
-		for (let i = 0; i < outputPromises.length; i += batchSize) {
-			promiseBatches.push(outputPromises.slice(i, i + batchSize));
-		}
-
-		for (const batch of promiseBatches) {
-			const activatedBatch: Promise<void>[] = batch.map((func) => func());
-
-			await Promise.allSettled(activatedBatch);
-		}
+		await batchProcess({
+			promisedProcs: outputPromises,
+			batchSize: batchSize,
+			context: "SVG Optimizer",
+		});
 	}
 
 	async optimise(
 		svgImagePaths: string[],
 		destinationBasePath: string,
 	): Promise<void> {
-		const progressBar = new SingleBar(
-			{
-				format:
-					"[{bar}] {percentage}% | {value} of {total} ✅ | ⌛ Elapsed:{duration}s - ETA:{eta}s",
-				hideCursor: true,
-			},
-			Presets.shades_classic,
-		);
+		const progressBar = initProgressBar({ context: "Optimizing SVG" });
 		progressBar.start(svgImagePaths.length, 0);
 
 		const outputPromises: (() => Promise<void>)[] = svgImagePaths.map(
@@ -179,14 +172,7 @@ class _RasterizedImageWorker {
 			| WebpEncodeOptions,
 		destinationBasePath: string,
 	): Promise<void> {
-		const progressBar = new SingleBar(
-			{
-				format:
-					"[{bar}] {percentage}% | {value} of {total} ✅ | ⌛ Elapsed:{duration}s - ETA:{eta}s",
-				hideCursor: true,
-			},
-			Presets.shades_classic,
-		);
+		const progressBar = initProgressBar({ context: "Encoding Images" });
 		progressBar.start(imagePaths.length, 0);
 
 		//number of concurrent process.
@@ -298,13 +284,17 @@ export default class ImageWorker {
 		targetFormat: ImageWorkerOutputTypes,
 		destinationBasePath: string = this.#destPath,
 	): Promise<void> {
+		/* dumpRunTimeData({ data: imagePaths, context: "Image Paths" }); */
+
+		imagePaths = Array.from(new Set(imagePaths)); // keep unique image paths
+
 		process.on("SIGINT", () => {
 			terminate({
 				reason: "User interrupted encoding process. Shutting down....",
 			});
 		});
 
-		console.log(`\n[${currentTime()}] +++> ⏰ Image Encoding Started`);
+		console.log(`\n[${currentTime()}] +++> ⏰ Image Encoding Started\n`);
 
 		console.log(
 			`Number of ${

diff --git a/lib/core/imageset/imageset.lib/generators/nonsvg.ts b/lib/core/imageset/imageset.lib/generators/nonsvg.ts
@@ -121,7 +121,7 @@ export default class RasterisedImageSetGenerator {
 					.then(resolve)
 					.catch((err) => {
 						reject(
-							`Error while generating a image of ${targetWidth}, at ${baseImagePath}\n${err}`,
+							`\nError while generating a image of ${targetWidth},\nat ${baseImagePath}\n${err}`,
 						);
 					});
 			} else {
@@ -133,7 +133,7 @@ export default class RasterisedImageSetGenerator {
 						.then(resolve)
 						.catch((err) => {
 							reject(
-								`Error while generating a image of ${targetWidth}, at ${baseImagePath}\n${err}`,
+								`\nError while generating a image of ${targetWidth},\nat ${baseImagePath}\n${err}`,
 							);
 						});
 				} else {

diff --git a/lib/core/imageset/imageset.lib/htmlparser.ts b/lib/core/imageset/imageset.lib/htmlparser.ts
@@ -9,7 +9,7 @@ import {
 	ScreenSizesRecordType,
 	SrcRecordType,
 } from "../../../types";
-import { terminate } from "../../../utils";
+import { batchProcess, initProgressBar, terminate } from "../../../utils";
 import { getImageSizes } from "./cssrender";
 
 export default class HTMLParser {
@@ -151,34 +151,26 @@ export default class HTMLParser {
 		htmlFiles: string[],
 		batchSize: number = 2,
 	): Promise<ImageTagsRecord[]> {
-		const htmlParsePromises: (() => Promise<ImageTagsRecord>)[] =
-			htmlFiles.map(
-				(htmlFile) => (): Promise<ImageTagsRecord> =>
-					this.#_extractImagesRecord(htmlFile),
-			);
-
-		const recordTable: ImageTagsRecord[] = [];
+		const progressBar = initProgressBar({
+			context: "Extracting Image Records",
+		});
 
-		const promiseBatches: (() => Promise<ImageTagsRecord>)[][] = [];
+		progressBar.start(htmlFiles.length, 0);
 
-		/* Batching promises */
-		for (let i = 0; i < htmlParsePromises.length; i += batchSize) {
-			promiseBatches.push(htmlParsePromises.slice(i, i + batchSize));
-		}
-
-		/* Activating batches */
-		for (const batch of promiseBatches) {
-			const activatedBatch: Promise<ImageTagsRecord>[] = batch.map(
-				(func) => func(),
-			);
+		const htmlParsePromises: (() => Promise<ImageTagsRecord>)[] =
+			htmlFiles.map((htmlFile) => async (): Promise<ImageTagsRecord> => {
+				const imageTagsRecord = await this.#_extractImagesRecord(htmlFile);
+				progressBar.increment();
+				return imageTagsRecord;
+			});
 
-			try {
-				const records = await Promise.all(activatedBatch);
-				recordTable.push(...records);
-			} catch (err) {
-				console.log(err);
-			}
-		}
+		const recordTable: ImageTagsRecord[] = await batchProcess({
+			promisedProcs: htmlParsePromises,
+			batchSize: batchSize,
+			context: "Image Records Extractor",
+		});
+		progressBar.stop();
+		console.log("");
 
 		return recordTable;
 	}

diff --git a/lib/core/imageset/imageset.lib/transformer.ts b/lib/core/imageset/imageset.lib/transformer.ts
@@ -18,7 +18,7 @@ import {
 	PictureTagMakerResponse,
 	SrcRecordType,
 } from "../../../types";
-import { terminate, writeContent } from "../../../utils";
+import { batchProcess, writeContent } from "../../../utils";
 
 export default class ImgTagTransformer {
 	#screenSizes;
@@ -289,31 +289,11 @@ export default class ImgTagTransformer {
 				}),
 		);
 
-		const promiseBatches = [];
-
-		for (let i = 0; i < promises.length; i += batchSize) {
-			promiseBatches.push(promises.slice(i, i + batchSize));
-		}
-
-		const transformedHtmls: ImgTagTransResponse[] = [];
-
-		for (const batch of promiseBatches) {
-			const activatedBatch: Promise<ImgTagTransResponse>[] = batch.map(
-				(func) => func(),
-			);
-
-			try {
-				const batchResult: ImgTagTransResponse[] = await Promise.all(
-					activatedBatch,
-				);
-
-				transformedHtmls.push(...batchResult);
-			} catch (err) {
-				terminate({
-					reason: `Batch process failed at sub-transformer\n${err}`,
-				});
-			}
-		}
+		const transformedHtmls: ImgTagTransResponse[] = await batchProcess({
+			promisedProcs: promises,
+			batchSize: batchSize,
+			context: "Image Tag Transformers",
+		});
 
 		return transformedHtmls;
 	}
@@ -368,24 +348,11 @@ export default class ImgTagTransformer {
 				}),
 		);
 
-		const promiseBatches: (() => Promise<void>)[][] = [];
-
-		/* Batching promises */
-		for (let i = 0; i < promises.length; i += batchSize) {
-			promiseBatches.push(promises.slice(i, i + batchSize));
-		}
-		/* Activating batches */
-		for (const batch of promiseBatches) {
-			const activatedBatch: Promise<void>[] = batch.map((func) => func());
-
-			try {
-				await Promise.all(activatedBatch);
-			} catch (err) {
-				terminate({
-					reason: `Batch process failed at transformer\n${err}`,
-				});
-			}
-		}
+		await batchProcess({
+			promisedProcs: promises,
+			batchSize: batchSize,
+			context: "transform(), while writing outputs",
+		});
 
 		return {
 			linkedVideos: linkedVideos,