chore: do some stuff on mdx-to-md converter

Author: A-Najmabadi

.gitignore

@@ -12,6 +12,9 @@
 
 all-links-llms.txt
 
+# mdx-to-md manifest (tracks processed files)
+public/llms/.mdx-to-md-manifest.json
+
 # testing
 /coverage
 

mdx-to-md-converter/src/index.ts

@@ -25,6 +25,7 @@ import {
   getAbsolutePaths,
   findMdxFilesWithoutMd
 } from './modules/gitChanges';
+import { ManifestManager } from './modules/manifestManager';
 
 import fs from 'fs';
 import path from 'path';
@@ -155,6 +156,8 @@ async function main() {
     const outputDir = path.join(projectRoot, 'public', 'llms');
     const allLinksPath = path.join(projectRoot, 'public', 'all-links-llms.txt');
 
+    const manifest = new ManifestManager(outputDir);
+    
     console.log(`Searching for MDX files in: ${srcPagesPath}`);
 
     const processAll = shouldProcessAllFiles();
@@ -251,6 +254,8 @@ async function main() {
         fs.writeFileSync(outputFilePath, '\ufeff' + finalMdContent, { encoding: 'utf8' });
         console.log(`✓ Saved: ${outputFilePath}`);
 
+        manifest.markAsProcessed(filePath, outputFilePath);
+        
         const urlPath = `llms/${mdFileName.replace(/\\/g, '/')}`;
         const url = `https://docs.liara.ir/${urlPath}`;
 

mdx-to-md-converter/src/modules/gitChanges.ts

@@ -1,6 +1,7 @@
 import { execSync } from 'child_process';
 import fs from 'fs';
 import path from 'path';
+import { ManifestManager } from './manifestManager';
 
 export interface GitChanges {
   modified: string[];
@@ -126,7 +127,8 @@ export function getAbsolutePaths(relativePaths: string[], basePath: string): str
 
 
 export function findMdxFilesWithoutMd(srcPagesPath: string, outputDir: string): string[] {
-  const missingMdFiles: string[] = [];
+  const filesToProcess: string[] = [];
+  const manifest = new ManifestManager(outputDir);
 
   function traverseMdxFiles(currentDir: string) {
     const items = fs.readdirSync(currentDir);
@@ -143,8 +145,8 @@ export function findMdxFilesWithoutMd(srcPagesPath: string, outputDir: string):
         const mdFileName = relativePath.replace(/\.mdx$/, '.md');
         const mdFilePath = path.join(outputDir, mdFileName);
 
-        if (!fs.existsSync(mdFilePath)) {
-          missingMdFiles.push(relativePath);
+        if (manifest.needsProcessing(fullPath, mdFilePath)) {
+          filesToProcess.push(relativePath);
         }
       }
     }
@@ -154,5 +156,5 @@ export function findMdxFilesWithoutMd(srcPagesPath: string, outputDir: string):
     traverseMdxFiles(srcPagesPath);
   }
 
-  return missingMdFiles;
+  return filesToProcess;
 }

mdx-to-md-converter/src/modules/manifestManager.ts

@@ -0,0 +1,146 @@
+import fs from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+
+interface ManifestEntry {
+  mdxPath: string;
+  mdPath: string;
+  mdxHash: string;
+  lastProcessed: string;
+}
+
+interface Manifest {
+  version: string;
+  entries: Record<string, ManifestEntry>;
+}
+
+const MANIFEST_VERSION = '1.0.0';
+
+export class ManifestManager {
+  private manifestPath: string;
+  private manifest: Manifest;
+
+  constructor(outputDir: string) {
+    this.manifestPath = path.join(outputDir, '.mdx-to-md-manifest.json');
+    this.manifest = this.loadManifest();
+  }
+
+  private loadManifest(): Manifest {
+    if (fs.existsSync(this.manifestPath)) {
+      try {
+        const content = fs.readFileSync(this.manifestPath, 'utf-8');
+        return JSON.parse(content);
+      } catch (error) {
+        console.warn('Failed to load manifest, creating new one:', error);
+      }
+    }
+    
+    return {
+      version: MANIFEST_VERSION,
+      entries: {}
+    };
+  }
+
+  private saveManifest(): void {
+    try {
+      fs.writeFileSync(
+        this.manifestPath,
+        JSON.stringify(this.manifest, null, 2),
+        'utf-8'
+      );
+    } catch (error) {
+      console.error('Failed to save manifest:', error);
+    }
+  }
+
+  /**
+   * Calculate MD5 hash of a file
+   */
+  private calculateFileHash(filePath: string): string {
+    try {
+      const content = fs.readFileSync(filePath, 'utf-8');
+      return crypto.createHash('md5').update(content).digest('hex');
+    } catch (error) {
+      return '';
+    }
+  }
+
+  /**
+   * Check if an MDX file needs to be processed
+   * Returns true if:
+   * - File is not in manifest
+   * - File hash has changed
+   * - Corresponding MD file doesn't exist
+   */
+  needsProcessing(mdxPath: string, mdPath: string): boolean {
+    const entry = this.manifest.entries[mdxPath];
+    
+    // If not in manifest, needs processing
+    if (!entry) {
+      return true;
+    }
+    
+    // If MD file doesn't exist, needs processing
+    if (!fs.existsSync(mdPath)) {
+      return true;
+    }
+    
+    // If MDX file hash has changed, needs processing
+    const currentHash = this.calculateFileHash(mdxPath);
+    if (currentHash !== entry.mdxHash) {
+      return true;
+    }
+    
+    return false;
+  }
+
+  /**
+   * Mark a file as processed
+   */
+  markAsProcessed(mdxPath: string, mdPath: string): void {
+    const mdxHash = this.calculateFileHash(mdxPath);
+    
+    this.manifest.entries[mdxPath] = {
+      mdxPath,
+      mdPath,
+      mdxHash,
+      lastProcessed: new Date().toISOString()
+    };
+    
+    this.saveManifest();
+  }
+
+  /**
+   * Remove entry from manifest (when MDX file is deleted)
+   */
+  removeEntry(mdxPath: string): void {
+    delete this.manifest.entries[mdxPath];
+    this.saveManifest();
+  }
+
+  /**
+   * Get all entries
+   */
+  getAllEntries(): Record<string, ManifestEntry> {
+    return this.manifest.entries;
+  }
+
+  /**
+   * Clean up entries for files that no longer exist
+   */
+  cleanup(existingMdxFiles: string[]): void {
+    const existingSet = new Set(existingMdxFiles);
+    let changed = false;
+    
+    for (const mdxPath in this.manifest.entries) {
+      if (!existingSet.has(mdxPath)) {
+        delete this.manifest.entries[mdxPath];
+        changed = true;
+      }
+    }
+    
+    if (changed) {
+      this.saveManifest();
+    }
+  }
+}