Convert Storybooks

.gitignore

@@ -1,3 +1,4 @@
+test-results
 *.swp
 .DS_Store
 node_modules

config/index.d.ts

@@ -72,6 +72,10 @@ export type BookConfig = {
         };
     }[];
     bookTabs?: BookTabsConfig;
+    pageIllustrations?: {
+        num: number;
+        filename: string;
+    }[];
 };
 
 export type BookCollectionConfig = {

convert/convertBooks.ts

@@ -40,11 +40,34 @@ function displayBookId(bcId: string, bookId: string) {
 function replaceVideoTags(text: string, _bcId: string, _bookId: string): string {
     return text.replace(/\\video (.*)/g, '\\zvideo-s |id="$1"\\*\\zvideo-e\\*');
 }
+function replacePStyleTags(text: string, _bcId: string, _bookId: string): string {
+    return text.replace(/\\(p_[^\s]+)/g, '\\m \\zstyle |id="$1"\\*');
+}
+function replaceCStyleTags(text: string, _bcId: string, _bookId: string): string {
+    return text.replace(/\\(c_[^\s]+)(.*?)\\\1\*/g, '\\zcstyle-s |id="$1"\\*$2\\zcstyle-e\\*');
+}
+/**
+ * Convert list tags to milestones
+ */
+export function transformLists(text: string, _bcId: string, _bookId: string): string {
+    text = transformZuliTags(text);
+    text = transformZoliTags(text);
+    text = transformZonTags(text);
+    return text;
+}
+
+function transformZuliTags(usfm: string): string {
+    return usfm.replace(/(\\zuli\d+)/g, '\\nb $1\\*');
+}
+
+function transformZoliTags(usfm: string): string {
+    return usfm.replace(/(\\zoli\d+)/g, '\\nb $1\\*');
+}
 
-// This is the start of supporting story books, but it still fails if there is no chapter.
-function replacePageTags(text: string, _bcId: string, _bookId: string): string {
-    return text.replace(/\\page (.*)/g, '\\zpage-s |id="$1"\\*\\zpage-e\\*');
+function transformZonTags(usfm: string): string {
+    return usfm.replace(/(\\zon\d+)\s(\d+)/g, '$1 |start="$2"\\*');
 }
+
 function loadGlossary(collection: any, dataDir: string): string[] {
     const glossary: string[] = [];
     for (const book of collection.books) {
@@ -275,7 +298,9 @@ type FilterFunction = (text: string, bcId: string, bookId: string) => string;
 const usfmFilterFunctions: FilterFunction[] = [
     removeStrongNumberReferences,
     replaceVideoTags,
-    replacePageTags,
+    replacePStyleTags,
+    replaceCStyleTags,
+    transformLists,
     convertMarkdownsToMilestones,
     encodeJmpLinks,
     handleNoCaptionFigures,
@@ -291,7 +316,8 @@ function applyFilters(
     text: string,
     filterFunctions: FilterFunction[],
     bcId: string,
-    bookId: string
+    bookId: string,
+    bookType?: string
 ): string {
     let filteredText = text;
     for (const filterFn of filterFunctions) {
@@ -322,7 +348,7 @@ type ConvertBookContext = {
     bcId: string;
 };
 
-const unsupportedBookTypes = ['story', 'songs', 'audio-only', 'bloom-player', 'quiz', 'undefined'];
+const unsupportedBookTypes = ['songs', 'audio-only', 'bloom-player', 'quiz', 'undefined'];
 export async function convertBooks(
     dataDir: string,
     scriptureConfig: ScriptureConfig,
@@ -394,7 +420,6 @@ export async function convertBooks(
         for (const book of collection.books) {
             let bookConverted = false;
             switch (book.type) {
-                case 'story':
                 case 'songs':
                 case 'audio-only':
                 case 'bloom-player':
@@ -415,6 +440,7 @@ export async function convertBooks(
                     });
                     displayBookId(context.bcId, book.id);
                     break;
+                case 'story':
                 default:
                     bookConverted = true;
                     if (book.format === 'html') {
@@ -702,7 +728,7 @@ function convertScriptureBook(
     function processBookContent(resolve: () => void, err: any, content: string) {
         //process.stdout.write(`processBookContent: bookId:${book.id}, error:${err}\n`);
         if (err) throw err;
-        content = applyFilters(content, usfmFilterFunctions, context.bcId, book.id);
+        content = applyFilters(content, usfmFilterFunctions, context.bcId, book.id, book.type);
         if (context.scriptureConfig.traits['has-glossary']) {
             content = verifyGlossaryEntries(content, bcGlossary);
         }
@@ -780,8 +806,22 @@ function convertScriptureBook(
                     const filePath = path.join(context.dataDir, 'books', context.bcId, file);
                     fileContents.push(fs.readFileSync(filePath, 'utf-8'));
                 });
+                // Collect the file contents into a single document
+                let usfm: string;
 
-                processBookContent(resolve, null, fileContents.join(''));
+                if (book.type === 'story') {
+                    // The first file contains meta-content (id, title, etc)
+                    usfm = fileContents[0];
+
+                    // Subsequent files represent storybook pages.
+                    // SAB deletes the \page tags. Replace them with chapter tags.
+                    for (let i = 1; i < fileContents.length; i++) {
+                        usfm += `\\c ${i} ${fileContents[i]}`;
+                    }
+                } else {
+                    usfm = fileContents.join('');
+                }
+                processBookContent(resolve, null, usfm);
             }
         })
     );

convert/convertConfig.ts

@@ -518,6 +518,7 @@ export function parseBookCollections(document: Document, verbose: number) {
             if (verbose >= 2) console.log(`. book: ${book.id}`);
             const audio: BookCollectionAudioConfig[] = [];
             let chaptersLabels: { [key: string]: string } | undefined;
+            let pageIllustrations: { num: number; filename: string }[] = [];
             for (const page of book.getElementsByTagName('page')) {
                 if (verbose >= 2) console.log(`.. page: ${page.attributes[0].value}`);
                 const char = page.attributes.getNamedItem('char')?.value;
@@ -529,6 +530,20 @@ export function parseBookCollections(document: Document, verbose: number) {
                     const chapterNum = page.attributes.getNamedItem('num')!.value;
                     chaptersLabels[chapterNum] = char;
                 }
+                const imageFileTag = page.getElementsByTagName('image-filename')[0];
+                if (imageFileTag) {
+                    pageIllustrations.push({
+                        num: Number(page.attributes.getNamedItem('num')?.value),
+                        filename: book.getElementsByTagName('images')[0]
+                            ? tag.id +
+                              '-' +
+                              book.attributes.getNamedItem('id')!.value +
+                              '-' +
+                              imageFileTag.innerHTML
+                            : imageFileTag.innerHTML
+                    });
+                }
+
                 const audioTag = page.getElementsByTagName('audio')[0];
                 if (!audioTag) continue;
                 const fTag = audioTag.getElementsByTagName('f')[0];
@@ -709,7 +724,8 @@ export function parseBookCollections(document: Document, verbose: number) {
                 style,
                 styles,
                 footer,
-                bookTabs
+                bookTabs,
+                pageIllustrations
             });
             if (verbose >= 3) console.log(`.... book: `, JSON.stringify(books[0]));
         }

convert/runTests.ts

@@ -0,0 +1,14 @@
+import { spawn } from 'node:child_process';
+
+const args = process.argv.slice(2).join(' ');
+console.log('playwright test ' + args);
+spawn('playwright test ' + args, {
+    shell: true,
+    stdio: 'inherit'
+}).on('close', (code) => {
+    console.log('vitest ' + args);
+    spawn('vitest ' + args, {
+        shell: true,
+        stdio: 'inherit'
+    });
+});

convert/tests/sab/convertConfigSAB.test.ts

@@ -109,9 +109,11 @@ if (programType === 'DAB') {
                 expect(bkk.chaptersN).not.toSatisfy((r) => r === '' || r === undefined);
                 expect(bkk.id).not.toSatisfy((r) => r === '' || r === undefined);
                 expect(bkk.name).not.toSatisfy((r) => r === '' || r === undefined);
-                expect(bkk.section).not.toSatisfy((r) => r === '' || r === undefined);
-                expect(bkk.testament).not.toSatisfy((r) => r === '' || r === undefined);
-                expect(bkk.abbreviation).not.toSatisfy((r) => r === '' || r === undefined);
+                if (bkk.type !== 'story') {
+                    expect(bkk.section).not.toSatisfy((r) => r === '' || r === undefined);
+                    expect(bkk.testament).not.toSatisfy((r) => r === '' || r === undefined);
+                    expect(bkk.abbreviation).not.toSatisfy((r) => r === '' || r === undefined);
+                }
                 expect(bkk.file).not.toSatisfy((r) => r === '' || r === undefined);
 
                 for (const audio in bkk.audio) {

convert/tests/sab/storybook.test.ts

@@ -0,0 +1,195 @@
+import { expect, test } from 'vitest';
+import { transformLists } from '../../convertBooks';
+
+function tokensOf(str: string) {
+    return str.split(/\s+/).filter((token) => token.length);
+}
+
+test('convert unordered list to milestones', () => {
+    const input = `
+        \\m Some content
+        \\zuli1 One
+        \\zuli1 Two \\zuli1 Three in a row!
+        \\b 
+    `;
+    const expected = `
+        \\m Some content
+        \\nb \\zuli1\\* One
+        \\nb \\zuli1\\* Two
+        \\nb \\zuli1\\* Three in a row!
+        \\b
+    `;
+    const result = transformLists(input, '', '');
+    expect(tokensOf(result)).toEqual(tokensOf(expected));
+});
+
+test('convert ordered list to milestones', () => {
+    const input = `
+        \\m Some content
+        \\zon1 10
+        \\zoli1 One
+        \\zoli1 Two \\zoli1 Three in a row!
+        \\b 
+    `;
+    const expected = `
+        \\m Some content
+        \\zon1 |start="10"\\*
+        \\nb \\zoli1\\* One
+        \\nb \\zoli1\\* Two
+        \\nb \\zoli1\\* Three in a row!
+        \\b
+    `;
+    const result = transformLists(input, '', '');
+    expect(tokensOf(result)).toEqual(tokensOf(expected));
+});
+
+test('convert multilevel unordered list to milestones', () => {
+    const input = `
+        \\c 2
+        \\b
+        \\zuli1 Old Testament
+            \\zuli2 Pentateuch
+                \\zuli3 Genesis
+                \\zuli3 Exodus
+                \\zuli3 Leviticus
+            \\zuli2 Joshua
+            \\zuli2 Judges
+        \\zuli1 New Testament
+            \\zuli2 Matthew
+            \\zuli2 Mark
+            \\zuli2 Luke
+            \\zuli2 John
+        \\zuli1 Glossary
+    `;
+    const expected = `
+        \\c 2
+        \\b
+        \\nb \\zuli1\\* Old Testament
+            \\nb \\zuli2\\* Pentateuch
+                \\nb \\zuli3\\* Genesis
+                \\nb \\zuli3\\* Exodus
+                \\nb \\zuli3\\* Leviticus
+            \\nb \\zuli2\\* Joshua
+            \\nb \\zuli2\\* Judges
+        \\nb \\zuli1\\* New Testament
+            \\nb \\zuli2\\* Matthew
+            \\nb \\zuli2\\* Mark
+            \\nb \\zuli2\\* Luke
+            \\nb \\zuli2\\* John
+        \\nb \\zuli1\\* Glossary
+    `;
+    const result = transformLists(input, '', '');
+    expect(tokensOf(result)).toEqual(tokensOf(expected));
+});
+
+test('convert multilevel ordered list to milestones', () => {
+    const input = `
+        \\m My List:
+        \\b
+        \\zon1 1
+        \\zoli1 Food
+            \\zon2 1
+            \\zoli2 Fruit
+                \\zon3 1
+                \\zoli3 Apples
+                \\zoli3 Bananas
+                \\zoli3 Pears
+            \\zoli2 Dessert
+                \\zoli3 Pie
+                \\zoli3 Cake
+                \\zoli3 Ice Cream
+        \\zoli1 Drinks
+            \\zoli2 Coffee
+            \\zoli2 Water
+            \\zoli2 Tea
+    `;
+    const expected = `
+        \\m My List:
+        \\b
+        \\zon1 |start="1"\\*
+        \\nb \\zoli1\\* Food
+            \\zon2 |start="1"\\*
+            \\nb \\zoli2\\* Fruit
+                \\zon3 |start="1"\\*
+                \\nb \\zoli3\\* Apples
+                \\nb \\zoli3\\* Bananas
+                \\nb \\zoli3\\* Pears
+            \\nb \\zoli2\\* Dessert
+                \\nb \\zoli3\\* Pie
+                \\nb \\zoli3\\* Cake
+                \\nb \\zoli3\\* Ice Cream
+        \\nb \\zoli1\\* Drinks
+            \\nb \\zoli2\\* Coffee
+            \\nb \\zoli2\\* Water
+            \\nb \\zoli2\\* Tea
+    `;
+    const result = transformLists(input, '', '');
+    expect(tokensOf(result)).toEqual(tokensOf(expected));
+});
+
+test('convert unordered list with formatting to milestones', () => {
+    const input = `
+        \\m Some content
+        \\zuli1 One
+        \\zuli1 \\bd Two \\bd*
+        \\zuli1 Three in a row!
+        \\b
+    `;
+    const expected = `
+        \\m Some content
+        \\nb \\zuli1\\* One
+        \\nb \\zuli1\\* \\bd Two \\bd*
+        \\nb \\zuli1\\* Three in a row!
+        \\b
+    `;
+    const result = transformLists(input, '', '');
+    expect(tokensOf(result)).toEqual(tokensOf(expected));
+});
+
+test('convert ordered list with formatting to milestones', () => {
+    const input = `
+        \\m Some content
+        \\zon1 10
+        \\zoli1 One
+        \\zoli1 \\bdit Two \\bdit*
+        \\zoli1 Three in a row!
+        \\b
+    `;
+    const expected = `
+        \\m Some content
+        \\zon1 |start="10"\\*
+        \\nb \\zoli1\\* One
+        \\nb \\zoli1\\* \\bdit Two \\bdit*
+        \\nb \\zoli1\\* Three in a row!
+        \\b
+    `;
+    const result = transformLists(input, '', '');
+    expect(tokensOf(result)).toEqual(tokensOf(expected));
+});
+
+test('convert multilevel ordered list with formatting to milestones', () => {
+    const input = `
+        \\m Some content
+        \\zon1 10
+        \\zoli1 One
+            \\zon2 3
+            \\zoli2 \\it sub-point 1 \\it*
+            \\zoli2 sub-point 2
+        \\zoli1 \\bdit Two \\bdit*
+        \\zoli1 Three in a row!
+        \\b
+    `;
+    const expected = `
+        \\m Some content
+        \\zon1 |start="10"\\*
+        \\nb \\zoli1\\* One
+            \\zon2 |start="3"\\*
+            \\nb \\zoli2\\* \\it sub-point 1 \\it*
+            \\nb \\zoli2\\* sub-point 2
+        \\nb \\zoli1\\* \\bdit Two \\bdit*
+        \\nb \\zoli1\\* Three in a row!
+        \\b
+    `;
+    const result = transformLists(input, '', '');
+    expect(tokensOf(result)).toEqual(tokensOf(expected));
+});