make bible

generateVersesJson.ts

/**
 * Script to generate verses.json from the scrollmapper Bible database.
 *
 * This script:
 * 1. Fetches the KJV CSV from GitHub
 * 2. Parses the CSV to extract book/chapter/verse structure
 * 3. Normalizes book names to our canonical format
 * 4. Outputs a compressed JSON with chapter ranges
 *
 * Run with: bun packages/db/scripts/generateVersesJson.ts
 */

import { writeFileSync } from 'node:fs'
import { resolve } from 'node:path'
import {
  BIBLE_BOOKS,
  type BibleBookName,
  type ChapterData,
  normalizeBookName,
} from '@preach-x/shared/verses'
import { Effect, Option, pipe } from 'effect'

const CSV_URL =
  'https://raw.githubusercontent.com/scrollmapper/bible_databases/master/formats/csv/KJV.csv'

type ParsedVerse = {
  book: string
  chapter: number
  verse: number
}

/**
 * Fetches and parses the CSV file from GitHub.
 */
const fetchAndParseCsv = Effect.gen(function* () {
  console.log(`Fetching CSV from ${CSV_URL}...`)

  const response = yield* Effect.tryPromise({
    try: () => fetch(CSV_URL),
    catch: (error) => new Error(`Failed to fetch CSV: ${error}`),
  })

  if (!response.ok) {
    return yield* Effect.fail(new Error(`HTTP error: ${response.status}`))
  }

  const text = yield* Effect.tryPromise({
    try: () => response.text(),
    catch: (error) => new Error(`Failed to read response: ${error}`),
  })

  console.log(`Parsing CSV...`)

  // Parse CSV (format: Book,Chapter,Verse,Text)
  // Lines may or may not be quoted, text may contain commas
  const lines = text.split('\n').slice(1) // Skip header
  const verses: ParsedVerse[] = []

  for (const line of lines) {
    if (!line.trim()) continue

    // Split by comma, but the format is: Book,Chapter,Verse,Text...
    // Book is never quoted, Chapter and Verse are numbers, Text may have commas
    // So we can safely split and take the first 3 parts
    const parts = line.split(',')
    if (parts.length >= 3) {
      const book = parts[0]?.trim()
      const chapter = parts[1]?.trim()
      const verse = parts[2]?.trim()

      if (book && chapter && verse) {
        const chapterNum = Number.parseInt(chapter, 10)
        const verseNum = Number.parseInt(verse, 10)

        if (!Number.isNaN(chapterNum) && !Number.isNaN(verseNum)) {
          verses.push({
            book,
            chapter: chapterNum,
            verse: verseNum,
          })
        }
      }
    }
  }

  console.log(`Parsed ${verses.length} verses`)
  return verses
})

/**
 * Groups verses by book and chapter, outputting chapter ranges.
 */
const groupIntoChapters = (verses: ParsedVerse[]): ChapterData[] => {
  // Group by book+chapter
  const chapters = new Map<string, { book: string; chapter: number; verses: number[] }>()

  for (const verse of verses) {
    const key = `${verse.book}:${verse.chapter}`
    const existing = chapters.get(key)
    if (existing) {
      existing.verses.push(verse.verse)
    } else {
      chapters.set(key, {
        book: verse.book,
        chapter: verse.chapter,
        verses: [verse.verse],
      })
    }
  }

  // Convert to ChapterData format
  const result: ChapterData[] = []

  for (const [, data] of chapters) {
    // Normalize book name
    const normalizedBook = pipe(
      normalizeBookName(data.book),
      Option.getOrElse(() => data.book as BibleBookName),
    )

    // Find book metadata
    const bookMeta = BIBLE_BOOKS.find((b) => b.name === normalizedBook)
    if (!bookMeta) {
      console.warn(`Unknown book: ${data.book} -> ${normalizedBook}`)
      continue
    }

    const sortedVerses = [...data.verses].sort((a, b) => a - b)
    const startVerse = sortedVerses[0] ?? 1
    const endVerse = sortedVerses[sortedVerses.length - 1] ?? 1

    result.push({
      book: normalizedBook,
      bookOrder: bookMeta.order,
      testament: bookMeta.testament,
      chapter: data.chapter,
      startVerse,
      endVerse,
    })
  }

  // Sort by book order, then chapter
  return result.sort((a, b) => {
    if (a.bookOrder !== b.bookOrder) {
      return a.bookOrder - b.bookOrder
    }
    return a.chapter - b.chapter
  })
}

/**
 * Main program
 */
const main = Effect.gen(function* () {
  const verses = yield* fetchAndParseCsv
  const chapters = groupIntoChapters(verses)

  console.log(`Generated ${chapters.length} chapter entries`)

  // Validate we have all 66 books
  const uniqueBooks = new Set(chapters.map((c) => c.book))
  console.log(`Found ${uniqueBooks.size} unique books`)

  if (uniqueBooks.size !== 66) {
    console.warn('Warning: Expected 66 books, but found', uniqueBooks.size)
  }

  // Count total verses
  const totalVerses = chapters.reduce((sum, c) => sum + (c.endVerse - c.startVerse + 1), 0)
  console.log(`Total verses: ${totalVerses}`)

  // Write to file
  const outputPath = resolve(import.meta.dir, '../data/verses.json')
  writeFileSync(outputPath, JSON.stringify(chapters, null, 2))
  console.log(`Wrote ${outputPath}`)
})

// Run
Effect.runPromise(main).catch(console.error)