find-affiliates.ts

import fs from "node:fs/promises"
import path from "node:path"
import { google } from "@ai-sdk/google"
import { getDomain, processBatchWithErrorHandling, sleep } from "@primoui/utils"
import { generateText, Output } from "ai"
import { z } from "zod"
import { env } from "~/env"
import { db } from "~/services/db"

const searchGoogle = async (query: string, retries = 3) => {
  if (!env.GOOGLE_SEARCH_API_KEY || !env.GOOGLE_SEARCH_ENGINE_ID) {
    throw new Error("Google Search API key and engine ID are required")
  }

  for (let i = 0; i < retries; i++) {
    try {
      const params = new URLSearchParams({
        key: env.GOOGLE_SEARCH_API_KEY,
        cx: env.GOOGLE_SEARCH_ENGINE_ID,
        q: query,
        num: "2",
      })

      const response = await fetch(`https://www.googleapis.com/customsearch/v1?${params}`)
      const data = await response.json()

      if (data.error) {
        throw new Error(data.error.message)
      }

      return data.items || []
    } catch (error) {
      if (i === retries - 1) throw error
      await sleep(2000 * (i + 1)) // Exponential backoff
    }
  }
  return []
}

const hasAffiliateProgram = async (websiteUrl: string, searchResults: any[], retries = 3) => {
  const model = google("gemini-2.5-pro-preview-05-06")

  for (let i = 0; i < retries; i++) {
    try {
      const schema = z.object({
        hasAffiliate: z.boolean().describe("Whether the website has an affiliate program"),
        confidence: z.number().min(0).max(1).describe("Confidence level in the decision"),
        reason: z.string().describe("Brief explanation of the decision"),
        affiliateUrl: z.string().describe("The URL of the affiliate program"),
      })

      const { output } = await generateText({
        model,
        output: Output.object({ schema }),
        system:
          "You are an expert at analyzing websites to determine if they have an affiliate program.",
        prompt: `
          Analyze these search results from the website ${websiteUrl} to determine if they have an affiliate program:
          ${searchResults
            .map(
              result => `
            Title: ${result.title}
            Snippet: ${result.snippet}
            URL: ${result.link}
          `,
            )
            .join("\n")}

          Based on these results, does the website have an affiliate program?
          Consider mentions of "affiliate", "partner program", "referral program", etc.
          Give the biggest priority to the title of the search result.
        `,
        temperature: 0.1,
      })

      return object
    } catch (error) {
      console.error(`Attempt ${i + 1} failed for ${websiteUrl}:`, error)
      if (i === retries - 1) return null
      await sleep(5000 * (i + 1)) // Longer delay for LLM
    }
  }
  return null
}

const ensureFile = async (filePath: string) => {
  try {
    await fs.access(filePath)
  } catch {
    await fs.writeFile(filePath, "[]", "utf-8")
  }
}

const readArrayFromFile = async <T>(filePath: string): Promise<T[]> => {
  const raw = await fs.readFile(filePath, "utf-8")
  return JSON.parse(raw) as T[]
}

const appendToArrayFile = async <T>(filePath: string, item: T) => {
  const arr = await readArrayFromFile<T>(filePath)
  arr.push(item)
  await fs.writeFile(filePath, JSON.stringify(arr, null, 2), "utf-8")
}

async function main() {
  const affiliateOutputPath = path.join(process.cwd(), "affiliates-programs.json")
  const searchOutputPath = path.join(process.cwd(), "affiliates-searches.json")

  await Promise.all([ensureFile(affiliateOutputPath), ensureFile(searchOutputPath)])

  const [tools, alternatives] = await Promise.all([
    db.tool.findMany({
      where: { OR: [{ affiliateUrl: null }, { affiliateUrl: "" }] },
      select: { id: true, name: true, websiteUrl: true },
      take: 500,
    }),
    db.alternative.findMany({
      where: { OR: [{ affiliateUrl: null }, { affiliateUrl: "" }] },
      select: { id: true, name: true, websiteUrl: true },
      take: 500,
    }),
  ])

  console.log(`Found ${tools.length} tools and ${alternatives.length} alternatives`)

  const items = [...tools, ...alternatives].filter(item => item.websiteUrl)

  const processor = async ({ name, websiteUrl }: (typeof items)[0]) => {
    console.log(`Processing ${name}...`)

    const hostname = getDomain(websiteUrl!)
    const results = await searchGoogle(`site:${hostname} "affiliate program"`)

    await appendToArrayFile(searchOutputPath, {
      alternativeName: name,
      websiteUrl,
      results,
    })

    if (results.length === 0) {
      console.log(`[-] No search results found for ${name}`)
      return null
    }

    await sleep(2000)

    const analysis = await hasAffiliateProgram(websiteUrl!, results)

    if (analysis?.hasAffiliate) {
      await appendToArrayFile(affiliateOutputPath, {
        name,
        websiteUrl,
        ...analysis,
      })
      console.log(`[+] Found affiliate program for ${name}`)
    } else {
      console.log(`[-] No affiliate program found for ${name}`)
    }

    await sleep(3000)
    return analysis
  }

  await processBatchWithErrorHandling(items, processor, {
    batchSize: 1,
    delay: 2000,
    onError: (error, item) => console.error(`Error processing ${item.name}:`, error),
  })

  const affiliateFileArr = await readArrayFromFile<any>(affiliateOutputPath)
  console.log(`\nFound ${affiliateFileArr.length} alternatives with affiliate programs`)
  console.log(`Affiliate results saved to ${affiliateOutputPath}`)
  console.log(`Search results saved to ${searchOutputPath}`)
}

main().catch(console.error)