automatically identify and remove duplicate items from your 1Password vault, complete with whitelisting, title cleaning, and persistent caching.

1password-douplicate-removed.js

#!/usr/bin/env node

/**
 * 1Password Duplicate Remover
 *
 * A CLI tool to identify and remove duplicate items from 1Password vaults.
 * Supports detailed logging, dry-run mode, and multiple matching strategies.
 *
 * @requires 1Password CLI (op) - https://1password.com/downloads/command-line/
 * @requires Node.js >=16
 */

const { execSync } = require("child_process");
const { URL } = require("url");
const fs = require("fs");
const path = require("path");

// ============================================================================
// Types & Constants
// ============================================================================

const KEY_STRATEGIES = {
  TITLE: "title",
  WEBSITE_USERNAME: "website-username",
};

// ============================================================================
// Configuration
// ============================================================================

const CONFIG = {
  DRY_RUN: false,
  MAX_DUPLICATE_KEYS: null, // null = unlimited
  CATEGORY_FILTER: null, // e.g., "Login", "Password", "CreditCard"
  USE_ARCHIVE: true,
  KEY_STRATEGY: KEY_STRATEGIES.WEBSITE_USERNAME, // "title" | "website-username"
  WHITELIST: [
    // Add URL patterns or domains to whitelist (items matching these will never be removed)
    // Examples:
    "localhost:3000",
    // "example.com",
    // "https://app.example.com",
  ],
};

// Cache file paths
const CACHE_DIR = path.join(__dirname, ".cache");
const DUPLICATE_CACHE_FILE = path.join(CACHE_DIR, "processed-duplicates.json");
const TITLE_CACHE_FILE = path.join(CACHE_DIR, "processed-titles.json");

// ============================================================================
// Cache Manager
// ============================================================================

class CacheManager {
  /**
   * Ensure cache directory exists
   */
  static ensureCacheDir() {
    if (!fs.existsSync(CACHE_DIR)) {
      fs.mkdirSync(CACHE_DIR, { recursive: true });
    }
  }

  /**
   * Load cache from file
   * @param {string} cacheFile - Path to cache file
   * @returns {Set<string>} Set of processed item IDs
   */
  static loadCache(cacheFile) {
    this.ensureCacheDir();

    if (!fs.existsSync(cacheFile)) {
      return new Set();
    }

    try {
      const data = fs.readFileSync(cacheFile, "utf-8");
      const cache = JSON.parse(data);
      return new Set(cache.processedItems || []);
    } catch (error) {
      console.warn(
        `  Warning: Could not load cache from ${cacheFile}: ${error.message}`,
      );
      return new Set();
    }
  }

  /**
   * Save cache to file
   * @param {string} cacheFile - Path to cache file
   * @param {Set<string>} cache - Set of processed item IDs
   */
  static saveCache(cacheFile, cache) {
    this.ensureCacheDir();

    try {
      const data = {
        processedItems: Array.from(cache),
        lastUpdated: new Date().toISOString(),
      };
      fs.writeFileSync(cacheFile, JSON.stringify(data, null, 2), "utf-8");
    } catch (error) {
      console.warn(
        `  Warning: Could not save cache to ${cacheFile}: ${error.message}`,
      );
    }
  }

  /**
   * Add item to cache
   * @param {Set<string>} cache - Cache set
   * @param {string} itemId - Item ID to add
   */
  static addToCache(cache, itemId) {
    cache.add(itemId);
  }

  /**
   * Add item to cache and save immediately
   * @param {Set<string>} cache - Cache set
   * @param {string} itemId - Item ID to add
   * @param {string} cacheFile - Path to cache file
   */
  static addToCacheAndSave(cache, itemId, cacheFile) {
    cache.add(itemId);
    this.saveCache(cacheFile, cache);
  }

  /**
   * Check if item is in cache
   * @param {Set<string>} cache - Cache set
   * @param {string} itemId - Item ID to check
   * @returns {boolean} True if item is in cache
   */
  static isInCache(cache, itemId) {
    return cache.has(itemId);
  }
}

// ============================================================================
// Utilities
// ============================================================================

class OnePasswordClient {
  /**
   * Execute 1Password CLI command
   * @param {string} command - The op command to execute
   * @returns {string} Command output
   * @throws {Error} If command fails
   */
  static exec(command) {
    try {
      return execSync(command, { encoding: "utf-8", stdio: "pipe" });
    } catch (error) {
      throw new Error(
        `1Password CLI command failed: ${command}\n${error.message}`,
      );
    }
  }

  /**
   * List all items from 1Password
   * @param {string|null} category - Optional category filter
   * @returns {Array<Object>} Array of item summaries
   */
  static listItems(category = null) {
    const command = category
      ? `op items list --categories ${category} --format=json`
      : `op items list --format=json`;

    const output = this.exec(command);
    return JSON.parse(output);
  }

  /**
   * Get full details for a specific item
   * @param {string} itemId - The item ID
   * @returns {Object} Full item details
   */
  static getItem(itemId) {
    const output = this.exec(`op item get ${itemId} --format=json`);
    return JSON.parse(output);
  }

  /**
   * Delete or archive an item
   * @param {string} itemId - The item ID
   * @param {boolean} archive - Whether to archive instead of delete
   * @returns {void}
   */
  static removeItem(itemId, archive = false) {
    const command = archive
      ? `op item delete ${itemId} --archive`
      : `op item delete ${itemId}`;
    this.exec(command);
  }

  /**
   * Update an item's title
   * @param {string} itemId - The item ID
   * @param {string} newTitle - The new title
   * @returns {void}
   */
  static updateItemTitle(itemId, newTitle) {
    // Use op item edit to update the title
    // Escape quotes in title and wrap in quotes
    const escapedTitle = newTitle.replace(/"/g, '\\"');
    this.exec(`op item edit ${itemId} --title="${escapedTitle}"`);
  }
}

class KeyGenerator {
  /**
   * Extract domain from URL
   * @param {string} url - The URL string
   * @returns {string|null} Extracted domain or null
   */
  static extractDomain(url) {
    if (!url) return null;

    try {
      const normalizedUrl = url.startsWith("http") ? url : `https://${url}`;
      const urlObj = new URL(normalizedUrl);
      return urlObj.hostname.replace(/^www\./, "");
    } catch {
      // Fallback regex extraction
      const match = url.match(/(?:https?:\/\/)?(?:www\.)?([^\/]+)/i);
      return match ? match[1] : null;
    }
  }

  /**
   * Get field value by label
   * @param {Array<Object>} fields - Array of field objects
   * @param {string} label - Field label to find
   * @returns {string|null} Field value or null
   */
  static getFieldValue(fields, label) {
    const field = fields.find((f) => f.label === label);
    return field?.value ?? null;
  }

  /**
   * Generate key for grouping items
   * @param {Object} item - Item summary object
   * @param {Object|null} itemDetails - Optional full item details
   * @param {string} strategy - Key generation strategy
   * @returns {string} Generated key
   */
  static generate(item, itemDetails = null, strategy = KEY_STRATEGIES.TITLE) {
    if (strategy === KEY_STRATEGIES.WEBSITE_USERNAME) {
      const details = itemDetails ?? item;
      const fields = details.fields ?? [];
      const urls = details.urls ?? [];

      const username = this.getFieldValue(fields, "username");
      const href = urls[0]?.href ?? urls[0] ?? null;
      const website = href ? this.extractDomain(href) : null;

      if (website && username) {
        return `${website}-${username}`;
      }
    }

    // Fallback to title
    return item.title ?? `item-${item.id}`;
  }
}

class WhitelistChecker {
  /**
   * Check if an item matches any whitelist pattern
   * @param {Object} item - Item object (can be summary or full details)
   * @param {Array<string>} whitelist - Array of whitelist patterns
   * @returns {boolean} True if item matches whitelist
   */
  static isWhitelisted(item, whitelist) {
    if (!whitelist || whitelist.length === 0) {
      return false;
    }

    const urls = item.urls ?? [];

    // Check all URLs in the item
    for (const urlObj of urls) {
      const url = urlObj.href ?? urlObj;
      if (!url) continue;

      // Extract domain from URL
      const domain = KeyGenerator.extractDomain(url);
      const fullUrl = url.toLowerCase();

      // Check against each whitelist pattern
      for (const pattern of whitelist) {
        const lowerPattern = pattern.toLowerCase();

        // Match full URL or domain
        if (
          fullUrl.includes(lowerPattern) ||
          (domain && domain.includes(lowerPattern))
        ) {
          return true;
        }
      }
    }

    // Also check title as fallback
    const title = item.title ?? "";
    if (title) {
      for (const pattern of whitelist) {
        if (title.toLowerCase().includes(pattern.toLowerCase())) {
          return true;
        }
      }
    }

    return false;
  }
}

class ItemComparator {
  /**
   * Find a field by label (case-insensitive)
   * @param {Array<Object>} fields - Array of field objects
   * @param {string} label - Field label to find
   * @returns {Object|null} Field object or null
   */
  static findFieldByLabel(fields, label) {
    if (!fields || fields.length === 0) return null;
    const lowerLabel = label.toLowerCase();
    return (
      fields.find(
        (f) =>
          f.label?.toLowerCase() === lowerLabel ||
          f.id?.toLowerCase() === lowerLabel,
      ) ?? null
    );
  }

  /**
   * Check if username and password fields match
   * @param {Array<Object>} fields1 - First item's fields
   * @param {Array<Object>} fields2 - Second item's fields
   * @returns {Object} Match result with usernameMatch and passwordMatch
   */
  static checkCredentialsMatch(fields1, fields2) {
    const username1 = this.findFieldByLabel(fields1, "username");
    const username2 = this.findFieldByLabel(fields2, "username");
    const password1 = this.findFieldByLabel(fields1, "password");
    const password2 = this.findFieldByLabel(fields2, "password");

    const usernameMatch =
      username1 && username2 && username1.value === username2.value;
    const passwordMatch =
      password1 && password2 && password1.value === password2.value;

    return {
      usernameMatch,
      passwordMatch,
      hasUsername: !!username1 && !!username2,
      hasPassword: !!password1 && !!password2,
    };
  }

  /**
   * Compare two items and return detailed comparison result
   * @param {Object} item1 - First item (kept item)
   * @param {Object} item2 - Second item (candidate duplicate)
   * @returns {Object} Comparison result
   */
  static compare(item1, item2) {
    const result = {
      isMatch: false,
      matchedFields: [],
      matchedUrls: [],
      fieldDifferences: [],
      urlDifferences: [],
      reason: "",
      credentialsMatch: false,
    };

    const fields1 = item1.fields ?? [];
    const fields2 = item2.fields ?? [];

    // Check credentials match first (username + password)
    const credentialsCheck = this.checkCredentialsMatch(fields1, fields2);
    result.credentialsMatch =
      credentialsCheck.usernameMatch && credentialsCheck.passwordMatch;

    // Compare URLs to show differences even when credentials match
    const urls1 = (item1.urls ?? []).map((url) => url.href ?? url).sort();
    const urls2 = (item2.urls ?? []).map((url) => url.href ?? url).sort();

    for (let i = 0; i < Math.max(urls1.length, urls2.length); i++) {
      const url1 = urls1[i];
      const url2 = urls2[i];
      if (url1 !== url2) {
        result.urlDifferences.push({
          keptUrl: url1 ?? "(none)",
          duplicateUrl: url2 ?? "(none)",
        });
      } else if (url1) {
        result.matchedUrls.push(url1);
      }
    }

    // If credentials match, consider it a duplicate even if URLs differ
    if (result.credentialsMatch) {
      result.isMatch = true;
      result.reason =
        "Username and password match - considered duplicate (URLs may differ)";
      return result;
    }

    // Otherwise, do full comparison
    // Check URL count (for strict matching)
    if (item1.urls?.length !== item2.urls?.length) {
      result.reason = `URL count mismatch: ${item1.urls?.length ?? 0} vs ${item2.urls?.length ?? 0}`;
      return result;
    }

    // Check field count
    if (fields1.length !== fields2.length) {
      result.reason = `Field count mismatch: ${fields1.length} vs ${fields2.length}`;
      return result;
    }

    const sortedFields1 = fields1.sort((a, b) => a.id.localeCompare(b.id));
    const sortedFields2 = fields2.sort((a, b) => a.id.localeCompare(b.id));

    // Compare fields
    for (let i = 0; i < sortedFields1.length; i++) {
      const field1 = sortedFields1[i];
      const field2 = sortedFields2[i];

      if (field1.id !== field2.id) {
        result.fieldDifferences.push({
          fieldId: `${field1.id} vs ${field2.id}`,
          label: `${field1.label ?? "N/A"} vs ${field2.label ?? "N/A"}`,
          keptValue: field1.value,
          duplicateValue: field2.value,
        });
      } else if (field1.value !== field2.value) {
        result.fieldDifferences.push({
          fieldId: field1.id,
          label: field1.label ?? "N/A",
          keptValue: field1.value,
          duplicateValue: field2.value,
        });
      } else {
        result.matchedFields.push({
          id: field1.id,
          label: field1.label,
          value: field1.value,
        });
      }
    }

    // URLs already compared above, skip duplicate comparison

    // Determine match
    if (
      result.fieldDifferences.length === 0 &&
      result.urlDifferences.length === 0
    ) {
      result.isMatch = true;
      result.reason = "All fields and URLs match";
    } else {
      result.reason = `Found ${result.fieldDifferences.length} field difference(s) and ${result.urlDifferences.length} URL difference(s)`;
    }

    return result;
  }
}

// ============================================================================
// Logging
// ============================================================================

class Logger {
  static SEPARATOR_LENGTH = 80;

  static separator() {
    console.log("\n" + "=".repeat(this.SEPARATOR_LENGTH));
  }

  static section(title) {
    console.log("\n" + "─".repeat(this.SEPARATOR_LENGTH));
    console.log(`  ${title}`);
    console.log("─".repeat(this.SEPARATOR_LENGTH));
  }

  static item(label, item) {
    console.log(`\n  ${label}:`);
    console.log(`    ID: ${item.id}`);
    console.log(`    Title: ${item.title ?? "N/A"}`);
    console.log(`    Created: ${item.createdAt ?? "N/A"}`);
    console.log(`    Updated: ${item.updatedAt ?? "N/A"}`);
  }

  static fields(fields, label = "Fields") {
    if (!fields || fields.length === 0) {
      console.log(`  ${label}: (none)`);
      return;
    }

    console.log(`  ${label} (${fields.length}):`);
    fields.forEach((field, idx) => {
      const value = field.value ?? "(empty)";
      const truncated =
        value.length > 60 ? `${value.substring(0, 60)}...` : value;
      console.log(
        `    ${idx + 1}. [${field.id}] ${field.label ?? "N/A"}: ${truncated}`,
      );
    });
  }

  static urls(urls, label = "URLs") {
    if (!urls || urls.length === 0) {
      console.log(`  ${label}: (none)`);
      return;
    }

    console.log(`  ${label} (${urls.length}):`);
    urls.forEach((url, idx) => {
      const href = url.href ?? url;
      console.log(`    ${idx + 1}. ${href}`);
    });
  }

  static comparison(comparison) {
    console.log("\n  Comparison Result:");
    if (comparison.isMatch) {
      if (comparison.credentialsMatch) {
        console.log(
          "    ✓ Username and password MATCH - duplicate will be removed (URLs differ but credentials are identical)",
        );
      } else {
        console.log("    ✓ Items are IDENTICAL - duplicate will be removed");
      }
    } else {
      console.log("    ✗ Items are DIFFERENT - both will be kept");
    }

    // Show credentials match info
    if (comparison.credentialsMatch) {
      console.log("\n    Credentials Match:");
      console.log("      ✓ Username: MATCH");
      console.log("      ✓ Password: MATCH");
    }

    if (comparison.fieldDifferences.length > 0) {
      console.log("\n    Field Differences:");
      comparison.fieldDifferences.forEach((diff) => {
        console.log(`      - [${diff.fieldId}] ${diff.label ?? "N/A"}:`);
        console.log(`          Kept: ${diff.keptValue ?? "(empty)"}`);
        console.log(`          Duplicate: ${diff.duplicateValue ?? "(empty)"}`);
      });
    }

    // Always show URL differences if they exist (even when credentials match)
    if (comparison.urlDifferences.length > 0) {
      console.log("\n    URL Differences:");
      comparison.urlDifferences.forEach((diff) => {
        console.log(`      - Kept: ${diff.keptUrl}`);
        console.log(`        Duplicate: ${diff.duplicateUrl}`);
      });
      if (comparison.credentialsMatch) {
        console.log(
          "      (Note: URLs differ but credentials match - duplicate will still be removed)",
        );
      }
    }

    if (comparison.matchedFields.length > 0) {
      console.log("\n    Matched Fields:");
      comparison.matchedFields.forEach((field) => {
        const value = field.value ?? "(empty)";
        const truncated =
          value.length > 50 ? `${value.substring(0, 50)}...` : value;
        console.log(
          `      ✓ [${field.id}] ${field.label ?? "N/A"}: ${truncated}`,
        );
      });
    }

    if (comparison.matchedUrls.length > 0) {
      console.log("\n    Matched URLs:");
      comparison.matchedUrls.forEach((url) => {
        console.log(`      ✓ ${url}`);
      });
    }
  }

  static action(action, itemId, dryRun = false) {
    const prefix = dryRun ? "[DRY RUN] " : "";
    console.log(`\n  ${prefix}${action}: ${itemId}`);
  }

  static config(config) {
    console.log("Configuration:");
    console.log(`  DRY_RUN: ${config.DRY_RUN}`);
    console.log(
      `  MAX_DUPLICATE_KEYS: ${config.MAX_DUPLICATE_KEYS ?? "unlimited"}`,
    );
    console.log(
      `  CATEGORY_FILTER: ${config.CATEGORY_FILTER ?? "all categories"}`,
    );
    console.log(`  USE_ARCHIVE: ${config.USE_ARCHIVE}`);
    console.log(`  KEY_STRATEGY: ${config.KEY_STRATEGY}`);
    console.log(
      `  WHITELIST: ${config.WHITELIST?.length > 0 ? config.WHITELIST.join(", ") : "none"}`,
    );
  }

  static summary(stats, config) {
    this.separator();
    console.log("\n" + "=".repeat(this.SEPARATOR_LENGTH));
    console.log("  SCRIPT SUMMARY");
    console.log("=".repeat(this.SEPARATOR_LENGTH));
    console.log(`  Total duplicate groups processed: ${stats.groupsProcessed}`);
    const actionWord = config.USE_ARCHIVE ? "archived" : "deleted";
    console.log(
      `  Total duplicates ${actionWord} (or to be ${actionWord} in dry run): ${stats.totalRemoved}`,
    );

    const mode = config.DRY_RUN ? "DRY RUN" : "LIVE";
    const action = config.USE_ARCHIVE ? "archived" : "deleted";
    console.log(
      `  Mode: ${mode} (${config.DRY_RUN ? "no items were actually" : "items were actually"} ${action})`,
    );
    console.log("=".repeat(this.SEPARATOR_LENGTH) + "\n");
  }
}

// ============================================================================
// Core Logic
// ============================================================================

class DuplicateProcessor {
  constructor(config) {
    this.config = config;
    this.stats = {
      groupsProcessed: 0,
      totalRemoved: 0,
    };
    // Load cache for processed items
    this.processedCache = CacheManager.loadCache(DUPLICATE_CACHE_FILE);
  }

  /**
   * Group items by generated key
   * @param {Array<Object>} items - Array of item summaries
   * @returns {Map<string, Array<Object>>} Map of keys to item arrays
   */
  groupItems(items) {
    const groups = new Map();

    for (const item of items) {
      const key = KeyGenerator.generate(item, null, this.config.KEY_STRATEGY);
      if (!groups.has(key)) {
        groups.set(key, []);
      }
      groups.get(key).push(item);
    }

    return groups;
  }

  /**
   * Filter groups to only those with duplicates
   * @param {Map<string, Array<Object>>} groups - Item groups
   * @returns {Array<[string, Array<Object>]>} Array of [key, items] tuples
   */
  getDuplicateGroups(groups) {
    return Array.from(groups.entries()).filter(([, items]) => items.length > 1);
  }

  /**
   * Load full details for all items in a group
   * @param {Array<Object>} items - Item summaries
   * @returns {Array<Object>} Items with full details
   */
  loadItemDetails(items) {
    return items.map((item) => {
      const details = OnePasswordClient.getItem(item.id);
      return { ...item, ...details };
    });
  }

  /**
   * Check if all items in a group are whitelisted (using summaries)
   * @param {Array<Object>} groupItems - Item summaries
   * @returns {boolean} True if all items are whitelisted
   */
  isGroupWhitelisted(groupItems) {
    if (!this.config.WHITELIST || this.config.WHITELIST.length === 0) {
      return false;
    }

    // Check if all items in the group match the whitelist
    return groupItems.every((item) =>
      WhitelistChecker.isWhitelisted(item, this.config.WHITELIST),
    );
  }

  /**
   * Process a single duplicate group
   * @param {string} key - Group key
   * @param {Array<Object>} groupItems - Items in the group
   * @param {number} groupIndex - Current group index
   * @param {number} totalGroups - Total number of groups
   * @returns {void}
   */
  processGroup(key, groupItems, groupIndex, totalGroups) {
    Logger.separator();
    Logger.section(
      `Duplicate Group ${groupIndex}/${totalGroups}: Key "${key}" (${groupItems.length} items)`,
    );

    // Filter out already processed items
    const unprocessedItems = groupItems.filter(
      (item) => !CacheManager.isInCache(this.processedCache, item.id),
    );

    if (unprocessedItems.length === 0) {
      console.log(
        `\n  ⚠️  All items in this group were already processed - skipping.`,
      );
      return;
    }

    if (unprocessedItems.length < groupItems.length) {
      const skipped = groupItems.length - unprocessedItems.length;
      console.log(
        `\n  ⚠️  Skipping ${skipped} already processed item(s), processing ${unprocessedItems.length} remaining item(s).`,
      );
    }

    // Early check: if all items in group are whitelisted, skip processing
    if (this.isGroupWhitelisted(unprocessedItems)) {
      console.log(
        `\n  ⚠️  All items in this group are whitelisted - skipping processing.`,
      );
      console.log(`  All ${unprocessedItems.length} item(s) will be kept.`);
      // Add whitelisted items to cache and save immediately
      unprocessedItems.forEach((item) => {
        CacheManager.addToCacheAndSave(
          this.processedCache,
          item.id,
          DUPLICATE_CACHE_FILE,
        );
      });
      return;
    }

    const itemsWithDetails = this.loadItemDetails(unprocessedItems);
    console.log(
      `\n  Loaded ${itemsWithDetails.length} item(s) for comparison.`,
    );

    const keptItems = [itemsWithDetails[0]];
    // Add first item to cache and save immediately
    CacheManager.addToCacheAndSave(
      this.processedCache,
      itemsWithDetails[0].id,
      DUPLICATE_CACHE_FILE,
    );
    Logger.item("KEEPING (First Item)", itemsWithDetails[0]);
    Logger.fields(itemsWithDetails[0].fields ?? [], "Fields");
    Logger.urls(itemsWithDetails[0].urls ?? [], "URLs");

    for (let i = 1; i < itemsWithDetails.length; i++) {
      const candidate = itemsWithDetails[i];
      Logger.separator();
      Logger.item(
        `COMPARING Item ${i + 1}/${itemsWithDetails.length - 1}`,
        candidate,
      );
      Logger.fields(candidate.fields ?? [], "Fields");
      Logger.urls(candidate.urls ?? [], "URLs");

      let isDuplicate = false;
      let matchedWith = null;
      let comparisonResult = null;

      // Compare against all kept items
      for (const keptItem of keptItems) {
        comparisonResult = ItemComparator.compare(keptItem, candidate);
        Logger.item("Compared Against (Kept Item)", keptItem);
        Logger.comparison(comparisonResult);

        if (comparisonResult.isMatch) {
          isDuplicate = true;
          matchedWith = keptItem;
          break;
        }
      }

      // Check if candidate is whitelisted
      const isCandidateWhitelisted = WhitelistChecker.isWhitelisted(
        candidate,
        this.config.WHITELIST,
      );

      if (isDuplicate) {
        // If candidate is whitelisted, keep it even if it's a duplicate
        if (isCandidateWhitelisted) {
          Logger.action(
            "KEEPING (whitelisted - will not be removed)",
            candidate.id,
            false,
          );
          keptItems.push(candidate);
          CacheManager.addToCacheAndSave(
            this.processedCache,
            candidate.id,
            DUPLICATE_CACHE_FILE,
          );
        } else {
          const action = this.config.USE_ARCHIVE ? "ARCHIVING" : "DELETING";
          Logger.action(
            `${action} duplicate (matched with ${matchedWith.id})`,
            candidate.id,
            this.config.DRY_RUN,
          );

          this.stats.totalRemoved++;

          if (!this.config.DRY_RUN) {
            OnePasswordClient.removeItem(candidate.id, this.config.USE_ARCHIVE);
          }
          // Add to cache after removal and save immediately
          CacheManager.addToCacheAndSave(
            this.processedCache,
            candidate.id,
            DUPLICATE_CACHE_FILE,
          );
        }
      } else {
        Logger.action(
          "KEEPING (different from all kept items)",
          candidate.id,
          false,
        );
        keptItems.push(candidate);
        CacheManager.addToCacheAndSave(
          this.processedCache,
          candidate.id,
          DUPLICATE_CACHE_FILE,
        );
        if (comparisonResult?.reason) {
          console.log(`    Reason: ${comparisonResult.reason}`);
        }
      }
    }

    const actionWord = this.config.USE_ARCHIVE ? "Archived" : "Deleted";
    console.log(
      `\n  Group Summary: Kept ${keptItems.length} item(s), ${actionWord} ${groupItems.length - keptItems.length} duplicate(s)`,
    );
  }

  /**
   * Process all duplicate groups
   * @returns {void}
   */
  process() {
    console.log("Starting 1Password Duplicate Remover...\n");
    Logger.config(this.config);
    console.log(
      `Loaded ${this.processedCache.size} previously processed item(s) from cache.`,
    );

    // Load items
    const items = OnePasswordClient.listItems(this.config.CATEGORY_FILTER);
    console.log(`\nLoaded ${items.length} item(s) from 1Password.`);

    if (this.config.KEY_STRATEGY === KEY_STRATEGIES.WEBSITE_USERNAME) {
      console.log(
        `\nNote: Using "website-username" key strategy. Items are initially grouped by title,`,
      );
      console.log(
        `but will be re-evaluated with full details. Items with same website+username but`,
      );
      console.log(
        `different titles may not be grouped initially, but exact duplicates will still be detected.`,
      );
    }

    // Group items
    const groups = this.groupItems(items);
    const duplicateGroups = this.getDuplicateGroups(groups);

    if (duplicateGroups.length === 0) {
      console.log("\n✓ No duplicate groups found!");
      return;
    }

    console.log(
      `\nFound ${duplicateGroups.length} duplicate group(s) to process.`,
    );

    // Apply limit
    const groupsToProcess =
      this.config.MAX_DUPLICATE_KEYS === null
        ? duplicateGroups
        : duplicateGroups.slice(0, this.config.MAX_DUPLICATE_KEYS);

    if (
      this.config.MAX_DUPLICATE_KEYS !== null &&
      duplicateGroups.length > this.config.MAX_DUPLICATE_KEYS
    ) {
      console.log(
        `Configured to process only the first ${this.config.MAX_DUPLICATE_KEYS} duplicate keys out of ${duplicateGroups.length}.`,
      );
    }

    // Process each group
    groupsToProcess.forEach(([key, groupItems], index) => {
      this.processGroup(key, groupItems, index + 1, groupsToProcess.length);
      this.stats.groupsProcessed++;
    });

    // Final cache sync (cache is already saved live, this is just a final sync)
    CacheManager.saveCache(DUPLICATE_CACHE_FILE, this.processedCache);
    console.log(
      `\nFinal cache sync: ${this.processedCache.size} processed item(s) in cache.`,
    );

    // Show summary
    Logger.summary(this.stats, this.config);
  }
}

// ============================================================================
// Title Cleaner
// ============================================================================

class TitleCleaner {
  constructor(config) {
    this.config = config;
    this.stats = {
      itemsProcessed: 0,
      titlesCleaned: 0,
    };
    // Load cache for processed items
    this.processedCache = CacheManager.loadCache(TITLE_CACHE_FILE);
  }

  /**
   * Extract clean title by removing username pattern like "Title (username)"
   * @param {string} title - The original title
   * @returns {Object} Object with cleaned title and whether it was changed
   */
  static cleanTitle(title) {
    if (!title || typeof title !== "string") {
      return { cleanedTitle: title, wasChanged: false };
    }

    // Pattern: <title> (<username>)
    // Match title ending with parentheses containing username/email
    // Only match if the title ends with a parentheses pattern
    const pattern = /^(.+?)\s*\(([^)]+)\)\s*$/;
    const match = title.match(pattern);

    if (match) {
      const cleanedTitle = match[1].trim();
      return {
        cleanedTitle,
        wasChanged: true,
        removedPart: `(${match[2]})`,
      };
    }

    return { cleanedTitle: title, wasChanged: false };
  }

  /**
   * Process all items and clean titles
   * @returns {void}
   */
  process() {
    Logger.separator();
    console.log("\n" + "=".repeat(Logger.SEPARATOR_LENGTH));
    console.log("  TITLE CLEANER PROCESS");
    console.log("=".repeat(Logger.SEPARATOR_LENGTH));
    console.log("\n  Cleaning titles with pattern: <title> (<username>)\n");
    console.log(
      `  Loaded ${this.processedCache.size} previously processed item(s) from cache.`,
    );

    // Load items
    const items = OnePasswordClient.listItems(this.config.CATEGORY_FILTER);
    console.log(`  Loaded ${items.length} item(s) from 1Password.\n`);

    if (items.length === 0) {
      console.log("  ✓ No items to process!");
      return;
    }

    // Filter out already processed items
    const unprocessedItems = items.filter(
      (item) => !CacheManager.isInCache(this.processedCache, item.id),
    );

    if (unprocessedItems.length === 0) {
      console.log("  ✓ All items were already processed!");
      return;
    }

    if (unprocessedItems.length < items.length) {
      const skipped = items.length - unprocessedItems.length;
      console.log(
        `  ⚠️  Skipping ${skipped} already processed item(s), processing ${unprocessedItems.length} remaining item(s).\n`,
      );
    }

    // Process each item
    for (let i = 0; i < unprocessedItems.length; i++) {
      const item = unprocessedItems[i];
      const originalTitle = item.title ?? "";

      if (!originalTitle) {
        continue;
      }

      const { cleanedTitle, wasChanged, removedPart } =
        TitleCleaner.cleanTitle(originalTitle);

      this.stats.itemsProcessed++;

      if (wasChanged) {
        console.log(`  [${i + 1}/${unprocessedItems.length}] Cleaning title:`);
        console.log(`    Original: "${originalTitle}"`);
        console.log(`    Cleaned:  "${cleanedTitle}"`);
        console.log(`    Removed:  "${removedPart}"`);

        if (this.config.DRY_RUN) {
          console.log(`    [DRY RUN] Would update item ${item.id}`);
        } else {
          try {
            OnePasswordClient.updateItemTitle(item.id, cleanedTitle);
            console.log(`    ✓ Updated item ${item.id}`);
          } catch (error) {
            console.error(
              `    ✗ Failed to update item ${item.id}: ${error.message}`,
            );
          }
        }

        this.stats.titlesCleaned++;
        // Add to cache after processing and save immediately
        CacheManager.addToCacheAndSave(
          this.processedCache,
          item.id,
          TITLE_CACHE_FILE,
        );
        console.log("");
      } else {
        // Add to cache even if title wasn't changed (already processed) and save immediately
        CacheManager.addToCacheAndSave(
          this.processedCache,
          item.id,
          TITLE_CACHE_FILE,
        );
      }
    }

    // Final cache sync (cache is already saved live, this is just a final sync)
    CacheManager.saveCache(TITLE_CACHE_FILE, this.processedCache);
    console.log(
      `\nFinal cache sync: ${this.processedCache.size} processed item(s) in cache.`,
    );

    // Show summary
    Logger.separator();
    console.log("\n" + "=".repeat(Logger.SEPARATOR_LENGTH));
    console.log("  TITLE CLEANER SUMMARY");
    console.log("=".repeat(Logger.SEPARATOR_LENGTH));
    console.log(`  Total items processed: ${this.stats.itemsProcessed}`);
    console.log(`  Titles cleaned: ${this.stats.titlesCleaned}`);

    const mode = this.config.DRY_RUN ? "DRY RUN" : "LIVE";
    console.log(
      `  Mode: ${mode} (${this.config.DRY_RUN ? "no titles were actually" : "titles were actually"} updated)`,
    );
    console.log("=".repeat(Logger.SEPARATOR_LENGTH) + "\n");
  }
}

// ============================================================================
// Main Entry Point
// ============================================================================

function main() {
  try {
    // Step 1: Process duplicates
    const processor = new DuplicateProcessor(CONFIG);
    processor.process();

    // Step 2: Clean titles (separate process)
    // const titleCleaner = new TitleCleaner(CONFIG);
    // titleCleaner.process();
  } catch (error) {
    console.error("\n❌ Error:", error.message);
    process.exit(1);
  }
}

if (require.main === module) {
  main();
}

module.exports = { DuplicateProcessor, CONFIG };