Recursively scan all executables in a folder and generate IDA databases in parallel

README.md

Recursively scan all executables (PE, ELF and MachO!) in a folder and generate IDA databases in parallel

Usage

node batch.js [path]

batch.js

const fs = require('fs').promises;
const path = require('path');
const os = require('os');

const { spawn } = require('child_process');
const { EventEmitter } = require('events');


async function* walk(dir) {
  const ignored = new Set('id0,id1,nam,log,til,idb,i64,js,py'.split(','));
  const files = await fs.readdir(dir);

  for (file of files) {
    const ext = path.extname(file).toLowerCase();
    if (ignored.has(ext)) {
      continue;
    }

    const joint = path.join(dir, file);
    try {
      const stat = await fs.stat(joint);
      if (stat.isDirectory()) {
        yield* walk(joint);
      } else {
        yield joint;
      }
    } catch(_) {
      continue;
    }
  }
}

async function check64bit(filename) {  
  const FAT_MAGIC = 0xcafebabe,
    FAT_CIGAM = 0xbebafeca,
    MH_MAGIC = 0xfeedface,
    MH_CIGAM = 0xcefaedfe,
    MH_MAGIC_64 = 0xfeedfacf,
    MH_CIGAM_64 = 0xcffaedfe;

  const IMAGE_FILE_MACHINE_I386 = 0x14c,
    IMAGE_FILE_MACHINE_AMD64 = 0x8664,
    IMAGE_FILE_MACHINE_ARM = 0x1c0,
    IMAGE_FILE_MACHINE_ARM64 = 0xaa64;

  const handle = await fs.open(filename);
  const buffer = Buffer.allocUnsafe(4);

  try {
    const { bytesRead } = await handle.read(buffer, 0, 4);
    if (bytesRead < 4) throw new RangeError('invalid file size');

    const magic = buffer.readUInt32LE();
    if (new Set([FAT_MAGIC, FAT_CIGAM, MH_MAGIC_64, MH_CIGAM_64]).has(magic)) {
      return 64;
    } else if (new Set([MH_MAGIC, MH_CIGAM]).has(magic)) {
      return 32;
    } else if (buffer.slice(0, 2).compare(Buffer.from('MZ')) === 0) {
      await handle.read(buffer, 0, 2, 0x3c);
      const offset = buffer.readUInt16LE();
      await handle.read(buffer, 0, 4, offset);
      if (buffer.compare(Buffer.from('PE\x00\x00')) !== 0) throw new Error('invalid COFF magic');
      await handle.read(buffer, 0, 2, offset + 4);

      // little endian only
      const cpuMagic = buffer.readUInt16LE();

      if ([IMAGE_FILE_MACHINE_I386, IMAGE_FILE_MACHINE_ARM].includes(cpuMagic)) return 64;
      if ([IMAGE_FILE_MACHINE_AMD64, IMAGE_FILE_MACHINE_ARM64].includes(cpuMagic)) return 32;

      throw new Error('Unknown CPU');
    } else if (buffer.compare(Buffer.from('\x7fELF')) === 0) {
      await handle.read(buffer, 0, 1, 4);
      const val = buffer.readUInt8();
      if ([1, 2].includes(val)) return val * 32;
      throw new Error('invalid EI_CLASS');
    } else {
      throw new Error('Unknown executable format');
    }
  } finally {
    await handle.close();
  }
}

class Parallel extends EventEmitter {
  constructor(count) {
    super();
    this.max = count || os.cpus().length;
    this.pool = new Set();

    process.on('SIGINT', () => {
      this.removeAllListeners();
      this.pool.forEach(p => p.kill());
      process.exit();
    });
  }

  spawn(cmd, filename) {
    const name = path.basename(filename);
    const py = path.join(__dirname, 'idc.py');
    const child = spawn(cmd, ['-c', '-A', `-S${py}`, `-L${filename}.log`, `-o${filename}`, filename]);
    this.pool.add(child);
    child.on('close', (code) => {
      this.pool.delete(child);
      this.emit('finish', { name, code });
    });
    child.on('error', () => {

    });
    return child;
  }

  async consume(name) {
    let bits
    try {
      bits = await check64bit(name);
    } catch(e) {
      return;
    }
    
    const cmd = bits == 64 ? 'idat64': 'idat';
    const ext = bits == 64 ? 'i64' : 'idb';
    const st = await fs.stat(`${name}.${ext}`).catch(e => {});
    if (st) return;
    if (this.pool.size >= this.max) {
      await new Promise(resolve => this.once('finish', resolve)).catch(e => console.error(e));
    }
    try {
      this.spawn(cmd, name);
    } catch(e) {
      console.log(e);
    }
  }
}

async function main(cwd) {
  const job = new Parallel();
  for await (let name of walk(cwd)) {
    await job.consume(name);
  }
}

main(process.argv[2] || '.');

idc.py

import idc


def main():
    # todo: add your analyzer
    idc.auto_wait()
    idc.qexit(0)


if __name__ == "__main__":
    main()

Forgot to mention that you need to idat and idat64 to $PATH