Subprocess with stdout and stderr control

subprocessHelper.py

import subprocess
import logging
import threading
import Queue
import time

def queue_stream(queue, stream):
    with stream:
        for line in iter(stream.readline, b''):
            report = line.rstrip()
            if report != "": #fuck off empty lines.
                queue.put(report)

#This blocks while trying to stream stdout and stderr to logging/files
def run_process(logPrefix, cmd, workingDirectory=None, logFilePath=None):
    logFile = None
    if logFilePath:
        logFile = open(logFilePath, "w")

    try:
        logging.debug("Running process: %s", cmd)
        p = subprocess.Popen(
            cmd,
            stdout = subprocess.PIPE,
            stderr = subprocess.PIPE,
            bufsize = 1, #Only buffer on line so we dont have to wait for all the thigns.
            cwd = workingDirectory
        )
        outQueue = Queue.Queue()
        errQueue = Queue.Queue()

        outThread = threading.Thread(target = queue_stream, args = (outQueue, p.stdout))
        outThread.start()

        errThread = threading.Thread(target = queue_stream, args = (errQueue, p.stderr))
        errThread.start()

        #go until the process returns and both queues are not empty
        #this way we don't miss any logs
        while p.poll() is None or not outQueue.empty() or not errQueue.empty():
            try:
                while not outQueue.empty(): #Drain the queue or else the process will outrun us
                    outLine = outQueue.get_nowait()
                    logging.debug('%s: %s' % (logPrefix, outLine))
                    if logFile:
                        logFile.write(outLine)
                    outQueue.task_done()
            except Queue.Empty:
                pass

            try:
                while not errQueue.empty(): #Drain the queue or else the process will outrun us
                    errLine = errQueue.get_nowait()
                    logging.error('%s: %s' % (logPrefix, errLine))
                    if logFile:
                        logFile.write(errLine)
                    errQueue.task_done()
            except Queue.Empty:
                pass
            time.sleep(0.1) #don't gobble CPU

        outThread.join(10)
        errThread.join(10)
    finally:
        if logFile:
            logFile.close()

    return p