A handy recipe I like to use, utilising python queue's and threads (via threading module), create a bunch of threads and process them with what is in the queue.

Uses a global var to know how long it's been since the last thread executed anything of use to know when to start shutting down the worker threads.

Depending on resources you can build a pool of threads to eat its way thru the jobs in your queue, this example loads 5000 'jobs' into the queue with 50 threads polling in non-blocking mode for a job to perform off the queue stack.

#!/usr/bin/python                                                                                                         

import time
import threading
import Queue    
import StringIO       

my_queue = Queue.Queue()
time_of_last_run = time.time()

class queue_runner(threading.Thread):
  """Threaded Queue runner for testing things"""
  def __init__(self, my_queue):      
      threading.Thread.__init__(self)           
      self.my_queue = my_queue                
                                                
  def run(self):                                
    global time_of_last_run                     

    while True:
      try:     
        my_id = self.my_queue.get(True, 2)
      except:                               
        if time.time() - time_of_last_run > 3:
          return                              
      else:                                   
        if my_id:                            
          time_of_last_run = time.time()
          # do processing stuff here      
          time.sleep(1)
          self.my_queue.task_done()

def main():
  global time_of_last_run
  time_of_last_run = time.time()
  
  if True:
    #spawn a pool of place worker threads
    for i in range(50):
      p = queue_runner(my_queue)
      p.setDaemon(True)
      p.start()

    #now load some arbitrary jobs into the queue
    for i in range(5000):
       my_queue.put(i)

    print "Running..."
    while time.time() - time_of_last_run < 3:
      print ">> qsize %s threadCount %s" % (my_queue.qsize(),threading.activeCount())
      time.sleep(2)

    time.sleep(4)
    my_queue.join()

if __name__ == "__main__":
  main()