Celery Multiple Queues Setup

celerybeat_schedule

from celery import crontab


CELERYBEAT_SCHEDULE = {

    'long_task': {
        'task': 'proj.app.tasks.LongTask',
        'schedule': crontab(minute='*/3')
    },

    'fast_task': {
        'task': 'proj.app3.tasks.fast_task',
        'schedule': crontab(minute='*/2')
    },
}

in_terminals

terminal 1:

$ celery -A proj worker -Q default -l debug -n default_worker


terminal 2:

$ celery -A proj worker -Q long -l debug -n long_worker


terminal 3:

$ celery -A proj beat -l debug

Procfile

web:  run-program gunicorn arena.wsgi

celery_beat: run-program celery -A arena beat -l info

celery1: run-program celery -A arena worker -Q default -l info --purge -n default_worker

celery2: run-program celery -A arena worker -Q feeds -l info --purge -n feeds_worker

settings

CELERY_TIMEZONE = TIME_ZONE
CELERY_ACCEPT_CONTENT = ['json', 'pickle']
CELERY_TASK_RESULT_EXPIRES = 60  # 1 mins
CELERYD_CONCURRENCY = 2
CELERYD_MAX_TASKS_PER_CHILD = 4
CELERYD_PREFETCH_MULTIPLIER = 1
CELERY_DEFAULT_QUEUE = 'default'
CELERY_DEFAULT_EXCHANGE_TYPE = 'topic'
CELERY_DEFAULT_ROUTING_KEY = 'default'

CELERY_QUEUES = (
    Queue('default', Exchange('default'), routing_key='default'),
    Queue('long', Exchange('long'), routing_key='long_tasks'),
)
CELERY_ROUTES = {
    'proj.app.tasks.LongTask': {
        'queue': 'long',
        'routing_key': 'long_tasks',
    },
   'proj.app2.tasks.another_long_task': {
        'queue': 'long',
        'routing_key': 'long_tasks'
   }
}

tasks

import celery


class LongTask(celery.Task):
  def run(self, *args, **kwargs):
    # do lotsa cool stuff here
    
    
@celery.task
def another_long_task():
  # do some other cool stuff here for a very long time
  
  
@celery.task
def fast_task():
  print 'i am real fast'